# Copyright 2015 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. """The datastore models for graph data. The Chromium project uses Buildbot to run its performance tests, and the structure of the data for the Performance Dashboard reflects this. Metadata about tests are structured in a hierarchy of Master, Bot, and Test entities. Master and Bot entities represent Buildbot masters and builders respectively, and Test entities represent groups of results, or individual data series. For example, entities might be structured as follows: Master: ChromiumPerf Bot: win7 Test: page_cycler.moz Test: times Test: page_load_time Test: page_load_time_ref Test: www.amazon.com Test: www.bing.com Test: commit_charge Test: ref Test: www.amazon.com Test: www.bing.com The graph data points are represented by Row entities. Each Row entity contains a revision and value, which are its X and Y values on a graph, and any other metadata associated with an individual performance test result. The keys of the Row entities for a particular data series are start with a TestContainer key, instead of a Test key. This way, the Row entities for each data series are in a different "entity group". This allows a faster rate of putting data in the datastore for many series at once. For example, Row entities are organized like this: TestContainer: ChromiumPerf/win7/page_cycler.moz/times/page_load_time Row: revision 12345, value 2.5 Row: revision 12346, value 2.0 Row: revision 12347, value 2.1 TestContainer: ChromiumPerf/win7/page_cycler.moz/times/page_load_time_ref Row: revision 12345, value 2.4 Row: revision 12346, value 2.0 Row: revision 12347, value 2.2 TestContainer: ChromiumPerf/win7/page_cycler.moz/commit_charge Row: revision 12345, value 10 Row: revision 12346, value 12 Row: revision 12347, value 11 IMPORTANT: If you add new kinds to this file, you must also add them to the Daily Backup url in cron.yaml in order for them to be properly backed up. See: https://developers.google.com/appengine/articles/scheduled_backups """ import logging from google.appengine.ext import ndb from dashboard import layered_cache from dashboard import utils from dashboard.models import anomaly from dashboard.models import anomaly_config from dashboard.models import internal_only_model from dashboard.models import sheriff as sheriff_module from dashboard.models import stoppage_alert as stoppage_alert_module # Maximum level of nested tests. MAX_TEST_ANCESTORS = 10 # Keys to the datastore-based cache. See stored_object. LIST_TESTS_SUBTEST_CACHE_KEY = 'list_tests_get_tests_new_%s_%s_%s' _MAX_STRING_LENGTH = 500 class Master(internal_only_model.InternalOnlyModel): """Information about the Buildbot master. Masters are keyed by name, e.g. 'ChromiumGPU' or 'ChromiumPerf'. All Bot entities that are Buildbot slaves of one master are children of one Master entity in the datastore. """ # Master has no properties; the name of the master is the ID. class Bot(internal_only_model.InternalOnlyModel): """Information about a Buildbot slave that runs perf tests. Bots are keyed by name, e.g. 'xp-release-dual-core'. A Bot entity contains information about whether the tests are only viewable to internal users, and each bot has a parent that is a Master entity. A Bot is be the ancestor of the Test entities that run on it. """ internal_only = ndb.BooleanProperty(default=False, indexed=True) class Test(internal_only_model.CreateHookInternalOnlyModel): """A Test entity is a node in a hierarchy of tests. A Test entity can represent a specific series of results which will be plotted on a graph, or it can represent a group of such series of results, or both. A Test entity that the property has_rows set to True corresponds to a trace on a graph, and the parent Test for a group of such tests corresponds to a graph with several traces. A parent Test for that test would correspond to a group of related graphs. Top-level Tests (also known as test suites) are parented by a Bot. Tests are keyed by name, and they also contain other metadata such as description and units. NOTE: If you remove any properties from Test, they should be added to the TEST_EXCLUDE_PROPERTIES list in migrate_test_names.py. """ internal_only = ndb.BooleanProperty(default=False, indexed=True) # Sheriff rotation for this test. Rotations are specified by regular # expressions that can be edited at /edit_sheriffs. sheriff = ndb.KeyProperty(kind=sheriff_module.Sheriff, indexed=True) # There is a default anomaly threshold config (in anomaly.py), and it can # be overridden for a group of tests by using /edit_sheriffs. overridden_anomaly_config = ndb.KeyProperty( kind=anomaly_config.AnomalyConfig, indexed=True) # Keep track of what direction is an improvement for this graph so we can # filter out alerts on regressions. improvement_direction = ndb.IntegerProperty( default=anomaly.UNKNOWN, choices=[ anomaly.UP, anomaly.DOWN, anomaly.UNKNOWN ], indexed=False ) # Units of the child Rows of this Test, or None if there are no child Rows. units = ndb.StringProperty(indexed=False) # The last alerted revision is used to avoid duplicate alerts. last_alerted_revision = ndb.IntegerProperty(indexed=False) # Whether or not the test has child rows. Set by hook on Row class put. has_rows = ndb.BooleanProperty(default=False, indexed=True) # If there is a currently a StoppageAlert that indicates that data hasn't # been received for some time, then will be set. Otherwise, it is None. stoppage_alert = ndb.KeyProperty( kind=stoppage_alert_module.StoppageAlert, indexed=True) # A test is marked "deprecated" if no new points have been received for # a long time; these tests should usually not be listed. deprecated = ndb.BooleanProperty(default=False, indexed=True) # For top-level test entities, this is a list of sub-tests that are checked # for alerts (i.e. they have a sheriff). For other tests, this is empty. monitored = ndb.KeyProperty(repeated=True, indexed=True) # Description of what the test measures. description = ndb.TextProperty(indexed=True) # Source code location of the test. Optional. code = ndb.StringProperty(indexed=False, repeated=True) # Command to run the test. Optional. command_line = ndb.StringProperty(indexed=False) # Computed properties are treated like member variables, so they have # lowercase names, even though they look like methods to pylint. # pylint: disable=invalid-name @ndb.ComputedProperty def bot(self): # pylint: disable=invalid-name """Immediate parent Bot entity, or None if this is not a test suite.""" parent = self.key.parent() if parent.kind() == 'Bot': return parent return None @ndb.ComputedProperty def parent_test(self): # pylint: disable=invalid-name """Immediate parent Test entity, or None if this is a test suite.""" parent = self.key.parent() if parent.kind() == 'Test': return parent return None @property def test_path(self): """Slash-separated list of key parts, 'master/bot/suite/chart/...'.""" return utils.TestPath(self.key) @ndb.ComputedProperty def master_name(self): return self.key.pairs()[0][1] @ndb.ComputedProperty def bot_name(self): return self.key.pairs()[1][1] @ndb.ComputedProperty def suite_name(self): return self.key.pairs()[2][1] @ndb.ComputedProperty def test_part1_name(self): pairs = self.key.pairs() if len(pairs) < 4: return '' return self.key.pairs()[3][1] @ndb.ComputedProperty def test_part2_name(self): pairs = self.key.pairs() if len(pairs) < 5: return '' return self.key.pairs()[4][1] @ndb.ComputedProperty def test_part3_name(self): pairs = self.key.pairs() if len(pairs) < 6: return '' return self.key.pairs()[5][1] @ndb.ComputedProperty def test_part4_name(self): pairs = self.key.pairs() if len(pairs) < 7: return '' return self.key.pairs()[6][0] @classmethod def _GetMasterBotSuite(cls, key): while key and key.parent(): if key.parent().kind() == 'Bot': if not key.parent().parent(): return None return (key.parent().parent().string_id(), key.parent().string_id(), key.string_id()) key = key.parent() return None def __init__(self, *args, **kwargs): # Indexed StringProperty has a maximum length. If this length is exceeded, # then an error will be thrown in ndb.Model.__init__. # Truncate the "description" property if necessary. description = kwargs.get('description') or '' kwargs['description'] = description[:_MAX_STRING_LENGTH] super(Test, self).__init__(*args, **kwargs) def _pre_put_hook(self): """This method is called before a Test is put into the datastore. Here, we check the sheriffs and anomaly configs to make sure they are current. We also update the monitored list of the test suite. """ # Set the sheriff to the first sheriff (alphabetically by sheriff name) # that has a test pattern that matches this test. self.sheriff = None for sheriff_entity in sheriff_module.Sheriff.query().fetch(): for pattern in sheriff_entity.patterns: if utils.TestMatchesPattern(self, pattern): self.sheriff = sheriff_entity.key if self.sheriff: break # If this Test is monitored, add it to the monitored list of its test suite. # A test is be monitored iff it has a sheriff, and monitored tests are # tracked in the monitored list of a test suite Test entity. test_suite = ndb.Key(*self.key.flat()[:6]).get() if self.sheriff: if test_suite and self.key not in test_suite.monitored: test_suite.monitored.append(self.key) test_suite.put() elif test_suite and self.key in test_suite.monitored: test_suite.monitored.remove(self.key) test_suite.put() # Set the anomaly threshold config to the first one that has a test pattern # that matches this test, if there is one. Anomaly configs are sorted by # name, so that a config with a name that comes earlier lexicographically # is considered higher-priority. self.overridden_anomaly_config = None anomaly_configs = anomaly_config.AnomalyConfig.query().fetch() anomaly_configs.sort(key=lambda config: config.key.string_id()) for anomaly_config_entity in anomaly_configs: for pattern in anomaly_config_entity.patterns: if utils.TestMatchesPattern(self, pattern): self.overridden_anomaly_config = anomaly_config_entity.key if self.overridden_anomaly_config: break def CreateCallback(self): """Called when the entity is first saved.""" if self.key.parent().kind() != 'Bot': layered_cache.Delete( LIST_TESTS_SUBTEST_CACHE_KEY % self._GetMasterBotSuite(self.key)) @classmethod # pylint: disable=unused-argument def _pre_delete_hook(cls, key): if key.parent() and key.parent().kind() != 'Bot': layered_cache.Delete( LIST_TESTS_SUBTEST_CACHE_KEY % Test._GetMasterBotSuite(key)) class LastAddedRevision(ndb.Model): """Represents the last added revision for a test path. The reason this property is separated from Test entity is to avoid contention issues (Frequent update of entity within the same group). This property is updated very frequent in /add_point. """ revision = ndb.IntegerProperty(indexed=False) class Row(internal_only_model.InternalOnlyModel, ndb.Expando): """A Row represents one data point. A Row has a revision and a value, which are the X and Y values, respectively. Each Row belongs to one Test, along with all of the other Row entities that it is plotted with. Rows are keyed by revision. In addition to the properties defined below, Row entities may also have other properties which specify additional supplemental data. These are called "supplemental columns", and should have the following prefixes: d_: A data point, such as d_1st_run or d_50th_percentile. FloatProperty. r_: Revision such as r_webkit or r_v8. StringProperty, limited to 25 characters, '0-9' and '.'. a_: Annotation such as a_chrome_bugid or a_gasp_anomaly. StringProperty. """ # Don't index by default (only explicitly indexed properties are indexed). _default_indexed = False internal_only = ndb.BooleanProperty(default=False, indexed=True) # The parent_test is the key of the Test entity that this Row belongs to. @ndb.ComputedProperty def parent_test(self): # pylint: disable=invalid-name # The Test entity that a Row belongs to isn't actually its parent in the # datastore. Rather, the parent key of each Row contains a test path, which # contains the information necessary to get the actual Test key. return utils.TestKey(self.key.parent().string_id()) # Points in each graph are sorted by "revision". This is usually a Chromium # SVN version number, but it might also be any other integer, as long as # newer points have higher numbers. @ndb.ComputedProperty def revision(self): # pylint: disable=invalid-name return self.key.integer_id() # The time the revision was added to the dashboard is tracked in order # to too many points from being added in a short period of time, which would # indicate an error or malicious code. timestamp = ndb.DateTimeProperty(auto_now_add=True, indexed=True) # The Y-value at this point. value = ndb.FloatProperty(indexed=True) # The standard deviation at this point. Optional. error = ndb.FloatProperty(indexed=False) def _pre_put_hook(self): """Sets the has_rows property of the parent test before putting this Row. This isn't atomic because the parent_test put() and Row put() don't happen in the same transaction. But in practice it shouldn't be an issue because the parent test will get more points as the test runs. """ parent_test = self.parent_test.get() # If the Test pointed to by parent_test is not valid, that indicates # that a Test entity was not properly created in add_point. if not parent_test: parent_key = self.key.parent() logging.warning('Row put without valid Test. Parent key: %s', parent_key) return if not parent_test.has_rows: parent_test.has_rows = True parent_test.put()