1 /* 2 * Copyright 2021 The gRPC Authors 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package io.grpc.xds; 18 19 import static com.google.common.truth.Truth.assertThat; 20 21 import com.google.common.collect.Iterables; 22 import io.grpc.Status; 23 import io.grpc.internal.FakeClock; 24 import io.grpc.xds.LoadStatsManager2.ClusterDropStats; 25 import io.grpc.xds.LoadStatsManager2.ClusterLocalityStats; 26 import io.grpc.xds.Stats.ClusterStats; 27 import io.grpc.xds.Stats.DroppedRequests; 28 import io.grpc.xds.Stats.UpstreamLocalityStats; 29 import java.util.List; 30 import java.util.Objects; 31 import java.util.concurrent.TimeUnit; 32 import javax.annotation.Nullable; 33 import org.junit.Test; 34 import org.junit.runner.RunWith; 35 import org.junit.runners.JUnit4; 36 37 /** 38 * Unit tests for {@link LoadStatsManager2}. 39 */ 40 @RunWith(JUnit4.class) 41 public class LoadStatsManager2Test { 42 private static final String CLUSTER_NAME1 = "cluster-foo.googleapis.com"; 43 private static final String CLUSTER_NAME2 = "cluster-bar.googleapis.com"; 44 private static final String EDS_SERVICE_NAME1 = "backend-service-foo.googleapis.com"; 45 private static final String EDS_SERVICE_NAME2 = "backend-service-bar.googleapis.com"; 46 private static final Locality LOCALITY1 = 47 Locality.create("test_region1", "test_zone1", "test_subzone1"); 48 private static final Locality LOCALITY2 = 49 Locality.create("test_region2", "test_zone2", "test_subzone2"); 50 private static final Locality LOCALITY3 = 51 Locality.create("test_region3", "test_zone3", "test_subzone3"); 52 53 private final FakeClock fakeClock = new FakeClock(); 54 private final LoadStatsManager2 loadStatsManager = 55 new LoadStatsManager2(fakeClock.getStopwatchSupplier()); 56 57 @Test recordAndGetReport()58 public void recordAndGetReport() { 59 ClusterDropStats dropCounter1 = loadStatsManager.getClusterDropStats( 60 CLUSTER_NAME1, EDS_SERVICE_NAME1); 61 ClusterDropStats dropCounter2 = loadStatsManager.getClusterDropStats( 62 CLUSTER_NAME1, EDS_SERVICE_NAME2); 63 ClusterLocalityStats loadCounter1 = loadStatsManager.getClusterLocalityStats( 64 CLUSTER_NAME1, EDS_SERVICE_NAME1, LOCALITY1); 65 ClusterLocalityStats loadCounter2 = loadStatsManager.getClusterLocalityStats( 66 CLUSTER_NAME1, EDS_SERVICE_NAME1, LOCALITY2); 67 ClusterLocalityStats loadCounter3 = loadStatsManager.getClusterLocalityStats( 68 CLUSTER_NAME2, null, LOCALITY3); 69 dropCounter1.recordDroppedRequest("lb"); 70 dropCounter1.recordDroppedRequest("throttle"); 71 for (int i = 0; i < 19; i++) { 72 loadCounter1.recordCallStarted(); 73 } 74 fakeClock.forwardTime(5L, TimeUnit.SECONDS); 75 dropCounter2.recordDroppedRequest(); 76 loadCounter1.recordCallFinished(Status.OK); 77 for (int i = 0; i < 9; i++) { 78 loadCounter2.recordCallStarted(); 79 } 80 loadCounter2.recordCallFinished(Status.UNAVAILABLE); 81 fakeClock.forwardTime(10L, TimeUnit.SECONDS); 82 loadCounter3.recordCallStarted(); 83 List<ClusterStats> allStats = loadStatsManager.getAllClusterStatsReports(); 84 assertThat(allStats).hasSize(3); // three cluster:edsServiceName 85 86 ClusterStats stats1 = findClusterStats(allStats, CLUSTER_NAME1, EDS_SERVICE_NAME1); 87 assertThat(stats1.loadReportIntervalNano()).isEqualTo(TimeUnit.SECONDS.toNanos(5L + 10L)); 88 assertThat(stats1.droppedRequestsList()).hasSize(2); 89 assertThat(findDroppedRequestCount(stats1.droppedRequestsList(), "lb")).isEqualTo(1L); 90 assertThat(findDroppedRequestCount(stats1.droppedRequestsList(), "throttle")).isEqualTo(1L); 91 assertThat(stats1.totalDroppedRequests()).isEqualTo(1L + 1L); 92 assertThat(stats1.upstreamLocalityStatsList()).hasSize(2); // two localities 93 UpstreamLocalityStats loadStats1 = 94 findLocalityStats(stats1.upstreamLocalityStatsList(), LOCALITY1); 95 assertThat(loadStats1.totalIssuedRequests()).isEqualTo(19L); 96 assertThat(loadStats1.totalSuccessfulRequests()).isEqualTo(1L); 97 assertThat(loadStats1.totalErrorRequests()).isEqualTo(0L); 98 assertThat(loadStats1.totalRequestsInProgress()).isEqualTo(19L - 1L); 99 100 UpstreamLocalityStats loadStats2 = 101 findLocalityStats(stats1.upstreamLocalityStatsList(), LOCALITY2); 102 assertThat(loadStats2.totalIssuedRequests()).isEqualTo(9L); 103 assertThat(loadStats2.totalSuccessfulRequests()).isEqualTo(0L); 104 assertThat(loadStats2.totalErrorRequests()).isEqualTo(1L); 105 assertThat(loadStats2.totalRequestsInProgress()).isEqualTo(9L - 1L); 106 107 ClusterStats stats2 = findClusterStats(allStats, CLUSTER_NAME1, EDS_SERVICE_NAME2); 108 assertThat(stats2.loadReportIntervalNano()).isEqualTo(TimeUnit.SECONDS.toNanos(5L + 10L)); 109 assertThat(stats2.droppedRequestsList()).isEmpty(); // no categorized drops 110 assertThat(stats2.totalDroppedRequests()).isEqualTo(1L); 111 assertThat(stats2.upstreamLocalityStatsList()).isEmpty(); // no per-locality stats 112 113 ClusterStats stats3 = findClusterStats(allStats, CLUSTER_NAME2, null); 114 assertThat(stats3.loadReportIntervalNano()).isEqualTo(TimeUnit.SECONDS.toNanos(5L + 10L)); 115 assertThat(stats3.droppedRequestsList()).isEmpty(); 116 assertThat(stats3.totalDroppedRequests()).isEqualTo(0L); // no drops recorded 117 assertThat(stats3.upstreamLocalityStatsList()).hasSize(1); // one localities 118 UpstreamLocalityStats loadStats3 = 119 Iterables.getOnlyElement(stats3.upstreamLocalityStatsList()); 120 assertThat(loadStats3.totalIssuedRequests()).isEqualTo(1L); 121 assertThat(loadStats3.totalSuccessfulRequests()).isEqualTo(0L); 122 assertThat(loadStats3.totalErrorRequests()).isEqualTo(0L); 123 assertThat(loadStats3.totalRequestsInProgress()).isEqualTo(1L); 124 125 fakeClock.forwardTime(3L, TimeUnit.SECONDS); 126 List<ClusterStats> clusterStatsList = loadStatsManager.getClusterStatsReports(CLUSTER_NAME1); 127 assertThat(clusterStatsList).hasSize(2); 128 stats1 = findClusterStats(clusterStatsList, CLUSTER_NAME1, EDS_SERVICE_NAME1); 129 assertThat(stats1.loadReportIntervalNano()).isEqualTo(TimeUnit.SECONDS.toNanos(3L)); 130 assertThat(stats1.droppedRequestsList()).isEmpty(); 131 assertThat(stats1.totalDroppedRequests()).isEqualTo(0L); // no new drops recorded 132 assertThat(stats1.upstreamLocalityStatsList()).hasSize(2); // two localities 133 loadStats1 = findLocalityStats(stats1.upstreamLocalityStatsList(), LOCALITY1); 134 assertThat(loadStats1.totalIssuedRequests()).isEqualTo(0L); 135 assertThat(loadStats1.totalSuccessfulRequests()).isEqualTo(0L); 136 assertThat(loadStats1.totalErrorRequests()).isEqualTo(0L); 137 assertThat(loadStats1.totalRequestsInProgress()).isEqualTo(18L); // still in-progress 138 loadStats2 = findLocalityStats(stats1.upstreamLocalityStatsList(), LOCALITY2); 139 assertThat(loadStats2.totalIssuedRequests()).isEqualTo(0L); 140 assertThat(loadStats2.totalSuccessfulRequests()).isEqualTo(0L); 141 assertThat(loadStats2.totalErrorRequests()).isEqualTo(0L); 142 assertThat(loadStats2.totalRequestsInProgress()).isEqualTo(8L); // still in-progress 143 144 stats2 = findClusterStats(clusterStatsList, CLUSTER_NAME1, EDS_SERVICE_NAME2); 145 assertThat(stats2.loadReportIntervalNano()).isEqualTo(TimeUnit.SECONDS.toNanos(3L)); 146 assertThat(stats2.droppedRequestsList()).isEmpty(); 147 assertThat(stats2.totalDroppedRequests()).isEqualTo(0L); // no new drops recorded 148 assertThat(stats2.upstreamLocalityStatsList()).isEmpty(); // no per-locality stats 149 } 150 151 @Test sharedDropCounterStatsAggregation()152 public void sharedDropCounterStatsAggregation() { 153 ClusterDropStats ref1 = loadStatsManager.getClusterDropStats( 154 CLUSTER_NAME1, EDS_SERVICE_NAME1); 155 ClusterDropStats ref2 = loadStatsManager.getClusterDropStats( 156 CLUSTER_NAME1, EDS_SERVICE_NAME1); 157 ref1.recordDroppedRequest("lb"); 158 ref2.recordDroppedRequest("throttle"); 159 ref1.recordDroppedRequest(); 160 ref2.recordDroppedRequest(); 161 162 ClusterStats stats = Iterables.getOnlyElement( 163 loadStatsManager.getClusterStatsReports(CLUSTER_NAME1)); 164 assertThat(stats.droppedRequestsList()).hasSize(2); 165 assertThat(findDroppedRequestCount(stats.droppedRequestsList(), "lb")).isEqualTo(1L); 166 assertThat(findDroppedRequestCount(stats.droppedRequestsList(), "throttle")).isEqualTo(1L); 167 assertThat(stats.totalDroppedRequests()).isEqualTo(4L); // 2 cagetorized + 2 uncategoized 168 } 169 170 @Test dropCounterDelayedDeletionAfterReported()171 public void dropCounterDelayedDeletionAfterReported() { 172 ClusterDropStats counter = loadStatsManager.getClusterDropStats( 173 CLUSTER_NAME1, EDS_SERVICE_NAME1); 174 counter.recordDroppedRequest("lb"); 175 ClusterStats stats = Iterables.getOnlyElement( 176 loadStatsManager.getClusterStatsReports(CLUSTER_NAME1)); 177 assertThat(stats.droppedRequestsList()).hasSize(1); 178 assertThat(Iterables.getOnlyElement(stats.droppedRequestsList()).droppedCount()) 179 .isEqualTo(1L); 180 assertThat(stats.totalDroppedRequests()).isEqualTo(1L); 181 182 counter.release(); 183 stats = Iterables.getOnlyElement(loadStatsManager.getClusterStatsReports(CLUSTER_NAME1)); 184 assertThat(stats.droppedRequestsList()).isEmpty(); 185 assertThat(stats.totalDroppedRequests()).isEqualTo(0L); 186 187 assertThat(loadStatsManager.getClusterStatsReports(CLUSTER_NAME1)).isEmpty(); 188 } 189 190 @Test sharedLoadCounterStatsAggregation()191 public void sharedLoadCounterStatsAggregation() { 192 ClusterLocalityStats ref1 = loadStatsManager.getClusterLocalityStats( 193 CLUSTER_NAME1, EDS_SERVICE_NAME1, LOCALITY1); 194 ClusterLocalityStats ref2 = loadStatsManager.getClusterLocalityStats( 195 CLUSTER_NAME1, EDS_SERVICE_NAME1, LOCALITY1); 196 ref1.recordCallStarted(); 197 ref1.recordCallFinished(Status.OK); 198 ref2.recordCallStarted(); 199 ref2.recordCallStarted(); 200 ref2.recordCallFinished(Status.UNAVAILABLE); 201 202 ClusterStats stats = Iterables.getOnlyElement( 203 loadStatsManager.getClusterStatsReports(CLUSTER_NAME1)); 204 UpstreamLocalityStats localityStats = 205 Iterables.getOnlyElement(stats.upstreamLocalityStatsList()); 206 assertThat(localityStats.totalIssuedRequests()).isEqualTo(1L + 2L); 207 assertThat(localityStats.totalSuccessfulRequests()).isEqualTo(1L); 208 assertThat(localityStats.totalErrorRequests()).isEqualTo(1L); 209 assertThat(localityStats.totalRequestsInProgress()).isEqualTo(1L + 2L - 1L - 1L); 210 } 211 212 @Test loadCounterDelayedDeletionAfterAllInProgressRequestsReported()213 public void loadCounterDelayedDeletionAfterAllInProgressRequestsReported() { 214 ClusterLocalityStats counter = loadStatsManager.getClusterLocalityStats( 215 CLUSTER_NAME1, EDS_SERVICE_NAME1, LOCALITY1); 216 counter.recordCallStarted(); 217 counter.recordCallStarted(); 218 219 ClusterStats stats = Iterables.getOnlyElement( 220 loadStatsManager.getClusterStatsReports(CLUSTER_NAME1)); 221 UpstreamLocalityStats localityStats = 222 Iterables.getOnlyElement(stats.upstreamLocalityStatsList()); 223 assertThat(localityStats.totalIssuedRequests()).isEqualTo(2L); 224 assertThat(localityStats.totalSuccessfulRequests()).isEqualTo(0L); 225 assertThat(localityStats.totalErrorRequests()).isEqualTo(0L); 226 assertThat(localityStats.totalRequestsInProgress()).isEqualTo(2L); 227 228 // release the counter, but requests still in-flight 229 counter.release(); 230 stats = Iterables.getOnlyElement(loadStatsManager.getClusterStatsReports(CLUSTER_NAME1)); 231 localityStats = Iterables.getOnlyElement(stats.upstreamLocalityStatsList()); 232 assertThat(localityStats.totalIssuedRequests()).isEqualTo(0L); 233 assertThat(localityStats.totalSuccessfulRequests()).isEqualTo(0L); 234 assertThat(localityStats.totalErrorRequests()).isEqualTo(0L); 235 assertThat(localityStats.totalRequestsInProgress()) 236 .isEqualTo(2L); // retained by in-flight calls 237 238 counter.recordCallFinished(Status.OK); 239 counter.recordCallFinished(Status.UNAVAILABLE); 240 stats = Iterables.getOnlyElement(loadStatsManager.getClusterStatsReports(CLUSTER_NAME1)); 241 localityStats = Iterables.getOnlyElement(stats.upstreamLocalityStatsList()); 242 assertThat(localityStats.totalIssuedRequests()).isEqualTo(0L); 243 assertThat(localityStats.totalSuccessfulRequests()).isEqualTo(1L); 244 assertThat(localityStats.totalErrorRequests()).isEqualTo(1L); 245 assertThat(localityStats.totalRequestsInProgress()).isEqualTo(0L); 246 247 assertThat(loadStatsManager.getClusterStatsReports(CLUSTER_NAME1)).isEmpty(); 248 } 249 250 @Nullable findClusterStats( List<ClusterStats> statsList, String cluster, @Nullable String edsServiceName)251 private static ClusterStats findClusterStats( 252 List<ClusterStats> statsList, String cluster, @Nullable String edsServiceName) { 253 for (ClusterStats stats : statsList) { 254 if (stats.clusterName().equals(cluster) 255 && Objects.equals(stats.clusterServiceName(), edsServiceName)) { 256 return stats; 257 } 258 } 259 return null; 260 } 261 262 @Nullable findLocalityStats( List<UpstreamLocalityStats> localityStatsList, Locality locality)263 private static UpstreamLocalityStats findLocalityStats( 264 List<UpstreamLocalityStats> localityStatsList, Locality locality) { 265 for (UpstreamLocalityStats stats : localityStatsList) { 266 if (stats.locality().equals(locality)) { 267 return stats; 268 } 269 } 270 return null; 271 } 272 findDroppedRequestCount( List<DroppedRequests> droppedRequestsLists, String category)273 private static long findDroppedRequestCount( 274 List<DroppedRequests> droppedRequestsLists, String category) { 275 DroppedRequests drop = null; 276 for (DroppedRequests stats : droppedRequestsLists) { 277 if (stats.category().equals(category)) { 278 drop = stats; 279 } 280 } 281 assertThat(drop).isNotNull(); 282 return drop.droppedCount(); 283 } 284 } 285