1 /* 2 * Copyright 2020 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.google.android.iwlan; 18 19 import android.content.Context; 20 import android.net.ipsec.ike.exceptions.IkeProtocolException; 21 import android.os.Handler; 22 import android.os.HandlerThread; 23 import android.os.Looper; 24 import android.os.Message; 25 import android.support.annotation.IntDef; 26 import android.support.annotation.NonNull; 27 import android.telephony.DataFailCause; 28 import android.telephony.TelephonyManager; 29 import android.telephony.data.DataService; 30 import android.text.TextUtils; 31 import android.util.Log; 32 33 import com.android.internal.annotations.VisibleForTesting; 34 35 import org.json.JSONArray; 36 import org.json.JSONException; 37 import org.json.JSONObject; 38 39 import java.io.BufferedReader; 40 import java.io.FileDescriptor; 41 import java.io.IOException; 42 import java.io.InputStream; 43 import java.io.InputStreamReader; 44 import java.io.PrintWriter; 45 import java.util.ArrayList; 46 import java.util.Arrays; 47 import java.util.Calendar; 48 import java.util.Date; 49 import java.util.HashMap; 50 import java.util.HashSet; 51 import java.util.List; 52 import java.util.Map; 53 import java.util.Set; 54 import java.util.concurrent.ConcurrentHashMap; 55 import java.util.concurrent.TimeUnit; 56 57 public class ErrorPolicyManager { 58 59 /** 60 * This type is not to be used in config. This is only used internally to catch errors in 61 * parsing the error type. 62 */ 63 private static final int UNKNOWN_ERROR_TYPE = -1; 64 65 /** 66 * This value represents that the error tye is to be used as a fallback to represent all the 67 * errors. 68 */ 69 private static final int FALLBACK_ERROR_TYPE = 1; 70 71 /** 72 * This value represents rest of the errors that are not defined above. ErrorDetails should 73 * mention the specific error. If it doesn't not - the policy will be used as a fallback global 74 * policy. Currently Supported ErrorDetails "IO_EXCEPTION" "TIMEOUT_EXCEPTION" 75 * "SERVER_SELECTION_FAILED" "TUNNEL_TRANSFORM_FAILED" 76 */ 77 private static final int GENERIC_ERROR_TYPE = 2; 78 79 /** 80 * This value represents IKE Protocol Error/Notify Error. 81 * 82 * @see <a href="https://tools.ietf.org/html/rfc4306#section-3.10.1">RFC 4306,Internet Key 83 * Exchange (IKEv2) Protocol </a> for global errors and carrier specific requirements for 84 * other carrier specific error codes. ErrorDetails defined for this type is always in 85 * numeric form representing the error codes. Examples: "24", "9000-9050" 86 */ 87 private static final int IKE_PROTOCOL_ERROR_TYPE = 3; 88 89 @IntDef({UNKNOWN_ERROR_TYPE, FALLBACK_ERROR_TYPE, GENERIC_ERROR_TYPE, IKE_PROTOCOL_ERROR_TYPE}) 90 @interface ErrorPolicyErrorType {}; 91 92 private static final String[] GENERIC_ERROR_DETAIL_STRINGS = { 93 "*", 94 "IO_EXCEPTION", 95 "TIMEOUT_EXCEPTION", 96 "SERVER_SELECTION_FAILED", 97 "TUNNEL_TRANSFORM_FAILED" 98 }; 99 100 /** Private IKEv2 notify message types. As defined in TS 124 302 (section 8.1.2.2) */ 101 private static final int IKE_PROTOCOL_ERROR_PDN_CONNECTION_REJECTION = 8192; 102 103 private static final int IKE_PROTOCOL_ERROR_MAX_CONNECTION_REACHED = 8193; 104 private static final int IKE_PROTOCOL_ERROR_SEMANTIC_ERROR_IN_THE_TFT_OPERATION = 8241; 105 private static final int IKE_PROTOCOL_ERROR_SYNTACTICAL_ERROR_IN_THE_TFT_OPERATION = 8242; 106 private static final int IKE_PROTOCOL_ERROR_SEMANTIC_ERRORS_IN_PACKET_FILTERS = 8244; 107 private static final int IKE_PROTOCOL_ERROR_SYNTACTICAL_ERRORS_IN_PACKET_FILTERS = 8245; 108 private static final int IKE_PROTOCOL_ERROR_NON_3GPP_ACCESS_TO_EPC_NOT_ALLOWED = 9000; 109 private static final int IKE_PROTOCOL_ERROR_USER_UNKNOWN = 9001; 110 private static final int IKE_PROTOCOL_ERROR_NO_APN_SUBSCRIPTION = 9002; 111 private static final int IKE_PROTOCOL_ERROR_AUTHORIZATION_REJECTED = 9003; 112 private static final int IKE_PROTOCOL_ERROR_ILLEGAL_ME = 9006; 113 private static final int IKE_PROTOCOL_ERROR_NETWORK_FAILURE = 10500; 114 private static final int IKE_PROTOCOL_ERROR_RAT_TYPE_NOT_ALLOWED = 11001; 115 private static final int IKE_PROTOCOL_ERROR_IMEI_NOT_ACCEPTED = 11005; 116 private static final int IKE_PROTOCOL_ERROR_PLMN_NOT_ALLOWED = 11011; 117 private static final int IKE_PROTOCOL_ERROR_UNAUTHENTICATED_EMERGENCY_NOT_SUPPORTED = 11055; 118 119 @IntDef({ 120 IKE_PROTOCOL_ERROR_PDN_CONNECTION_REJECTION, 121 IKE_PROTOCOL_ERROR_MAX_CONNECTION_REACHED, 122 IKE_PROTOCOL_ERROR_SEMANTIC_ERROR_IN_THE_TFT_OPERATION, 123 IKE_PROTOCOL_ERROR_SYNTACTICAL_ERROR_IN_THE_TFT_OPERATION, 124 IKE_PROTOCOL_ERROR_SEMANTIC_ERRORS_IN_PACKET_FILTERS, 125 IKE_PROTOCOL_ERROR_SYNTACTICAL_ERRORS_IN_PACKET_FILTERS, 126 IKE_PROTOCOL_ERROR_NON_3GPP_ACCESS_TO_EPC_NOT_ALLOWED, 127 IKE_PROTOCOL_ERROR_USER_UNKNOWN, 128 IKE_PROTOCOL_ERROR_NO_APN_SUBSCRIPTION, 129 IKE_PROTOCOL_ERROR_AUTHORIZATION_REJECTED, 130 IKE_PROTOCOL_ERROR_ILLEGAL_ME, 131 IKE_PROTOCOL_ERROR_NETWORK_FAILURE, 132 IKE_PROTOCOL_ERROR_RAT_TYPE_NOT_ALLOWED, 133 IKE_PROTOCOL_ERROR_IMEI_NOT_ACCEPTED, 134 IKE_PROTOCOL_ERROR_PLMN_NOT_ALLOWED, 135 IKE_PROTOCOL_ERROR_UNAUTHENTICATED_EMERGENCY_NOT_SUPPORTED 136 }) 137 @interface IkeProtocolErrorType {}; 138 139 private final String LOG_TAG; 140 141 private static Map<Integer, ErrorPolicyManager> mInstances = new ConcurrentHashMap<>(); 142 private Context mContext; 143 private int mSlotId; 144 145 // Policies read from defaultiwlanerrorconfig.json 146 // String APN as key to identify the ErrorPolicies associated with it. 147 private final Map<String, List<ErrorPolicy>> mDefaultPolicies = new HashMap<>(); 148 149 // Policies read from CarrierConfig 150 // String APN as key to identify the ErrorPolicies associated with it. 151 private Map<String, List<ErrorPolicy>> mCarrierConfigPolicies = new HashMap<>(); 152 153 // String APN as key to identify the ErrorInfo associated with that APN 154 private Map<String, ErrorInfo> mLastErrorForApn = new HashMap<>(); 155 156 // List of current Unthrottling events registered with IwlanEventListener 157 private Set<Integer> mUnthrottlingEvents; 158 159 private ErrorStats mErrorStats = new ErrorStats(); 160 161 private HandlerThread mHandlerThread; 162 @VisibleForTesting Handler mHandler; 163 164 private int carrierId = TelephonyManager.UNKNOWN_CARRIER_ID; 165 166 private String carrierConfigErrorPolicyString; 167 168 @VisibleForTesting 169 static final String KEY_ERROR_POLICY_CONFIG_STRING = "iwlan.key_error_policy_config_string"; 170 171 /** 172 * Returns ErrorPolicyManager instance for the subId 173 * 174 * @param context 175 * @param slotId 176 */ getInstance(@onNull Context context, int slotId)177 public static ErrorPolicyManager getInstance(@NonNull Context context, int slotId) { 178 return mInstances.computeIfAbsent(slotId, k -> new ErrorPolicyManager(context, slotId)); 179 } 180 181 /** 182 * Release or reset the instance. 183 * 184 * @param context 185 * @param slotId 186 */ releaseInstance()187 public void releaseInstance() { 188 Log.d(LOG_TAG, "Release Instance with slotId: " + mSlotId); 189 IwlanEventListener.getInstance(mContext, mSlotId).removeEventListener(mHandler); 190 mHandlerThread.quit(); 191 mInstances.remove(mSlotId); 192 } 193 194 /** 195 * Updates the last error details and returns the retry time. Return value is -1, which should 196 * be ignored, when the error is IwlanError.NO_ERROR. 197 * 198 * @param apn apn name for which the error happened 199 * @param iwlanError Error 200 * @return retry time. 0 = immediate retry, -1 = fail and n = retry after n seconds 201 */ reportIwlanError(String apn, IwlanError iwlanError)202 public synchronized long reportIwlanError(String apn, IwlanError iwlanError) { 203 // Fail by default 204 long retryTime = -1; 205 206 if (iwlanError.getErrorType() == IwlanError.NO_ERROR) { 207 Log.d(LOG_TAG, "reportIwlanError: NO_ERROR"); 208 mLastErrorForApn.remove(apn); 209 return retryTime; 210 } 211 mErrorStats.update(apn, iwlanError); 212 213 // remove the entry with the same error if it has back off time 214 if (mLastErrorForApn.containsKey(apn) 215 && mLastErrorForApn.get(apn).getError().equals(iwlanError) 216 && mLastErrorForApn.get(apn).isBackOffTimeValid()) { 217 mLastErrorForApn.remove(apn); 218 } 219 if (!mLastErrorForApn.containsKey(apn) 220 || !mLastErrorForApn.get(apn).getError().equals(iwlanError)) { 221 Log.d(LOG_TAG, "Doesn't match to the previous error" + iwlanError.toString()); 222 ErrorPolicy policy = findErrorPolicy(apn, iwlanError); 223 ErrorInfo errorInfo = new ErrorInfo(iwlanError, policy); 224 mLastErrorForApn.put(apn, errorInfo); 225 } 226 retryTime = mLastErrorForApn.get(apn).updateCurrentRetryTime(); 227 return retryTime; 228 } 229 230 /** 231 * Updates the last error details with back off time. Return value is -1, which should be 232 * ignored, when the error is IwlanError.NO_ERROR. 233 * 234 * @param apn apn name for which the error happened 235 * @param iwlanError Error 236 * @param long backOffTime in seconds 237 * @return retry time which is the backoff time. -1 if it is NO_ERROR 238 */ reportIwlanError(String apn, IwlanError iwlanError, long backOffTime)239 public synchronized long reportIwlanError(String apn, IwlanError iwlanError, long backOffTime) { 240 // Fail by default 241 long retryTime = -1; 242 243 if (iwlanError.getErrorType() == IwlanError.NO_ERROR) { 244 Log.d(LOG_TAG, "reportIwlanError: NO_ERROR"); 245 mLastErrorForApn.remove(apn); 246 return retryTime; 247 } 248 mErrorStats.update(apn, iwlanError); 249 250 // remove the entry with the same error if it doesn't have back off time. 251 if (mLastErrorForApn.containsKey(apn) 252 && mLastErrorForApn.get(apn).getError().equals(iwlanError) 253 && !mLastErrorForApn.get(apn).isBackOffTimeValid()) { 254 mLastErrorForApn.remove(apn); 255 } 256 retryTime = backOffTime; 257 if (!mLastErrorForApn.containsKey(apn) 258 || !mLastErrorForApn.get(apn).getError().equals(iwlanError)) { 259 Log.d(LOG_TAG, "Doesn't match to the previous error" + iwlanError.toString()); 260 ErrorPolicy policy = findErrorPolicy(apn, iwlanError); 261 ErrorInfo errorInfo = new ErrorInfo(iwlanError, policy, backOffTime); 262 mLastErrorForApn.put(apn, errorInfo); 263 } else { 264 ErrorInfo info = mLastErrorForApn.get(apn); 265 info.setBackOffTime(backOffTime); 266 } 267 return retryTime; 268 } 269 270 /** 271 * Checks whether we can bring up Epdg Tunnel - Based on lastErrorForApn 272 * 273 * @param apn apn for which tunnel bring up needs to be checked 274 * @return true if tunnel can be brought up, false otherwise 275 */ canBringUpTunnel(String apn)276 public synchronized boolean canBringUpTunnel(String apn) { 277 boolean ret = true; 278 if (mLastErrorForApn.containsKey(apn)) { 279 ret = mLastErrorForApn.get(apn).canBringUpTunnel(); 280 } 281 Log.d(LOG_TAG, "canBringUpTunnel: " + ret); 282 return ret; 283 } 284 285 // TODO: Modify framework/base/Android.bp to get access to Annotation.java to use 286 // @DataFailureCause 287 // annotation as return type here. (after moving to aosp?) 288 /** 289 * Returns the DataFailCause based on the lastErrorForApn 290 * 291 * @param apn apn name for which DataFailCause is needed 292 * @return DataFailCause corresponding to the error for the apn 293 */ getDataFailCause(String apn)294 public synchronized int getDataFailCause(String apn) { 295 296 if (!mLastErrorForApn.containsKey(apn)) { 297 return DataFailCause.NONE; 298 } 299 IwlanError error = mLastErrorForApn.get(apn).getError(); 300 int ret = DataFailCause.ERROR_UNSPECIFIED; 301 if (error.getErrorType() == IwlanError.EPDG_SELECTOR_SERVER_SELECTION_FAILED) { 302 ret = DataFailCause.IWLAN_DNS_RESOLUTION_NAME_FAILURE; 303 } else if (error.getErrorType() == IwlanError.IKE_INTERNAL_IO_EXCEPTION) { 304 ret = DataFailCause.IWLAN_IKEV2_MSG_TIMEOUT; 305 } else if (error.getErrorType() == IwlanError.IKE_PROTOCOL_EXCEPTION) { 306 Exception exception = error.getException(); 307 if (exception != null && exception instanceof IkeProtocolException) { 308 int protocolErrorType = ((IkeProtocolException) exception).getErrorType(); 309 switch (protocolErrorType) { 310 case IkeProtocolException.ERROR_TYPE_AUTHENTICATION_FAILED: 311 ret = DataFailCause.IWLAN_IKEV2_AUTH_FAILURE; 312 break; 313 case IKE_PROTOCOL_ERROR_PDN_CONNECTION_REJECTION: 314 ret = DataFailCause.IWLAN_PDN_CONNECTION_REJECTION; 315 break; 316 case IKE_PROTOCOL_ERROR_MAX_CONNECTION_REACHED: 317 ret = DataFailCause.IWLAN_MAX_CONNECTION_REACHED; 318 break; 319 case IKE_PROTOCOL_ERROR_SEMANTIC_ERROR_IN_THE_TFT_OPERATION: 320 ret = DataFailCause.IWLAN_SEMANTIC_ERROR_IN_THE_TFT_OPERATION; 321 break; 322 case IKE_PROTOCOL_ERROR_SYNTACTICAL_ERROR_IN_THE_TFT_OPERATION: 323 ret = DataFailCause.IWLAN_SYNTACTICAL_ERROR_IN_THE_TFT_OPERATION; 324 break; 325 case IKE_PROTOCOL_ERROR_SEMANTIC_ERRORS_IN_PACKET_FILTERS: 326 ret = DataFailCause.IWLAN_SEMANTIC_ERRORS_IN_PACKET_FILTERS; 327 break; 328 case IKE_PROTOCOL_ERROR_SYNTACTICAL_ERRORS_IN_PACKET_FILTERS: 329 ret = DataFailCause.IWLAN_SYNTACTICAL_ERRORS_IN_PACKET_FILTERS; 330 break; 331 case IKE_PROTOCOL_ERROR_NON_3GPP_ACCESS_TO_EPC_NOT_ALLOWED: 332 ret = DataFailCause.IWLAN_NON_3GPP_ACCESS_TO_EPC_NOT_ALLOWED; 333 break; 334 case IKE_PROTOCOL_ERROR_USER_UNKNOWN: 335 ret = DataFailCause.IWLAN_USER_UNKNOWN; 336 break; 337 case IKE_PROTOCOL_ERROR_NO_APN_SUBSCRIPTION: 338 ret = DataFailCause.IWLAN_NO_APN_SUBSCRIPTION; 339 break; 340 case IKE_PROTOCOL_ERROR_AUTHORIZATION_REJECTED: 341 ret = DataFailCause.IWLAN_AUTHORIZATION_REJECTED; 342 break; 343 case IKE_PROTOCOL_ERROR_ILLEGAL_ME: 344 ret = DataFailCause.IWLAN_ILLEGAL_ME; 345 break; 346 case IKE_PROTOCOL_ERROR_NETWORK_FAILURE: 347 ret = DataFailCause.IWLAN_NETWORK_FAILURE; 348 break; 349 case IKE_PROTOCOL_ERROR_RAT_TYPE_NOT_ALLOWED: 350 ret = DataFailCause.IWLAN_RAT_TYPE_NOT_ALLOWED; 351 break; 352 case IKE_PROTOCOL_ERROR_IMEI_NOT_ACCEPTED: 353 ret = DataFailCause.IWLAN_IMEI_NOT_ACCEPTED; 354 break; 355 case IKE_PROTOCOL_ERROR_PLMN_NOT_ALLOWED: 356 ret = DataFailCause.IWLAN_PLMN_NOT_ALLOWED; 357 break; 358 case IKE_PROTOCOL_ERROR_UNAUTHENTICATED_EMERGENCY_NOT_SUPPORTED: 359 ret = DataFailCause.IWLAN_UNAUTHENTICATED_EMERGENCY_NOT_SUPPORTED; 360 break; 361 default: 362 ret = DataFailCause.IWLAN_NETWORK_FAILURE; 363 break; 364 } 365 } 366 } 367 return ret; 368 } 369 370 /** 371 * Returns the current retryTime based on the lastErrorForApn 372 * 373 * @param apn apn name for which curren retry time is needed 374 * @return long current retry time in milliseconds 375 */ getCurrentRetryTimeMs(String apn)376 public synchronized long getCurrentRetryTimeMs(String apn) { 377 if (!mLastErrorForApn.containsKey(apn)) { 378 return -1; 379 } 380 return mLastErrorForApn.get(apn).getCurrentRetryTime(); 381 } 382 383 /** 384 * Returns the last error for that apn 385 * 386 * @param apn apn name 387 * @return IwlanError or null if there is no error 388 */ getLastError(String apn)389 public synchronized IwlanError getLastError(String apn) { 390 if (mLastErrorForApn.containsKey(apn)) { 391 return mLastErrorForApn.get(apn).getError(); 392 } 393 return new IwlanError(IwlanError.NO_ERROR); 394 } 395 logErrorPolicies()396 public void logErrorPolicies() { 397 Log.d(LOG_TAG, "mCarrierConfigPolicies:"); 398 for (Map.Entry<String, List<ErrorPolicy>> entry : mCarrierConfigPolicies.entrySet()) { 399 Log.d(LOG_TAG, "Apn: " + entry.getKey()); 400 for (ErrorPolicy policy : entry.getValue()) { 401 policy.log(); 402 } 403 } 404 Log.d(LOG_TAG, "mDefaultPolicies:"); 405 for (Map.Entry<String, List<ErrorPolicy>> entry : mDefaultPolicies.entrySet()) { 406 Log.d(LOG_TAG, "Apn: " + entry.getKey()); 407 for (ErrorPolicy policy : entry.getValue()) { 408 policy.log(); 409 } 410 } 411 } 412 dump(FileDescriptor fd, PrintWriter pw, String[] args)413 public synchronized void dump(FileDescriptor fd, PrintWriter pw, String[] args) { 414 pw.println("---- ErrorPolicyManager ----"); 415 for (Map.Entry<String, ErrorInfo> entry : mLastErrorForApn.entrySet()) { 416 pw.print("APN: " + entry.getKey() + " IwlanError: " + entry.getValue().getError()); 417 pw.println(" currentRetryTime: " + entry.getValue().getCurrentRetryTime()); 418 } 419 pw.println(mErrorStats); 420 pw.println("----------------------------"); 421 } 422 ErrorPolicyManager(Context context, int slotId)423 private ErrorPolicyManager(Context context, int slotId) { 424 mContext = context; 425 mSlotId = slotId; 426 LOG_TAG = ErrorPolicyManager.class.getSimpleName() + "[" + slotId + "]"; 427 428 initHandler(); 429 430 // read from default error policy config file 431 try { 432 mDefaultPolicies.putAll(readErrorPolicies(new JSONArray(getDefaultJSONConfig()))); 433 } catch (IOException | JSONException | IllegalArgumentException e) { 434 throw new AssertionError(e); 435 } 436 437 carrierConfigErrorPolicyString = null; 438 readFromCarrierConfig(IwlanHelper.getCarrierId(mContext, mSlotId)); 439 updateUnthrottlingEvents(); 440 } 441 findErrorPolicy(String apn, IwlanError iwlanError)442 private ErrorPolicy findErrorPolicy(String apn, IwlanError iwlanError) { 443 ErrorPolicy policy = null; 444 445 if (mCarrierConfigPolicies.containsKey(apn)) { 446 policy = getPreferredErrorPolicy(mCarrierConfigPolicies.get(apn), iwlanError); 447 } 448 if (policy == null && mCarrierConfigPolicies.containsKey("*")) { 449 policy = getPreferredErrorPolicy(mCarrierConfigPolicies.get("*"), iwlanError); 450 } 451 if (policy == null && mDefaultPolicies.containsKey(apn)) { 452 policy = getPreferredErrorPolicy(mDefaultPolicies.get(apn), iwlanError); 453 } 454 if (policy == null && mDefaultPolicies.containsKey("*")) { 455 policy = getPreferredErrorPolicy(mDefaultPolicies.get("*"), iwlanError); 456 } else if (policy == null) { 457 // there should at least be one default policy defined in Default config 458 // that will apply to all errors. 459 logErrorPolicies(); 460 throw new AssertionError("no Default policy defined in the config"); 461 } 462 return policy; 463 } 464 getPreferredErrorPolicy( List<ErrorPolicy> errorPolicies, IwlanError iwlanError)465 private ErrorPolicy getPreferredErrorPolicy( 466 List<ErrorPolicy> errorPolicies, IwlanError iwlanError) { 467 468 ErrorPolicy selectedPolicy = null; 469 for (ErrorPolicy policy : errorPolicies) { 470 if (policy.match(iwlanError)) { 471 if (!policy.isFallback()) { 472 selectedPolicy = policy; 473 break; 474 } 475 if (selectedPolicy == null || policy.getErrorType() != GENERIC_ERROR_TYPE) { 476 selectedPolicy = policy; 477 } 478 } 479 } 480 return selectedPolicy; 481 } 482 initHandler()483 private void initHandler() { 484 mHandlerThread = new HandlerThread("ErrorPolicyManagerThread"); 485 mHandlerThread.start(); 486 mHandler = new EpmHandler(mHandlerThread.getLooper()); 487 } 488 getDefaultJSONConfig()489 private String getDefaultJSONConfig() throws IOException { 490 String str = ""; 491 StringBuilder stringBuilder = new StringBuilder(); 492 InputStream is = mContext.getAssets().open("defaultiwlanerrorconfig.json"); 493 BufferedReader reader = new BufferedReader(new InputStreamReader(is)); 494 while ((str = reader.readLine()) != null && str.length() > 0) { 495 // ignore the lines starting with '#' as they are intended to be 496 // comments 497 if (str.charAt(0) == '#') { 498 continue; 499 } 500 stringBuilder.append(str).append("\n"); 501 } 502 is.close(); 503 return stringBuilder.toString(); 504 } 505 readErrorPolicies(JSONArray apnArray)506 private Map<String, List<ErrorPolicy>> readErrorPolicies(JSONArray apnArray) 507 throws JSONException, IllegalArgumentException { 508 Map<String, List<ErrorPolicy>> errorPolicies = new HashMap<>(); 509 for (int i = 0; i < apnArray.length(); i++) { 510 JSONObject apnDetails = apnArray.getJSONObject(i); 511 512 String apnName = ((String) apnDetails.get("ApnName")).trim(); 513 JSONArray errorTypeArray = (JSONArray) apnDetails.get("ErrorTypes"); 514 515 for (int j = 0; j < errorTypeArray.length(); j++) { 516 JSONObject errorTypeObject = errorTypeArray.getJSONObject(j); 517 518 String errorTypeStr = ((String) errorTypeObject.get("ErrorType")).trim(); 519 JSONArray errorDetailArray = (JSONArray) errorTypeObject.get("ErrorDetails"); 520 int errorType = UNKNOWN_ERROR_TYPE; 521 522 if ((errorType = getErrorPolicyErrorType(errorTypeStr)) == UNKNOWN_ERROR_TYPE) { 523 throw new IllegalArgumentException("Unknown error type in the parsing"); 524 } 525 526 ErrorPolicy errorPolicy = 527 new ErrorPolicy( 528 errorType, 529 parseErrorDetails(errorType, errorDetailArray), 530 parseRetryArray((JSONArray) errorTypeObject.get("RetryArray")), 531 parseUnthrottlingEvents( 532 (JSONArray) errorTypeObject.get("UnthrottlingEvents"))); 533 534 errorPolicies.putIfAbsent(apnName, new ArrayList<ErrorPolicy>()); 535 errorPolicies.get(apnName).add(errorPolicy); 536 } 537 } 538 return errorPolicies; 539 } 540 parseRetryArray(JSONArray retryArray)541 private List<Integer> parseRetryArray(JSONArray retryArray) 542 throws JSONException, IllegalArgumentException { 543 List<Integer> ret = new ArrayList<>(); 544 for (int i = 0; i < retryArray.length(); i++) { 545 String retryTime = retryArray.getString(i).trim(); 546 547 // catch misplaced -1 retry times in the array. 548 // 1. if it is not placed at the last position in the array 549 // 2. if it is placed in the first position (catches the case where it is 550 // the only element. 551 if (retryTime.equals("-1") && (i != retryArray.length() - 1 || i == 0)) { 552 throw new IllegalArgumentException("Misplaced -1 in retry array"); 553 } 554 if (TextUtils.isDigitsOnly(retryTime) || retryTime.equals("-1")) { 555 ret.add(Integer.parseInt(retryTime)); 556 } else if (retryTime.contains("+r")) { 557 // randomized retry time 558 String[] times = retryTime.split("\\+r"); 559 if (times.length == 2 560 && TextUtils.isDigitsOnly(times[0]) 561 && TextUtils.isDigitsOnly(times[1])) { 562 ret.add( 563 Integer.parseInt(times[0]) 564 + (int) (Math.random() * Long.parseLong(times[1]))); 565 } else { 566 throw new IllegalArgumentException( 567 "Randomized Retry time is not in acceptable format"); 568 } 569 } else { 570 throw new IllegalArgumentException("Retry time is not in acceptable format"); 571 } 572 } 573 return ret; 574 } 575 parseUnthrottlingEvents(JSONArray unthrottlingEvents)576 private List<Integer> parseUnthrottlingEvents(JSONArray unthrottlingEvents) 577 throws JSONException, IllegalArgumentException { 578 List<Integer> ret = new ArrayList<>(); 579 for (int i = 0; i < unthrottlingEvents.length(); i++) { 580 int event = 581 IwlanEventListener.getUnthrottlingEvent(unthrottlingEvents.getString(i).trim()); 582 if (event == IwlanEventListener.UNKNOWN_EVENT) { 583 throw new IllegalArgumentException( 584 "Unexpected unthrottlingEvent " + unthrottlingEvents.getString(i)); 585 } 586 ret.add(event); 587 } 588 return ret; 589 } 590 parseErrorDetails(int errorType, JSONArray errorDetailArray)591 private List<String> parseErrorDetails(int errorType, JSONArray errorDetailArray) 592 throws JSONException, IllegalArgumentException { 593 List<String> ret = new ArrayList<>(); 594 boolean isValidErrorDetail = true; 595 596 for (int i = 0; i < errorDetailArray.length(); i++) { 597 String errorDetail = errorDetailArray.getString(i).trim(); 598 switch (errorType) { 599 case IKE_PROTOCOL_ERROR_TYPE: 600 isValidErrorDetail = verifyIkeProtocolErrorDetail(errorDetail); 601 break; 602 case GENERIC_ERROR_TYPE: 603 isValidErrorDetail = verifyGenericErrorDetail(errorDetail); 604 break; 605 } 606 if (!isValidErrorDetail) { 607 throw new IllegalArgumentException( 608 "Invalid ErrorDetail: " + errorDetail + " for ErrorType: " + errorType); 609 } 610 ret.add(errorDetail); 611 } 612 return ret; 613 } 614 615 /** Allowed formats are: number(Integer), range(Integers separated by -) and "*" */ verifyIkeProtocolErrorDetail(String errorDetailStr)616 private boolean verifyIkeProtocolErrorDetail(String errorDetailStr) { 617 boolean ret = true; 618 if (errorDetailStr.contains("-")) { 619 // verify range format 620 String rangeNumbers[] = errorDetailStr.split("-"); 621 if (rangeNumbers.length == 2) { 622 for (String range : rangeNumbers) { 623 if (!TextUtils.isDigitsOnly(range)) { 624 ret = false; 625 } 626 } 627 } else { 628 ret = false; 629 } 630 } else if (!errorDetailStr.equals("*") && !TextUtils.isDigitsOnly(errorDetailStr)) { 631 ret = false; 632 } 633 return ret; 634 } 635 636 /** 637 * Allowed strings are: "IO_EXCEPTION", "TIMEOUT_EXCEPTION", "SERVER_SELECTION_FAILED", 638 * "TUNNEL_TRANSFORM_FAILED" and "*" 639 */ verifyGenericErrorDetail(String errorDetailStr)640 private boolean verifyGenericErrorDetail(String errorDetailStr) { 641 boolean ret = false; 642 for (String str : GENERIC_ERROR_DETAIL_STRINGS) { 643 if (errorDetailStr.equals(str)) { 644 ret = true; 645 break; 646 } 647 } 648 return ret; 649 } 650 getErrorPolicyErrorType(String errorType)651 private @ErrorPolicyErrorType int getErrorPolicyErrorType(String errorType) { 652 int ret = UNKNOWN_ERROR_TYPE; 653 switch (errorType) { 654 case "IKE_PROTOCOL_ERROR_TYPE": 655 ret = IKE_PROTOCOL_ERROR_TYPE; 656 break; 657 case "GENERIC_ERROR_TYPE": 658 ret = GENERIC_ERROR_TYPE; 659 break; 660 case "*": 661 ret = FALLBACK_ERROR_TYPE; 662 break; 663 } 664 return ret; 665 } 666 getAllUnthrottlingEvents()667 private synchronized Set<Integer> getAllUnthrottlingEvents() { 668 Set<Integer> events = new HashSet<>(); 669 for (Map.Entry<String, List<ErrorPolicy>> entry : mCarrierConfigPolicies.entrySet()) { 670 List<ErrorPolicy> errorPolicies = entry.getValue(); 671 for (ErrorPolicy errorPolicy : errorPolicies) { 672 events.addAll(errorPolicy.mUnthrottlingEvents); 673 } 674 } 675 for (Map.Entry<String, List<ErrorPolicy>> entry : mDefaultPolicies.entrySet()) { 676 List<ErrorPolicy> errorPolicies = entry.getValue(); 677 for (ErrorPolicy errorPolicy : errorPolicies) { 678 events.addAll(errorPolicy.mUnthrottlingEvents); 679 } 680 } 681 events.add(IwlanEventListener.CARRIER_CONFIG_CHANGED_EVENT); 682 return events; 683 } 684 685 /** 686 * This method is called once on initialization of this class And is also called from handler on 687 * CARRIER_CONFIG_CHANGED event. There is no race condition between both as we register for the 688 * events after the calling this method. 689 */ readFromCarrierConfig(int currentCarrierId)690 private synchronized void readFromCarrierConfig(int currentCarrierId) { 691 String carrierConfigErrorPolicy = 692 (String) IwlanHelper.getConfig(KEY_ERROR_POLICY_CONFIG_STRING, mContext, mSlotId); 693 if (carrierConfigErrorPolicy == null) { 694 Log.e(LOG_TAG, "ErrorPolicy from Carrier Config is NULL"); 695 return; 696 } 697 try { 698 Map<String, List<ErrorPolicy>> errorPolicies = 699 readErrorPolicies(new JSONArray(carrierConfigErrorPolicy)); 700 if (errorPolicies.size() > 0) { 701 carrierConfigErrorPolicyString = carrierConfigErrorPolicy; 702 carrierId = currentCarrierId; 703 mCarrierConfigPolicies.clear(); 704 mCarrierConfigPolicies.putAll(errorPolicies); 705 } 706 } catch (JSONException | IllegalArgumentException e) { 707 Log.e( 708 LOG_TAG, 709 "Unable to parse the ErrorPolicy from CarrierConfig\n" 710 + carrierConfigErrorPolicy); 711 if (mCarrierConfigPolicies != null) { 712 mCarrierConfigPolicies.clear(); 713 } 714 carrierConfigErrorPolicyString = null; 715 e.printStackTrace(); 716 } 717 } 718 updateUnthrottlingEvents()719 private void updateUnthrottlingEvents() { 720 Set<Integer> registerEvents, unregisterEvents; 721 unregisterEvents = mUnthrottlingEvents; 722 registerEvents = getAllUnthrottlingEvents(); 723 mUnthrottlingEvents = getAllUnthrottlingEvents(); 724 725 if (registerEvents != null && unregisterEvents != null) { 726 registerEvents.removeAll(unregisterEvents); 727 unregisterEvents.removeAll(mUnthrottlingEvents); 728 } 729 730 if (registerEvents != null) { 731 IwlanEventListener.getInstance(mContext, mSlotId) 732 .addEventListener(new ArrayList<Integer>(registerEvents), mHandler); 733 } 734 if (unregisterEvents != null) { 735 IwlanEventListener.getInstance(mContext, mSlotId) 736 .removeEventListener(new ArrayList<Integer>(unregisterEvents), mHandler); 737 } 738 Log.d( 739 LOG_TAG, 740 "UnthrottlingEvents: " 741 + (mUnthrottlingEvents != null 742 ? Arrays.toString(mUnthrottlingEvents.toArray()) 743 : "null")); 744 } 745 unthrottleLastErrorOnEvent(int event)746 private synchronized void unthrottleLastErrorOnEvent(int event) { 747 Log.d(LOG_TAG, "unthrottleLastErrorOnEvent: " + event); 748 if (event == IwlanEventListener.CARRIER_CONFIG_CHANGED_EVENT) { 749 mLastErrorForApn.clear(); 750 return; 751 } 752 String apn; 753 for (Map.Entry<String, ErrorInfo> entry : mLastErrorForApn.entrySet()) { 754 ErrorPolicy errorPolicy = entry.getValue().getErrorPolicy(); 755 if (errorPolicy.canUnthrottle(event)) { 756 apn = entry.getKey(); 757 mLastErrorForApn.remove(apn); 758 DataService.DataServiceProvider provider = 759 IwlanDataService.getDataServiceProvider(mSlotId); 760 if (provider != null) { 761 provider.notifyApnUnthrottled(apn); 762 } 763 Log.d(LOG_TAG, "unthrottled error for: " + apn); 764 } 765 } 766 } 767 768 @VisibleForTesting getErrorStats()769 ErrorStats getErrorStats() { 770 return mErrorStats; 771 } 772 773 class ErrorPolicy { 774 @ErrorPolicyErrorType int mErrorType; 775 List<String> mErrorDetails; 776 List<Integer> mRetryArray; 777 List<Integer> mUnthrottlingEvents; 778 ErrorPolicy( @rrorPolicyErrorType int errorType, List<String> errorDetails, List<Integer> retryArray, List<Integer> unthrottlingEvents)779 ErrorPolicy( 780 @ErrorPolicyErrorType int errorType, 781 List<String> errorDetails, 782 List<Integer> retryArray, 783 List<Integer> unthrottlingEvents) { 784 mErrorType = errorType; 785 mErrorDetails = errorDetails; 786 mRetryArray = retryArray; 787 mUnthrottlingEvents = unthrottlingEvents; 788 } 789 getRetryTime(int index)790 long getRetryTime(int index) { 791 long retryTime = -1; 792 if (mRetryArray.size() > 0) { 793 // If the index is greater than or equal to the last element's index 794 // and if the last item in the retryArray is "-1" use the retryTime 795 // of the element before the last element to repeat the element. 796 if (index >= mRetryArray.size() - 1 797 && mRetryArray.get(mRetryArray.size() - 1) == -1L) { 798 index = mRetryArray.size() - 2; 799 } 800 if (index >= 0 && index < mRetryArray.size()) { 801 retryTime = mRetryArray.get(index); 802 } 803 } 804 805 // retryTime -1 represents indefinite failure. In that case 806 // return time that represents 1 day to not retry for that day. 807 if (retryTime == -1L) { 808 retryTime = TimeUnit.DAYS.toSeconds(1); 809 } 810 return retryTime; 811 } 812 813 @ErrorPolicyErrorType getErrorType()814 int getErrorType() { 815 return mErrorType; 816 } 817 canUnthrottle(int event)818 synchronized boolean canUnthrottle(int event) { 819 return mUnthrottlingEvents.contains(event); 820 } 821 match(IwlanError iwlanError)822 boolean match(IwlanError iwlanError) { 823 // Generic by default to match to generic policy. 824 String iwlanErrorDetail = "*"; 825 if (mErrorType == FALLBACK_ERROR_TYPE) { 826 return true; 827 } else if (mErrorType == IKE_PROTOCOL_ERROR_TYPE 828 && iwlanError.getErrorType() == IwlanError.IKE_PROTOCOL_EXCEPTION) { 829 IkeProtocolException exception = (IkeProtocolException) iwlanError.getException(); 830 iwlanErrorDetail = String.valueOf(exception.getErrorType()); 831 } else if (mErrorType == GENERIC_ERROR_TYPE) { 832 iwlanErrorDetail = getGenericErrorDetailString(iwlanError); 833 if (iwlanErrorDetail.equals("UNKNOWN")) { 834 return false; 835 } 836 } else { 837 return false; 838 } 839 840 boolean ret = false; 841 for (String errorDetail : mErrorDetails) { 842 if (mErrorType == IKE_PROTOCOL_ERROR_TYPE 843 && iwlanError.getErrorType() == IwlanError.IKE_PROTOCOL_EXCEPTION 844 && errorDetail.contains("-")) { 845 // error detail is stored in range format. 846 // ErrorPolicyManager#verifyIkeProtocolErrorDetail will make sure that 847 // this is stored correctly in "min-max" format. 848 String range[] = errorDetail.split("-"); 849 int min = Integer.parseInt(range[0]); 850 int max = Integer.parseInt(range[1]); 851 int error = Integer.parseInt(iwlanErrorDetail); 852 if (error >= min && error <= max) { 853 ret = true; 854 break; 855 } 856 } else if (errorDetail.equals(iwlanErrorDetail) || errorDetail.equals("*")) { 857 ret = true; 858 break; 859 } 860 } 861 return ret; 862 } 863 log()864 void log() { 865 Log.d(LOG_TAG, "ErrorType: " + mErrorType); 866 Log.d(LOG_TAG, "ErrorDetail: " + Arrays.toString(mErrorDetails.toArray())); 867 Log.d(LOG_TAG, "RetryArray: " + Arrays.toString(mRetryArray.toArray())); 868 Log.d(LOG_TAG, "unthrottlingEvents: " + Arrays.toString(mUnthrottlingEvents.toArray())); 869 } 870 isFallback()871 boolean isFallback() { 872 if ((mErrorType == FALLBACK_ERROR_TYPE) 873 || (mErrorDetails.size() == 1 && mErrorDetails.get(0).equals("*"))) { 874 return true; 875 } 876 return false; 877 } 878 getGenericErrorDetailString(IwlanError iwlanError)879 String getGenericErrorDetailString(IwlanError iwlanError) { 880 String ret = "UNKNOWN"; 881 switch (iwlanError.getErrorType()) { 882 case IwlanError.IKE_INTERNAL_IO_EXCEPTION: 883 ret = "IO_EXCEPTION"; 884 break; 885 case IwlanError.EPDG_SELECTOR_SERVER_SELECTION_FAILED: 886 ret = "SERVER_SELECTION_FAILED"; 887 break; 888 case IwlanError.TUNNEL_TRANSFORM_FAILED: 889 ret = "TUNNEL_TRANSFORM_FAILED"; 890 break; 891 // TODO: Add TIMEOUT_EXCEPTION processing 892 } 893 return ret; 894 } 895 } 896 897 class ErrorInfo { 898 IwlanError mError; 899 ErrorPolicy mErrorPolicy; 900 int mCurrentRetryIndex; 901 long mLastErrorTime; 902 boolean mIsBackOffTimeValid = false; 903 long mBackOffTime; 904 ErrorInfo(IwlanError error, ErrorPolicy errorPolicy)905 ErrorInfo(IwlanError error, ErrorPolicy errorPolicy) { 906 mError = error; 907 mErrorPolicy = errorPolicy; 908 mCurrentRetryIndex = -1; 909 mLastErrorTime = new Date().getTime(); 910 } 911 ErrorInfo(IwlanError error, ErrorPolicy errorPolicy, long backOffTime)912 ErrorInfo(IwlanError error, ErrorPolicy errorPolicy, long backOffTime) { 913 mError = error; 914 mErrorPolicy = errorPolicy; 915 mCurrentRetryIndex = -1; 916 mIsBackOffTimeValid = true; 917 mBackOffTime = backOffTime; 918 mLastErrorTime = new Date().getTime(); 919 } 920 921 /** 922 * Updates the current retry index and returns the retry time at new index position and also 923 * updates mLastErrorTime to current time. returns -1 if the index is out of bounds 924 */ updateCurrentRetryTime()925 long updateCurrentRetryTime() { 926 if (mErrorPolicy == null) { 927 return -1; 928 } 929 long time = mErrorPolicy.getRetryTime(++mCurrentRetryIndex); 930 mLastErrorTime = new Date().getTime(); 931 Log.d(LOG_TAG, "Current RetryArray index: " + mCurrentRetryIndex + " time: " + time); 932 return time; 933 } 934 935 /** 936 * Return the current retry time without changing the index. returns -1 if the index is out 937 * of bounds. 938 */ getCurrentRetryTime()939 long getCurrentRetryTime() { 940 long time = -1; 941 942 if (mIsBackOffTimeValid) { 943 time = TimeUnit.SECONDS.toMillis(mBackOffTime); 944 } else if (mErrorPolicy == null) { 945 return time; 946 } else { 947 time = TimeUnit.SECONDS.toMillis(mErrorPolicy.getRetryTime(mCurrentRetryIndex)); 948 } 949 long currentTime = new Date().getTime(); 950 time = Math.max(0, time - (currentTime - mLastErrorTime)); 951 Log.d( 952 LOG_TAG, 953 "Current RetryArray index: " + mCurrentRetryIndex + " and time: " + time); 954 return time; 955 } 956 isBackOffTimeValid()957 boolean isBackOffTimeValid() { 958 return mIsBackOffTimeValid; 959 } 960 setBackOffTime(long backOffTime)961 void setBackOffTime(long backOffTime) { 962 mBackOffTime = backOffTime; 963 mLastErrorTime = new Date().getTime(); 964 } 965 canBringUpTunnel()966 boolean canBringUpTunnel() { 967 long retryTime; 968 boolean ret = true; 969 970 if (mIsBackOffTimeValid) { 971 retryTime = TimeUnit.SECONDS.toMillis(mBackOffTime); 972 } else if (mErrorPolicy == null) { 973 return ret; 974 } else { 975 retryTime = 976 TimeUnit.SECONDS.toMillis(mErrorPolicy.getRetryTime(mCurrentRetryIndex)); 977 } 978 long currentTime = new Date().getTime(); 979 long timeDifference = currentTime - mLastErrorTime; 980 if (timeDifference < retryTime) { 981 ret = false; 982 } 983 return ret; 984 } 985 getErrorPolicy()986 ErrorPolicy getErrorPolicy() { 987 return mErrorPolicy; 988 } 989 getError()990 IwlanError getError() { 991 return mError; 992 } 993 } 994 isValidCarrierConfigChangedEvent(int currentCarrierId)995 private boolean isValidCarrierConfigChangedEvent(int currentCarrierId) { 996 String errorPolicyConfig = 997 (String) IwlanHelper.getConfig(KEY_ERROR_POLICY_CONFIG_STRING, mContext, mSlotId); 998 boolean isValidEvent = 999 (currentCarrierId != carrierId) 1000 || (carrierConfigErrorPolicyString == null) 1001 || (errorPolicyConfig != null 1002 && !carrierConfigErrorPolicyString.equals(errorPolicyConfig)); 1003 return isValidEvent; 1004 } 1005 1006 private final class EpmHandler extends Handler { 1007 private final String TAG = EpmHandler.class.getSimpleName(); 1008 1009 @Override handleMessage(Message msg)1010 public void handleMessage(Message msg) { 1011 Log.d(TAG, "msg.what = " + msg.what); 1012 switch (msg.what) { 1013 case IwlanEventListener.CARRIER_CONFIG_CHANGED_EVENT: 1014 Log.d(TAG, "On CARRIER_CONFIG_CHANGED_EVENT"); 1015 int currentCarrierId = IwlanHelper.getCarrierId(mContext, mSlotId); 1016 if (isValidCarrierConfigChangedEvent(currentCarrierId)) { 1017 Log.d(TAG, "Unthrottle last error and read from carrier config"); 1018 unthrottleLastErrorOnEvent(IwlanEventListener.CARRIER_CONFIG_CHANGED_EVENT); 1019 readFromCarrierConfig(currentCarrierId); 1020 updateUnthrottlingEvents(); 1021 } 1022 break; 1023 case IwlanEventListener.APM_ENABLE_EVENT: 1024 case IwlanEventListener.APM_DISABLE_EVENT: 1025 case IwlanEventListener.WIFI_DISABLE_EVENT: 1026 case IwlanEventListener.WIFI_CALLING_DISABLE_EVENT: 1027 unthrottleLastErrorOnEvent(msg.what); 1028 break; 1029 default: 1030 Log.d(TAG, "Unknown message received!"); 1031 break; 1032 } 1033 } 1034 EpmHandler(Looper looper)1035 EpmHandler(Looper looper) { 1036 super(looper); 1037 } 1038 } 1039 1040 @VisibleForTesting 1041 class ErrorStats { 1042 @VisibleForTesting Map<String, Map<String, Long>> mStats = new HashMap<>(); 1043 private Date mStartTime; 1044 private int mStatCount = 0; 1045 private final int APN_COUNT_MAX = 10; 1046 private final int ERROR_COUNT_MAX = 1000; 1047 ErrorStats()1048 ErrorStats() { 1049 mStartTime = Calendar.getInstance().getTime(); 1050 mStatCount = 0; 1051 } 1052 update(String apn, IwlanError error)1053 void update(String apn, IwlanError error) { 1054 if (mStats.size() >= APN_COUNT_MAX || mStatCount >= ERROR_COUNT_MAX) { 1055 reset(); 1056 } 1057 if (!mStats.containsKey(apn)) { 1058 mStats.put(apn, new HashMap<String, Long>()); 1059 } 1060 Map<String, Long> errorMap = mStats.get(apn); 1061 String errorString = error.toString(); 1062 if (!errorMap.containsKey(errorString)) { 1063 errorMap.put(errorString, 0L); 1064 } 1065 long count = errorMap.get(errorString); 1066 errorMap.put(errorString, ++count); 1067 mStats.put(apn, errorMap); 1068 mStatCount++; 1069 } 1070 reset()1071 void reset() { 1072 mStartTime = Calendar.getInstance().getTime(); 1073 mStats = new HashMap<String, Map<String, Long>>(); 1074 mStatCount = 0; 1075 } 1076 1077 @Override toString()1078 public String toString() { 1079 StringBuilder sb = new StringBuilder(); 1080 sb.append("mStartTime: " + mStartTime); 1081 sb.append("\nErrorStats"); 1082 for (Map.Entry<String, Map<String, Long>> entry : mStats.entrySet()) { 1083 sb.append("\n\tApn: " + entry.getKey()); 1084 for (Map.Entry<String, Long> errorEntry : entry.getValue().entrySet()) { 1085 sb.append("\n\t " + errorEntry.getKey() + " : " + errorEntry.getValue()); 1086 } 1087 } 1088 return sb.toString(); 1089 } 1090 } 1091 } 1092