/* * Copyright (C) 2012 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.android.test.hwuicompare; import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; import java.util.ArrayList; import java.util.Comparator; import java.util.HashMap; import java.util.TreeSet; import org.json.JSONException; import org.json.JSONObject; import android.os.Bundle; import android.os.Environment; import android.os.Trace; import android.util.Log; import android.widget.ImageView; import android.widget.Toast; public class AutomaticActivity extends CompareActivity { private static final String LOG_TAG = "AutomaticActivity"; private static final float ERROR_DISPLAY_THRESHOLD = 0.01f; protected static final boolean DRAW_BITMAPS = false; /** * Threshold of error change required to consider a test regressed/improved */ private static final float ERROR_CHANGE_THRESHOLD = 0.001f; private static final float[] ERROR_CUTOFFS = { 0, 0.005f, 0.01f, 0.02f, 0.05f, 0.1f, 0.25f, 0.5f, 1f, 2f }; private final float[] mErrorRates = new float[ERROR_CUTOFFS.length]; private float mTotalTests = 0; private float mTotalError = 0; private int mTestsRegressed = 0; private int mTestsImproved = 0; private ImageView mSoftwareImageView = null; private ImageView mHardwareImageView = null; public abstract static class FinalCallback { abstract void report(String name, float value); void complete() {}; } private final ArrayList mFinalCallbacks = new ArrayList(); Runnable mRunnable = new Runnable() { @Override public void run() { loadBitmaps(); if (mSoftwareBitmap == null || mHardwareBitmap == null) { Log.e(LOG_TAG, "bitmap is null"); return; } if (DRAW_BITMAPS) { mSoftwareImageView.setImageBitmap(mSoftwareBitmap); mHardwareImageView.setImageBitmap(mHardwareBitmap); } Trace.traceBegin(Trace.TRACE_TAG_ALWAYS, "calculateError"); float error = mErrorCalculator.calcErrorRS(mSoftwareBitmap, mHardwareBitmap); Trace.traceEnd(Trace.TRACE_TAG_ALWAYS); final String[] modifierNames = DisplayModifier.getLastAppliedModifications(); handleError(modifierNames, error); if (DisplayModifier.step()) { finishTest(); } else { mHardwareView.invalidate(); if (DRAW_BITMAPS) { mSoftwareImageView.invalidate(); mHardwareImageView.invalidate(); } } mHandler.removeCallbacks(mRunnable); } }; @Override protected void onPause() { super.onPause(); mHandler.removeCallbacks(mRunnable); }; @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); setContentView(R.layout.automatic_layout); mSoftwareImageView = findViewById(R.id.software_image_view); mHardwareImageView = findViewById(R.id.hardware_image_view); onCreateCommon(mRunnable); beginTest(); } private static class TestResult { TestResult(String label, float error) { mLabel = label; mTotalError = error; mCount = 1; } public void addInto(float error) { mTotalError += error; mCount++; } public float getAverage() { return mTotalError / mCount; } final String mLabel; float mTotalError; int mCount; } JSONObject mOutputJson = null; JSONObject mInputJson = null; final HashMap mModifierResults = new HashMap(); final HashMap mIndividualResults = new HashMap(); final HashMap mModifierDiffResults = new HashMap(); final HashMap mIndividualDiffResults = new HashMap(); private void beginTest() { mFinalCallbacks.add(new FinalCallback() { @Override void report(String name, float value) { Log.d(LOG_TAG, name + " " + value); }; }); File inputFile = new File(Environment.getExternalStorageDirectory(), "CanvasCompareInput.json"); if (inputFile.exists() && inputFile.canRead() && inputFile.length() > 0) { try { FileInputStream inputStream = new FileInputStream(inputFile); Log.d(LOG_TAG, "Parsing input file..."); StringBuffer content = new StringBuffer((int)inputFile.length()); byte[] buffer = new byte[1024]; while (inputStream.read(buffer) != -1) { content.append(new String(buffer)); } mInputJson = new JSONObject(content.toString()); inputStream.close(); Log.d(LOG_TAG, "Parsed input file with " + mInputJson.length() + " entries"); } catch (JSONException e) { Log.e(LOG_TAG, "error parsing input json", e); } catch (IOException e) { Log.e(LOG_TAG, "error reading input json from sd", e); } } mOutputJson = new JSONObject(); } private static void logTestResultHash(String label, HashMap map) { Log.d(LOG_TAG, "---------------"); Log.d(LOG_TAG, label + ":"); Log.d(LOG_TAG, "---------------"); TreeSet set = new TreeSet(new Comparator() { @Override public int compare(TestResult lhs, TestResult rhs) { if (lhs == rhs) return 0; // don't need to worry about complex equality int cmp = Float.compare(lhs.getAverage(), rhs.getAverage()); if (cmp != 0) { return cmp; } return lhs.mLabel.compareTo(rhs.mLabel); } }); for (TestResult t : map.values()) { set.add(t); } for (TestResult t : set.descendingSet()) { if (Math.abs(t.getAverage()) > ERROR_DISPLAY_THRESHOLD) { Log.d(LOG_TAG, String.format("%2.4f : %s", t.getAverage(), t.mLabel)); } } Log.d(LOG_TAG, ""); } private void finishTest() { for (FinalCallback c : mFinalCallbacks) { c.report("averageError", (mTotalError / mTotalTests)); for (int i = 1; i < ERROR_CUTOFFS.length; i++) { c.report(String.format("tests with error over %1.3f", ERROR_CUTOFFS[i]), mErrorRates[i]); } if (mInputJson != null) { c.report("tests regressed", mTestsRegressed); c.report("tests improved", mTestsImproved); } c.complete(); } try { if (mOutputJson != null) { String outputString = mOutputJson.toString(4); File outputFile = new File(Environment.getExternalStorageDirectory(), "CanvasCompareOutput.json"); FileOutputStream outputStream = new FileOutputStream(outputFile); outputStream.write(outputString.getBytes()); outputStream.close(); Log.d(LOG_TAG, "Saved output file with " + mOutputJson.length() + " entries"); } } catch (JSONException e) { Log.e(LOG_TAG, "error during JSON stringify", e); } catch (IOException e) { Log.e(LOG_TAG, "error storing JSON output on sd", e); } logTestResultHash("Modifier change vs previous", mModifierDiffResults); logTestResultHash("Invidual test change vs previous", mIndividualDiffResults); logTestResultHash("Modifier average test results", mModifierResults); logTestResultHash("Individual test results", mIndividualResults); Toast.makeText(getApplicationContext(), "done!", Toast.LENGTH_SHORT).show(); finish(); } /** * Inserts the error value into all TestResult objects, associated with each of its modifiers */ private static void addForAllModifiers(String fullName, float error, String[] modifierNames, HashMap modifierResults) { for (String modifierName : modifierNames) { TestResult r = modifierResults.get(fullName); if (r == null) { modifierResults.put(modifierName, new TestResult(modifierName, error)); } else { r.addInto(error); } } } private void handleError(final String[] modifierNames, final float error) { String fullName = ""; for (String s : modifierNames) { fullName = fullName.concat("." + s); } fullName = fullName.substring(1); float deltaError = 0; if (mInputJson != null) { try { deltaError = error - (float)mInputJson.getDouble(fullName); } catch (JSONException e) { Log.w(LOG_TAG, "Warning: unable to read from input json", e); } if (deltaError > ERROR_CHANGE_THRESHOLD) mTestsRegressed++; if (deltaError < -ERROR_CHANGE_THRESHOLD) mTestsImproved++; mIndividualDiffResults.put(fullName, new TestResult(fullName, deltaError)); addForAllModifiers(fullName, deltaError, modifierNames, mModifierDiffResults); } mIndividualResults.put(fullName, new TestResult(fullName, error)); addForAllModifiers(fullName, error, modifierNames, mModifierResults); try { if (mOutputJson != null) { mOutputJson.put(fullName, error); } } catch (JSONException e) { Log.e(LOG_TAG, "exception during JSON recording", e); mOutputJson = null; } for (int i = 0; i < ERROR_CUTOFFS.length; i++) { if (error <= ERROR_CUTOFFS[i]) break; mErrorRates[i]++; } mTotalError += error; mTotalTests++; } @Override protected boolean forceRecreateBitmaps() { // disable, unless needed for drawing into imageviews return DRAW_BITMAPS; } // FOR TESTING public void setFinalCallback(FinalCallback c) { mFinalCallbacks.add(c); } }