tests/test_expectations/GPUTestExpectationsParser_unittest.cpp

//
// Copyright 2019 The ANGLE Project Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
//
// GPUTestExpectationsParser_unittest.cpp: Unit tests for GPUTestExpectationsParser*
//

#include "tests/test_expectations/GPUTestExpectationsParser.h"
#include <gtest/gtest.h>
#include "tests/test_expectations/GPUTestConfig.h"

using namespace angle;

namespace
{

class GPUTestConfigTester : public GPUTestConfig
{
  public:
    GPUTestConfigTester()
    {
        mConditions.reset();
        mConditions[GPUTestConfig::kConditionWin]    = true;
        mConditions[GPUTestConfig::kConditionNVIDIA] = true;
        mConditions[GPUTestConfig::kConditionD3D11]  = true;
    }
};

// A correct entry with a test that's skipped on all platforms should not lead
// to any errors, and should properly return the expectation SKIP.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserSkip)
{
    GPUTestConfigTester config;
    std::string line =
        R"(100 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
    GPUTestExpectationsParser parser;
    EXPECT_TRUE(parser.loadTestExpectations(config, line));
    EXPECT_TRUE(parser.getErrorMessages().empty());
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestSkip);
}

// A correct entry with a test that's failed on all platforms should not lead
// to any errors, and should properly return the expectation FAIL.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserFail)
{
    GPUTestConfigTester config;
    std::string line =
        R"(100 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = FAIL)";
    GPUTestExpectationsParser parser;
    EXPECT_TRUE(parser.loadTestExpectations(config, line));
    EXPECT_TRUE(parser.getErrorMessages().empty());
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestFail);
}

// A correct entry with a test that's passed on all platforms should not lead
// to any errors, and should properly return the expectation PASS.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserPass)
{
    GPUTestConfigTester config;
    std::string line =
        R"(100 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = PASS)";
    GPUTestExpectationsParser parser;
    EXPECT_TRUE(parser.loadTestExpectations(config, line));
    EXPECT_TRUE(parser.getErrorMessages().empty());
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestPass);
}

// A correct entry with a test that's timed out on all platforms should not lead
// to any errors, and should properly return the expectation TIMEOUT.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserTimeout)
{
    GPUTestConfigTester config;
    std::string line =
        R"(100 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = TIMEOUT)";
    GPUTestExpectationsParser parser;
    EXPECT_TRUE(parser.loadTestExpectations(config, line));
    EXPECT_TRUE(parser.getErrorMessages().empty());
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestTimeout);
}

// A correct entry with a test that's flaky on all platforms should not lead
// to any errors, and should properly return the expectation FLAKY.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserFlaky)
{
    GPUTestConfigTester config;
    std::string line =
        R"(100 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = FLAKY)";
    GPUTestExpectationsParser parser;
    EXPECT_TRUE(parser.loadTestExpectations(config, line));
    EXPECT_TRUE(parser.getErrorMessages().empty());
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestFlaky);
}

// A correct entry with a test that's skipped on windows should not lead
// to any errors, and should properly return the expectation SKIP on this
// tester.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserSingleLineWin)
{
    GPUTestConfigTester config;
    std::string line =
        R"(100 WIN : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
    GPUTestExpectationsParser parser;
    EXPECT_TRUE(parser.loadTestExpectations(config, line));
    EXPECT_TRUE(parser.getErrorMessages().empty());
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestSkip);
}

// A correct entry with a test that's skipped on windows/NVIDIA should not lead
// to any errors, and should properly return the expectation SKIP on this
// tester.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserSingleLineWinNVIDIA)
{
    GPUTestConfigTester config;
    std::string line =
        R"(100 WIN NVIDIA : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
    GPUTestExpectationsParser parser;
    EXPECT_TRUE(parser.loadTestExpectations(config, line));
    EXPECT_TRUE(parser.getErrorMessages().empty());
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestSkip);
}

// A correct entry with a test that's skipped on windows/NVIDIA/D3D11 should not
// lead to any errors, and should properly return the expectation SKIP on this
// tester.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserSingleLineWinNVIDIAD3D11)
{
    GPUTestConfigTester config;
    std::string line =
        R"(100 WIN NVIDIA D3D11 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
    GPUTestExpectationsParser parser;
    EXPECT_TRUE(parser.loadTestExpectations(config, line));
    EXPECT_TRUE(parser.getErrorMessages().empty());
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestSkip);
}

// Same as GPUTestExpectationsParserSingleLineWinNVIDIAD3D11, but verifying that the order
// of these conditions doesn't matter
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserSingleLineWinNVIDIAD3D11OtherOrder)
{
    GPUTestConfigTester config;
    std::string line =
        R"(100 D3D11 NVIDIA WIN : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
    GPUTestExpectationsParser parser;
    EXPECT_TRUE(parser.loadTestExpectations(config, line));
    EXPECT_TRUE(parser.getErrorMessages().empty());
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestSkip);
}

// A correct entry with a test that's skipped on mac should not lead
// to any errors, and should default to PASS on this tester (windows).
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserSingleLineMac)
{
    GPUTestConfigTester config;
    std::string line =
        R"(100 MAC : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
    GPUTestExpectationsParser parser;
    EXPECT_TRUE(parser.loadTestExpectations(config, line));
    EXPECT_TRUE(parser.getErrorMessages().empty());
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestPass);
}

// A correct entry with a test that has conflicting entries should not lead
// to any errors, and should default to PASS.
// (https:anglebug.com/3368) In the future, this condition should cause an
// error
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserSingleLineConflict)
{
    GPUTestConfigTester config;
    std::string line =
        R"(100 WIN MAC : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
    GPUTestExpectationsParser parser;
    EXPECT_TRUE(parser.loadTestExpectations(config, line));
    EXPECT_TRUE(parser.getErrorMessages().empty());
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestPass);
}

// A line without a bug ID should return an error and not add the expectation.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserMissingBugId)
{
    GPUTestConfigTester config;
    std::string line = R"( : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
    GPUTestExpectationsParser parser;
    EXPECT_FALSE(parser.loadTestExpectations(config, line));
    EXPECT_EQ(parser.getErrorMessages().size(), 1u);
    if (parser.getErrorMessages().size() >= 1)
    {
        EXPECT_EQ(parser.getErrorMessages()[0], "Line 1 : entry with wrong format");
    }
    // Default behavior is to let missing tests pass
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestPass);
}

// A line without a bug ID should return an error and not add the expectation, (even if
// the line contains conditions that might be mistaken for a bug id)
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserMissingBugIdWithConditions)
{
    GPUTestConfigTester config;
    std::string line =
        R"(WIN D3D11 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
    GPUTestExpectationsParser parser;
    EXPECT_FALSE(parser.loadTestExpectations(config, line));
    EXPECT_EQ(parser.getErrorMessages().size(), 1u);
    if (parser.getErrorMessages().size() >= 1)
    {
        EXPECT_EQ(parser.getErrorMessages()[0], "Line 1 : entry with wrong format");
    }
    // Default behavior is to let missing tests pass
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestPass);
}

// A line without a colon should return an error and not add the expectation.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserMissingColon)
{
    GPUTestConfigTester config;
    std::string line = R"(100 dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
    GPUTestExpectationsParser parser;
    EXPECT_FALSE(parser.loadTestExpectations(config, line));
    EXPECT_EQ(parser.getErrorMessages().size(), 1u);
    if (parser.getErrorMessages().size() >= 1)
    {
        EXPECT_EQ(parser.getErrorMessages()[0], "Line 1 : entry with wrong format");
    }
    // Default behavior is to let missing tests pass
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestPass);
}

// A wild character (*) at the end of a line should match any expectations that are a subset of that
// line. It should not greedily match to omany expectations that aren't in that subset.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserWildChar)
{
    GPUTestConfigTester config;
    std::string line = R"(100 : dEQP-GLES31.functional.layout_binding.ubo.* = SKIP)";
    GPUTestExpectationsParser parser;
    EXPECT_TRUE(parser.loadTestExpectations(config, line));
    EXPECT_TRUE(parser.getErrorMessages().empty());
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestSkip);
    // Also ensure the wild char is not too wild, only covers tests that are more specific
    EXPECT_EQ(parser.getTestExpectation(
                  "dEQP-GLES31.functional.program_interface_query.transform_feedback_varying."
                  "resource_list.vertex_fragment.builtin_gl_position"),
              GPUTestExpectationsParser::kGpuTestPass);
}

// A line without an equals should return an error and not add the expectation.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserMissingEquals)
{
    GPUTestConfigTester config;
    std::string line = R"(100 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max SKIP)";
    GPUTestExpectationsParser parser;
    EXPECT_FALSE(parser.loadTestExpectations(config, line));
    EXPECT_EQ(parser.getErrorMessages().size(), 1u);
    if (parser.getErrorMessages().size() >= 1)
    {
        EXPECT_EQ(parser.getErrorMessages()[0], "Line 1 : entry with wrong format");
    }
    // Default behavior is to let missing tests pass
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestPass);
}

// A line without an expectation should return an error and not add the expectation.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserMissingExpectation)
{
    GPUTestConfigTester config;
    std::string line = R"(100 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max =)";
    GPUTestExpectationsParser parser;
    EXPECT_FALSE(parser.loadTestExpectations(config, line));
    EXPECT_EQ(parser.getErrorMessages().size(), 1u);
    if (parser.getErrorMessages().size() >= 1)
    {
        EXPECT_EQ(parser.getErrorMessages()[0], "Line 1 : entry with wrong format");
    }
    // Default behavior is to let missing tests pass
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestPass);
}

// A line with an expectation that doesn't exist should return an error and not add the expectation.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserInvalidExpectation)
{
    GPUTestConfigTester config;
    std::string line =
        R"(100 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = WRONG)";
    GPUTestExpectationsParser parser;
    EXPECT_FALSE(parser.loadTestExpectations(config, line));
    EXPECT_EQ(parser.getErrorMessages().size(), 1u);
    if (parser.getErrorMessages().size() >= 1)
    {
        EXPECT_EQ(parser.getErrorMessages()[0], "Line 1 : entry with wrong format");
    }
    // Default behavior is to let missing tests pass
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestPass);
}

// ChromeOS is reserved as a token, but doesn't actually check any conditions. Any tokens that
// do not check conditions should return an error and not add the expectation
// (https://anglebug.com/3363) Remove/update this test when ChromeOS is supported
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserUnimplementedCondition)
{
    GPUTestConfigTester config;
    std::string line =
        R"(100 CHROMEOS : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
    GPUTestExpectationsParser parser;
    EXPECT_FALSE(parser.loadTestExpectations(config, line));
    EXPECT_EQ(parser.getErrorMessages().size(), 1u);
    if (parser.getErrorMessages().size() >= 1)
    {
        EXPECT_EQ(parser.getErrorMessages()[0],
                  "Line 1 : entry invalid, likely unimplemented modifiers");
    }
    // Default behavior is to let missing tests pass
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestPass);
}

// If a line starts with a comment, it's ignored and should not be added to the list.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserComment)
{
    GPUTestConfigTester config;
    std::string line =
        R"(//100 : dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max = SKIP)";
    GPUTestExpectationsParser parser;
    EXPECT_TRUE(parser.loadTestExpectations(config, line));
    EXPECT_TRUE(parser.getErrorMessages().empty());
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestPass);
}

// A misspelled expectation should not be matched from getTestExpectation, and should lead to an
// unused expectation when later queried.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserMisspelledExpectation)
{
    GPUTestConfigTester config;
    std::string line =
        R"(100 : dEQP-GLES31.functionaal.layout_binding.ubo.* = SKIP)";  // "functionaal"
    GPUTestExpectationsParser parser;
    EXPECT_TRUE(parser.loadTestExpectations(config, line));
    EXPECT_TRUE(parser.getErrorMessages().empty());
    // Default behavior is to let missing tests pass
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestPass);
    EXPECT_EQ(parser.getUnusedExpectationsMessages().size(), 1u);
    if (parser.getUnusedExpectationsMessages().size() >= 1)
    {
        EXPECT_EQ(parser.getUnusedExpectationsMessages()[0], "Line 1: expectation was unused.");
    }
}

// Wild characters that match groups of expectations can be overridden with more specific lines.
// The parse should still compute correctly which lines were used and which were unused.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserOverrideExpectation)
{
    GPUTestConfigTester config;
    // Fail all layout_binding tests, but skip the layout_binding.ubo subset.
    std::string line = R"(100 : dEQP-GLES31.functional.layout_binding.* = FAIL
100 : dEQP-GLES31.functional.layout_binding.ubo.* = SKIP)";
    GPUTestExpectationsParser parser;
    EXPECT_TRUE(parser.loadTestExpectations(config, line));
    EXPECT_TRUE(parser.getErrorMessages().empty());
    // Default behavior is to let missing tests pass
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestSkip);
    // The FAIL expectation was unused because it was overridden.
    EXPECT_EQ(parser.getUnusedExpectationsMessages().size(), 1u);
    if (parser.getUnusedExpectationsMessages().size() >= 1)
    {
        EXPECT_EQ(parser.getUnusedExpectationsMessages()[0], "Line 1: expectation was unused.");
    }
    // Now try a test that doesn't match the override criteria
    EXPECT_EQ(parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.image.test"),
              GPUTestExpectationsParser::kGpuTestFail);
    EXPECT_TRUE(parser.getUnusedExpectationsMessages().empty());
}

// This test is the same as GPUTestExpectationsParserOverrideExpectation, but verifying the order
// doesn't matter when overriding.
TEST(GPUTestExpectationsParserTest, GPUTestExpectationsParserOverrideExpectationOtherOrder)
{
    GPUTestConfigTester config;
    // Fail all layout_binding tests, but skip the layout_binding.ubo subset.
    std::string line = R"(100 : dEQP-GLES31.functional.layout_binding.ubo.* = SKIP
100 : dEQP-GLES31.functional.layout_binding.* = FAIL)";
    GPUTestExpectationsParser parser;
    EXPECT_TRUE(parser.loadTestExpectations(config, line));
    EXPECT_TRUE(parser.getErrorMessages().empty());
    // Default behavior is to let missing tests pass
    EXPECT_EQ(
        parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.ubo.vertex_binding_max"),
        GPUTestExpectationsParser::kGpuTestSkip);
    // The FAIL expectation was unused because it was overridden.
    EXPECT_EQ(parser.getUnusedExpectationsMessages().size(), 1u);
    if (parser.getUnusedExpectationsMessages().size() >= 1)
    {
        EXPECT_EQ(parser.getUnusedExpectationsMessages()[0], "Line 2: expectation was unused.");
    }
    // Now try a test that doesn't match the override criteria
    EXPECT_EQ(parser.getTestExpectation("dEQP-GLES31.functional.layout_binding.image.test"),
              GPUTestExpectationsParser::kGpuTestFail);
    EXPECT_TRUE(parser.getUnusedExpectationsMessages().empty());
}

}  // anonymous namespace