1 /* 2 * Copyright (c) 2020 Network New Technologies Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.networknt.schema; 18 19 import com.fasterxml.jackson.databind.ObjectMapper; 20 import com.networknt.schema.SpecVersion.VersionFlag; 21 import com.networknt.schema.serialization.JsonMapperFactory; 22 import com.networknt.schema.suite.TestCase; 23 import com.networknt.schema.suite.TestSource; 24 import com.networknt.schema.suite.TestSpec; 25 26 import org.junit.jupiter.api.AssertionFailureBuilder; 27 import org.junit.jupiter.api.DynamicNode; 28 import org.opentest4j.AssertionFailedError; 29 30 import java.io.IOException; 31 import java.io.UncheckedIOException; 32 import java.nio.file.Files; 33 import java.nio.file.Path; 34 import java.nio.file.Paths; 35 import java.util.Collections; 36 import java.util.List; 37 import java.util.Optional; 38 import java.util.Set; 39 import java.util.stream.Collectors; 40 import java.util.stream.Stream; 41 42 import static com.networknt.schema.SpecVersionDetector.detectVersion; 43 import static org.junit.jupiter.api.Assumptions.abort; 44 import static org.junit.jupiter.api.DynamicContainer.dynamicContainer; 45 import static org.junit.jupiter.api.DynamicTest.dynamicTest; 46 47 public abstract class AbstractJsonSchemaTestSuite extends HTTPServiceSupport { 48 49 50 protected ObjectMapper mapper = JsonMapperFactory.getInstance(); 51 toForwardSlashPath(Path file)52 private static String toForwardSlashPath(Path file) { 53 return file.toString().replace('\\', '/'); 54 } 55 executeTest(JsonSchema schema, TestSpec testSpec)56 private static void executeTest(JsonSchema schema, TestSpec testSpec) { 57 Set<ValidationMessage> errors = schema.validate(testSpec.getData(), OutputFormat.DEFAULT, (executionContext, validationContext) -> { 58 if (testSpec.getTestCase().getSource().getPath().getParent().toString().endsWith("format")) { 59 executionContext.getExecutionConfig().setFormatAssertionsEnabled(true); 60 } 61 }); 62 63 if (testSpec.isValid()) { 64 if (!errors.isEmpty()) { 65 String msg = new StringBuilder("Expected success") 66 .append("\n description: ") 67 .append(testSpec.getDescription()) 68 .append("\n schema: ") 69 .append(schema) 70 .append("\n data: ") 71 .append(testSpec.getData()) 72 .toString(); 73 74 AssertionFailedError t = AssertionFailureBuilder.assertionFailure() 75 .message(msg) 76 .reason(errors.stream().map(ValidationMessage::getMessage).collect(Collectors.joining("\n ", "\n errors:\n ", ""))) 77 .build(); 78 t.setStackTrace(new StackTraceElement[0]); 79 throw t; 80 } 81 } else { 82 if (errors.isEmpty()) { 83 String msg = new StringBuilder("Expected failure") 84 .append("\n description: ") 85 .append(testSpec.getDescription()) 86 .append("\n schema: ") 87 .append(schema) 88 .append("\n data: ") 89 .append(testSpec.getData()) 90 .toString(); 91 92 AssertionFailedError t = AssertionFailureBuilder.assertionFailure() 93 .message(msg) 94 .build(); 95 t.setStackTrace(new StackTraceElement[0]); 96 throw t; 97 } 98 } 99 100 // Expected Validation Messages need not be exactly same as actual errors. 101 // This code checks if expected validation message is subset of actual errors 102 Set<String> actual = errors.stream().map(ValidationMessage::getMessage).collect(Collectors.toSet()); 103 Set<String> expected = testSpec.getValidationMessages(); 104 expected.removeAll(actual); 105 if (!expected.isEmpty()) { 106 String msg = new StringBuilder("Expected Validation Messages") 107 .append("\n description: ") 108 .append(testSpec.getDescription()) 109 .append("\n schema: ") 110 .append(schema) 111 .append("\n data: ") 112 .append(testSpec.getData()) 113 .append(actual.stream().collect(Collectors.joining("\n ", "\n errors:\n ", ""))) 114 .toString(); 115 116 AssertionFailedError t = AssertionFailureBuilder.assertionFailure() 117 .message(msg) 118 .reason(expected.stream().collect(Collectors.joining("\n ", "\n expected:\n ", ""))) 119 .build(); 120 t.setStackTrace(new StackTraceElement[0]); 121 throw t; 122 } 123 } 124 unsupportedMetaSchema(TestCase testCase)125 private static Iterable<? extends DynamicNode> unsupportedMetaSchema(TestCase testCase) { 126 return Collections.singleton( 127 dynamicTest("Detected an unsupported schema", () -> { 128 String schema = testCase.getSchema().asText(); 129 AssertionFailedError t = AssertionFailureBuilder.assertionFailure() 130 .message("Detected an unsupported schema: " + schema) 131 .reason("Future and custom meta-schemas are not supported") 132 .build(); 133 t.setStackTrace(new StackTraceElement[0]); 134 throw t; 135 }) 136 ); 137 } 138 createTests(VersionFlag defaultVersion, String basePath)139 protected Stream<DynamicNode> createTests(VersionFlag defaultVersion, String basePath) { 140 return findTestCases(basePath) 141 .stream() 142 .peek(System.out::println) 143 .flatMap(path -> buildContainers(defaultVersion, path)); 144 } 145 enabled(@uppressWarnings"unused") Path path)146 protected boolean enabled(@SuppressWarnings("unused") Path path) { 147 return true; 148 } 149 reason(@uppressWarnings"unused") Path path)150 protected Optional<String> reason(@SuppressWarnings("unused") Path path) { 151 return Optional.empty(); 152 } 153 buildContainers(VersionFlag defaultVersion, Path path)154 private Stream<DynamicNode> buildContainers(VersionFlag defaultVersion, Path path) { 155 boolean disabled = !enabled(path); 156 String reason = reason(path).orElse("Unknown"); 157 return TestSource.loadFrom(path, disabled, reason) 158 .map(testSource -> buildContainer(defaultVersion, testSource)) 159 .orElse(Stream.empty()); 160 } 161 buildContainer(VersionFlag defaultVersion, TestSource testSource)162 private Stream<DynamicNode> buildContainer(VersionFlag defaultVersion, TestSource testSource) { 163 return testSource.getTestCases().stream().map(testCase -> buildContainer(defaultVersion, testCase)); 164 } 165 buildContainer(VersionFlag defaultVersion, TestCase testCase)166 private DynamicNode buildContainer(VersionFlag defaultVersion, TestCase testCase) { 167 try { 168 JsonSchemaFactory validatorFactory = buildValidatorFactory(defaultVersion, testCase); 169 170 return dynamicContainer(testCase.getDisplayName(), testCase.getTests().stream().map(testSpec -> { 171 return buildTest(validatorFactory, testSpec); 172 })); 173 } catch (JsonSchemaException e) { 174 String msg = e.getMessage(); 175 if (msg.endsWith("' is unrecognizable schema")) { 176 return dynamicContainer(testCase.getDisplayName(), unsupportedMetaSchema(testCase)); 177 } 178 throw e; 179 } 180 } 181 182 private JsonSchemaFactory buildValidatorFactory(VersionFlag defaultVersion, TestCase testCase) { 183 if (testCase.isDisabled()) return null; 184 185 VersionFlag specVersion = detectVersion(testCase.getSchema(), testCase.getSpecification(), defaultVersion, false); 186 JsonSchemaFactory base = JsonSchemaFactory.getInstance(specVersion); 187 return JsonSchemaFactory 188 .builder(base) 189 .jsonMapper(this.mapper) 190 .schemaMappers(schemaMappers -> schemaMappers 191 .mapPrefix("https://", "http://") 192 .mapPrefix("http://json-schema.org", "resource:")) 193 .build(); 194 } 195 196 private DynamicNode buildTest(JsonSchemaFactory validatorFactory, TestSpec testSpec) { 197 if (testSpec.isDisabled()) { 198 return dynamicTest(testSpec.getDescription(), () -> abortAndReset(testSpec.getReason())); 199 } 200 201 // Configure the schemaValidator to set typeLoose's value based on the test file, 202 // if test file do not contains typeLoose flag, use default value: false. 203 @SuppressWarnings("deprecation") boolean typeLoose = testSpec.isTypeLoose(); 204 205 SchemaValidatorsConfig config = new SchemaValidatorsConfig(); 206 config.setTypeLoose(typeLoose); 207 config.setEcma262Validator(TestSpec.RegexKind.JDK != testSpec.getRegex()); 208 testSpec.getStrictness().forEach(config::setStrict); 209 210 if (testSpec.getConfig() != null) { 211 if (testSpec.getConfig().containsKey("isCustomMessageSupported")) { 212 config.setCustomMessageSupported((Boolean) testSpec.getConfig().get("isCustomMessageSupported")); 213 } 214 if (testSpec.getConfig().containsKey("readOnly")) { 215 config.setReadOnly((Boolean) testSpec.getConfig().get("readOnly")); 216 } 217 if (testSpec.getConfig().containsKey("writeOnly")) { 218 config.setWriteOnly((Boolean) testSpec.getConfig().get("writeOnly")); 219 } 220 } 221 222 SchemaLocation testCaseFileUri = SchemaLocation.of("classpath:" + toForwardSlashPath(testSpec.getTestCase().getSpecification())); 223 JsonSchema schema = validatorFactory.getSchema(testCaseFileUri, testSpec.getTestCase().getSchema(), config); 224 225 return dynamicTest(testSpec.getDescription(), () -> executeAndReset(schema, testSpec)); 226 } 227 228 private void abortAndReset(String reason) { 229 try { 230 abort(reason); 231 } finally { 232 cleanup(); 233 } 234 } 235 236 private void executeAndReset(JsonSchema schema, TestSpec testSpec) { 237 try { 238 executeTest(schema, testSpec); 239 } finally { 240 cleanup(); 241 } 242 } 243 244 private List<Path> findTestCases(String basePath) { 245 try (Stream<Path> paths = Files.walk(Paths.get(basePath))) { 246 return paths 247 .filter(path -> path.toString().endsWith(".json")) 248 .collect(Collectors.toList()); 249 } catch (IOException e) { 250 throw new UncheckedIOException(e); 251 } 252 } 253 254 } 255