1 /** 2 * Copyright (c) 2008, SnakeYAML 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 * in compliance with the License. You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software distributed under the License 10 * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express 11 * or implied. See the License for the specific language governing permissions and limitations under 12 * the License. 13 */ 14 package org.pyyaml; 15 16 import java.io.File; 17 import java.io.FileInputStream; 18 import java.io.FileNotFoundException; 19 import java.io.IOException; 20 import java.io.InputStream; 21 import java.util.ArrayList; 22 import java.util.Collections; 23 import java.util.HashMap; 24 import java.util.List; 25 import java.util.Map; 26 import org.yaml.snakeyaml.reader.StreamReader; 27 import org.yaml.snakeyaml.reader.UnicodeReader; 28 import org.yaml.snakeyaml.scanner.Scanner; 29 import org.yaml.snakeyaml.scanner.ScannerImpl; 30 import org.yaml.snakeyaml.tokens.StreamEndToken; 31 import org.yaml.snakeyaml.tokens.StreamStartToken; 32 import org.yaml.snakeyaml.tokens.Token; 33 34 /** 35 * imported from PyYAML 36 */ 37 public class PyTokensTest extends PyImportTest { 38 testTokens()39 public void testTokens() throws FileNotFoundException { 40 Map<Token.ID, String> replaces = new HashMap<Token.ID, String>(); 41 replaces.put(Token.ID.Directive, "%"); 42 replaces.put(Token.ID.DocumentStart, "---"); 43 replaces.put(Token.ID.DocumentEnd, "..."); 44 replaces.put(Token.ID.Alias, "*"); 45 replaces.put(Token.ID.Anchor, "&"); 46 replaces.put(Token.ID.Tag, "!"); 47 replaces.put(Token.ID.Scalar, "_"); 48 replaces.put(Token.ID.BlockSequenceStart, "[["); 49 replaces.put(Token.ID.BlockMappingStart, "{{"); 50 replaces.put(Token.ID.BlockEnd, "]}"); 51 replaces.put(Token.ID.FlowSequenceStart, "["); 52 replaces.put(Token.ID.FlowSequenceEnd, "]"); 53 replaces.put(Token.ID.FlowMappingStart, "{"); 54 replaces.put(Token.ID.FlowMappingEnd, "}"); 55 replaces.put(Token.ID.BlockEntry, ","); 56 replaces.put(Token.ID.FlowEntry, ","); 57 replaces.put(Token.ID.Key, "?"); 58 replaces.put(Token.ID.Value, ":"); 59 // 60 File[] tokensFiles = getStreamsByExtension(".tokens"); 61 assertTrue("No test files found.", tokensFiles.length > 0); 62 for (int i = 0; i < tokensFiles.length; i++) { 63 String name = tokensFiles[i].getName(); 64 int position = name.lastIndexOf('.'); 65 String dataName = name.substring(0, position) + ".data"; 66 // 67 String tokenFileData = getResource(name); 68 String[] split = tokenFileData.split("\\s+"); 69 List<String> tokens2 = new ArrayList<String>(); 70 Collections.addAll(tokens2, split); 71 // 72 List<String> tokens1 = new ArrayList<String>(); 73 StreamReader reader = 74 new StreamReader(new UnicodeReader(new FileInputStream(getFileByName(dataName)))); 75 Scanner scanner = new ScannerImpl(reader); 76 try { 77 while (scanner.checkToken()) { 78 Token token = scanner.getToken(); 79 if (!(token instanceof StreamStartToken || token instanceof StreamEndToken)) { 80 String replacement = replaces.get(token.getTokenId()); 81 tokens1.add(replacement); 82 } 83 } 84 assertEquals(tokenFileData, tokens1.size(), tokens2.size()); 85 assertEquals(tokens1, tokens2); 86 } catch (RuntimeException e) { 87 System.out.println("File name: \n" + tokensFiles[i].getName()); 88 String data = getResource(tokensFiles[i].getName()); 89 System.out.println("Data: \n" + data); 90 System.out.println("Tokens:"); 91 for (String token : tokens1) { 92 System.out.println(token); 93 } 94 fail("Cannot scan: " + tokensFiles[i]); 95 } 96 } 97 } 98 testScanner()99 public void testScanner() throws IOException { 100 File[] files = getStreamsByExtension(".data", true); 101 assertTrue("No test files found.", files.length > 0); 102 for (File file : files) { 103 List<String> tokens = new ArrayList<String>(); 104 InputStream input = new FileInputStream(file); 105 StreamReader reader = new StreamReader(new UnicodeReader(input)); 106 Scanner scanner = new ScannerImpl(reader); 107 try { 108 while (scanner.checkToken()) { 109 Token token = scanner.getToken(); 110 tokens.add(token.getClass().getName()); 111 } 112 } catch (RuntimeException e) { 113 System.out.println("File name: \n" + file.getName()); 114 String data = getResource(file.getName()); 115 System.out.println("Data: \n" + data); 116 System.out.println("Tokens:"); 117 for (String token : tokens) { 118 System.out.println(token); 119 } 120 fail("Cannot scan: " + file + "; " + e.getMessage()); 121 } finally { 122 input.close(); 123 } 124 } 125 } 126 } 127