1 /* 2 * Copyright (C) 2018 Google, Inc. 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 package com.google.escapevelocity; 17 18 import static com.google.escapevelocity.Node.emptyNode; 19 20 import com.google.escapevelocity.DirectiveNode.ForEachNode; 21 import com.google.escapevelocity.DirectiveNode.IfNode; 22 import com.google.escapevelocity.DirectiveNode.MacroCallNode; 23 import com.google.escapevelocity.DirectiveNode.SetNode; 24 import com.google.escapevelocity.TokenNode.CommentTokenNode; 25 import com.google.escapevelocity.TokenNode.ElseIfTokenNode; 26 import com.google.escapevelocity.TokenNode.ElseTokenNode; 27 import com.google.escapevelocity.TokenNode.EndTokenNode; 28 import com.google.escapevelocity.TokenNode.EofNode; 29 import com.google.escapevelocity.TokenNode.ForEachTokenNode; 30 import com.google.escapevelocity.TokenNode.IfOrElseIfTokenNode; 31 import com.google.escapevelocity.TokenNode.IfTokenNode; 32 import com.google.escapevelocity.TokenNode.MacroDefinitionTokenNode; 33 import com.google.escapevelocity.TokenNode.NestedTokenNode; 34 import com.google.common.base.CharMatcher; 35 import com.google.common.collect.ImmutableList; 36 import com.google.common.collect.ImmutableSet; 37 import com.google.common.collect.Iterables; 38 import java.util.Map; 39 import java.util.Set; 40 import java.util.TreeMap; 41 42 /** 43 * The second phase of parsing. See {@link Parser#parse()} for a description of the phases and why 44 * we need them. 45 * 46 * @author emcmanus@google.com (Éamonn McManus) 47 */ 48 class Reparser { 49 private static final ImmutableSet<Class<? extends TokenNode>> END_SET = 50 ImmutableSet.<Class<? extends TokenNode>>of(EndTokenNode.class); 51 private static final ImmutableSet<Class<? extends TokenNode>> EOF_SET = 52 ImmutableSet.<Class<? extends TokenNode>>of(EofNode.class); 53 private static final ImmutableSet<Class<? extends TokenNode>> ELSE_ELSE_IF_END_SET = 54 ImmutableSet.<Class<? extends TokenNode>>of( 55 ElseTokenNode.class, ElseIfTokenNode.class, EndTokenNode.class); 56 57 /** 58 * The nodes that make up the input sequence. Nodes are removed one by one from this list as 59 * parsing proceeds. At any time, {@link #currentNode} is the node being examined. 60 */ 61 private final ImmutableList<Node> nodes; 62 63 /** 64 * The index of the node we are currently looking at while parsing. 65 */ 66 private int nodeIndex; 67 68 /** 69 * Macros are removed from the input as they are found. They do not appear in the output parse 70 * tree. Macro definitions are not executed in place but are all applied before template rendering 71 * starts. This means that a macro can be referenced before it is defined. 72 */ 73 private final Map<String, Macro> macros; 74 Reparser(ImmutableList<Node> nodes)75 Reparser(ImmutableList<Node> nodes) { 76 this(nodes, new TreeMap<>()); 77 } 78 Reparser(ImmutableList<Node> nodes, Map<String, Macro> macros)79 private Reparser(ImmutableList<Node> nodes, Map<String, Macro> macros) { 80 this.nodes = removeSpaceBeforeSet(nodes); 81 this.nodeIndex = 0; 82 this.macros = macros; 83 } 84 reparse()85 Template reparse() { 86 Node root = reparseNodes(); 87 linkMacroCalls(); 88 return new Template(root); 89 } 90 reparseNodes()91 private Node reparseNodes() { 92 return parseTo(EOF_SET, new EofNode((String) null, 1)); 93 } 94 95 /** 96 * Returns a copy of the given list where spaces have been moved where appropriate after {@code 97 * #set}. This hack is needed to match what appears to be special treatment in Apache Velocity of 98 * spaces before {@code #set} directives. If you have <i>thing</i> <i>whitespace</i> {@code #set}, 99 * then the whitespace is deleted if the <i>thing</i> is a comment ({@code ##...\n}); a reference 100 * ({@code $x} or {@code $x.foo} etc); a macro definition; or another {@code #set}. 101 */ removeSpaceBeforeSet(ImmutableList<Node> nodes)102 private static ImmutableList<Node> removeSpaceBeforeSet(ImmutableList<Node> nodes) { 103 assert Iterables.getLast(nodes) instanceof EofNode; 104 // Since the last node is EofNode, the i + 1 and i + 2 accesses below are safe. 105 ImmutableList.Builder<Node> newNodes = ImmutableList.builder(); 106 for (int i = 0; i < nodes.size(); i++) { 107 Node nodeI = nodes.get(i); 108 newNodes.add(nodeI); 109 if (shouldDeleteSpaceBetweenThisAndSet(nodeI) 110 && isWhitespaceLiteral(nodes.get(i + 1)) 111 && nodes.get(i + 2) instanceof SetNode) { 112 // Skip the space. 113 i++; 114 } 115 } 116 return newNodes.build(); 117 } 118 shouldDeleteSpaceBetweenThisAndSet(Node node)119 private static boolean shouldDeleteSpaceBetweenThisAndSet(Node node) { 120 return node instanceof CommentTokenNode 121 || node instanceof ReferenceNode 122 || node instanceof SetNode 123 || node instanceof MacroDefinitionTokenNode; 124 } 125 isWhitespaceLiteral(Node node)126 private static boolean isWhitespaceLiteral(Node node) { 127 if (node instanceof ConstantExpressionNode) { 128 Object constant = node.evaluate(null); 129 return constant instanceof String && CharMatcher.whitespace().matchesAllOf((String) constant); 130 } 131 return false; 132 } 133 134 /** 135 * Parse subtrees until one of the token types in {@code stopSet} is encountered. 136 * If this is the top level, {@code stopSet} will include {@link EofNode} so parsing will stop 137 * when it reaches the end of the input. Otherwise, if an {@code EofNode} is encountered it is an 138 * error because we have something like {@code #if} without {@code #end}. 139 * 140 * @param stopSet the kinds of tokens that will stop the parse. For example, if we are parsing 141 * after an {@code #if}, we will stop at any of {@code #else}, {@code #elseif}, 142 * or {@code #end}. 143 * @param forWhat the token that triggered this call, for example the {@code #if} whose 144 * {@code #end} etc we are looking for. 145 * 146 * @return a Node that is the concatenation of the parsed subtrees 147 */ parseTo(Set<Class<? extends TokenNode>> stopSet, TokenNode forWhat)148 private Node parseTo(Set<Class<? extends TokenNode>> stopSet, TokenNode forWhat) { 149 ImmutableList.Builder<Node> nodeList = ImmutableList.builder(); 150 while (true) { 151 Node currentNode = currentNode(); 152 if (stopSet.contains(currentNode.getClass())) { 153 break; 154 } 155 if (currentNode instanceof EofNode) { 156 throw new ParseException( 157 "Reached end of file while parsing " + forWhat.name(), 158 forWhat.resourceName, 159 forWhat.lineNumber); 160 } 161 Node parsed; 162 if (currentNode instanceof TokenNode) { 163 parsed = parseTokenNode(); 164 } else { 165 parsed = currentNode; 166 nextNode(); 167 } 168 nodeList.add(parsed); 169 } 170 return Node.cons(forWhat.resourceName, forWhat.lineNumber, nodeList.build()); 171 } 172 currentNode()173 private Node currentNode() { 174 return nodes.get(nodeIndex); 175 } 176 nextNode()177 private Node nextNode() { 178 Node currentNode = currentNode(); 179 if (currentNode instanceof EofNode) { 180 return currentNode; 181 } else { 182 nodeIndex++; 183 return currentNode(); 184 } 185 } 186 parseTokenNode()187 private Node parseTokenNode() { 188 TokenNode tokenNode = (TokenNode) currentNode(); 189 nextNode(); 190 if (tokenNode instanceof CommentTokenNode) { 191 return emptyNode(tokenNode.resourceName, tokenNode.lineNumber); 192 } else if (tokenNode instanceof IfTokenNode) { 193 return parseIfOrElseIf((IfTokenNode) tokenNode); 194 } else if (tokenNode instanceof ForEachTokenNode) { 195 return parseForEach((ForEachTokenNode) tokenNode); 196 } else if (tokenNode instanceof NestedTokenNode) { 197 return parseNested((NestedTokenNode) tokenNode); 198 } else if (tokenNode instanceof MacroDefinitionTokenNode) { 199 return parseMacroDefinition((MacroDefinitionTokenNode) tokenNode); 200 } else { 201 throw new IllegalArgumentException( 202 "Unexpected token: " + tokenNode.name() + " on line " + tokenNode.lineNumber); 203 } 204 } 205 parseForEach(ForEachTokenNode forEach)206 private Node parseForEach(ForEachTokenNode forEach) { 207 Node body = parseTo(END_SET, forEach); 208 nextNode(); // Skip #end 209 return new ForEachNode( 210 forEach.resourceName, forEach.lineNumber, forEach.var, forEach.collection, body); 211 } 212 parseIfOrElseIf(IfOrElseIfTokenNode ifOrElseIf)213 private Node parseIfOrElseIf(IfOrElseIfTokenNode ifOrElseIf) { 214 Node truePart = parseTo(ELSE_ELSE_IF_END_SET, ifOrElseIf); 215 Node falsePart; 216 Node token = currentNode(); 217 nextNode(); // Skip #else or #elseif (cond) or #end. 218 if (token instanceof EndTokenNode) { 219 falsePart = emptyNode(token.resourceName, token.lineNumber); 220 } else if (token instanceof ElseTokenNode) { 221 falsePart = parseTo(END_SET, ifOrElseIf); 222 nextNode(); // Skip #end 223 } else if (token instanceof ElseIfTokenNode) { 224 // We've seen #if (condition1) ... #elseif (condition2). currentToken is the first token 225 // after (condition2). We pretend that we've just seen #if (condition2) and parse out 226 // the remainder (which might have further #elseif and final #else). Then we pretend that 227 // we actually saw #if (condition1) ... #else #if (condition2) ...remainder ... #end #end. 228 falsePart = parseIfOrElseIf((ElseIfTokenNode) token); 229 } else { 230 throw new AssertionError(currentNode()); 231 } 232 return new IfNode( 233 ifOrElseIf.resourceName, ifOrElseIf.lineNumber, ifOrElseIf.condition, truePart, falsePart); 234 } 235 236 // This is a #parse("foo.vm") directive. We've already done the first phase of parsing on the 237 // contents of foo.vm. Now we need to do the second phase, and insert the result into the 238 // reparsed nodes. We can call Reparser recursively, but we must ensure that any macros found 239 // are added to the containing Reparser's macro definitions. parseNested(NestedTokenNode nested)240 private Node parseNested(NestedTokenNode nested) { 241 Reparser reparser = new Reparser(nested.nodes, this.macros); 242 return reparser.reparseNodes(); 243 } 244 parseMacroDefinition(MacroDefinitionTokenNode macroDefinition)245 private Node parseMacroDefinition(MacroDefinitionTokenNode macroDefinition) { 246 Node body = parseTo(END_SET, macroDefinition); 247 nextNode(); // Skip #end 248 if (!macros.containsKey(macroDefinition.name)) { 249 Macro macro = new Macro( 250 macroDefinition.lineNumber, macroDefinition.name, macroDefinition.parameterNames, body); 251 macros.put(macroDefinition.name, macro); 252 } 253 return emptyNode(macroDefinition.resourceName, macroDefinition.lineNumber); 254 } 255 linkMacroCalls()256 private void linkMacroCalls() { 257 for (Node node : nodes) { 258 if (node instanceof MacroCallNode) { 259 linkMacroCall((MacroCallNode) node); 260 } 261 } 262 } 263 linkMacroCall(MacroCallNode macroCall)264 private void linkMacroCall(MacroCallNode macroCall) { 265 Macro macro = macros.get(macroCall.name()); 266 if (macro == null) { 267 throw new ParseException( 268 "#" + macroCall.name() 269 + " is neither a standard directive nor a macro that has been defined", 270 macroCall.resourceName, 271 macroCall.lineNumber); 272 } 273 if (macro.parameterCount() != macroCall.argumentCount()) { 274 throw new ParseException( 275 "Wrong number of arguments to #" + macroCall.name() 276 + ": expected " + macro.parameterCount() 277 + ", got " + macroCall.argumentCount(), 278 macroCall.resourceName, 279 macroCall.lineNumber); 280 } 281 macroCall.setMacro(macro); 282 } 283 } 284