• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * [The "BSD license"]
3  *  Copyright (c) 2010 Terence Parr
4  *  All rights reserved.
5  *
6  *  Redistribution and use in source and binary forms, with or without
7  *  modification, are permitted provided that the following conditions
8  *  are met:
9  *  1. Redistributions of source code must retain the above copyright
10  *      notice, this list of conditions and the following disclaimer.
11  *  2. Redistributions in binary form must reproduce the above copyright
12  *      notice, this list of conditions and the following disclaimer in the
13  *      documentation and/or other materials provided with the distribution.
14  *  3. The name of the author may not be used to endorse or promote products
15  *      derived from this software without specific prior written permission.
16  *
17  *  THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18  *  IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19  *  OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20  *  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21  *  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22  *  NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23  *  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24  *  THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25  *  (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26  *  THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27  */
28 package org.antlr.codegen;
29 
30 
31 import org.antlr.Tool;
32 import org.antlr.analysis.DFA;
33 import org.antlr.analysis.*;
34 import org.antlr.grammar.v3.ANTLRLexer;
35 import org.antlr.grammar.v3.ANTLRParser;
36 import org.antlr.grammar.v3.ActionTranslator;
37 import org.antlr.grammar.v3.CodeGenTreeWalker;
38 import org.antlr.misc.BitSet;
39 import org.antlr.misc.*;
40 import org.antlr.runtime.*;
41 import org.antlr.runtime.tree.CommonTreeNodeStream;
42 import org.antlr.tool.*;
43 import org.stringtemplate.v4.*;
44 
45 import java.io.IOException;
46 import java.io.Writer;
47 import java.util.*;
48 
49 /** ANTLR's code generator.
50  *
51  *  Generate recognizers derived from grammars.  Language independence
52  *  achieved through the use of STGroup objects.  All output
53  *  strings are completely encapsulated in the group files such as Java.stg.
54  *  Some computations are done that are unused by a particular language.
55  *  This generator just computes and sets the values into the templates;
56  *  the templates are free to use or not use the information.
57  *
58  *  To make a new code generation target, define X.stg for language X
59  *  by copying from existing Y.stg most closely releated to your language;
60  *  e.g., to do CSharp.stg copy Java.stg.  The template group file has a
61  *  bunch of templates that are needed by the code generator.  You can add
62  *  a new target w/o even recompiling ANTLR itself.  The language=X option
63  *  in a grammar file dictates which templates get loaded/used.
64  *
65  *  Some language like C need both parser files and header files.  Java needs
66  *  to have a separate file for the cyclic DFA as ANTLR generates bytecodes
67  *  directly (which cannot be in the generated parser Java file).  To facilitate
68  *  this,
69  *
70  * cyclic can be in same file, but header, output must be searpate.  recognizer
71  *  is in outptufile.
72  */
73 public class CodeGenerator {
74 	/** When generating SWITCH statements, some targets might need to limit
75 	 *  the size (based upon the number of case labels).  Generally, this
76 	 *  limit will be hit only for lexers where wildcard in a UNICODE
77 	 *  vocabulary environment would generate a SWITCH with 65000 labels.
78 	 */
79 	public final static int MSCL_DEFAULT = 300;
80 	public static int MAX_SWITCH_CASE_LABELS = MSCL_DEFAULT;
81 	public final static int MSA_DEFAULT = 3;
82 	public static int MIN_SWITCH_ALTS = MSA_DEFAULT;
83 	public boolean GENERATE_SWITCHES_WHEN_POSSIBLE = true;
84 	public static boolean LAUNCH_ST_INSPECTOR = false;
85 	public final static int MADSI_DEFAULT = 60; // do lots of states inline (needed for expression rules)
86 	public static int MAX_ACYCLIC_DFA_STATES_INLINE = MADSI_DEFAULT;
87 
88 	public static String classpathTemplateRootDirectoryName =
89 		"org/antlr/codegen/templates";
90 
91 	/** Which grammar are we generating code for?  Each generator
92 	 *  is attached to a specific grammar.
93 	 */
94 	public Grammar grammar;
95 
96 	/** What language are we generating? */
97 	protected String language;
98 
99 	/** The target specifies how to write out files and do other language
100 	 *  specific actions.
101 	 */
102 	public Target target = null;
103 
104 	/** Where are the templates this generator should use to generate code? */
105 	protected STGroup templates;
106 
107 	/** The basic output templates without AST or templates stuff; this will be
108 	 *  the templates loaded for the language such as Java.stg *and* the Dbg
109 	 *  stuff if turned on.  This is used for generating syntactic predicates.
110 	 */
111 	protected STGroup baseTemplates;
112 
113 	protected ST recognizerST;
114 	protected ST outputFileST;
115 	protected ST headerFileST;
116 
117 	/** Used to create unique labels */
118 	protected int uniqueLabelNumber = 1;
119 
120 	/** A reference to the ANTLR tool so we can learn about output directories
121 	 *  and such.
122 	 */
123 	protected Tool tool;
124 
125 	/** Generate debugging event method calls */
126 	protected boolean debug;
127 
128 	/** Create a Tracer object and make the recognizer invoke this. */
129 	protected boolean trace;
130 
131 	/** Track runtime parsing information about decisions etc...
132 	 *  This requires the debugging event mechanism to work.
133 	 */
134 	protected boolean profile;
135 
136 	protected int lineWidth = 72;
137 
138 	/** I have factored out the generation of acyclic DFAs to separate class */
139 	public ACyclicDFACodeGenerator acyclicDFAGenerator =
140 		new ACyclicDFACodeGenerator(this);
141 
142 	/** I have factored out the generation of cyclic DFAs to separate class */
143 	/*
144 	public CyclicDFACodeGenerator cyclicDFAGenerator =
145 		new CyclicDFACodeGenerator(this);
146 		*/
147 
148 	public static final String VOCAB_FILE_EXTENSION = ".tokens";
149 	protected final static String vocabFilePattern =
150 		"<tokens:{it|<it.name>=<it.type>\n}>" +
151 		"<literals:{it|<it.name>=<it.type>\n}>";
152 
CodeGenerator(Tool tool, Grammar grammar, String language)153 	public CodeGenerator(Tool tool, Grammar grammar, String language) {
154 		this.tool = tool;
155 		this.grammar = grammar;
156 		this.language = language;
157 		target = loadLanguageTarget(language);
158 	}
159 
loadLanguageTarget(String language)160 	public static Target loadLanguageTarget(String language) {
161 		Target target = null;
162 		String targetName = "org.antlr.codegen."+language+"Target";
163 		try {
164 			Class<? extends Target> c = Class.forName(targetName).asSubclass(Target.class);
165 			target = (Target)c.newInstance();
166 		}
167 		catch (ClassNotFoundException cnfe) {
168 			target = new Target(); // use default
169 		}
170 		catch (InstantiationException ie) {
171 			ErrorManager.error(ErrorManager.MSG_CANNOT_CREATE_TARGET_GENERATOR,
172 							   targetName,
173 							   ie);
174 		}
175 		catch (IllegalAccessException cnfe) {
176 			ErrorManager.error(ErrorManager.MSG_CANNOT_CREATE_TARGET_GENERATOR,
177 							   targetName,
178 							   cnfe);
179 		}
180 		return target;
181 	}
182 
183 	/** load the main language.stg template group file */
loadTemplates(String language)184 	public void loadTemplates(String language) {
185 		String langDir = classpathTemplateRootDirectoryName+"/"+language;
186 		STGroup coreTemplates = new ToolSTGroupFile(langDir+"/"+language+".stg");
187 		baseTemplates = coreTemplates;
188 
189 		// dynamically add subgroups that act like filters to apply to
190 		// their supergroup.  E.g., Java:Dbg:AST:ASTParser::ASTDbg.
191 		String outputOption = (String)grammar.getOption("output");
192 		if ( outputOption!=null && outputOption.equals("AST") ) {
193 			if ( debug && grammar.type!=Grammar.LEXER ) {
194 				STGroup dbgTemplates = new ToolSTGroupFile(langDir+"/Dbg.stg");
195 				dbgTemplates.importTemplates(coreTemplates);
196 				baseTemplates = dbgTemplates;
197 				STGroup astTemplates = new ToolSTGroupFile(langDir+"/AST.stg");
198 				astTemplates.importTemplates(dbgTemplates);
199 				STGroup astParserTemplates;
200 				if ( grammar.type==Grammar.TREE_PARSER ) {
201 					astParserTemplates = new ToolSTGroupFile(langDir+"/ASTTreeParser.stg");
202 					astParserTemplates.importTemplates(astTemplates);
203 				}
204 				else {
205 					astParserTemplates = new ToolSTGroupFile(langDir+"/ASTParser.stg");
206 					astParserTemplates.importTemplates(astTemplates);
207 				}
208 				STGroup astDbgTemplates = new ToolSTGroupFile(langDir+"/ASTDbg.stg");
209 				astDbgTemplates.importTemplates(astParserTemplates);
210 				templates = astDbgTemplates;
211 				dbgTemplates.iterateAcrossValues = true; // ST v3 compatibility with Maps
212 				astDbgTemplates.iterateAcrossValues = true;
213 				astParserTemplates.iterateAcrossValues = true;
214 			}
215 			else {
216 				STGroup astTemplates = new ToolSTGroupFile(langDir+"/AST.stg");
217 				astTemplates.importTemplates(coreTemplates);
218 				STGroup astParserTemplates;
219 				if ( grammar.type==Grammar.TREE_PARSER ) {
220 					astParserTemplates = new ToolSTGroupFile(langDir+"/ASTTreeParser.stg");
221 					astParserTemplates.importTemplates(astTemplates);
222 				}
223 				else {
224 					astParserTemplates = new ToolSTGroupFile(langDir+"/ASTParser.stg");
225 					astParserTemplates.importTemplates(astTemplates);
226 				}
227 				templates = astParserTemplates;
228 				astTemplates.iterateAcrossValues = true; // ST v3 compatibility with Maps
229 				astParserTemplates.iterateAcrossValues = true;
230 			}
231 		}
232 		else if ( outputOption!=null && outputOption.equals("template") ) {
233 			if ( debug && grammar.type!=Grammar.LEXER ) {
234 				STGroup dbgTemplates = new ToolSTGroupFile(langDir+"/Dbg.stg");
235 				dbgTemplates.importTemplates(coreTemplates);
236 				baseTemplates = dbgTemplates;
237 				STGroup stTemplates = new ToolSTGroupFile(langDir+"/ST.stg");
238 				stTemplates.importTemplates(dbgTemplates);
239 				templates = stTemplates;
240 				dbgTemplates.iterateAcrossValues = true;
241 			}
242 			else {
243 				STGroup stTemplates = new ToolSTGroupFile(langDir+"/ST.stg");
244 				stTemplates.importTemplates(coreTemplates);
245 				templates = stTemplates;
246 			}
247 			templates.iterateAcrossValues = true; // ST v3 compatibility with Maps
248 		}
249 		else if ( debug && grammar.type!=Grammar.LEXER ) {
250 			STGroup dbgTemplates = new ToolSTGroupFile(langDir+"/Dbg.stg");
251 			dbgTemplates.importTemplates(coreTemplates);
252 			templates = dbgTemplates;
253 			baseTemplates = templates;
254 			baseTemplates.iterateAcrossValues = true; // ST v3 compatibility with Maps
255 		}
256 		else {
257 			templates = coreTemplates;
258 			coreTemplates.iterateAcrossValues = true; // ST v3 compatibility with Maps
259 		}
260 	}
261 
262 	/** Given the grammar to which we are attached, walk the AST associated
263 	 *  with that grammar to create NFAs.  Then create the DFAs for all
264 	 *  decision points in the grammar by converting the NFAs to DFAs.
265 	 *  Finally, walk the AST again to generate code.
266 	 *
267 	 *  Either 1 or 2 files are written:
268 	 *
269 	 * 		recognizer: the main parser/lexer/treewalker item
270 	 * 		header file: language like C/C++ need extern definitions
271 	 *
272 	 *  The target, such as JavaTarget, dictates which files get written.
273 	 */
genRecognizer()274 	public ST genRecognizer() {
275 		//System.out.println("### generate "+grammar.name+" recognizer");
276 		// LOAD OUTPUT TEMPLATES
277 		loadTemplates(language);
278 		if ( templates==null ) {
279 			return null;
280 		}
281 
282 		// CREATE NFA FROM GRAMMAR, CREATE DFA FROM NFA
283 		if ( ErrorManager.doNotAttemptAnalysis() ) {
284 			return null;
285 		}
286 		target.performGrammarAnalysis(this, grammar);
287 
288 
289 		// some grammar analysis errors will not yield reliable DFA
290 		if ( ErrorManager.doNotAttemptCodeGen() ) {
291 			return null;
292 		}
293 
294 		// OPTIMIZE DFA
295 		DFAOptimizer optimizer = new DFAOptimizer(grammar);
296 		optimizer.optimize();
297 
298 		// OUTPUT FILE (contains recognizerST)
299 		outputFileST = templates.getInstanceOf("outputFile");
300 
301 		// HEADER FILE
302 		if ( templates.isDefined("headerFile") ) {
303 			headerFileST = templates.getInstanceOf("headerFile");
304 		}
305 		else {
306 			// create a dummy to avoid null-checks all over code generator
307 			headerFileST = new ST(templates,"xyz");
308 			headerFileST.add("cyclicDFAs", (Object)null); // it normally sees this from outputFile
309 			//headerFileST.impl.name = "dummy-header-file";
310 		}
311 
312 		boolean filterMode = grammar.getOption("filter")!=null &&
313 							  grammar.getOption("filter").equals("true");
314         boolean canBacktrack = grammar.getSyntacticPredicates()!=null ||
315                                grammar.composite.getRootGrammar().atLeastOneBacktrackOption ||
316                                filterMode;
317 
318         // TODO: move this down further because generating the recognizer
319 		// alters the model with info on who uses predefined properties etc...
320 		// The actions here might refer to something.
321 
322 		// The only two possible output files are available at this point.
323 		// Verify action scopes are ok for target and dump actions into output
324 		// Templates can say <actions.parser.header> for example.
325 		Map<String, Map<String, Object>> actions = grammar.getActions();
326 		verifyActionScopesOkForTarget(actions);
327 		// translate $x::y references
328 		translateActionAttributeReferences(actions);
329 
330         ST gateST = templates.getInstanceOf("actionGate");
331         if ( filterMode ) {
332             // if filtering, we need to set actions to execute at backtracking
333             // level 1 not 0.
334             gateST = templates.getInstanceOf("filteringActionGate");
335         }
336         grammar.setSynPredGateIfNotAlready(gateST);
337 
338         headerFileST.add("actions", actions);
339 		outputFileST.add("actions", actions);
340 
341 		headerFileST.add("buildTemplate", grammar.buildTemplate());
342 		outputFileST.add("buildTemplate", grammar.buildTemplate());
343 		headerFileST.add("buildAST", grammar.buildAST());
344 		outputFileST.add("buildAST", grammar.buildAST());
345 
346 		outputFileST.add("rewriteMode", grammar.rewriteMode());
347 		headerFileST.add("rewriteMode", grammar.rewriteMode());
348 
349 		outputFileST.add("backtracking", canBacktrack);
350 		headerFileST.add("backtracking", canBacktrack);
351 		// turn on memoize attribute at grammar level so we can create ruleMemo.
352 		// each rule has memoize attr that hides this one, indicating whether
353 		// it needs to save results
354 		String memoize = (String)grammar.getOption("memoize");
355 		outputFileST.add("memoize",
356 						 (grammar.atLeastOneRuleMemoizes ||
357 						  memoize != null && memoize.equals("true") &&
358 						  canBacktrack));
359 		headerFileST.add("memoize",
360 						 (grammar.atLeastOneRuleMemoizes ||
361 						  memoize != null && memoize.equals("true") &&
362 						  canBacktrack));
363 
364 
365 		outputFileST.add("trace", trace);
366 		headerFileST.add("trace", trace);
367 
368 		outputFileST.add("profile", profile);
369 		headerFileST.add("profile", profile);
370 
371 		// RECOGNIZER
372 		if ( grammar.type==Grammar.LEXER ) {
373 			recognizerST = templates.getInstanceOf("lexer");
374 			outputFileST.add("LEXER", true);
375 			headerFileST.add("LEXER", true);
376 			recognizerST.add("filterMode",
377 							 filterMode);
378 		}
379 		else if ( grammar.type==Grammar.PARSER ||
380 			grammar.type==Grammar.COMBINED )
381 		{
382 			recognizerST = templates.getInstanceOf("parser");
383 			outputFileST.add("PARSER", true);
384 			headerFileST.add("PARSER", true);
385 		}
386 		else {
387 			recognizerST = templates.getInstanceOf("treeParser");
388 			outputFileST.add("TREE_PARSER", true);
389 			headerFileST.add("TREE_PARSER", true);
390             recognizerST.add("filterMode",
391 							 filterMode);
392 		}
393 		outputFileST.add("recognizer", recognizerST);
394 		headerFileST.add("recognizer", recognizerST);
395 		outputFileST.add("actionScope",
396 						 grammar.getDefaultActionScope(grammar.type));
397 		headerFileST.add("actionScope",
398 						 grammar.getDefaultActionScope(grammar.type));
399 
400 		String targetAppropriateFileNameString =
401 			target.getTargetStringLiteralFromString(grammar.getFileName());
402 		outputFileST.add("fileName", targetAppropriateFileNameString);
403 		headerFileST.add("fileName", targetAppropriateFileNameString);
404 		outputFileST.add("ANTLRVersion", tool.VERSION);
405 		headerFileST.add("ANTLRVersion", tool.VERSION);
406 		outputFileST.add("generatedTimestamp", Tool.getCurrentTimeStamp());
407 		headerFileST.add("generatedTimestamp", Tool.getCurrentTimeStamp());
408 
409 		// GENERATE RECOGNIZER
410 		// Walk the AST holding the input grammar, this time generating code
411 		// Decisions are generated by using the precomputed DFAs
412 		// Fill in the various templates with data
413 		CodeGenTreeWalker gen = new CodeGenTreeWalker(new CommonTreeNodeStream(grammar.getGrammarTree()));
414 		try {
415 			gen.grammar_(
416 						grammar,
417 						recognizerST,
418 						outputFileST,
419 						headerFileST);
420 		}
421 		catch (RecognitionException re) {
422 			ErrorManager.error(ErrorManager.MSG_BAD_AST_STRUCTURE,
423 							   re);
424 		}
425 
426 		genTokenTypeConstants(recognizerST);
427 		genTokenTypeConstants(outputFileST);
428 		genTokenTypeConstants(headerFileST);
429 
430 		if ( grammar.type!=Grammar.LEXER ) {
431 			genTokenTypeNames(recognizerST);
432 			genTokenTypeNames(outputFileST);
433 			genTokenTypeNames(headerFileST);
434 		}
435 
436 		// Now that we know what synpreds are used, we can set into template
437 		Set<String> synpredNames = null;
438 		if ( grammar.synPredNamesUsedInDFA.size()>0 ) {
439 			synpredNames = grammar.synPredNamesUsedInDFA;
440 		}
441 		outputFileST.add("synpreds", synpredNames);
442 		headerFileST.add("synpreds", synpredNames);
443 
444 		// all recognizers can see Grammar object
445 		recognizerST.add("grammar", grammar);
446 
447 		// do not render templates to disk if errors occurred
448 		if ( ErrorManager.getErrorState().errors > 0 ) {
449 			return null;
450 		}
451 
452 		if (LAUNCH_ST_INSPECTOR) {
453 			outputFileST.inspect();
454 			if ( templates.isDefined("headerFile") ) headerFileST.inspect();
455 		}
456 
457 		// WRITE FILES
458 		try {
459 			target.genRecognizerFile(tool,this,grammar,outputFileST);
460 			if ( templates.isDefined("headerFile") ) {
461 				ST extST = templates.getInstanceOf("headerFileExtension");
462 				target.genRecognizerHeaderFile(tool,this,grammar,headerFileST,extST.render());
463 			}
464 			// write out the vocab interchange file; used by antlr,
465 			// does not change per target
466 			ST tokenVocabSerialization = genTokenVocabOutput();
467 			String vocabFileName = getVocabFileName();
468 			if ( vocabFileName!=null ) {
469 				write(tokenVocabSerialization, vocabFileName);
470 			}
471 			//System.out.println(outputFileST.getDOTForDependencyGraph(false));
472 		}
473 		catch (IOException ioe) {
474 			ErrorManager.error(ErrorManager.MSG_CANNOT_WRITE_FILE, ioe);
475 		}
476 		/*
477 		System.out.println("num obj.prop refs: "+ ASTExpr.totalObjPropRefs);
478 		System.out.println("num reflection lookups: "+ ASTExpr.totalReflectionLookups);
479 		*/
480 
481 		return outputFileST;
482 	}
483 
484 	/** Some targets will have some extra scopes like C++ may have
485 	 *  '@headerfile:name {action}' or something.  Make sure the
486 	 *  target likes the scopes in action table.
487 	 */
verifyActionScopesOkForTarget(Map<String, Map<String, Object>> actions)488 	protected void verifyActionScopesOkForTarget(Map<String, Map<String, Object>> actions) {
489 		for (Map.Entry<String, Map<String, Object>> entry : actions.entrySet()) {
490 			String scope = entry.getKey();
491 			if ( !target.isValidActionScope(grammar.type, scope) ) {
492 				// get any action from the scope to get error location
493 				Map<String, Object> scopeActions = entry.getValue();
494 				GrammarAST actionAST =
495 					(GrammarAST)scopeActions.values().iterator().next();
496 				ErrorManager.grammarError(
497 					ErrorManager.MSG_INVALID_ACTION_SCOPE,grammar,
498 					actionAST.getToken(),scope,
499 					grammar.getGrammarTypeString());
500 			}
501 		}
502 	}
503 
504 	/** Actions may reference $x::y attributes, call translateAction on
505 	 *  each action and replace that action in the Map.
506 	 */
translateActionAttributeReferences(Map<String, Map<String, Object>> actions)507 	protected void translateActionAttributeReferences(Map<String, Map<String, Object>> actions) {
508 		for (Map.Entry<String, Map<String, Object>> entry : actions.entrySet()) {
509 			Map<String, Object> scopeActions = entry.getValue();
510 			translateActionAttributeReferencesForSingleScope(null,scopeActions);
511 		}
512 	}
513 
514 	/** Use for translating rule @init{...} actions that have no scope */
translateActionAttributeReferencesForSingleScope( Rule r, Map<String, Object> scopeActions)515 	public void translateActionAttributeReferencesForSingleScope(
516 		Rule r,
517 		Map<String, Object> scopeActions)
518 	{
519 		String ruleName=null;
520 		if ( r!=null ) {
521 			ruleName = r.name;
522 		}
523 		for (Map.Entry<String, Object> entry : scopeActions.entrySet()) {
524 			String name = entry.getKey();
525 			GrammarAST actionAST = (GrammarAST)entry.getValue();
526 			List<?> chunks = translateAction(ruleName,actionAST);
527 			scopeActions.put(name, chunks); // replace with translation
528 		}
529 	}
530 
531 	/** Error recovery in ANTLR recognizers.
532 	 *
533 	 *  Based upon original ideas:
534 	 *
535 	 *  Algorithms + Data Structures = Programs by Niklaus Wirth
536 	 *
537 	 *  and
538 	 *
539 	 *  A note on error recovery in recursive descent parsers:
540 	 *  http://portal.acm.org/citation.cfm?id=947902.947905
541 	 *
542 	 *  Later, Josef Grosch had some good ideas:
543 	 *  Efficient and Comfortable Error Recovery in Recursive Descent Parsers:
544 	 *  ftp://www.cocolab.com/products/cocktail/doca4.ps/ell.ps.zip
545 	 *
546 	 *  Like Grosch I implemented local FOLLOW sets that are combined at run-time
547 	 *  upon error to avoid parsing overhead.
548 	 */
generateLocalFOLLOW(GrammarAST referencedElementNode, String referencedElementName, String enclosingRuleName, int elementIndex)549 	public void generateLocalFOLLOW(GrammarAST referencedElementNode,
550 									String referencedElementName,
551 									String enclosingRuleName,
552 									int elementIndex)
553 	{
554 		/*
555 		System.out.println("compute FOLLOW "+grammar.name+"."+referencedElementNode.toString()+
556 						 " for "+referencedElementName+"#"+elementIndex +" in "+
557 						 enclosingRuleName+
558 						 " line="+referencedElementNode.getLine());
559 						 */
560 		NFAState followingNFAState = referencedElementNode.followingNFAState;
561 		LookaheadSet follow = null;
562 		if ( followingNFAState!=null ) {
563 			// compute follow for this element and, as side-effect, track
564 			// the rule LOOK sensitivity.
565 			follow = grammar.FIRST(followingNFAState);
566 		}
567 
568 		if ( follow==null ) {
569 			ErrorManager.internalError("no follow state or cannot compute follow");
570 			follow = new LookaheadSet();
571 		}
572 		if ( follow.member(Label.EOF) ) {
573 			// TODO: can we just remove?  Seems needed here:
574 			// compilation_unit : global_statement* EOF
575 			// Actually i guess we resync to EOF regardless
576 			follow.remove(Label.EOF);
577 		}
578 		//System.out.println(" "+follow);
579 
580         List<Integer> tokenTypeList;
581         long[] words;
582 		if ( follow.tokenTypeSet==null ) {
583 			words = new long[1];
584             tokenTypeList = new ArrayList<Integer>();
585         }
586 		else {
587 			BitSet bits = BitSet.of(follow.tokenTypeSet);
588 			words = bits.toPackedArray();
589             tokenTypeList = follow.tokenTypeSet.toList();
590         }
591 		// use the target to convert to hex strings (typically)
592 		String[] wordStrings = new String[words.length];
593 		for (int j = 0; j < words.length; j++) {
594 			long w = words[j];
595 			wordStrings[j] = target.getTarget64BitStringFromValue(w);
596 		}
597 		recognizerST.addAggr("bitsets.{name,inName,bits,tokenTypes,tokenIndex}",
598 							 referencedElementName,
599 							 enclosingRuleName,
600 							 wordStrings,
601 							 tokenTypeList,
602 							 Utils.integer(elementIndex));
603 		outputFileST.addAggr("bitsets.{name,inName,bits,tokenTypes,tokenIndex}",
604 							 referencedElementName,
605 							 enclosingRuleName,
606 							 wordStrings,
607 							 tokenTypeList,
608 							 Utils.integer(elementIndex));
609 		headerFileST.addAggr("bitsets.{name,inName,bits,tokenTypes,tokenIndex}",
610 							 referencedElementName,
611 							 enclosingRuleName,
612 							 wordStrings,
613 							 tokenTypeList,
614 							 Utils.integer(elementIndex));
615 	}
616 
617 	// L O O K A H E A D  D E C I S I O N  G E N E R A T I O N
618 
619 	/** Generate code that computes the predicted alt given a DFA.  The
620 	 *  recognizerST can be either the main generated recognizerTemplate
621 	 *  for storage in the main parser file or a separate file.  It's up to
622 	 *  the code that ultimately invokes the codegen.g grammar rule.
623 	 *
624 	 *  Regardless, the output file and header file get a copy of the DFAs.
625 	 */
genLookaheadDecision(ST recognizerST, DFA dfa)626 	public ST genLookaheadDecision(ST recognizerST,
627 								   DFA dfa)
628 	{
629 		ST decisionST;
630 		// If we are doing inline DFA and this one is acyclic and LL(*)
631 		// I have to check for is-non-LL(*) because if non-LL(*) the cyclic
632 		// check is not done by DFA.verify(); that is, verify() avoids
633 		// doesStateReachAcceptState() if non-LL(*)
634 		if ( dfa.canInlineDecision() ) {
635 			decisionST =
636 				acyclicDFAGenerator.genFixedLookaheadDecision(getTemplates(), dfa);
637 		}
638 		else {
639 			// generate any kind of DFA here (cyclic or acyclic)
640 			dfa.createStateTables(this);
641 			outputFileST.add("cyclicDFAs", dfa);
642 			headerFileST.add("cyclicDFAs", dfa);
643 			decisionST = templates.getInstanceOf("dfaDecision");
644 			String description = dfa.getNFADecisionStartState().getDescription();
645 			description = target.getTargetStringLiteralFromString(description);
646 			if ( description!=null ) {
647 				decisionST.add("description", description);
648 			}
649 			decisionST.add("decisionNumber",
650 						   Utils.integer(dfa.getDecisionNumber()));
651 		}
652 		return decisionST;
653 	}
654 
655 	/** A special state is huge (too big for state tables) or has a predicated
656 	 *  edge.  Generate a simple if-then-else.  Cannot be an accept state as
657 	 *  they have no emanating edges.  Don't worry about switch vs if-then-else
658 	 *  because if you get here, the state is super complicated and needs an
659 	 *  if-then-else.  This is used by the new DFA scheme created June 2006.
660 	 */
generateSpecialState(DFAState s)661 	public ST generateSpecialState(DFAState s) {
662 		ST stateST;
663 		stateST = templates.getInstanceOf("cyclicDFAState");
664 		stateST.add("needErrorClause", true);
665 		stateST.add("semPredState",
666 					s.isResolvedWithPredicates());
667 		stateST.add("stateNumber", s.stateNumber);
668 		stateST.add("decisionNumber", s.dfa.decisionNumber);
669 
670 		boolean foundGatedPred = false;
671 		ST eotST = null;
672 		for (int i = 0; i < s.getNumberOfTransitions(); i++) {
673 			Transition edge = s.transition(i);
674 			ST edgeST;
675 			if ( edge.label.getAtom()==Label.EOT ) {
676 				// this is the default clause; has to held until last
677 				edgeST = templates.getInstanceOf("eotDFAEdge");
678 				stateST.remove("needErrorClause");
679 				eotST = edgeST;
680 			}
681 			else {
682 				edgeST = templates.getInstanceOf("cyclicDFAEdge");
683 				ST exprST =
684 					genLabelExpr(templates,edge,1);
685 				edgeST.add("labelExpr", exprST);
686 			}
687 			edgeST.add("edgeNumber", Utils.integer(i + 1));
688 			edgeST.add("targetStateNumber",
689 					   Utils.integer(edge.target.stateNumber));
690 			// stick in any gated predicates for any edge if not already a pred
691 			if ( !edge.label.isSemanticPredicate() ) {
692 				DFAState t = (DFAState)edge.target;
693 				SemanticContext preds =	t.getGatedPredicatesInNFAConfigurations();
694 				if ( preds!=null ) {
695 					foundGatedPred = true;
696 					ST predST = preds.genExpr(this,
697 														  getTemplates(),
698 														  t.dfa);
699 					edgeST.add("predicates", predST.render());
700 				}
701 			}
702 			if ( edge.label.getAtom()!=Label.EOT ) {
703 				stateST.add("edges", edgeST);
704 			}
705 		}
706 		if ( foundGatedPred ) {
707 			// state has >= 1 edge with a gated pred (syn or sem)
708 			// must rewind input first, set flag.
709 			stateST.add("semPredState", foundGatedPred);
710 		}
711 		if ( eotST!=null ) {
712 			stateST.add("edges", eotST);
713 		}
714 		return stateST;
715 	}
716 
717 	/** Generate an expression for traversing an edge. */
genLabelExpr(STGroup templates, Transition edge, int k)718 	protected ST genLabelExpr(STGroup templates,
719 										  Transition edge,
720 										  int k)
721 	{
722 		Label label = edge.label;
723 		if ( label.isSemanticPredicate() ) {
724 			return genSemanticPredicateExpr(templates, edge);
725 		}
726 		if ( label.isSet() ) {
727 			return genSetExpr(templates, label.getSet(), k, true);
728 		}
729 		// must be simple label
730 		ST eST = templates.getInstanceOf("lookaheadTest");
731 		eST.add("atom", getTokenTypeAsTargetLabel(label.getAtom()));
732 		eST.add("atomAsInt", Utils.integer(label.getAtom()));
733 		eST.add("k", Utils.integer(k));
734 		return eST;
735 	}
736 
genSemanticPredicateExpr(STGroup templates, Transition edge)737 	protected ST genSemanticPredicateExpr(STGroup templates,
738 													  Transition edge)
739 	{
740 		DFA dfa = ((DFAState)edge.target).dfa; // which DFA are we in
741 		Label label = edge.label;
742 		SemanticContext semCtx = label.getSemanticContext();
743 		return semCtx.genExpr(this,templates,dfa);
744 	}
745 
746 	/** For intervals such as [3..3, 30..35], generate an expression that
747 	 *  tests the lookahead similar to LA(1)==3 || (LA(1)&gt;=30&amp;&amp;LA(1)&lt;=35)
748 	 */
genSetExpr(STGroup templates, IntSet set, int k, boolean partOfDFA)749 	public ST genSetExpr(STGroup templates,
750 									 IntSet set,
751 									 int k,
752 									 boolean partOfDFA)
753 	{
754 		if ( !(set instanceof IntervalSet) ) {
755 			throw new IllegalArgumentException("unable to generate expressions for non IntervalSet objects");
756 		}
757 		IntervalSet iset = (IntervalSet)set;
758 		if ( iset.getIntervals()==null || iset.getIntervals().isEmpty() ) {
759 			ST emptyST = new ST(templates, "");
760 			emptyST.impl.name = "empty-set-expr";
761 			return emptyST;
762 		}
763 		String testSTName = "lookaheadTest";
764 		String testRangeSTName = "lookaheadRangeTest";
765 		String testSetSTName = "lookaheadSetTest";
766 		String varSTName = "lookaheadVarName";
767 		if ( !partOfDFA ) {
768 			testSTName = "isolatedLookaheadTest";
769 			testRangeSTName = "isolatedLookaheadRangeTest";
770 			testSetSTName = "isolatedLookaheadSetTest";
771 			varSTName = "isolatedLookaheadVarName";
772 		}
773 		ST setST = templates.getInstanceOf("setTest");
774 		// If the SetTest template exists, separate the ranges:
775 		// flatten the small ones into one list and make that a range,
776 		// and leave the others as they are.
777 		if ( templates.isDefined(testSetSTName) ) {
778 			// Flatten the IntervalSet into a list of integers.
779 			ST sST = templates.getInstanceOf(testSetSTName);
780 			Iterator<Interval> iter = iset.getIntervals().iterator();
781 			int rangeNumber = 1;
782 			while (iter.hasNext()) {
783 				Interval I = iter.next();
784 				int a = I.a;
785 				int b = I.b;
786 				// Not flattening the large ranges helps us avoid making a
787 				// set that contains 90% of Unicode when we could just use
788 				// a simple range like (LA(1)>=123 && LA(1)<=65535).
789 				// This flattens all ranges of length 4 or less.
790 				if (b - a < 4) {
791 					for (int i = a; i <= b; i++) {
792 						sST.add("values", getTokenTypeAsTargetLabel(i));
793 						sST.add("valuesAsInt", Utils.integer(i));
794 					}
795 				} else {
796 					ST eST = templates.getInstanceOf(testRangeSTName);
797 					eST.add("lower", getTokenTypeAsTargetLabel(a));
798 					eST.add("lowerAsInt", Utils.integer(a));
799 					eST.add("upper", getTokenTypeAsTargetLabel(b));
800 					eST.add("upperAsInt", Utils.integer(b));
801 					eST.add("rangeNumber", Utils.integer(rangeNumber));
802 					eST.add("k", Utils.integer(k));
803 					setST.add("ranges", eST);
804 					rangeNumber++;
805 				}
806 			}
807 			sST.add("k", Utils.integer(k));
808 			setST.add("ranges", sST);
809 			return setST;
810 		}
811 		Iterator<Interval> iter = iset.getIntervals().iterator();
812 		int rangeNumber = 1;
813 		while (iter.hasNext()) {
814 			Interval I = iter.next();
815 			int a = I.a;
816 			int b = I.b;
817 			ST eST;
818 			if ( a==b ) {
819 				eST = templates.getInstanceOf(testSTName);
820 				eST.add("atom", getTokenTypeAsTargetLabel(a));
821 				eST.add("atomAsInt", Utils.integer(a));
822 				//eST.add("k",Utils.integer(k));
823 			}
824 			else {
825 				eST = templates.getInstanceOf(testRangeSTName);
826 				eST.add("lower", getTokenTypeAsTargetLabel(a));
827 				eST.add("lowerAsInt", Utils.integer(a));
828 				eST.add("upper", getTokenTypeAsTargetLabel(b));
829 				eST.add("upperAsInt", Utils.integer(b));
830 				eST.add("rangeNumber", Utils.integer(rangeNumber));
831 			}
832 			eST.add("k", Utils.integer(k));
833 			setST.add("ranges", eST);
834 			rangeNumber++;
835 		}
836 		return setST;
837 	}
838 
839 	// T O K E N  D E F I N I T I O N  G E N E R A T I O N
840 
841 	/** Set attributes tokens and literals attributes in the incoming
842 	 *  code template.  This is not the token vocab interchange file, but
843 	 *  rather a list of token type ID needed by the recognizer.
844 	 */
genTokenTypeConstants(ST code)845 	protected void genTokenTypeConstants(ST code) {
846 		// make constants for the token types
847 		for (String tokenID : grammar.getTokenIDs()) {
848 			int tokenType = grammar.getTokenType(tokenID);
849 			if ( tokenType==Label.EOF ||
850 				 tokenType>=Label.MIN_TOKEN_TYPE )
851 			{
852 				// don't do FAUX labels 'cept EOF
853 				code.addAggr("tokens.{name,type}", tokenID, Utils.integer(tokenType));
854 			}
855 		}
856 	}
857 
858 	/** Generate a token names table that maps token type to a printable
859 	 *  name: either the label like INT or the literal like "begin".
860 	 */
genTokenTypeNames(ST code)861 	protected void genTokenTypeNames(ST code) {
862 		for (int t=Label.MIN_TOKEN_TYPE; t<=grammar.getMaxTokenType(); t++) {
863 			String tokenName = grammar.getTokenDisplayName(t);
864 			if ( tokenName!=null ) {
865 				tokenName=target.getTargetStringLiteralFromString(tokenName, true);
866 				code.add("tokenNames", tokenName);
867 			}
868 		}
869 	}
870 
871 	/** Get a meaningful name for a token type useful during code generation.
872 	 *  Literals without associated names are converted to the string equivalent
873 	 *  of their integer values. Used to generate x==ID and x==34 type comparisons
874 	 *  etc...  Essentially we are looking for the most obvious way to refer
875 	 *  to a token type in the generated code.  If in the lexer, return the
876 	 *  char literal translated to the target language.  For example, ttype=10
877 	 *  will yield '\n' from the getTokenDisplayName method.  That must
878 	 *  be converted to the target languages literals.  For most C-derived
879 	 *  languages no translation is needed.
880 	 */
getTokenTypeAsTargetLabel(int ttype)881 	public String getTokenTypeAsTargetLabel(int ttype) {
882 		if ( grammar.type==Grammar.LEXER ) {
883 			String name = grammar.getTokenDisplayName(ttype);
884 			return target.getTargetCharLiteralFromANTLRCharLiteral(this,name);
885 		}
886 		return target.getTokenTypeAsTargetLabel(this,ttype);
887 	}
888 
889 	/** Generate a token vocab file with all the token names/types.  For example:
890 	 *  ID=7
891 	 *  FOR=8
892 	 *  'for'=8
893 	 *
894 	 *  This is independent of the target language; used by antlr internally
895 	 */
genTokenVocabOutput()896 	protected ST genTokenVocabOutput() {
897 		ST vocabFileST = new ST(vocabFilePattern);
898 		vocabFileST.add("literals",(Object)null); // "define" literals arg
899 		vocabFileST.add("tokens",(Object)null);
900 		vocabFileST.impl.name = "vocab-file";
901 		// make constants for the token names
902 		for (String tokenID : grammar.getTokenIDs()) {
903 			int tokenType = grammar.getTokenType(tokenID);
904 			if ( tokenType>=Label.MIN_TOKEN_TYPE ) {
905 				vocabFileST.addAggr("tokens.{name,type}", tokenID, Utils.integer(tokenType));
906 			}
907 		}
908 
909 		// now dump the strings
910 		for (String literal : grammar.getStringLiterals()) {
911 			int tokenType = grammar.getTokenType(literal);
912 			if ( tokenType>=Label.MIN_TOKEN_TYPE ) {
913 				vocabFileST.addAggr("tokens.{name,type}", literal, Utils.integer(tokenType));
914 			}
915 		}
916 
917 		return vocabFileST;
918 	}
919 
translateAction(String ruleName, GrammarAST actionTree)920 	public List<? extends Object> translateAction(String ruleName,
921 								GrammarAST actionTree)
922 	{
923 		if ( actionTree.getType()==ANTLRParser.ARG_ACTION ) {
924 			return translateArgAction(ruleName, actionTree);
925 		}
926 		ActionTranslator translator = new ActionTranslator(this,ruleName,actionTree);
927 		List<Object> chunks = translator.translateToChunks();
928 		chunks = target.postProcessAction(chunks, actionTree.token);
929 		return chunks;
930 	}
931 
932 	/** Translate an action like [3,"foo",a[3]] and return a List of the
933 	 *  translated actions.  Because actions are themselves translated to a list
934 	 *  of chunks, must cat together into a ST&gt;.  Don't translate
935 	 *  to strings early as we need to eval templates in context.
936 	 */
translateArgAction(String ruleName, GrammarAST actionTree)937 	public List<ST> translateArgAction(String ruleName,
938 										   GrammarAST actionTree)
939 	{
940 		String actionText = actionTree.token.getText();
941 		List<String> args = getListOfArgumentsFromAction(actionText,',');
942 		List<ST> translatedArgs = new ArrayList<ST>();
943 		for (String arg : args) {
944 			if ( arg!=null ) {
945 				Token actionToken =
946 					new CommonToken(ANTLRParser.ACTION,arg);
947 				ActionTranslator translator =
948 					new ActionTranslator(this,ruleName,
949 											  actionToken,
950 											  actionTree.outerAltNum);
951 				List<Object> chunks = translator.translateToChunks();
952 				chunks = target.postProcessAction(chunks, actionToken);
953 				ST catST = new ST(templates, "<chunks>");
954 				catST.add("chunks", chunks);
955 				translatedArgs.add(catST);
956 			}
957 		}
958 		if ( translatedArgs.isEmpty() ) {
959 			return null;
960 		}
961 		return translatedArgs;
962 	}
963 
getListOfArgumentsFromAction(String actionText, int separatorChar)964 	public static List<String> getListOfArgumentsFromAction(String actionText,
965 															int separatorChar)
966 	{
967 		List<String> args = new ArrayList<String>();
968 		getListOfArgumentsFromAction(actionText, 0, -1, separatorChar, args);
969 		return args;
970 	}
971 
972 	/** Given an arg action like
973 	 *
974 	 *  [x, (*a).foo(21,33), 3.2+1, '\n',
975 	 *  "a,oo\nick", {bl, "fdkj"eck}, ["cat\n,", x, 43]]
976 	 *
977 	 *  convert to a list of arguments.  Allow nested square brackets etc...
978 	 *  Set separatorChar to ';' or ',' or whatever you want.
979 	 */
getListOfArgumentsFromAction(String actionText, int start, int targetChar, int separatorChar, List<String> args)980 	public static int getListOfArgumentsFromAction(String actionText,
981 												   int start,
982 												   int targetChar,
983 												   int separatorChar,
984 												   List<String> args)
985 	{
986 		if ( actionText==null ) {
987 			return -1;
988 		}
989 		actionText = actionText.replaceAll("//.*\n", "");
990 		int n = actionText.length();
991 		//System.out.println("actionText@"+start+"->"+(char)targetChar+"="+actionText.substring(start,n));
992 		int p = start;
993 		int last = p;
994 		while ( p<n && actionText.charAt(p)!=targetChar ) {
995 			int c = actionText.charAt(p);
996 			switch ( c ) {
997 				case '\'' :
998 					p++;
999 					while ( p<n && actionText.charAt(p)!='\'' ) {
1000 						if ( actionText.charAt(p)=='\\' && (p+1)<n &&
1001 							 actionText.charAt(p+1)=='\'' )
1002 						{
1003 							p++; // skip escaped quote
1004 						}
1005 						p++;
1006 					}
1007 					p++;
1008 					break;
1009 				case '"' :
1010 					p++;
1011 					while ( p<n && actionText.charAt(p)!='\"' ) {
1012 						if ( actionText.charAt(p)=='\\' && (p+1)<n &&
1013 							 actionText.charAt(p+1)=='\"' )
1014 						{
1015 							p++; // skip escaped quote
1016 						}
1017 						p++;
1018 					}
1019 					p++;
1020 					break;
1021 				case '(' :
1022 					p = getListOfArgumentsFromAction(actionText,p+1,')',separatorChar,args);
1023 					break;
1024 				case '{' :
1025 					p = getListOfArgumentsFromAction(actionText,p+1,'}',separatorChar,args);
1026 					break;
1027 				case '<' :
1028 					if ( actionText.indexOf('>',p+1)>=p ) {
1029 						// do we see a matching '>' ahead?  if so, hope it's a generic
1030 						// and not less followed by expr with greater than
1031 						p = getListOfArgumentsFromAction(actionText,p+1,'>',separatorChar,args);
1032 					}
1033 					else {
1034 						p++; // treat as normal char
1035 					}
1036 					break;
1037 				case '[' :
1038 					p = getListOfArgumentsFromAction(actionText,p+1,']',separatorChar,args);
1039 					break;
1040 				default :
1041 					if ( c==separatorChar && targetChar==-1 ) {
1042 						String arg = actionText.substring(last, p);
1043 						//System.out.println("arg="+arg);
1044 						args.add(arg.trim());
1045 						last = p+1;
1046 					}
1047 					p++;
1048 					break;
1049 			}
1050 		}
1051 		if ( targetChar==-1 && p<=n ) {
1052 			String arg = actionText.substring(last, p).trim();
1053 			//System.out.println("arg="+arg);
1054 			if ( arg.length()>0 ) {
1055 				args.add(arg.trim());
1056 			}
1057 		}
1058 		p++;
1059 		return p;
1060 	}
1061 
1062 	/** Given a template constructor action like %foo(a={...}) in
1063 	 *  an action, translate it to the appropriate template constructor
1064 	 *  from the templateLib. This translates a *piece* of the action.
1065 	 */
translateTemplateConstructor(String ruleName, int outerAltNum, Token actionToken, String templateActionText)1066 	public ST translateTemplateConstructor(String ruleName,
1067 													   int outerAltNum,
1068 													   Token actionToken,
1069 													   String templateActionText)
1070 	{
1071 		// first, parse with antlr.g
1072 		//System.out.println("translate template: "+templateActionText);
1073 		ANTLRLexer lexer = new ANTLRLexer(new ANTLRStringStream(templateActionText));
1074 		lexer.setFileName(grammar.getFileName());
1075 		ANTLRParser parser = ANTLRParser.createParser(new CommonTokenStream(lexer));
1076 		parser.setFileName(grammar.getFileName());
1077 		ANTLRParser.rewrite_template_return parseResult = null;
1078 		try {
1079 			parseResult = parser.rewrite_template();
1080 		}
1081 		catch (RecognitionException re) {
1082 			ErrorManager.grammarError(ErrorManager.MSG_INVALID_TEMPLATE_ACTION,
1083 										  grammar,
1084 										  actionToken,
1085 										  templateActionText);
1086 		}
1087 		catch (Exception tse) {
1088 			ErrorManager.internalError("can't parse template action",tse);
1089 		}
1090 		GrammarAST rewriteTree = parseResult.getTree();
1091 
1092 		// then translate via codegen.g
1093 		CodeGenTreeWalker gen = new CodeGenTreeWalker(new CommonTreeNodeStream(rewriteTree));
1094 		gen.init(grammar);
1095 		gen.setCurrentRuleName(ruleName);
1096 		gen.setOuterAltNum(outerAltNum);
1097 		ST st = null;
1098 		try {
1099 			st = gen.rewrite_template();
1100 		}
1101 		catch (RecognitionException re) {
1102 			ErrorManager.error(ErrorManager.MSG_BAD_AST_STRUCTURE,
1103 							   re);
1104 		}
1105 		return st;
1106 	}
1107 
1108 
issueInvalidScopeError(String x, String y, Rule enclosingRule, Token actionToken, int outerAltNum)1109 	public void issueInvalidScopeError(String x,
1110 									   String y,
1111 									   Rule enclosingRule,
1112 									   Token actionToken,
1113 									   int outerAltNum)
1114 	{
1115 		//System.out.println("error $"+x+"::"+y);
1116 		Rule r = grammar.getRule(x);
1117 		AttributeScope scope = grammar.getGlobalScope(x);
1118 		if ( scope==null ) {
1119 			if ( r!=null ) {
1120 				scope = r.ruleScope; // if not global, might be rule scope
1121 			}
1122 		}
1123 		if ( scope==null ) {
1124 			ErrorManager.grammarError(ErrorManager.MSG_UNKNOWN_DYNAMIC_SCOPE,
1125 										  grammar,
1126 										  actionToken,
1127 										  x);
1128 		}
1129 		else if ( scope.getAttribute(y)==null ) {
1130 			ErrorManager.grammarError(ErrorManager.MSG_UNKNOWN_DYNAMIC_SCOPE_ATTRIBUTE,
1131 										  grammar,
1132 										  actionToken,
1133 										  x,
1134 										  y);
1135 		}
1136 	}
1137 
issueInvalidAttributeError(String x, String y, Rule enclosingRule, Token actionToken, int outerAltNum)1138 	public void issueInvalidAttributeError(String x,
1139 										   String y,
1140 										   Rule enclosingRule,
1141 										   Token actionToken,
1142 										   int outerAltNum)
1143 	{
1144 		//System.out.println("error $"+x+"."+y);
1145 		if ( enclosingRule==null ) {
1146 			// action not in a rule
1147 			ErrorManager.grammarError(ErrorManager.MSG_ATTRIBUTE_REF_NOT_IN_RULE,
1148 										  grammar,
1149 										  actionToken,
1150 										  x,
1151 										  y);
1152 			return;
1153 		}
1154 
1155 		// action is in a rule
1156 		Grammar.LabelElementPair label = enclosingRule.getRuleLabel(x);
1157 
1158 		if ( label!=null || enclosingRule.getRuleRefsInAlt(x, outerAltNum)!=null ) {
1159 			// $rulelabel.attr or $ruleref.attr; must be unknown attr
1160 			String refdRuleName = x;
1161 			if ( label!=null ) {
1162 				refdRuleName = enclosingRule.getRuleLabel(x).referencedRuleName;
1163 			}
1164 			Rule refdRule = grammar.getRule(refdRuleName);
1165 			AttributeScope scope = refdRule.getAttributeScope(y);
1166 			if ( scope==null ) {
1167 				ErrorManager.grammarError(ErrorManager.MSG_UNKNOWN_RULE_ATTRIBUTE,
1168 										  grammar,
1169 										  actionToken,
1170 										  refdRuleName,
1171 										  y);
1172 			}
1173 			else if ( scope.isParameterScope ) {
1174 				ErrorManager.grammarError(ErrorManager.MSG_INVALID_RULE_PARAMETER_REF,
1175 										  grammar,
1176 										  actionToken,
1177 										  refdRuleName,
1178 										  y);
1179 			}
1180 			else if ( scope.isDynamicRuleScope ) {
1181 				ErrorManager.grammarError(ErrorManager.MSG_INVALID_RULE_SCOPE_ATTRIBUTE_REF,
1182 										  grammar,
1183 										  actionToken,
1184 										  refdRuleName,
1185 										  y);
1186 			}
1187 		}
1188 
1189 	}
1190 
issueInvalidAttributeError(String x, Rule enclosingRule, Token actionToken, int outerAltNum)1191 	public void issueInvalidAttributeError(String x,
1192 										   Rule enclosingRule,
1193 										   Token actionToken,
1194 										   int outerAltNum)
1195 	{
1196 		//System.out.println("error $"+x);
1197 		if ( enclosingRule==null ) {
1198 			// action not in a rule
1199 			ErrorManager.grammarError(ErrorManager.MSG_ATTRIBUTE_REF_NOT_IN_RULE,
1200 										  grammar,
1201 										  actionToken,
1202 										  x);
1203 			return;
1204 		}
1205 
1206 		// action is in a rule
1207 		Grammar.LabelElementPair label = enclosingRule.getRuleLabel(x);
1208 		AttributeScope scope = enclosingRule.getAttributeScope(x);
1209 
1210 		if ( label!=null ||
1211 			 enclosingRule.getRuleRefsInAlt(x, outerAltNum)!=null ||
1212 			 enclosingRule.name.equals(x) )
1213 		{
1214 			ErrorManager.grammarError(ErrorManager.MSG_ISOLATED_RULE_SCOPE,
1215 										  grammar,
1216 										  actionToken,
1217 										  x);
1218 		}
1219 		else if ( scope!=null && scope.isDynamicRuleScope ) {
1220 			ErrorManager.grammarError(ErrorManager.MSG_ISOLATED_RULE_ATTRIBUTE,
1221 										  grammar,
1222 										  actionToken,
1223 										  x);
1224 		}
1225 		else {
1226 			ErrorManager.grammarError(ErrorManager.MSG_UNKNOWN_SIMPLE_ATTRIBUTE,
1227 									  grammar,
1228 									  actionToken,
1229 									  x);
1230 		}
1231 	}
1232 
1233 	// M I S C
1234 
getTemplates()1235 	public STGroup getTemplates() {
1236 		return templates;
1237 	}
1238 
getBaseTemplates()1239 	public STGroup getBaseTemplates() {
1240 		return baseTemplates;
1241 	}
1242 
setDebug(boolean debug)1243 	public void setDebug(boolean debug) {
1244 		this.debug = debug;
1245 	}
1246 
setTrace(boolean trace)1247 	public void setTrace(boolean trace) {
1248 		this.trace = trace;
1249 	}
1250 
setProfile(boolean profile)1251 	public void setProfile(boolean profile) {
1252 		this.profile = profile;
1253 		if ( profile ) {
1254 			setDebug(true); // requires debug events
1255 		}
1256 	}
1257 
getRecognizerST()1258 	public ST getRecognizerST() {
1259 		return outputFileST;
1260 	}
1261 
1262 	/** Generate TParser.java and TLexer.java from T.g if combined, else
1263 	 *  just use T.java as output regardless of type.
1264 	 */
getRecognizerFileName(String name, int type)1265 	public String getRecognizerFileName(String name, int type) {
1266 		ST extST = templates.getInstanceOf("codeFileExtension");
1267 		String recognizerName = grammar.getRecognizerName();
1268 		return recognizerName+extST.render();
1269 		/*
1270 		String suffix = "";
1271 		if ( type==Grammar.COMBINED ||
1272 			 (type==Grammar.LEXER && !grammar.implicitLexer) )
1273 		{
1274 			suffix = Grammar.grammarTypeToFileNameSuffix[type];
1275 		}
1276 		return name+suffix+extST.toString();
1277 		*/
1278 	}
1279 
1280 	/** What is the name of the vocab file generated for this grammar?
1281 	 *  Returns null if no .tokens file should be generated.
1282 	 */
getVocabFileName()1283 	public String getVocabFileName() {
1284 		if ( grammar.isBuiltFromString() ) {
1285 			return null;
1286 		}
1287 		return grammar.name+VOCAB_FILE_EXTENSION;
1288 	}
1289 
write(ST code, String fileName)1290 	public void write(ST code, String fileName) throws IOException {
1291 		//long start = System.currentTimeMillis();
1292 		Writer w = tool.getOutputFile(grammar, fileName);
1293 		// Write the output to a StringWriter
1294 		STWriter wr = new AutoIndentWriter(w);
1295 		wr.setLineWidth(lineWidth);
1296 		code.write(wr);
1297 		w.close();
1298 		//long stop = System.currentTimeMillis();
1299 		//System.out.println("render time for "+fileName+": "+(int)(stop-start)+"ms");
1300 	}
1301 
1302 	/** You can generate a switch rather than if-then-else for a DFA state
1303 	 *  if there are no semantic predicates and the number of edge label
1304 	 *  values is small enough; e.g., don't generate a switch for a state
1305 	 *  containing an edge label such as 20..52330 (the resulting byte codes
1306 	 *  would overflow the method 65k limit probably).
1307 	 */
canGenerateSwitch(DFAState s)1308 	protected boolean canGenerateSwitch(DFAState s) {
1309 		if ( !GENERATE_SWITCHES_WHEN_POSSIBLE ) {
1310 			return false;
1311 		}
1312 		int size = 0;
1313 		for (int i = 0; i < s.getNumberOfTransitions(); i++) {
1314 			Transition edge = s.transition(i);
1315 			if ( edge.label.isSemanticPredicate() ) {
1316 				return false;
1317 			}
1318 			// can't do a switch if the edges are going to require predicates
1319 			if ( edge.label.getAtom()==Label.EOT ) {
1320 				int EOTPredicts = ((DFAState)edge.target).getUniquelyPredictedAlt();
1321 				if ( EOTPredicts==NFA.INVALID_ALT_NUMBER ) {
1322 					// EOT target has to be a predicate then; no unique alt
1323 					return false;
1324 				}
1325 			}
1326 			// if target is a state with gated preds, we need to use preds on
1327 			// this edge then to reach it.
1328 			if ( ((DFAState)edge.target).getGatedPredicatesInNFAConfigurations()!=null ) {
1329 				return false;
1330 			}
1331 			size += edge.label.getSet().size();
1332 		}
1333 		if ( s.getNumberOfTransitions()<MIN_SWITCH_ALTS ||
1334 			 size>MAX_SWITCH_CASE_LABELS ) {
1335 			return false;
1336 		}
1337 		return true;
1338 	}
1339 
1340 	/** Create a label to track a token / rule reference's result.
1341 	 *  Technically, this is a place where I break model-view separation
1342 	 *  as I am creating a variable name that could be invalid in a
1343 	 *  target language, however, label ::= &lt;ID&gt;&lt;INT&gt; is probably ok in
1344 	 *  all languages we care about.
1345 	 */
createUniqueLabel(String name)1346 	public String createUniqueLabel(String name) {
1347 		return new StringBuffer()
1348 			.append(name).append(uniqueLabelNumber++).toString();
1349 	}
1350 }
1351