1// Copyright 2017 The Bazel Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package syntax_test 6 7import ( 8 "bufio" 9 "bytes" 10 "fmt" 11 "go/build" 12 "io/ioutil" 13 "path/filepath" 14 "reflect" 15 "strings" 16 "testing" 17 18 "go.starlark.net/internal/chunkedfile" 19 "go.starlark.net/starlarktest" 20 "go.starlark.net/syntax" 21) 22 23func TestExprParseTrees(t *testing.T) { 24 for _, test := range []struct { 25 input, want string 26 }{ 27 {`print(1)`, 28 `(CallExpr Fn=print Args=(1))`}, 29 {"print(1)\n", 30 `(CallExpr Fn=print Args=(1))`}, 31 {`x + 1`, 32 `(BinaryExpr X=x Op=+ Y=1)`}, 33 {`[x for x in y]`, 34 `(Comprehension Body=x Clauses=((ForClause Vars=x X=y)))`}, 35 {`[x for x in (a if b else c)]`, 36 `(Comprehension Body=x Clauses=((ForClause Vars=x X=(ParenExpr X=(CondExpr Cond=b True=a False=c)))))`}, 37 {`x[i].f(42)`, 38 `(CallExpr Fn=(DotExpr X=(IndexExpr X=x Y=i) Name=f) Args=(42))`}, 39 {`x.f()`, 40 `(CallExpr Fn=(DotExpr X=x Name=f))`}, 41 {`x+y*z`, 42 `(BinaryExpr X=x Op=+ Y=(BinaryExpr X=y Op=* Y=z))`}, 43 {`x%y-z`, 44 `(BinaryExpr X=(BinaryExpr X=x Op=% Y=y) Op=- Y=z)`}, 45 {`a + b not in c`, 46 `(BinaryExpr X=(BinaryExpr X=a Op=+ Y=b) Op=not in Y=c)`}, 47 {`lambda x, *args, **kwargs: None`, 48 `(LambdaExpr Params=(x (UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)) Body=None)`}, 49 {`{"one": 1}`, 50 `(DictExpr List=((DictEntry Key="one" Value=1)))`}, 51 {`a[i]`, 52 `(IndexExpr X=a Y=i)`}, 53 {`a[i:]`, 54 `(SliceExpr X=a Lo=i)`}, 55 {`a[:j]`, 56 `(SliceExpr X=a Hi=j)`}, 57 {`a[::]`, 58 `(SliceExpr X=a)`}, 59 {`a[::k]`, 60 `(SliceExpr X=a Step=k)`}, 61 {`[]`, 62 `(ListExpr)`}, 63 {`[1]`, 64 `(ListExpr List=(1))`}, 65 {`[1,]`, 66 `(ListExpr List=(1))`}, 67 {`[1, 2]`, 68 `(ListExpr List=(1 2))`}, 69 {`()`, 70 `(TupleExpr)`}, 71 {`(4,)`, 72 `(ParenExpr X=(TupleExpr List=(4)))`}, 73 {`(4)`, 74 `(ParenExpr X=4)`}, 75 {`(4, 5)`, 76 `(ParenExpr X=(TupleExpr List=(4 5)))`}, 77 {`1, 2, 3`, 78 `(TupleExpr List=(1 2 3))`}, 79 {`1, 2,`, 80 `unparenthesized tuple with trailing comma`}, 81 {`{}`, 82 `(DictExpr)`}, 83 {`{"a": 1}`, 84 `(DictExpr List=((DictEntry Key="a" Value=1)))`}, 85 {`{"a": 1,}`, 86 `(DictExpr List=((DictEntry Key="a" Value=1)))`}, 87 {`{"a": 1, "b": 2}`, 88 `(DictExpr List=((DictEntry Key="a" Value=1) (DictEntry Key="b" Value=2)))`}, 89 {`{x: y for (x, y) in z}`, 90 `(Comprehension Curly Body=(DictEntry Key=x Value=y) Clauses=((ForClause Vars=(ParenExpr X=(TupleExpr List=(x y))) X=z)))`}, 91 {`{x: y for a in b if c}`, 92 `(Comprehension Curly Body=(DictEntry Key=x Value=y) Clauses=((ForClause Vars=a X=b) (IfClause Cond=c)))`}, 93 {`-1 + +2`, 94 `(BinaryExpr X=(UnaryExpr Op=- X=1) Op=+ Y=(UnaryExpr Op=+ X=2))`}, 95 {`"foo" + "bar"`, 96 `(BinaryExpr X="foo" Op=+ Y="bar")`}, 97 {`-1 * 2`, // prec(unary -) > prec(binary *) 98 `(BinaryExpr X=(UnaryExpr Op=- X=1) Op=* Y=2)`}, 99 {`-x[i]`, // prec(unary -) < prec(x[i]) 100 `(UnaryExpr Op=- X=(IndexExpr X=x Y=i))`}, 101 {`a | b & c | d`, // prec(|) < prec(&) 102 `(BinaryExpr X=(BinaryExpr X=a Op=| Y=(BinaryExpr X=b Op=& Y=c)) Op=| Y=d)`}, 103 {`a or b and c or d`, 104 `(BinaryExpr X=(BinaryExpr X=a Op=or Y=(BinaryExpr X=b Op=and Y=c)) Op=or Y=d)`}, 105 {`a and b or c and d`, 106 `(BinaryExpr X=(BinaryExpr X=a Op=and Y=b) Op=or Y=(BinaryExpr X=c Op=and Y=d))`}, 107 {`f(1, x=y)`, 108 `(CallExpr Fn=f Args=(1 (BinaryExpr X=x Op== Y=y)))`}, 109 {`f(*args, **kwargs)`, 110 `(CallExpr Fn=f Args=((UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)))`}, 111 {`lambda *args, *, x=1, **kwargs: 0`, 112 `(LambdaExpr Params=((UnaryExpr Op=* X=args) (UnaryExpr Op=*) (BinaryExpr X=x Op== Y=1) (UnaryExpr Op=** X=kwargs)) Body=0)`}, 113 {`lambda *, a, *b: 0`, 114 `(LambdaExpr Params=((UnaryExpr Op=*) a (UnaryExpr Op=* X=b)) Body=0)`}, 115 {`a if b else c`, 116 `(CondExpr Cond=b True=a False=c)`}, 117 {`a and not b`, 118 `(BinaryExpr X=a Op=and Y=(UnaryExpr Op=not X=b))`}, 119 {`[e for x in y if cond1 if cond2]`, 120 `(Comprehension Body=e Clauses=((ForClause Vars=x X=y) (IfClause Cond=cond1) (IfClause Cond=cond2)))`}, // github.com/google/skylark/issues/53 121 } { 122 e, err := syntax.ParseExpr("foo.star", test.input, 0) 123 var got string 124 if err != nil { 125 got = stripPos(err) 126 } else { 127 got = treeString(e) 128 } 129 if test.want != got { 130 t.Errorf("parse `%s` = %s, want %s", test.input, got, test.want) 131 } 132 } 133} 134 135func TestStmtParseTrees(t *testing.T) { 136 for _, test := range []struct { 137 input, want string 138 }{ 139 {`print(1)`, 140 `(ExprStmt X=(CallExpr Fn=print Args=(1)))`}, 141 {`return 1, 2`, 142 `(ReturnStmt Result=(TupleExpr List=(1 2)))`}, 143 {`return`, 144 `(ReturnStmt)`}, 145 {`for i in "abc": break`, 146 `(ForStmt Vars=i X="abc" Body=((BranchStmt Token=break)))`}, 147 {`for i in "abc": continue`, 148 `(ForStmt Vars=i X="abc" Body=((BranchStmt Token=continue)))`}, 149 {`for x, y in z: pass`, 150 `(ForStmt Vars=(TupleExpr List=(x y)) X=z Body=((BranchStmt Token=pass)))`}, 151 {`if True: pass`, 152 `(IfStmt Cond=True True=((BranchStmt Token=pass)))`}, 153 {`if True: break`, 154 `(IfStmt Cond=True True=((BranchStmt Token=break)))`}, 155 {`if True: continue`, 156 `(IfStmt Cond=True True=((BranchStmt Token=continue)))`}, 157 {`if True: pass 158else: 159 pass`, 160 `(IfStmt Cond=True True=((BranchStmt Token=pass)) False=((BranchStmt Token=pass)))`}, 161 {"if a: pass\nelif b: pass\nelse: pass", 162 `(IfStmt Cond=a True=((BranchStmt Token=pass)) False=((IfStmt Cond=b True=((BranchStmt Token=pass)) False=((BranchStmt Token=pass)))))`}, 163 {`x, y = 1, 2`, 164 `(AssignStmt Op== LHS=(TupleExpr List=(x y)) RHS=(TupleExpr List=(1 2)))`}, 165 {`x[i] = 1`, 166 `(AssignStmt Op== LHS=(IndexExpr X=x Y=i) RHS=1)`}, 167 {`x.f = 1`, 168 `(AssignStmt Op== LHS=(DotExpr X=x Name=f) RHS=1)`}, 169 {`(x, y) = 1`, 170 `(AssignStmt Op== LHS=(ParenExpr X=(TupleExpr List=(x y))) RHS=1)`}, 171 {`load("", "a", b="c")`, 172 `(LoadStmt Module="" From=(a c) To=(a b))`}, 173 {`if True: load("", "a", b="c")`, // load needn't be at toplevel 174 `(IfStmt Cond=True True=((LoadStmt Module="" From=(a c) To=(a b))))`}, 175 {`def f(x, *args, **kwargs): 176 pass`, 177 `(DefStmt Name=f Params=(x (UnaryExpr Op=* X=args) (UnaryExpr Op=** X=kwargs)) Body=((BranchStmt Token=pass)))`}, 178 {`def f(**kwargs, *args): pass`, 179 `(DefStmt Name=f Params=((UnaryExpr Op=** X=kwargs) (UnaryExpr Op=* X=args)) Body=((BranchStmt Token=pass)))`}, 180 {`def f(a, b, c=d): pass`, 181 `(DefStmt Name=f Params=(a b (BinaryExpr X=c Op== Y=d)) Body=((BranchStmt Token=pass)))`}, 182 {`def f(a, b=c, d): pass`, 183 `(DefStmt Name=f Params=(a (BinaryExpr X=b Op== Y=c) d) Body=((BranchStmt Token=pass)))`}, // TODO(adonovan): fix this 184 {`def f(): 185 def g(): 186 pass 187 pass 188def h(): 189 pass`, 190 `(DefStmt Name=f Body=((DefStmt Name=g Body=((BranchStmt Token=pass))) (BranchStmt Token=pass)))`}, 191 {"f();g()", 192 `(ExprStmt X=(CallExpr Fn=f))`}, 193 {"f();", 194 `(ExprStmt X=(CallExpr Fn=f))`}, 195 {"f();g()\n", 196 `(ExprStmt X=(CallExpr Fn=f))`}, 197 {"f();\n", 198 `(ExprStmt X=(CallExpr Fn=f))`}, 199 } { 200 f, err := syntax.Parse("foo.star", test.input, 0) 201 if err != nil { 202 t.Errorf("parse `%s` failed: %v", test.input, stripPos(err)) 203 continue 204 } 205 if got := treeString(f.Stmts[0]); test.want != got { 206 t.Errorf("parse `%s` = %s, want %s", test.input, got, test.want) 207 } 208 } 209} 210 211// TestFileParseTrees tests sequences of statements, and particularly 212// handling of indentation, newlines, line continuations, and blank lines. 213func TestFileParseTrees(t *testing.T) { 214 for _, test := range []struct { 215 input, want string 216 }{ 217 {`x = 1 218print(x)`, 219 `(AssignStmt Op== LHS=x RHS=1) 220(ExprStmt X=(CallExpr Fn=print Args=(x)))`}, 221 {"if cond:\n\tpass", 222 `(IfStmt Cond=cond True=((BranchStmt Token=pass)))`}, 223 {"if cond:\n\tpass\nelse:\n\tpass", 224 `(IfStmt Cond=cond True=((BranchStmt Token=pass)) False=((BranchStmt Token=pass)))`}, 225 {`def f(): 226 pass 227pass 228 229pass`, 230 `(DefStmt Name=f Body=((BranchStmt Token=pass))) 231(BranchStmt Token=pass) 232(BranchStmt Token=pass)`}, 233 {`pass; pass`, 234 `(BranchStmt Token=pass) 235(BranchStmt Token=pass)`}, 236 {"pass\npass", 237 `(BranchStmt Token=pass) 238(BranchStmt Token=pass)`}, 239 {"pass\n\npass", 240 `(BranchStmt Token=pass) 241(BranchStmt Token=pass)`}, 242 {`x = (1 + 2432)`, 244 `(AssignStmt Op== LHS=x RHS=(ParenExpr X=(BinaryExpr X=1 Op=+ Y=2)))`}, 245 {`x = 1 \ 246+ 2`, 247 `(AssignStmt Op== LHS=x RHS=(BinaryExpr X=1 Op=+ Y=2))`}, 248 } { 249 f, err := syntax.Parse("foo.star", test.input, 0) 250 if err != nil { 251 t.Errorf("parse `%s` failed: %v", test.input, stripPos(err)) 252 continue 253 } 254 var buf bytes.Buffer 255 for i, stmt := range f.Stmts { 256 if i > 0 { 257 buf.WriteByte('\n') 258 } 259 writeTree(&buf, reflect.ValueOf(stmt)) 260 } 261 if got := buf.String(); test.want != got { 262 t.Errorf("parse `%s` = %s, want %s", test.input, got, test.want) 263 } 264 } 265} 266 267// TestCompoundStmt tests handling of REPL-style compound statements. 268func TestCompoundStmt(t *testing.T) { 269 for _, test := range []struct { 270 input, want string 271 }{ 272 // blank lines 273 {"\n", 274 ``}, 275 {" \n", 276 ``}, 277 {"# comment\n", 278 ``}, 279 // simple statement 280 {"1\n", 281 `(ExprStmt X=1)`}, 282 {"print(1)\n", 283 `(ExprStmt X=(CallExpr Fn=print Args=(1)))`}, 284 {"1;2;3;\n", 285 `(ExprStmt X=1)(ExprStmt X=2)(ExprStmt X=3)`}, 286 {"f();g()\n", 287 `(ExprStmt X=(CallExpr Fn=f))(ExprStmt X=(CallExpr Fn=g))`}, 288 {"f();\n", 289 `(ExprStmt X=(CallExpr Fn=f))`}, 290 {"f(\n\n\n\n\n\n\n)\n", 291 `(ExprStmt X=(CallExpr Fn=f))`}, 292 // complex statements 293 {"def f():\n pass\n\n", 294 `(DefStmt Name=f Body=((BranchStmt Token=pass)))`}, 295 {"if cond:\n pass\n\n", 296 `(IfStmt Cond=cond True=((BranchStmt Token=pass)))`}, 297 // Even as a 1-liner, the following blank line is required. 298 {"if cond: pass\n\n", 299 `(IfStmt Cond=cond True=((BranchStmt Token=pass)))`}, 300 // github.com/google/starlark-go/issues/121 301 {"a; b; c\n", 302 `(ExprStmt X=a)(ExprStmt X=b)(ExprStmt X=c)`}, 303 {"a; b c\n", 304 `invalid syntax`}, 305 } { 306 307 // Fake readline input from string. 308 // The ! suffix, which would cause a parse error, 309 // tests that the parser doesn't read more than necessary. 310 sc := bufio.NewScanner(strings.NewReader(test.input + "!")) 311 readline := func() ([]byte, error) { 312 if sc.Scan() { 313 return []byte(sc.Text() + "\n"), nil 314 } 315 return nil, sc.Err() 316 } 317 318 var got string 319 f, err := syntax.ParseCompoundStmt("foo.star", readline) 320 if err != nil { 321 got = stripPos(err) 322 } else { 323 for _, stmt := range f.Stmts { 324 got += treeString(stmt) 325 } 326 } 327 if test.want != got { 328 t.Errorf("parse `%s` = %s, want %s", test.input, got, test.want) 329 } 330 } 331} 332 333func stripPos(err error) string { 334 s := err.Error() 335 if i := strings.Index(s, ": "); i >= 0 { 336 s = s[i+len(": "):] // strip file:line:col 337 } 338 return s 339} 340 341// treeString prints a syntax node as a parenthesized tree. 342// Idents are printed as foo and Literals as "foo" or 42. 343// Structs are printed as (type name=value ...). 344// Only non-empty fields are shown. 345func treeString(n syntax.Node) string { 346 var buf bytes.Buffer 347 writeTree(&buf, reflect.ValueOf(n)) 348 return buf.String() 349} 350 351func writeTree(out *bytes.Buffer, x reflect.Value) { 352 switch x.Kind() { 353 case reflect.String, reflect.Int, reflect.Bool: 354 fmt.Fprintf(out, "%v", x.Interface()) 355 case reflect.Ptr, reflect.Interface: 356 if elem := x.Elem(); elem.Kind() == 0 { 357 out.WriteString("nil") 358 } else { 359 writeTree(out, elem) 360 } 361 case reflect.Struct: 362 switch v := x.Interface().(type) { 363 case syntax.Literal: 364 switch v.Token { 365 case syntax.STRING: 366 fmt.Fprintf(out, "%q", v.Value) 367 case syntax.BYTES: 368 fmt.Fprintf(out, "b%q", v.Value) 369 case syntax.INT: 370 fmt.Fprintf(out, "%d", v.Value) 371 } 372 return 373 case syntax.Ident: 374 out.WriteString(v.Name) 375 return 376 } 377 fmt.Fprintf(out, "(%s", strings.TrimPrefix(x.Type().String(), "syntax.")) 378 for i, n := 0, x.NumField(); i < n; i++ { 379 f := x.Field(i) 380 if f.Type() == reflect.TypeOf(syntax.Position{}) { 381 continue // skip positions 382 } 383 name := x.Type().Field(i).Name 384 if name == "commentsRef" { 385 continue // skip comments fields 386 } 387 if f.Type() == reflect.TypeOf(syntax.Token(0)) { 388 fmt.Fprintf(out, " %s=%s", name, f.Interface()) 389 continue 390 } 391 392 switch f.Kind() { 393 case reflect.Slice: 394 if n := f.Len(); n > 0 { 395 fmt.Fprintf(out, " %s=(", name) 396 for i := 0; i < n; i++ { 397 if i > 0 { 398 out.WriteByte(' ') 399 } 400 writeTree(out, f.Index(i)) 401 } 402 out.WriteByte(')') 403 } 404 continue 405 case reflect.Ptr, reflect.Interface: 406 if f.IsNil() { 407 continue 408 } 409 case reflect.Int: 410 if f.Int() != 0 { 411 fmt.Fprintf(out, " %s=%d", name, f.Int()) 412 } 413 continue 414 case reflect.Bool: 415 if f.Bool() { 416 fmt.Fprintf(out, " %s", name) 417 } 418 continue 419 } 420 fmt.Fprintf(out, " %s=", name) 421 writeTree(out, f) 422 } 423 fmt.Fprintf(out, ")") 424 default: 425 fmt.Fprintf(out, "%T", x.Interface()) 426 } 427} 428 429func TestParseErrors(t *testing.T) { 430 filename := starlarktest.DataFile("syntax", "testdata/errors.star") 431 for _, chunk := range chunkedfile.Read(filename, t) { 432 _, err := syntax.Parse(filename, chunk.Source, 0) 433 switch err := err.(type) { 434 case nil: 435 // ok 436 case syntax.Error: 437 chunk.GotError(int(err.Pos.Line), err.Msg) 438 default: 439 t.Error(err) 440 } 441 chunk.Done() 442 } 443} 444 445func TestFilePortion(t *testing.T) { 446 // Imagine that the Starlark file or expression print(x.f) is extracted 447 // from the middle of a file in some hypothetical template language; 448 // see https://github.com/google/starlark-go/issues/346. For example: 449 // -- 450 // {{loop x seq}} 451 // {{print(x.f)}} 452 // {{end}} 453 // -- 454 fp := syntax.FilePortion{Content: []byte("print(x.f)"), FirstLine: 2, FirstCol: 4} 455 file, err := syntax.Parse("foo.template", fp, 0) 456 if err != nil { 457 t.Fatal(err) 458 } 459 span := fmt.Sprint(file.Stmts[0].Span()) 460 want := "foo.template:2:4 foo.template:2:14" 461 if span != want { 462 t.Errorf("wrong span: got %q, want %q", span, want) 463 } 464} 465 466// dataFile is the same as starlarktest.DataFile. 467// We make a copy to avoid a dependency cycle. 468var dataFile = func(pkgdir, filename string) string { 469 return filepath.Join(build.Default.GOPATH, "src/go.starlark.net", pkgdir, filename) 470} 471 472func BenchmarkParse(b *testing.B) { 473 filename := dataFile("syntax", "testdata/scan.star") 474 b.StopTimer() 475 data, err := ioutil.ReadFile(filename) 476 if err != nil { 477 b.Fatal(err) 478 } 479 b.StartTimer() 480 481 for i := 0; i < b.N; i++ { 482 _, err := syntax.Parse(filename, data, 0) 483 if err != nil { 484 b.Fatal(err) 485 } 486 } 487} 488