1 /*
2 * ECMA Test 262 Runner for QuickJS
3 *
4 * Copyright (c) 2017-2020 Fabrice Bellard
5 * Copyright (c) 2017-2020 Charlie Gordon
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a copy
8 * of this software and associated documentation files (the "Software"), to deal
9 * in the Software without restriction, including without limitation the rights
10 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 * copies of the Software, and to permit persons to whom the Software is
12 * furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice shall be included in
15 * all copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23 * THE SOFTWARE.
24 */
25 #include <stdlib.h>
26 #include <stdio.h>
27 #include <stdarg.h>
28 #include <inttypes.h>
29 #include <string.h>
30 #include <assert.h>
31 #include <ctype.h>
32 #include <unistd.h>
33 #include <errno.h>
34 #include <time.h>
35 #include <dirent.h>
36 #include <ftw.h>
37
38 #include "cutils.h"
39 #include "list.h"
40 #include "quickjs-libc.h"
41
42 /* enable test262 thread support to test SharedArrayBuffer and Atomics */
43 #define CONFIG_AGENT
44
45 #define CMD_NAME "run-test262"
46
47 typedef struct namelist_t {
48 char **array;
49 int count;
50 int size;
51 unsigned int sorted : 1;
52 } namelist_t;
53
54 namelist_t test_list;
55 namelist_t exclude_list;
56 namelist_t exclude_dir_list;
57
58 FILE *outfile;
59 enum test_mode_t {
60 TEST_DEFAULT_NOSTRICT, /* run tests as nostrict unless test is flagged as strictonly */
61 TEST_DEFAULT_STRICT, /* run tests as strict unless test is flagged as nostrict */
62 TEST_NOSTRICT, /* run tests as nostrict, skip strictonly tests */
63 TEST_STRICT, /* run tests as strict, skip nostrict tests */
64 TEST_ALL, /* run tests in both strict and nostrict, unless restricted by spec */
65 } test_mode = TEST_DEFAULT_NOSTRICT;
66 int skip_async;
67 int skip_module;
68 int new_style;
69 int dump_memory;
70 int stats_count;
71 JSMemoryUsage stats_all, stats_avg, stats_min, stats_max;
72 char *stats_min_filename;
73 char *stats_max_filename;
74 int verbose;
75 char *harness_dir;
76 char *harness_exclude;
77 char *harness_features;
78 char *harness_skip_features;
79 char *error_filename;
80 char *error_file;
81 FILE *error_out;
82 char *report_filename;
83 int update_errors;
84 int test_count, test_failed, test_index, test_skipped, test_excluded;
85 int new_errors, changed_errors, fixed_errors;
86 int async_done;
87
88 void warning(const char *, ...) __attribute__((__format__(__printf__, 1, 2)));
89 void fatal(int, const char *, ...) __attribute__((__format__(__printf__, 2, 3)));
90
warning(const char * fmt,...)91 void warning(const char *fmt, ...)
92 {
93 va_list ap;
94
95 fflush(stdout);
96 fprintf(stderr, "%s: ", CMD_NAME);
97 va_start(ap, fmt);
98 vfprintf(stderr, fmt, ap);
99 va_end(ap);
100 fputc('\n', stderr);
101 }
102
fatal(int errcode,const char * fmt,...)103 void fatal(int errcode, const char *fmt, ...)
104 {
105 va_list ap;
106
107 fflush(stdout);
108 fprintf(stderr, "%s: ", CMD_NAME);
109 va_start(ap, fmt);
110 vfprintf(stderr, fmt, ap);
111 va_end(ap);
112 fputc('\n', stderr);
113
114 exit(errcode);
115 }
116
perror_exit(int errcode,const char * s)117 void perror_exit(int errcode, const char *s)
118 {
119 fflush(stdout);
120 fprintf(stderr, "%s: ", CMD_NAME);
121 perror(s);
122 exit(errcode);
123 }
124
strdup_len(const char * str,int len)125 char *strdup_len(const char *str, int len)
126 {
127 char *p = malloc(len + 1);
128 memcpy(p, str, len);
129 p[len] = '\0';
130 return p;
131 }
132
str_equal(const char * a,const char * b)133 static inline int str_equal(const char *a, const char *b) {
134 return !strcmp(a, b);
135 }
136
str_append(char ** pp,const char * sep,const char * str)137 char *str_append(char **pp, const char *sep, const char *str) {
138 char *res, *p;
139 size_t len = 0;
140 p = *pp;
141 if (p) {
142 len = strlen(p) + strlen(sep);
143 }
144 res = malloc(len + strlen(str) + 1);
145 if (p) {
146 strcpy(res, p);
147 strcat(res, sep);
148 }
149 strcpy(res + len, str);
150 free(p);
151 return *pp = res;
152 }
153
str_strip(char * p)154 char *str_strip(char *p)
155 {
156 size_t len = strlen(p);
157 while (len > 0 && isspace((unsigned char)p[len - 1]))
158 p[--len] = '\0';
159 while (isspace((unsigned char)*p))
160 p++;
161 return p;
162 }
163
has_prefix(const char * str,const char * prefix)164 int has_prefix(const char *str, const char *prefix)
165 {
166 return !strncmp(str, prefix, strlen(prefix));
167 }
168
skip_prefix(const char * str,const char * prefix)169 char *skip_prefix(const char *str, const char *prefix)
170 {
171 int i;
172 for (i = 0;; i++) {
173 if (prefix[i] == '\0') { /* skip the prefix */
174 str += i;
175 break;
176 }
177 if (str[i] != prefix[i])
178 break;
179 }
180 return (char *)str;
181 }
182
get_basename(const char * filename)183 char *get_basename(const char *filename)
184 {
185 char *p;
186
187 p = strrchr(filename, '/');
188 if (!p)
189 return NULL;
190 return strdup_len(filename, p - filename);
191 }
192
compose_path(const char * path,const char * name)193 char *compose_path(const char *path, const char *name)
194 {
195 int path_len, name_len;
196 char *d, *q;
197
198 if (!path || path[0] == '\0' || *name == '/') {
199 d = strdup(name);
200 } else {
201 path_len = strlen(path);
202 name_len = strlen(name);
203 d = malloc(path_len + 1 + name_len + 1);
204 if (d) {
205 q = d;
206 memcpy(q, path, path_len);
207 q += path_len;
208 if (path[path_len - 1] != '/')
209 *q++ = '/';
210 memcpy(q, name, name_len + 1);
211 }
212 }
213 return d;
214 }
215
namelist_cmp(const char * a,const char * b)216 int namelist_cmp(const char *a, const char *b)
217 {
218 /* compare strings in modified lexicographical order */
219 for (;;) {
220 int ca = (unsigned char)*a++;
221 int cb = (unsigned char)*b++;
222 if (isdigit(ca) && isdigit(cb)) {
223 int na = ca - '0';
224 int nb = cb - '0';
225 while (isdigit(ca = (unsigned char)*a++))
226 na = na * 10 + ca - '0';
227 while (isdigit(cb = (unsigned char)*b++))
228 nb = nb * 10 + cb - '0';
229 if (na < nb)
230 return -1;
231 if (na > nb)
232 return +1;
233 }
234 if (ca < cb)
235 return -1;
236 if (ca > cb)
237 return +1;
238 if (ca == '\0')
239 return 0;
240 }
241 }
242
namelist_cmp_indirect(const void * a,const void * b)243 int namelist_cmp_indirect(const void *a, const void *b)
244 {
245 return namelist_cmp(*(const char **)a, *(const char **)b);
246 }
247
namelist_sort(namelist_t * lp)248 void namelist_sort(namelist_t *lp)
249 {
250 int i, count;
251 if (lp->count > 1) {
252 qsort(lp->array, lp->count, sizeof(*lp->array), namelist_cmp_indirect);
253 /* remove duplicates */
254 for (count = i = 1; i < lp->count; i++) {
255 if (namelist_cmp(lp->array[count - 1], lp->array[i]) == 0) {
256 free(lp->array[i]);
257 } else {
258 lp->array[count++] = lp->array[i];
259 }
260 }
261 lp->count = count;
262 }
263 lp->sorted = 1;
264 }
265
namelist_find(namelist_t * lp,const char * name)266 int namelist_find(namelist_t *lp, const char *name)
267 {
268 int a, b, m, cmp;
269
270 if (!lp->sorted) {
271 namelist_sort(lp);
272 }
273 for (a = 0, b = lp->count; a < b;) {
274 m = a + (b - a) / 2;
275 cmp = namelist_cmp(lp->array[m], name);
276 if (cmp < 0)
277 a = m + 1;
278 else if (cmp > 0)
279 b = m;
280 else
281 return m;
282 }
283 return -1;
284 }
285
namelist_add(namelist_t * lp,const char * base,const char * name)286 void namelist_add(namelist_t *lp, const char *base, const char *name)
287 {
288 char *s;
289
290 s = compose_path(base, name);
291 if (!s)
292 goto fail;
293 if (lp->count == lp->size) {
294 size_t newsize = lp->size + (lp->size >> 1) + 4;
295 char **a = realloc(lp->array, sizeof(lp->array[0]) * newsize);
296 if (!a)
297 goto fail;
298 lp->array = a;
299 lp->size = newsize;
300 }
301 lp->array[lp->count] = s;
302 lp->count++;
303 return;
304 fail:
305 fatal(1, "allocation failure\n");
306 }
307
namelist_load(namelist_t * lp,const char * filename)308 void namelist_load(namelist_t *lp, const char *filename)
309 {
310 char buf[1024];
311 char *base_name;
312 FILE *f;
313
314 f = fopen(filename, "rb");
315 if (!f) {
316 perror_exit(1, filename);
317 }
318 base_name = get_basename(filename);
319
320 while (fgets(buf, sizeof(buf), f) != NULL) {
321 char *p = str_strip(buf);
322 if (*p == '#' || *p == ';' || *p == '\0')
323 continue; /* line comment */
324
325 namelist_add(lp, base_name, p);
326 }
327 free(base_name);
328 fclose(f);
329 }
330
namelist_add_from_error_file(namelist_t * lp,const char * file)331 void namelist_add_from_error_file(namelist_t *lp, const char *file)
332 {
333 const char *p, *p0;
334 char *pp;
335
336 for (p = file; (p = strstr(p, ".js:")) != NULL; p++) {
337 for (p0 = p; p0 > file && p0[-1] != '\n'; p0--)
338 continue;
339 pp = strdup_len(p0, p + 3 - p0);
340 namelist_add(lp, NULL, pp);
341 free(pp);
342 }
343 }
344
namelist_free(namelist_t * lp)345 void namelist_free(namelist_t *lp)
346 {
347 while (lp->count > 0) {
348 free(lp->array[--lp->count]);
349 }
350 free(lp->array);
351 lp->array = NULL;
352 lp->size = 0;
353 }
354
add_test_file(const char * filename,const struct stat * ptr,int flag)355 static int add_test_file(const char *filename, const struct stat *ptr, int flag)
356 {
357 namelist_t *lp = &test_list;
358 if (has_suffix(filename, ".js") && !has_suffix(filename, "_FIXTURE.js"))
359 namelist_add(lp, NULL, filename);
360 return 0;
361 }
362
363 /* find js files from the directory tree and sort the list */
enumerate_tests(const char * path)364 static void enumerate_tests(const char *path)
365 {
366 namelist_t *lp = &test_list;
367 int start = lp->count;
368 ftw(path, add_test_file, 100);
369 qsort(lp->array + start, lp->count - start, sizeof(*lp->array),
370 namelist_cmp_indirect);
371 }
372
js_print(JSContext * ctx,JSValueConst this_val,int argc,JSValueConst * argv)373 static JSValue js_print(JSContext *ctx, JSValueConst this_val,
374 int argc, JSValueConst *argv)
375 {
376 int i;
377 const char *str;
378
379 if (outfile) {
380 for (i = 0; i < argc; i++) {
381 if (i != 0)
382 fputc(' ', outfile);
383 str = JS_ToCString(ctx, argv[i]);
384 if (!str)
385 return JS_EXCEPTION;
386 if (!strcmp(str, "Test262:AsyncTestComplete")) {
387 async_done++;
388 } else if (strstart(str, "Test262:AsyncTestFailure", NULL)) {
389 async_done = 2; /* force an error */
390 }
391 fputs(str, outfile);
392 JS_FreeCString(ctx, str);
393 }
394 fputc('\n', outfile);
395 }
396 return JS_UNDEFINED;
397 }
398
js_detachArrayBuffer(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)399 static JSValue js_detachArrayBuffer(JSContext *ctx, JSValue this_val,
400 int argc, JSValue *argv)
401 {
402 JS_DetachArrayBuffer(ctx, argv[0]);
403 return JS_UNDEFINED;
404 }
405
js_evalScript(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)406 static JSValue js_evalScript(JSContext *ctx, JSValue this_val,
407 int argc, JSValue *argv)
408 {
409 const char *str;
410 size_t len;
411 JSValue ret;
412 str = JS_ToCStringLen(ctx, &len, argv[0]);
413 if (!str)
414 return JS_EXCEPTION;
415 ret = JS_Eval(ctx, str, len, "<evalScript>", JS_EVAL_TYPE_GLOBAL);
416 JS_FreeCString(ctx, str);
417 return ret;
418 }
419
420 #ifdef CONFIG_AGENT
421
422 #include <pthread.h>
423
424 typedef struct {
425 struct list_head link;
426 pthread_t tid;
427 char *script;
428 JSValue broadcast_func;
429 BOOL broadcast_pending;
430 JSValue broadcast_sab; /* in the main context */
431 uint8_t *broadcast_sab_buf;
432 size_t broadcast_sab_size;
433 int32_t broadcast_val;
434 } Test262Agent;
435
436 typedef struct {
437 struct list_head link;
438 char *str;
439 } AgentReport;
440
441 static JSValue add_helpers1(JSContext *ctx);
442 static void add_helpers(JSContext *ctx);
443
444 static pthread_mutex_t agent_mutex = PTHREAD_MUTEX_INITIALIZER;
445 static pthread_cond_t agent_cond = PTHREAD_COND_INITIALIZER;
446 /* list of Test262Agent.link */
447 static struct list_head agent_list = LIST_HEAD_INIT(agent_list);
448
449 static pthread_mutex_t report_mutex = PTHREAD_MUTEX_INITIALIZER;
450 /* list of AgentReport.link */
451 static struct list_head report_list = LIST_HEAD_INIT(report_list);
452
agent_start(void * arg)453 static void *agent_start(void *arg)
454 {
455 Test262Agent *agent = arg;
456 JSRuntime *rt;
457 JSContext *ctx;
458 JSValue ret_val;
459 int ret;
460
461 rt = JS_NewRuntime();
462 if (rt == NULL) {
463 fatal(1, "JS_NewRuntime failure");
464 }
465 ctx = JS_NewContext(rt);
466 if (ctx == NULL) {
467 JS_FreeRuntime(rt);
468 fatal(1, "JS_NewContext failure");
469 }
470 JS_SetContextOpaque(ctx, agent);
471 JS_SetRuntimeInfo(rt, "agent");
472 JS_SetCanBlock(rt, TRUE);
473
474 add_helpers(ctx);
475 ret_val = JS_Eval(ctx, agent->script, strlen(agent->script),
476 "<evalScript>", JS_EVAL_TYPE_GLOBAL);
477 free(agent->script);
478 agent->script = NULL;
479 if (JS_IsException(ret_val))
480 js_std_dump_error(ctx);
481 JS_FreeValue(ctx, ret_val);
482
483 for(;;) {
484 JSContext *ctx1;
485 ret = JS_ExecutePendingJob(JS_GetRuntime(ctx), &ctx1);
486 if (ret < 0) {
487 js_std_dump_error(ctx);
488 break;
489 } else if (ret == 0) {
490 if (JS_IsUndefined(agent->broadcast_func)) {
491 break;
492 } else {
493 JSValue args[2];
494
495 pthread_mutex_lock(&agent_mutex);
496 while (!agent->broadcast_pending) {
497 pthread_cond_wait(&agent_cond, &agent_mutex);
498 }
499
500 agent->broadcast_pending = FALSE;
501 pthread_cond_signal(&agent_cond);
502
503 pthread_mutex_unlock(&agent_mutex);
504
505 args[0] = JS_NewArrayBuffer(ctx, agent->broadcast_sab_buf,
506 agent->broadcast_sab_size,
507 NULL, NULL, TRUE);
508 args[1] = JS_NewInt32(ctx, agent->broadcast_val);
509 ret_val = JS_Call(ctx, agent->broadcast_func, JS_UNDEFINED,
510 2, (JSValueConst *)args);
511 JS_FreeValue(ctx, args[0]);
512 JS_FreeValue(ctx, args[1]);
513 if (JS_IsException(ret_val))
514 js_std_dump_error(ctx);
515 JS_FreeValue(ctx, ret_val);
516 JS_FreeValue(ctx, agent->broadcast_func);
517 agent->broadcast_func = JS_UNDEFINED;
518 }
519 }
520 }
521 JS_FreeValue(ctx, agent->broadcast_func);
522
523 JS_FreeContext(ctx);
524 JS_FreeRuntime(rt);
525 return NULL;
526 }
527
js_agent_start(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)528 static JSValue js_agent_start(JSContext *ctx, JSValue this_val,
529 int argc, JSValue *argv)
530 {
531 const char *script;
532 Test262Agent *agent;
533
534 if (JS_GetContextOpaque(ctx) != NULL)
535 return JS_ThrowTypeError(ctx, "cannot be called inside an agent");
536
537 script = JS_ToCString(ctx, argv[0]);
538 if (!script)
539 return JS_EXCEPTION;
540 agent = malloc(sizeof(*agent));
541 memset(agent, 0, sizeof(*agent));
542 agent->broadcast_func = JS_UNDEFINED;
543 agent->broadcast_sab = JS_UNDEFINED;
544 agent->script = strdup(script);
545 JS_FreeCString(ctx, script);
546 list_add_tail(&agent->link, &agent_list);
547 pthread_create(&agent->tid, NULL, agent_start, agent);
548 return JS_UNDEFINED;
549 }
550
js_agent_free(JSContext * ctx)551 static void js_agent_free(JSContext *ctx)
552 {
553 struct list_head *el, *el1;
554 Test262Agent *agent;
555
556 list_for_each_safe(el, el1, &agent_list) {
557 agent = list_entry(el, Test262Agent, link);
558 pthread_join(agent->tid, NULL);
559 JS_FreeValue(ctx, agent->broadcast_sab);
560 list_del(&agent->link);
561 free(agent);
562 }
563 }
564
js_agent_leaving(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)565 static JSValue js_agent_leaving(JSContext *ctx, JSValue this_val,
566 int argc, JSValue *argv)
567 {
568 Test262Agent *agent = JS_GetContextOpaque(ctx);
569 if (!agent)
570 return JS_ThrowTypeError(ctx, "must be called inside an agent");
571 /* nothing to do */
572 return JS_UNDEFINED;
573 }
574
is_broadcast_pending(void)575 static BOOL is_broadcast_pending(void)
576 {
577 struct list_head *el;
578 Test262Agent *agent;
579 list_for_each(el, &agent_list) {
580 agent = list_entry(el, Test262Agent, link);
581 if (agent->broadcast_pending)
582 return TRUE;
583 }
584 return FALSE;
585 }
586
js_agent_broadcast(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)587 static JSValue js_agent_broadcast(JSContext *ctx, JSValue this_val,
588 int argc, JSValue *argv)
589 {
590 JSValueConst sab = argv[0];
591 struct list_head *el;
592 Test262Agent *agent;
593 uint8_t *buf;
594 size_t buf_size;
595 int32_t val;
596
597 if (JS_GetContextOpaque(ctx) != NULL)
598 return JS_ThrowTypeError(ctx, "cannot be called inside an agent");
599
600 buf = JS_GetArrayBuffer(ctx, &buf_size, sab);
601 if (!buf)
602 return JS_EXCEPTION;
603 if (JS_ToInt32(ctx, &val, argv[1]))
604 return JS_EXCEPTION;
605
606 /* broadcast the values and wait until all agents have started
607 calling their callbacks */
608 pthread_mutex_lock(&agent_mutex);
609 list_for_each(el, &agent_list) {
610 agent = list_entry(el, Test262Agent, link);
611 agent->broadcast_pending = TRUE;
612 /* the shared array buffer is used by the thread, so increment
613 its refcount */
614 agent->broadcast_sab = JS_DupValue(ctx, sab);
615 agent->broadcast_sab_buf = buf;
616 agent->broadcast_sab_size = buf_size;
617 agent->broadcast_val = val;
618 }
619 pthread_cond_broadcast(&agent_cond);
620
621 while (is_broadcast_pending()) {
622 pthread_cond_wait(&agent_cond, &agent_mutex);
623 }
624 pthread_mutex_unlock(&agent_mutex);
625 return JS_UNDEFINED;
626 }
627
js_agent_receiveBroadcast(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)628 static JSValue js_agent_receiveBroadcast(JSContext *ctx, JSValue this_val,
629 int argc, JSValue *argv)
630 {
631 Test262Agent *agent = JS_GetContextOpaque(ctx);
632 if (!agent)
633 return JS_ThrowTypeError(ctx, "must be called inside an agent");
634 if (!JS_IsFunction(ctx, argv[0]))
635 return JS_ThrowTypeError(ctx, "expecting function");
636 JS_FreeValue(ctx, agent->broadcast_func);
637 agent->broadcast_func = JS_DupValue(ctx, argv[0]);
638 return JS_UNDEFINED;
639 }
640
js_agent_sleep(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)641 static JSValue js_agent_sleep(JSContext *ctx, JSValue this_val,
642 int argc, JSValue *argv)
643 {
644 uint32_t duration;
645 if (JS_ToUint32(ctx, &duration, argv[0]))
646 return JS_EXCEPTION;
647 usleep(duration * 1000);
648 return JS_UNDEFINED;
649 }
650
get_clock_ms(void)651 static int64_t get_clock_ms(void)
652 {
653 struct timespec ts;
654 clock_gettime(CLOCK_MONOTONIC, &ts);
655 return (uint64_t)ts.tv_sec * 1000 + (ts.tv_nsec / 1000000);
656 }
657
js_agent_monotonicNow(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)658 static JSValue js_agent_monotonicNow(JSContext *ctx, JSValue this_val,
659 int argc, JSValue *argv)
660 {
661 return JS_NewInt64(ctx, get_clock_ms());
662 }
663
js_agent_getReport(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)664 static JSValue js_agent_getReport(JSContext *ctx, JSValue this_val,
665 int argc, JSValue *argv)
666 {
667 AgentReport *rep;
668 JSValue ret;
669
670 pthread_mutex_lock(&report_mutex);
671 if (list_empty(&report_list)) {
672 rep = NULL;
673 } else {
674 rep = list_entry(report_list.next, AgentReport, link);
675 list_del(&rep->link);
676 }
677 pthread_mutex_unlock(&report_mutex);
678 if (rep) {
679 ret = JS_NewString(ctx, rep->str);
680 free(rep->str);
681 free(rep);
682 } else {
683 ret = JS_NULL;
684 }
685 return ret;
686 }
687
js_agent_report(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)688 static JSValue js_agent_report(JSContext *ctx, JSValue this_val,
689 int argc, JSValue *argv)
690 {
691 const char *str;
692 AgentReport *rep;
693
694 str = JS_ToCString(ctx, argv[0]);
695 if (!str)
696 return JS_EXCEPTION;
697 rep = malloc(sizeof(*rep));
698 rep->str = strdup(str);
699 JS_FreeCString(ctx, str);
700
701 pthread_mutex_lock(&report_mutex);
702 list_add_tail(&rep->link, &report_list);
703 pthread_mutex_unlock(&report_mutex);
704 return JS_UNDEFINED;
705 }
706
707 static const JSCFunctionListEntry js_agent_funcs[] = {
708 /* only in main */
709 JS_CFUNC_DEF("start", 1, js_agent_start ),
710 JS_CFUNC_DEF("getReport", 0, js_agent_getReport ),
711 JS_CFUNC_DEF("broadcast", 2, js_agent_broadcast ),
712 /* only in agent */
713 JS_CFUNC_DEF("report", 1, js_agent_report ),
714 JS_CFUNC_DEF("leaving", 0, js_agent_leaving ),
715 JS_CFUNC_DEF("receiveBroadcast", 1, js_agent_receiveBroadcast ),
716 /* in both */
717 JS_CFUNC_DEF("sleep", 1, js_agent_sleep ),
718 JS_CFUNC_DEF("monotonicNow", 0, js_agent_monotonicNow ),
719 };
720
js_new_agent(JSContext * ctx)721 static JSValue js_new_agent(JSContext *ctx)
722 {
723 JSValue agent;
724 agent = JS_NewObject(ctx);
725 JS_SetPropertyFunctionList(ctx, agent, js_agent_funcs,
726 countof(js_agent_funcs));
727 return agent;
728 }
729 #endif
730
js_createRealm(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)731 static JSValue js_createRealm(JSContext *ctx, JSValue this_val,
732 int argc, JSValue *argv)
733 {
734 JSContext *ctx1;
735 JSValue ret;
736
737 ctx1 = JS_NewContext(JS_GetRuntime(ctx));
738 if (!ctx1)
739 return JS_ThrowOutOfMemory(ctx);
740 ret = add_helpers1(ctx1);
741 /* ctx1 has a refcount so it stays alive */
742 JS_FreeContext(ctx1);
743 return ret;
744 }
745
js_IsHTMLDDA(JSContext * ctx,JSValue this_val,int argc,JSValue * argv)746 static JSValue js_IsHTMLDDA(JSContext *ctx, JSValue this_val,
747 int argc, JSValue *argv)
748 {
749 return JS_NULL;
750 }
751
add_helpers1(JSContext * ctx)752 static JSValue add_helpers1(JSContext *ctx)
753 {
754 JSValue global_obj;
755 JSValue obj262, obj;
756
757 global_obj = JS_GetGlobalObject(ctx);
758
759 JS_SetPropertyStr(ctx, global_obj, "print",
760 JS_NewCFunction(ctx, js_print, "print", 1));
761
762 /* $262 special object used by the tests */
763 obj262 = JS_NewObject(ctx);
764 JS_SetPropertyStr(ctx, obj262, "detachArrayBuffer",
765 JS_NewCFunction(ctx, js_detachArrayBuffer,
766 "detachArrayBuffer", 1));
767 JS_SetPropertyStr(ctx, obj262, "evalScript",
768 JS_NewCFunction(ctx, js_evalScript,
769 "evalScript", 1));
770 JS_SetPropertyStr(ctx, obj262, "codePointRange",
771 JS_NewCFunction(ctx, js_string_codePointRange,
772 "codePointRange", 2));
773 #ifdef CONFIG_AGENT
774 JS_SetPropertyStr(ctx, obj262, "agent", js_new_agent(ctx));
775 #endif
776
777 JS_SetPropertyStr(ctx, obj262, "global",
778 JS_DupValue(ctx, global_obj));
779 JS_SetPropertyStr(ctx, obj262, "createRealm",
780 JS_NewCFunction(ctx, js_createRealm,
781 "createRealm", 0));
782 obj = JS_NewCFunction(ctx, js_IsHTMLDDA, "IsHTMLDDA", 0);
783 JS_SetIsHTMLDDA(ctx, obj);
784 JS_SetPropertyStr(ctx, obj262, "IsHTMLDDA", obj);
785
786 JS_SetPropertyStr(ctx, global_obj, "$262", JS_DupValue(ctx, obj262));
787
788 JS_FreeValue(ctx, global_obj);
789 return obj262;
790 }
791
add_helpers(JSContext * ctx)792 static void add_helpers(JSContext *ctx)
793 {
794 JS_FreeValue(ctx, add_helpers1(ctx));
795 }
796
load_file(const char * filename,size_t * lenp)797 static char *load_file(const char *filename, size_t *lenp)
798 {
799 char *buf;
800 size_t buf_len;
801 buf = (char *)js_load_file(NULL, &buf_len, filename);
802 if (!buf)
803 perror_exit(1, filename);
804 if (lenp)
805 *lenp = buf_len;
806 return buf;
807 }
808
js_module_loader_test(JSContext * ctx,const char * module_name,void * opaque)809 static JSModuleDef *js_module_loader_test(JSContext *ctx,
810 const char *module_name, void *opaque)
811 {
812 size_t buf_len;
813 uint8_t *buf;
814 JSModuleDef *m;
815 JSValue func_val;
816
817 buf = js_load_file(ctx, &buf_len, module_name);
818 if (!buf) {
819 JS_ThrowReferenceError(ctx, "could not load module filename '%s'",
820 module_name);
821 return NULL;
822 }
823
824 /* compile the module */
825 func_val = JS_Eval(ctx, (char *)buf, buf_len, module_name,
826 JS_EVAL_TYPE_MODULE | JS_EVAL_FLAG_COMPILE_ONLY);
827 js_free(ctx, buf);
828 if (JS_IsException(func_val))
829 return NULL;
830 /* the module is already referenced, so we must free it */
831 m = JS_VALUE_GET_PTR(func_val);
832 JS_FreeValue(ctx, func_val);
833 return m;
834 }
835
is_line_sep(char c)836 int is_line_sep(char c)
837 {
838 return (c == '\0' || c == '\n' || c == '\r');
839 }
840
find_line(const char * str,const char * line)841 char *find_line(const char *str, const char *line)
842 {
843 if (str) {
844 const char *p;
845 int len = strlen(line);
846 for (p = str; (p = strstr(p, line)) != NULL; p += len + 1) {
847 if ((p == str || is_line_sep(p[-1])) && is_line_sep(p[len]))
848 return (char *)p;
849 }
850 }
851 return NULL;
852 }
853
is_word_sep(char c)854 int is_word_sep(char c)
855 {
856 return (c == '\0' || isspace((unsigned char)c) || c == ',');
857 }
858
find_word(const char * str,const char * word)859 char *find_word(const char *str, const char *word)
860 {
861 const char *p;
862 int len = strlen(word);
863 if (str && len) {
864 for (p = str; (p = strstr(p, word)) != NULL; p += len) {
865 if ((p == str || is_word_sep(p[-1])) && is_word_sep(p[len]))
866 return (char *)p;
867 }
868 }
869 return NULL;
870 }
871
872 /* handle exclude directories */
update_exclude_dirs(void)873 void update_exclude_dirs(void)
874 {
875 namelist_t *lp = &test_list;
876 namelist_t *ep = &exclude_list;
877 namelist_t *dp = &exclude_dir_list;
878 char *name;
879 int i, j, count;
880
881 /* split directpries from exclude_list */
882 for (count = i = 0; i < ep->count; i++) {
883 name = ep->array[i];
884 if (has_suffix(name, "/")) {
885 namelist_add(dp, NULL, name);
886 free(name);
887 } else {
888 ep->array[count++] = name;
889 }
890 }
891 ep->count = count;
892
893 namelist_sort(dp);
894
895 /* filter out excluded directories */
896 for (count = i = 0; i < lp->count; i++) {
897 name = lp->array[i];
898 for (j = 0; j < dp->count; j++) {
899 if (has_prefix(name, dp->array[j])) {
900 test_excluded++;
901 free(name);
902 name = NULL;
903 break;
904 }
905 }
906 if (name) {
907 lp->array[count++] = name;
908 }
909 }
910 lp->count = count;
911 }
912
load_config(const char * filename)913 void load_config(const char *filename)
914 {
915 char buf[1024];
916 FILE *f;
917 char *base_name;
918 enum {
919 SECTION_NONE = 0,
920 SECTION_CONFIG,
921 SECTION_EXCLUDE,
922 SECTION_FEATURES,
923 SECTION_TESTS,
924 } section = SECTION_NONE;
925 int lineno = 0;
926
927 f = fopen(filename, "rb");
928 if (!f) {
929 perror_exit(1, filename);
930 }
931 base_name = get_basename(filename);
932
933 while (fgets(buf, sizeof(buf), f) != NULL) {
934 char *p, *q;
935 lineno++;
936 p = str_strip(buf);
937 if (*p == '#' || *p == ';' || *p == '\0')
938 continue; /* line comment */
939
940 if (*p == "[]"[0]) {
941 /* new section */
942 p++;
943 p[strcspn(p, "]")] = '\0';
944 if (str_equal(p, "config"))
945 section = SECTION_CONFIG;
946 else if (str_equal(p, "exclude"))
947 section = SECTION_EXCLUDE;
948 else if (str_equal(p, "features"))
949 section = SECTION_FEATURES;
950 else if (str_equal(p, "tests"))
951 section = SECTION_TESTS;
952 else
953 section = SECTION_NONE;
954 continue;
955 }
956 q = strchr(p, '=');
957 if (q) {
958 /* setting: name=value */
959 *q++ = '\0';
960 q = str_strip(q);
961 }
962 switch (section) {
963 case SECTION_CONFIG:
964 if (!q) {
965 printf("%s:%d: syntax error\n", filename, lineno);
966 continue;
967 }
968 if (str_equal(p, "style")) {
969 new_style = str_equal(q, "new");
970 continue;
971 }
972 if (str_equal(p, "testdir")) {
973 char *testdir = compose_path(base_name, q);
974 enumerate_tests(testdir);
975 free(testdir);
976 continue;
977 }
978 if (str_equal(p, "harnessdir")) {
979 harness_dir = compose_path(base_name, q);
980 continue;
981 }
982 if (str_equal(p, "harnessexclude")) {
983 str_append(&harness_exclude, " ", q);
984 continue;
985 }
986 if (str_equal(p, "features")) {
987 str_append(&harness_features, " ", q);
988 continue;
989 }
990 if (str_equal(p, "skip-features")) {
991 str_append(&harness_skip_features, " ", q);
992 continue;
993 }
994 if (str_equal(p, "mode")) {
995 if (str_equal(q, "default") || str_equal(q, "default-nostrict"))
996 test_mode = TEST_DEFAULT_NOSTRICT;
997 else if (str_equal(q, "default-strict"))
998 test_mode = TEST_DEFAULT_STRICT;
999 else if (str_equal(q, "nostrict"))
1000 test_mode = TEST_NOSTRICT;
1001 else if (str_equal(q, "strict"))
1002 test_mode = TEST_STRICT;
1003 else if (str_equal(q, "all") || str_equal(q, "both"))
1004 test_mode = TEST_ALL;
1005 else
1006 fatal(2, "unknown test mode: %s", q);
1007 continue;
1008 }
1009 if (str_equal(p, "strict")) {
1010 if (str_equal(q, "skip") || str_equal(q, "no"))
1011 test_mode = TEST_NOSTRICT;
1012 continue;
1013 }
1014 if (str_equal(p, "nostrict")) {
1015 if (str_equal(q, "skip") || str_equal(q, "no"))
1016 test_mode = TEST_STRICT;
1017 continue;
1018 }
1019 if (str_equal(p, "async")) {
1020 skip_async = !str_equal(q, "yes");
1021 continue;
1022 }
1023 if (str_equal(p, "module")) {
1024 skip_module = !str_equal(q, "yes");
1025 continue;
1026 }
1027 if (str_equal(p, "verbose")) {
1028 verbose = str_equal(q, "yes");
1029 continue;
1030 }
1031 if (str_equal(p, "errorfile")) {
1032 error_filename = compose_path(base_name, q);
1033 continue;
1034 }
1035 if (str_equal(p, "excludefile")) {
1036 char *path = compose_path(base_name, q);
1037 namelist_load(&exclude_list, path);
1038 free(path);
1039 continue;
1040 }
1041 if (str_equal(p, "reportfile")) {
1042 report_filename = compose_path(base_name, q);
1043 continue;
1044 }
1045 case SECTION_EXCLUDE:
1046 namelist_add(&exclude_list, base_name, p);
1047 break;
1048 case SECTION_FEATURES:
1049 if (!q || str_equal(q, "yes"))
1050 str_append(&harness_features, " ", p);
1051 else
1052 str_append(&harness_skip_features, " ", p);
1053 break;
1054 case SECTION_TESTS:
1055 namelist_add(&test_list, base_name, p);
1056 break;
1057 default:
1058 /* ignore settings in other sections */
1059 break;
1060 }
1061 }
1062 fclose(f);
1063 free(base_name);
1064 }
1065
find_error(const char * filename,int * pline,int is_strict)1066 char *find_error(const char *filename, int *pline, int is_strict)
1067 {
1068 if (error_file) {
1069 size_t len = strlen(filename);
1070 const char *p, *q, *r;
1071 int line;
1072
1073 for (p = error_file; (p = strstr(p, filename)) != NULL; p += len) {
1074 if ((p == error_file || p[-1] == '\n' || p[-1] == '(') && p[len] == ':') {
1075 q = p + len;
1076 line = 1;
1077 if (*q == ':') {
1078 line = strtol(q + 1, (char**)&q, 10);
1079 if (*q == ':')
1080 q++;
1081 }
1082 while (*q == ' ') {
1083 q++;
1084 }
1085 /* check strict mode indicator */
1086 if (!strstart(q, "strict mode: ", &q) != !is_strict)
1087 continue;
1088 r = q = skip_prefix(q, "unexpected error: ");
1089 r += strcspn(r, "\n");
1090 while (r[0] == '\n' && r[1] && strncmp(r + 1, filename, 8)) {
1091 r++;
1092 r += strcspn(r, "\n");
1093 }
1094 if (pline)
1095 *pline = line;
1096 return strdup_len(q, r - q);
1097 }
1098 }
1099 }
1100 return NULL;
1101 }
1102
skip_comments(const char * str,int line,int * pline)1103 int skip_comments(const char *str, int line, int *pline)
1104 {
1105 const char *p;
1106 int c;
1107
1108 p = str;
1109 while ((c = (unsigned char)*p++) != '\0') {
1110 if (isspace(c)) {
1111 if (c == '\n')
1112 line++;
1113 continue;
1114 }
1115 if (c == '/' && *p == '/') {
1116 while (*++p && *p != '\n')
1117 continue;
1118 continue;
1119 }
1120 if (c == '/' && *p == '*') {
1121 for (p += 1; *p; p++) {
1122 if (*p == '\n') {
1123 line++;
1124 continue;
1125 }
1126 if (*p == '*' && p[1] == '/') {
1127 p += 2;
1128 break;
1129 }
1130 }
1131 continue;
1132 }
1133 break;
1134 }
1135 if (pline)
1136 *pline = line;
1137
1138 return p - str;
1139 }
1140
longest_match(const char * str,const char * find,int pos,int * ppos,int line,int * pline)1141 int longest_match(const char *str, const char *find, int pos, int *ppos, int line, int *pline)
1142 {
1143 int len, maxlen;
1144
1145 maxlen = 0;
1146
1147 if (*find) {
1148 const char *p;
1149 for (p = str + pos; *p; p++) {
1150 if (*p == *find) {
1151 for (len = 1; p[len] && p[len] == find[len]; len++)
1152 continue;
1153 if (len > maxlen) {
1154 maxlen = len;
1155 if (ppos)
1156 *ppos = p - str;
1157 if (pline)
1158 *pline = line;
1159 if (!find[len])
1160 break;
1161 }
1162 }
1163 if (*p == '\n')
1164 line++;
1165 }
1166 }
1167 return maxlen;
1168 }
1169
eval_buf(JSContext * ctx,const char * buf,size_t buf_len,const char * filename,int is_test,int is_negative,const char * error_type,FILE * outfile,int eval_flags,int is_async)1170 static int eval_buf(JSContext *ctx, const char *buf, size_t buf_len,
1171 const char *filename, int is_test, int is_negative,
1172 const char *error_type, FILE *outfile, int eval_flags,
1173 int is_async)
1174 {
1175 JSValue res_val, exception_val;
1176 int ret, error_line, pos, pos_line;
1177 BOOL is_error, has_error_line;
1178 const char *error_name;
1179
1180 pos = skip_comments(buf, 1, &pos_line);
1181 error_line = pos_line;
1182 has_error_line = FALSE;
1183 exception_val = JS_UNDEFINED;
1184 error_name = NULL;
1185
1186 async_done = 0; /* counter of "Test262:AsyncTestComplete" messages */
1187
1188 res_val = JS_Eval(ctx, buf, buf_len, filename, eval_flags);
1189
1190 if (is_async && !JS_IsException(res_val)) {
1191 JS_FreeValue(ctx, res_val);
1192 for(;;) {
1193 JSContext *ctx1;
1194 ret = JS_ExecutePendingJob(JS_GetRuntime(ctx), &ctx1);
1195 if (ret < 0) {
1196 res_val = JS_EXCEPTION;
1197 break;
1198 } else if (ret == 0) {
1199 /* test if the test called $DONE() once */
1200 if (async_done != 1) {
1201 res_val = JS_ThrowTypeError(ctx, "$DONE() not called");
1202 } else {
1203 res_val = JS_UNDEFINED;
1204 }
1205 break;
1206 }
1207 }
1208 }
1209
1210 if (JS_IsException(res_val)) {
1211 exception_val = JS_GetException(ctx);
1212 is_error = JS_IsError(ctx, exception_val);
1213 /* XXX: should get the filename and line number */
1214 if (outfile) {
1215 if (!is_error)
1216 fprintf(outfile, "%sThrow: ", (eval_flags & JS_EVAL_FLAG_STRICT) ?
1217 "strict mode: " : "");
1218 js_print(ctx, JS_NULL, 1, &exception_val);
1219 }
1220 if (is_error) {
1221 JSValue name, stack;
1222 const char *stack_str;
1223
1224 name = JS_GetPropertyStr(ctx, exception_val, "name");
1225 error_name = JS_ToCString(ctx, name);
1226 stack = JS_GetPropertyStr(ctx, exception_val, "stack");
1227 if (!JS_IsUndefined(stack)) {
1228 stack_str = JS_ToCString(ctx, stack);
1229 if (stack_str) {
1230 const char *p;
1231 int len;
1232
1233 if (outfile)
1234 fprintf(outfile, "%s", stack_str);
1235
1236 len = strlen(filename);
1237 p = strstr(stack_str, filename);
1238 if (p != NULL && p[len] == ':') {
1239 error_line = atoi(p + len + 1);
1240 has_error_line = TRUE;
1241 }
1242 JS_FreeCString(ctx, stack_str);
1243 }
1244 }
1245 JS_FreeValue(ctx, stack);
1246 JS_FreeValue(ctx, name);
1247 }
1248 if (is_negative) {
1249 ret = 0;
1250 if (error_type) {
1251 char *error_class;
1252 const char *msg;
1253
1254 msg = JS_ToCString(ctx, exception_val);
1255 error_class = strdup_len(msg, strcspn(msg, ":"));
1256 if (!str_equal(error_class, error_type))
1257 ret = -1;
1258 free(error_class);
1259 JS_FreeCString(ctx, msg);
1260 }
1261 } else {
1262 ret = -1;
1263 }
1264 } else {
1265 if (is_negative)
1266 ret = -1;
1267 else
1268 ret = 0;
1269 }
1270
1271 if (verbose && is_test) {
1272 JSValue msg_val = JS_UNDEFINED;
1273 const char *msg = NULL;
1274 int s_line;
1275 char *s = find_error(filename, &s_line, eval_flags & JS_EVAL_FLAG_STRICT);
1276 const char *strict_mode = (eval_flags & JS_EVAL_FLAG_STRICT) ? "strict mode: " : "";
1277
1278 if (!JS_IsUndefined(exception_val)) {
1279 msg_val = JS_ToString(ctx, exception_val);
1280 msg = JS_ToCString(ctx, msg_val);
1281 }
1282 if (is_negative) { // expect error
1283 if (ret == 0) {
1284 if (msg && s &&
1285 (str_equal(s, "expected error") ||
1286 strstart(s, "unexpected error type:", NULL) ||
1287 str_equal(s, msg))) { // did not have error yet
1288 if (!has_error_line) {
1289 longest_match(buf, msg, pos, &pos, pos_line, &error_line);
1290 }
1291 printf("%s:%d: %sOK, now has error %s\n",
1292 filename, error_line, strict_mode, msg);
1293 fixed_errors++;
1294 }
1295 } else {
1296 if (!s) { // not yet reported
1297 if (msg) {
1298 fprintf(error_out, "%s:%d: %sunexpected error type: %s\n",
1299 filename, error_line, strict_mode, msg);
1300 } else {
1301 fprintf(error_out, "%s:%d: %sexpected error\n",
1302 filename, error_line, strict_mode);
1303 }
1304 new_errors++;
1305 }
1306 }
1307 } else { // should not have error
1308 if (msg) {
1309 if (!s || !str_equal(s, msg)) {
1310 if (!has_error_line) {
1311 char *p = skip_prefix(msg, "Test262 Error: ");
1312 if (strstr(p, "Test case returned non-true value!")) {
1313 longest_match(buf, "runTestCase", pos, &pos, pos_line, &error_line);
1314 } else {
1315 longest_match(buf, p, pos, &pos, pos_line, &error_line);
1316 }
1317 }
1318 fprintf(error_out, "%s:%d: %s%s%s\n", filename, error_line, strict_mode,
1319 error_file ? "unexpected error: " : "", msg);
1320
1321 if (s && (!str_equal(s, msg) || error_line != s_line)) {
1322 printf("%s:%d: %sprevious error: %s\n", filename, s_line, strict_mode, s);
1323 changed_errors++;
1324 } else {
1325 new_errors++;
1326 }
1327 }
1328 } else {
1329 if (s) {
1330 printf("%s:%d: %sOK, fixed error: %s\n", filename, s_line, strict_mode, s);
1331 fixed_errors++;
1332 }
1333 }
1334 }
1335 JS_FreeValue(ctx, msg_val);
1336 JS_FreeCString(ctx, msg);
1337 free(s);
1338 }
1339 JS_FreeCString(ctx, error_name);
1340 JS_FreeValue(ctx, exception_val);
1341 JS_FreeValue(ctx, res_val);
1342 return ret;
1343 }
1344
eval_file(JSContext * ctx,const char * base,const char * p,int eval_flags)1345 static int eval_file(JSContext *ctx, const char *base, const char *p,
1346 int eval_flags)
1347 {
1348 char *buf;
1349 size_t buf_len;
1350 char *filename = compose_path(base, p);
1351
1352 buf = load_file(filename, &buf_len);
1353 if (!buf) {
1354 warning("cannot load %s", filename);
1355 goto fail;
1356 }
1357 if (eval_buf(ctx, buf, buf_len, filename, FALSE, FALSE, NULL, stderr,
1358 eval_flags, FALSE)) {
1359 warning("error evaluating %s", filename);
1360 goto fail;
1361 }
1362 free(buf);
1363 free(filename);
1364 return 0;
1365
1366 fail:
1367 free(buf);
1368 free(filename);
1369 return 1;
1370 }
1371
extract_desc(const char * buf,char style)1372 char *extract_desc(const char *buf, char style)
1373 {
1374 const char *p, *desc_start;
1375 char *desc;
1376 int len;
1377
1378 p = buf;
1379 while (*p != '\0') {
1380 if (p[0] == '/' && p[1] == '*' && p[2] == style && p[3] != '/') {
1381 p += 3;
1382 desc_start = p;
1383 while (*p != '\0' && (p[0] != '*' || p[1] != '/'))
1384 p++;
1385 if (*p == '\0') {
1386 warning("Expecting end of desc comment");
1387 return NULL;
1388 }
1389 len = p - desc_start;
1390 desc = malloc(len + 1);
1391 memcpy(desc, desc_start, len);
1392 desc[len] = '\0';
1393 return desc;
1394 } else {
1395 p++;
1396 }
1397 }
1398 return NULL;
1399 }
1400
find_tag(char * desc,const char * tag,int * state)1401 static char *find_tag(char *desc, const char *tag, int *state)
1402 {
1403 char *p;
1404 p = strstr(desc, tag);
1405 if (p) {
1406 p += strlen(tag);
1407 *state = 0;
1408 }
1409 return p;
1410 }
1411
get_option(char ** pp,int * state)1412 static char *get_option(char **pp, int *state)
1413 {
1414 char *p, *p0, *option = NULL;
1415 if (*pp) {
1416 for (p = *pp;; p++) {
1417 switch (*p) {
1418 case '[':
1419 *state += 1;
1420 continue;
1421 case ']':
1422 *state -= 1;
1423 if (*state > 0)
1424 continue;
1425 p = NULL;
1426 break;
1427 case ' ':
1428 case '\t':
1429 case '\r':
1430 case ',':
1431 case '-':
1432 continue;
1433 case '\n':
1434 if (*state > 0 || p[1] == ' ')
1435 continue;
1436 p = NULL;
1437 break;
1438 case '\0':
1439 p = NULL;
1440 break;
1441 default:
1442 p0 = p;
1443 p += strcspn(p0, " \t\r\n,]");
1444 option = strdup_len(p0, p - p0);
1445 break;
1446 }
1447 break;
1448 }
1449 *pp = p;
1450 }
1451 return option;
1452 }
1453
update_stats(JSRuntime * rt,const char * filename)1454 void update_stats(JSRuntime *rt, const char *filename) {
1455 JSMemoryUsage stats;
1456 JS_ComputeMemoryUsage(rt, &stats);
1457 if (stats_count++ == 0) {
1458 stats_avg = stats_all = stats_min = stats_max = stats;
1459 stats_min_filename = strdup(filename);
1460 stats_max_filename = strdup(filename);
1461 } else {
1462 if (stats_max.malloc_size < stats.malloc_size) {
1463 stats_max = stats;
1464 free(stats_max_filename);
1465 stats_max_filename = strdup(filename);
1466 }
1467 if (stats_min.malloc_size > stats.malloc_size) {
1468 stats_min = stats;
1469 free(stats_min_filename);
1470 stats_min_filename = strdup(filename);
1471 }
1472
1473 #define update(f) stats_avg.f = (stats_all.f += stats.f) / stats_count
1474 update(malloc_count);
1475 update(malloc_size);
1476 update(memory_used_count);
1477 update(memory_used_size);
1478 update(atom_count);
1479 update(atom_size);
1480 update(str_count);
1481 update(str_size);
1482 update(obj_count);
1483 update(obj_size);
1484 update(prop_count);
1485 update(prop_size);
1486 update(shape_count);
1487 update(shape_size);
1488 update(js_func_count);
1489 update(js_func_size);
1490 update(js_func_code_size);
1491 update(js_func_pc2line_count);
1492 update(js_func_pc2line_size);
1493 update(c_func_count);
1494 update(array_count);
1495 update(fast_array_count);
1496 update(fast_array_elements);
1497 }
1498 #undef update
1499 }
1500
run_test_buf(const char * filename,char * harness,namelist_t * ip,char * buf,size_t buf_len,const char * error_type,int eval_flags,BOOL is_negative,BOOL is_async,BOOL can_block)1501 int run_test_buf(const char *filename, char *harness, namelist_t *ip,
1502 char *buf, size_t buf_len, const char* error_type,
1503 int eval_flags, BOOL is_negative, BOOL is_async,
1504 BOOL can_block)
1505 {
1506 JSRuntime *rt;
1507 JSContext *ctx;
1508 int i, ret;
1509
1510 rt = JS_NewRuntime();
1511 if (rt == NULL) {
1512 fatal(1, "JS_NewRuntime failure");
1513 }
1514 ctx = JS_NewContext(rt);
1515 if (ctx == NULL) {
1516 JS_FreeRuntime(rt);
1517 fatal(1, "JS_NewContext failure");
1518 }
1519 JS_SetRuntimeInfo(rt, filename);
1520
1521 JS_SetCanBlock(rt, can_block);
1522
1523 /* loader for ES6 modules */
1524 JS_SetModuleLoaderFunc(rt, NULL, js_module_loader_test, NULL);
1525
1526 add_helpers(ctx);
1527
1528 for (i = 0; i < ip->count; i++) {
1529 if (eval_file(ctx, harness, ip->array[i],
1530 JS_EVAL_TYPE_GLOBAL | JS_EVAL_FLAG_STRIP)) {
1531 fatal(1, "error including %s for %s", ip->array[i], filename);
1532 }
1533 }
1534
1535 ret = eval_buf(ctx, buf, buf_len, filename, TRUE, is_negative,
1536 error_type, outfile, eval_flags, is_async);
1537 ret = (ret != 0);
1538
1539 if (dump_memory) {
1540 update_stats(rt, filename);
1541 }
1542 #ifdef CONFIG_AGENT
1543 js_agent_free(ctx);
1544 #endif
1545 JS_FreeContext(ctx);
1546 JS_FreeRuntime(rt);
1547
1548 test_count++;
1549 if (ret) {
1550 test_failed++;
1551 if (outfile) {
1552 /* do not output a failure number to minimize diff */
1553 fprintf(outfile, " FAILED\n");
1554 }
1555 }
1556 return ret;
1557 }
1558
run_test(const char * filename,int index)1559 int run_test(const char *filename, int index)
1560 {
1561 char harnessbuf[1024];
1562 char *harness;
1563 char *buf;
1564 size_t buf_len;
1565 char *desc, *p;
1566 char *error_type;
1567 int ret, eval_flags, use_strict, use_nostrict;
1568 BOOL is_negative, is_nostrict, is_onlystrict, is_async, is_module, skip;
1569 BOOL can_block;
1570 namelist_t include_list = { 0 }, *ip = &include_list;
1571
1572 is_nostrict = is_onlystrict = is_negative = is_async = is_module = skip = FALSE;
1573 can_block = TRUE;
1574 error_type = NULL;
1575 buf = load_file(filename, &buf_len);
1576
1577 harness = harness_dir;
1578
1579 if (new_style) {
1580 if (!harness) {
1581 p = strstr(filename, "test/");
1582 if (p) {
1583 snprintf(harnessbuf, sizeof(harnessbuf), "%.*s%s",
1584 (int)(p - filename), filename, "harness");
1585 }
1586 harness = harnessbuf;
1587 }
1588 namelist_add(ip, NULL, "sta.js");
1589 namelist_add(ip, NULL, "assert.js");
1590 /* extract the YAML frontmatter */
1591 desc = extract_desc(buf, '-');
1592 if (desc) {
1593 char *ifile, *option;
1594 int state;
1595 p = find_tag(desc, "includes:", &state);
1596 if (p) {
1597 while ((ifile = get_option(&p, &state)) != NULL) {
1598 // skip unsupported harness files
1599 if (find_word(harness_exclude, ifile)) {
1600 skip |= 1;
1601 } else {
1602 namelist_add(ip, NULL, ifile);
1603 }
1604 free(ifile);
1605 }
1606 }
1607 p = find_tag(desc, "flags:", &state);
1608 if (p) {
1609 while ((option = get_option(&p, &state)) != NULL) {
1610 if (str_equal(option, "noStrict") ||
1611 str_equal(option, "raw")) {
1612 is_nostrict = TRUE;
1613 skip |= (test_mode == TEST_STRICT);
1614 }
1615 else if (str_equal(option, "onlyStrict")) {
1616 is_onlystrict = TRUE;
1617 skip |= (test_mode == TEST_NOSTRICT);
1618 }
1619 else if (str_equal(option, "async")) {
1620 is_async = TRUE;
1621 skip |= skip_async;
1622 }
1623 else if (str_equal(option, "module")) {
1624 is_module = TRUE;
1625 skip |= skip_module;
1626 }
1627 else if (str_equal(option, "CanBlockIsFalse")) {
1628 can_block = FALSE;
1629 }
1630 free(option);
1631 }
1632 }
1633 p = find_tag(desc, "negative:", &state);
1634 if (p) {
1635 /* XXX: should extract the phase */
1636 char *q = find_tag(p, "type:", &state);
1637 if (q) {
1638 while (isspace(*q))
1639 q++;
1640 error_type = strdup_len(q, strcspn(q, " \n"));
1641 }
1642 is_negative = TRUE;
1643 }
1644 p = find_tag(desc, "features:", &state);
1645 if (p) {
1646 while ((option = get_option(&p, &state)) != NULL) {
1647 if (find_word(harness_features, option)) {
1648 /* feature is enabled */
1649 } else if (find_word(harness_skip_features, option)) {
1650 /* skip disabled feature */
1651 skip |= 1;
1652 } else {
1653 /* feature is not listed: skip and warn */
1654 printf("%s:%d: unknown feature: %s\n", filename, 1, option);
1655 skip |= 1;
1656 }
1657 free(option);
1658 }
1659 }
1660 free(desc);
1661 }
1662 if (is_async)
1663 namelist_add(ip, NULL, "doneprintHandle.js");
1664 } else {
1665 char *ifile;
1666
1667 if (!harness) {
1668 p = strstr(filename, "test/");
1669 if (p) {
1670 snprintf(harnessbuf, sizeof(harnessbuf), "%.*s%s",
1671 (int)(p - filename), filename, "test/harness");
1672 }
1673 harness = harnessbuf;
1674 }
1675
1676 namelist_add(ip, NULL, "sta.js");
1677
1678 /* include extra harness files */
1679 for (p = buf; (p = strstr(p, "$INCLUDE(\"")) != NULL; p++) {
1680 p += 10;
1681 ifile = strdup_len(p, strcspn(p, "\""));
1682 // skip unsupported harness files
1683 if (find_word(harness_exclude, ifile)) {
1684 skip |= 1;
1685 } else {
1686 namelist_add(ip, NULL, ifile);
1687 }
1688 free(ifile);
1689 }
1690
1691 /* locate the old style configuration comment */
1692 desc = extract_desc(buf, '*');
1693 if (desc) {
1694 if (strstr(desc, "@noStrict")) {
1695 is_nostrict = TRUE;
1696 skip |= (test_mode == TEST_STRICT);
1697 }
1698 if (strstr(desc, "@onlyStrict")) {
1699 is_onlystrict = TRUE;
1700 skip |= (test_mode == TEST_NOSTRICT);
1701 }
1702 if (strstr(desc, "@negative")) {
1703 /* XXX: should extract the regex to check error type */
1704 is_negative = TRUE;
1705 }
1706 free(desc);
1707 }
1708 }
1709
1710 if (outfile && index >= 0) {
1711 fprintf(outfile, "%d: %s%s%s%s%s%s%s\n", index, filename,
1712 is_nostrict ? " @noStrict" : "",
1713 is_onlystrict ? " @onlyStrict" : "",
1714 is_async ? " async" : "",
1715 is_module ? " module" : "",
1716 is_negative ? " @negative" : "",
1717 skip ? " SKIPPED" : "");
1718 fflush(outfile);
1719 }
1720
1721 use_strict = use_nostrict = 0;
1722 /* XXX: should remove 'test_mode' or simplify it just to force
1723 strict or non strict mode for single file tests */
1724 switch (test_mode) {
1725 case TEST_DEFAULT_NOSTRICT:
1726 if (is_onlystrict)
1727 use_strict = 1;
1728 else
1729 use_nostrict = 1;
1730 break;
1731 case TEST_DEFAULT_STRICT:
1732 if (is_nostrict)
1733 use_nostrict = 1;
1734 else
1735 use_strict = 1;
1736 break;
1737 case TEST_NOSTRICT:
1738 if (!is_onlystrict)
1739 use_nostrict = 1;
1740 break;
1741 case TEST_STRICT:
1742 if (!is_nostrict)
1743 use_strict = 1;
1744 break;
1745 case TEST_ALL:
1746 if (is_module) {
1747 use_nostrict = 1;
1748 } else {
1749 if (!is_nostrict)
1750 use_strict = 1;
1751 if (!is_onlystrict)
1752 use_nostrict = 1;
1753 }
1754 break;
1755 }
1756
1757 if (skip || use_strict + use_nostrict == 0) {
1758 test_skipped++;
1759 ret = -2;
1760 } else {
1761 clock_t clocks;
1762
1763 if (is_module) {
1764 eval_flags = JS_EVAL_TYPE_MODULE;
1765 } else {
1766 eval_flags = JS_EVAL_TYPE_GLOBAL;
1767 }
1768 clocks = clock();
1769 ret = 0;
1770 if (use_nostrict) {
1771 ret = run_test_buf(filename, harness, ip, buf, buf_len,
1772 error_type, eval_flags, is_negative, is_async,
1773 can_block);
1774 }
1775 if (use_strict) {
1776 ret |= run_test_buf(filename, harness, ip, buf, buf_len,
1777 error_type, eval_flags | JS_EVAL_FLAG_STRICT,
1778 is_negative, is_async, can_block);
1779 }
1780 clocks = clock() - clocks;
1781 if (outfile && index >= 0 && clocks >= CLOCKS_PER_SEC / 10) {
1782 /* output timings for tests that take more than 100 ms */
1783 fprintf(outfile, " time: %d ms\n", (int)(clocks * 1000LL / CLOCKS_PER_SEC));
1784 }
1785 }
1786 namelist_free(&include_list);
1787 free(error_type);
1788 free(buf);
1789
1790 return ret;
1791 }
1792
1793 /* run a test when called by test262-harness+eshost */
run_test262_harness_test(const char * filename,BOOL is_module)1794 int run_test262_harness_test(const char *filename, BOOL is_module)
1795 {
1796 JSRuntime *rt;
1797 JSContext *ctx;
1798 char *buf;
1799 size_t buf_len;
1800 int eval_flags, ret_code, ret;
1801 JSValue res_val;
1802 BOOL can_block;
1803
1804 outfile = stdout; /* for js_print */
1805
1806 rt = JS_NewRuntime();
1807 if (rt == NULL) {
1808 fatal(1, "JS_NewRuntime failure");
1809 }
1810 ctx = JS_NewContext(rt);
1811 if (ctx == NULL) {
1812 JS_FreeRuntime(rt);
1813 fatal(1, "JS_NewContext failure");
1814 }
1815 JS_SetRuntimeInfo(rt, filename);
1816
1817 can_block = TRUE;
1818 JS_SetCanBlock(rt, can_block);
1819
1820 /* loader for ES6 modules */
1821 JS_SetModuleLoaderFunc(rt, NULL, js_module_loader_test, NULL);
1822
1823 add_helpers(ctx);
1824
1825 buf = load_file(filename, &buf_len);
1826
1827 if (is_module) {
1828 eval_flags = JS_EVAL_TYPE_MODULE;
1829 } else {
1830 eval_flags = JS_EVAL_TYPE_GLOBAL;
1831 }
1832 res_val = JS_Eval(ctx, buf, buf_len, filename, eval_flags);
1833 ret_code = 0;
1834 if (JS_IsException(res_val)) {
1835 js_std_dump_error(ctx);
1836 ret_code = 1;
1837 } else {
1838 JS_FreeValue(ctx, res_val);
1839 for(;;) {
1840 JSContext *ctx1;
1841 ret = JS_ExecutePendingJob(JS_GetRuntime(ctx), &ctx1);
1842 if (ret < 0) {
1843 js_std_dump_error(ctx1);
1844 ret_code = 1;
1845 } else if (ret == 0) {
1846 break;
1847 }
1848 }
1849 }
1850 free(buf);
1851 #ifdef CONFIG_AGENT
1852 js_agent_free(ctx);
1853 #endif
1854 JS_FreeContext(ctx);
1855 JS_FreeRuntime(rt);
1856 return ret_code;
1857 }
1858
1859 clock_t last_clock;
1860
show_progress(int force)1861 void show_progress(int force) {
1862 clock_t t = clock();
1863 if (force || !last_clock || (t - last_clock) > CLOCKS_PER_SEC / 20) {
1864 last_clock = t;
1865 /* output progress indicator: erase end of line and return to col 0 */
1866 fprintf(stderr, "%d/%d/%d\033[K\r",
1867 test_failed, test_count, test_skipped);
1868 fflush(stderr);
1869 }
1870 }
1871
1872 static int slow_test_threshold;
1873
run_test_dir_list(namelist_t * lp,int start_index,int stop_index)1874 void run_test_dir_list(namelist_t *lp, int start_index, int stop_index)
1875 {
1876 int i;
1877
1878 namelist_sort(lp);
1879 for (i = 0; i < lp->count; i++) {
1880 const char *p = lp->array[i];
1881 if (namelist_find(&exclude_list, p) >= 0) {
1882 test_excluded++;
1883 } else if (test_index < start_index) {
1884 test_skipped++;
1885 } else if (stop_index >= 0 && test_index > stop_index) {
1886 test_skipped++;
1887 } else {
1888 int ti;
1889 if (slow_test_threshold != 0) {
1890 ti = get_clock_ms();
1891 } else {
1892 ti = 0;
1893 }
1894 run_test(p, test_index);
1895 if (slow_test_threshold != 0) {
1896 ti = get_clock_ms() - ti;
1897 if (ti >= slow_test_threshold)
1898 fprintf(stderr, "\n%s (%d ms)\n", p, ti);
1899 }
1900 show_progress(FALSE);
1901 }
1902 test_index++;
1903 }
1904 show_progress(TRUE);
1905 }
1906
help(void)1907 void help(void)
1908 {
1909 printf("run-test262 version " CONFIG_VERSION "\n"
1910 "usage: run-test262 [options] {-f file ... | [dir_list] [index range]}\n"
1911 "-h help\n"
1912 "-a run tests in strict and nostrict modes\n"
1913 "-m print memory usage summary\n"
1914 "-n use new style harness\n"
1915 "-N run test prepared by test262-harness+eshost\n"
1916 "-s run tests in strict mode, skip @nostrict tests\n"
1917 "-E only run tests from the error file\n"
1918 "-u update error file\n"
1919 "-v verbose: output error messages\n"
1920 "-T duration display tests taking more than 'duration' ms\n"
1921 "-c file read configuration from 'file'\n"
1922 "-d dir run all test files in directory tree 'dir'\n"
1923 "-e file load the known errors from 'file'\n"
1924 "-f file execute single test from 'file'\n"
1925 "-r file set the report file name (default=none)\n"
1926 "-x file exclude tests listed in 'file'\n");
1927 exit(1);
1928 }
1929
get_opt_arg(const char * option,char * arg)1930 char *get_opt_arg(const char *option, char *arg)
1931 {
1932 if (!arg) {
1933 fatal(2, "missing argument for option %s", option);
1934 }
1935 return arg;
1936 }
1937
main(int argc,char ** argv)1938 int main(int argc, char **argv)
1939 {
1940 int optind, start_index, stop_index;
1941 BOOL is_dir_list;
1942 BOOL only_check_errors = FALSE;
1943 const char *filename;
1944 BOOL is_test262_harness = FALSE;
1945 BOOL is_module = FALSE;
1946
1947 #if !defined(_WIN32)
1948 /* Date tests assume California local time */
1949 setenv("TZ", "America/Los_Angeles", 1);
1950 #endif
1951
1952 /* cannot use getopt because we want to pass the command line to
1953 the script */
1954 optind = 1;
1955 is_dir_list = TRUE;
1956 while (optind < argc) {
1957 char *arg = argv[optind];
1958 if (*arg != '-')
1959 break;
1960 optind++;
1961 if (str_equal(arg, "-h")) {
1962 help();
1963 } else if (str_equal(arg, "-m")) {
1964 dump_memory++;
1965 } else if (str_equal(arg, "-n")) {
1966 new_style++;
1967 } else if (str_equal(arg, "-s")) {
1968 test_mode = TEST_STRICT;
1969 } else if (str_equal(arg, "-a")) {
1970 test_mode = TEST_ALL;
1971 } else if (str_equal(arg, "-u")) {
1972 update_errors++;
1973 } else if (str_equal(arg, "-v")) {
1974 verbose++;
1975 } else if (str_equal(arg, "-c")) {
1976 load_config(get_opt_arg(arg, argv[optind++]));
1977 } else if (str_equal(arg, "-d")) {
1978 enumerate_tests(get_opt_arg(arg, argv[optind++]));
1979 } else if (str_equal(arg, "-e")) {
1980 error_filename = get_opt_arg(arg, argv[optind++]);
1981 } else if (str_equal(arg, "-x")) {
1982 namelist_load(&exclude_list, get_opt_arg(arg, argv[optind++]));
1983 } else if (str_equal(arg, "-f")) {
1984 is_dir_list = FALSE;
1985 } else if (str_equal(arg, "-r")) {
1986 report_filename = get_opt_arg(arg, argv[optind++]);
1987 } else if (str_equal(arg, "-E")) {
1988 only_check_errors = TRUE;
1989 } else if (str_equal(arg, "-T")) {
1990 slow_test_threshold = atoi(get_opt_arg(arg, argv[optind++]));
1991 } else if (str_equal(arg, "-N")) {
1992 is_test262_harness = TRUE;
1993 } else if (str_equal(arg, "--module")) {
1994 is_module = TRUE;
1995 } else {
1996 fatal(1, "unknown option: %s", arg);
1997 break;
1998 }
1999 }
2000
2001 if (optind >= argc && !test_list.count)
2002 help();
2003
2004 if (is_test262_harness) {
2005 return run_test262_harness_test(argv[optind], is_module);
2006 }
2007
2008 error_out = stdout;
2009 if (error_filename) {
2010 error_file = load_file(error_filename, NULL);
2011 if (only_check_errors && error_file) {
2012 namelist_free(&test_list);
2013 namelist_add_from_error_file(&test_list, error_file);
2014 }
2015 if (update_errors) {
2016 free(error_file);
2017 error_file = NULL;
2018 error_out = fopen(error_filename, "w");
2019 if (!error_out) {
2020 perror_exit(1, error_filename);
2021 }
2022 }
2023 }
2024
2025 update_exclude_dirs();
2026
2027 if (is_dir_list) {
2028 if (optind < argc && !isdigit(argv[optind][0])) {
2029 filename = argv[optind++];
2030 namelist_load(&test_list, filename);
2031 }
2032 start_index = 0;
2033 stop_index = -1;
2034 if (optind < argc) {
2035 start_index = atoi(argv[optind++]);
2036 if (optind < argc) {
2037 stop_index = atoi(argv[optind++]);
2038 }
2039 }
2040 if (!report_filename || str_equal(report_filename, "none")) {
2041 outfile = NULL;
2042 } else if (str_equal(report_filename, "-")) {
2043 outfile = stdout;
2044 } else {
2045 outfile = fopen(report_filename, "wb");
2046 if (!outfile) {
2047 perror_exit(1, report_filename);
2048 }
2049 }
2050 run_test_dir_list(&test_list, start_index, stop_index);
2051
2052 if (outfile && outfile != stdout) {
2053 fclose(outfile);
2054 outfile = NULL;
2055 }
2056 } else {
2057 outfile = stdout;
2058 while (optind < argc) {
2059 run_test(argv[optind++], -1);
2060 }
2061 }
2062
2063 if (dump_memory) {
2064 if (dump_memory > 1 && stats_count > 1) {
2065 printf("\nMininum memory statistics for %s:\n\n", stats_min_filename);
2066 JS_DumpMemoryUsage(stdout, &stats_min, NULL);
2067 printf("\nMaximum memory statistics for %s:\n\n", stats_max_filename);
2068 JS_DumpMemoryUsage(stdout, &stats_max, NULL);
2069 }
2070 printf("\nAverage memory statistics for %d tests:\n\n", stats_count);
2071 JS_DumpMemoryUsage(stdout, &stats_avg, NULL);
2072 printf("\n");
2073 }
2074
2075 if (is_dir_list) {
2076 fprintf(stderr, "Result: %d/%d error%s",
2077 test_failed, test_count, test_count != 1 ? "s" : "");
2078 if (test_excluded)
2079 fprintf(stderr, ", %d excluded", test_excluded);
2080 if (test_skipped)
2081 fprintf(stderr, ", %d skipped", test_skipped);
2082 if (error_file) {
2083 if (new_errors)
2084 fprintf(stderr, ", %d new", new_errors);
2085 if (changed_errors)
2086 fprintf(stderr, ", %d changed", changed_errors);
2087 if (fixed_errors)
2088 fprintf(stderr, ", %d fixed", fixed_errors);
2089 }
2090 fprintf(stderr, "\n");
2091 }
2092
2093 if (error_out && error_out != stdout) {
2094 fclose(error_out);
2095 error_out = NULL;
2096 }
2097
2098 namelist_free(&test_list);
2099 namelist_free(&exclude_list);
2100 namelist_free(&exclude_dir_list);
2101 free(harness_dir);
2102 free(harness_features);
2103 free(harness_exclude);
2104 free(error_file);
2105
2106 return 0;
2107 }
2108