1 /*
2 * Copyright (C) 2008 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <dirent.h>
18 #include <errno.h>
19 #include <fcntl.h>
20 #include <limits.h>
21 #include <poll.h>
22 #include <signal.h>
23 #include <stdarg.h>
24 #include <stdio.h>
25 #include <stdlib.h>
26 #include <string.h>
27 #include <sys/inotify.h>
28 #include <sys/stat.h>
29 #include <sys/time.h>
30 #include <sys/wait.h>
31 #include <sys/klog.h>
32 #include <time.h>
33 #include <unistd.h>
34 #include <sys/prctl.h>
35
36 #include <cutils/debugger.h>
37 #include <cutils/properties.h>
38 #include <cutils/sockets.h>
39 #include <private/android_filesystem_config.h>
40
41 #include <selinux/android.h>
42
43 #include "dumpstate.h"
44
45 static const int64_t NANOS_PER_SEC = 1000000000;
46
47 /* list of native processes to include in the native dumps */
48 static const char* native_processes_to_dump[] = {
49 "/system/bin/drmserver",
50 "/system/bin/mediaserver",
51 "/system/bin/sdcard",
52 "/system/bin/surfaceflinger",
53 NULL,
54 };
55
for_each_userid(void (* func)(int),const char * header)56 void for_each_userid(void (*func)(int), const char *header) {
57 DIR *d;
58 struct dirent *de;
59
60 if (header) printf("\n------ %s ------\n", header);
61 func(0);
62
63 if (!(d = opendir("/data/system/users"))) {
64 printf("Failed to open /data/system/users (%s)\n", strerror(errno));
65 return;
66 }
67
68 while ((de = readdir(d))) {
69 int userid;
70 if (de->d_type != DT_DIR || !(userid = atoi(de->d_name))) {
71 continue;
72 }
73 func(userid);
74 }
75
76 closedir(d);
77 }
78
__for_each_pid(void (* helper)(int,const char *,void *),const char * header,void * arg)79 static void __for_each_pid(void (*helper)(int, const char *, void *), const char *header, void *arg) {
80 DIR *d;
81 struct dirent *de;
82
83 if (!(d = opendir("/proc"))) {
84 printf("Failed to open /proc (%s)\n", strerror(errno));
85 return;
86 }
87
88 printf("\n------ %s ------\n", header);
89 while ((de = readdir(d))) {
90 int pid;
91 int fd;
92 char cmdpath[255];
93 char cmdline[255];
94
95 if (!(pid = atoi(de->d_name))) {
96 continue;
97 }
98
99 sprintf(cmdpath,"/proc/%d/cmdline", pid);
100 memset(cmdline, 0, sizeof(cmdline));
101 if ((fd = open(cmdpath, O_RDONLY)) < 0) {
102 strcpy(cmdline, "N/A");
103 } else {
104 read(fd, cmdline, sizeof(cmdline) - 1);
105 close(fd);
106 }
107 helper(pid, cmdline, arg);
108 }
109
110 closedir(d);
111 }
112
for_each_pid_helper(int pid,const char * cmdline,void * arg)113 static void for_each_pid_helper(int pid, const char *cmdline, void *arg) {
114 for_each_pid_func *func = arg;
115 func(pid, cmdline);
116 }
117
for_each_pid(for_each_pid_func func,const char * header)118 void for_each_pid(for_each_pid_func func, const char *header) {
119 __for_each_pid(for_each_pid_helper, header, func);
120 }
121
for_each_tid_helper(int pid,const char * cmdline,void * arg)122 static void for_each_tid_helper(int pid, const char *cmdline, void *arg) {
123 DIR *d;
124 struct dirent *de;
125 char taskpath[255];
126 for_each_tid_func *func = arg;
127
128 sprintf(taskpath, "/proc/%d/task", pid);
129
130 if (!(d = opendir(taskpath))) {
131 printf("Failed to open %s (%s)\n", taskpath, strerror(errno));
132 return;
133 }
134
135 func(pid, pid, cmdline);
136
137 while ((de = readdir(d))) {
138 int tid;
139 int fd;
140 char commpath[255];
141 char comm[255];
142
143 if (!(tid = atoi(de->d_name))) {
144 continue;
145 }
146
147 if (tid == pid)
148 continue;
149
150 sprintf(commpath,"/proc/%d/comm", tid);
151 memset(comm, 0, sizeof(comm));
152 if ((fd = open(commpath, O_RDONLY)) < 0) {
153 strcpy(comm, "N/A");
154 } else {
155 char *c;
156 read(fd, comm, sizeof(comm) - 1);
157 close(fd);
158
159 c = strrchr(comm, '\n');
160 if (c) {
161 *c = '\0';
162 }
163 }
164 func(pid, tid, comm);
165 }
166
167 closedir(d);
168 }
169
for_each_tid(for_each_tid_func func,const char * header)170 void for_each_tid(for_each_tid_func func, const char *header) {
171 __for_each_pid(for_each_tid_helper, header, func);
172 }
173
show_wchan(int pid,int tid,const char * name)174 void show_wchan(int pid, int tid, const char *name) {
175 char path[255];
176 char buffer[255];
177 int fd;
178 char name_buffer[255];
179
180 memset(buffer, 0, sizeof(buffer));
181
182 sprintf(path, "/proc/%d/wchan", tid);
183 if ((fd = open(path, O_RDONLY)) < 0) {
184 printf("Failed to open '%s' (%s)\n", path, strerror(errno));
185 return;
186 }
187
188 if (read(fd, buffer, sizeof(buffer)) < 0) {
189 printf("Failed to read '%s' (%s)\n", path, strerror(errno));
190 goto out_close;
191 }
192
193 snprintf(name_buffer, sizeof(name_buffer), "%*s%s",
194 pid == tid ? 0 : 3, "", name);
195
196 printf("%-7d %-32s %s\n", tid, name_buffer, buffer);
197
198 out_close:
199 close(fd);
200 return;
201 }
202
do_dump_settings(int userid)203 void do_dump_settings(int userid) {
204 char title[255];
205 char dbpath[255];
206 char sql[255];
207 sprintf(title, "SYSTEM SETTINGS (user %d)", userid);
208 if (userid == 0) {
209 strcpy(dbpath, "/data/data/com.android.providers.settings/databases/settings.db");
210 strcpy(sql, "pragma user_version; select * from system; select * from secure; select * from global;");
211 } else {
212 sprintf(dbpath, "/data/system/users/%d/settings.db", userid);
213 strcpy(sql, "pragma user_version; select * from system; select * from secure;");
214 }
215 run_command(title, 20, SU_PATH, "root", "sqlite3", dbpath, sql, NULL);
216 return;
217 }
218
do_dmesg()219 void do_dmesg() {
220 printf("------ KERNEL LOG (dmesg) ------\n");
221 /* Get size of kernel buffer */
222 int size = klogctl(KLOG_SIZE_BUFFER, NULL, 0);
223 if (size <= 0) {
224 printf("Unexpected klogctl return value: %d\n\n", size);
225 return;
226 }
227 char *buf = (char *) malloc(size + 1);
228 if (buf == NULL) {
229 printf("memory allocation failed\n\n");
230 return;
231 }
232 int retval = klogctl(KLOG_READ_ALL, buf, size);
233 if (retval < 0) {
234 printf("klogctl failure\n\n");
235 free(buf);
236 return;
237 }
238 buf[retval] = '\0';
239 printf("%s\n\n", buf);
240 free(buf);
241 return;
242 }
243
do_showmap(int pid,const char * name)244 void do_showmap(int pid, const char *name) {
245 char title[255];
246 char arg[255];
247
248 sprintf(title, "SHOW MAP %d (%s)", pid, name);
249 sprintf(arg, "%d", pid);
250 run_command(title, 10, SU_PATH, "root", "showmap", arg, NULL);
251 }
252
253 /* prints the contents of a file */
dump_file(const char * title,const char * path)254 int dump_file(const char *title, const char *path) {
255 int fd = open(path, O_RDONLY);
256 if (fd < 0) {
257 int err = errno;
258 if (title) printf("------ %s (%s) ------\n", title, path);
259 printf("*** %s: %s\n", path, strerror(err));
260 if (title) printf("\n");
261 return -1;
262 }
263 return dump_file_from_fd(title, path, fd);
264 }
265
dump_file_from_fd(const char * title,const char * path,int fd)266 int dump_file_from_fd(const char *title, const char *path, int fd) {
267 char buffer[32768];
268
269 if (title) printf("------ %s (%s", title, path);
270
271 if (title) {
272 struct stat st;
273 if (memcmp(path, "/proc/", 6) && memcmp(path, "/sys/", 5) && !fstat(fd, &st)) {
274 char stamp[80];
275 time_t mtime = st.st_mtime;
276 strftime(stamp, sizeof(stamp), "%Y-%m-%d %H:%M:%S", localtime(&mtime));
277 printf(": %s", stamp);
278 }
279 printf(") ------\n");
280 }
281
282 int newline = 0;
283 for (;;) {
284 int ret = read(fd, buffer, sizeof(buffer));
285 if (ret > 0) {
286 newline = (buffer[ret - 1] == '\n');
287 ret = fwrite(buffer, ret, 1, stdout);
288 }
289 if (ret <= 0) break;
290 }
291 close(fd);
292
293 if (!newline) printf("\n");
294 if (title) printf("\n");
295 return 0;
296 }
297
nanotime()298 static int64_t nanotime() {
299 struct timespec ts;
300 clock_gettime(CLOCK_MONOTONIC, &ts);
301 return (int64_t)ts.tv_sec * NANOS_PER_SEC + ts.tv_nsec;
302 }
303
304 /* forks a command and waits for it to finish */
run_command(const char * title,int timeout_seconds,const char * command,...)305 int run_command(const char *title, int timeout_seconds, const char *command, ...) {
306 fflush(stdout);
307 int64_t start = nanotime();
308 pid_t pid = fork();
309
310 /* handle error case */
311 if (pid < 0) {
312 printf("*** fork: %s\n", strerror(errno));
313 return pid;
314 }
315
316 /* handle child case */
317 if (pid == 0) {
318 const char *args[1024] = {command};
319 size_t arg;
320
321 /* make sure the child dies when dumpstate dies */
322 prctl(PR_SET_PDEATHSIG, SIGKILL);
323
324 /* just ignore SIGPIPE, will go down with parent's */
325 struct sigaction sigact;
326 memset(&sigact, 0, sizeof(sigact));
327 sigact.sa_handler = SIG_IGN;
328 sigaction(SIGPIPE, &sigact, NULL);
329
330 va_list ap;
331 va_start(ap, command);
332 if (title) printf("------ %s (%s", title, command);
333 for (arg = 1; arg < sizeof(args) / sizeof(args[0]); ++arg) {
334 args[arg] = va_arg(ap, const char *);
335 if (args[arg] == NULL) break;
336 if (title) printf(" %s", args[arg]);
337 }
338 if (title) printf(") ------\n");
339 fflush(stdout);
340
341 execvp(command, (char**) args);
342 printf("*** exec(%s): %s\n", command, strerror(errno));
343 fflush(stdout);
344 _exit(-1);
345 }
346
347 /* handle parent case */
348 for (;;) {
349 int status;
350 pid_t p = waitpid(pid, &status, WNOHANG);
351 int64_t elapsed = nanotime() - start;
352 if (p == pid) {
353 if (WIFSIGNALED(status)) {
354 printf("*** %s: Killed by signal %d\n", command, WTERMSIG(status));
355 } else if (WIFEXITED(status) && WEXITSTATUS(status) > 0) {
356 printf("*** %s: Exit code %d\n", command, WEXITSTATUS(status));
357 }
358 if (title) printf("[%s: %.3fs elapsed]\n\n", command, (float)elapsed / NANOS_PER_SEC);
359 return status;
360 }
361
362 if (timeout_seconds && elapsed / NANOS_PER_SEC > timeout_seconds) {
363 printf("*** %s: Timed out after %.3fs (killing pid %d)\n", command, (float) elapsed / NANOS_PER_SEC, pid);
364 kill(pid, SIGTERM);
365 return -1;
366 }
367
368 usleep(100000); // poll every 0.1 sec
369 }
370 }
371
372 size_t num_props = 0;
373 static char* props[2000];
374
print_prop(const char * key,const char * name,void * user)375 static void print_prop(const char *key, const char *name, void *user) {
376 (void) user;
377 if (num_props < sizeof(props) / sizeof(props[0])) {
378 char buf[PROPERTY_KEY_MAX + PROPERTY_VALUE_MAX + 10];
379 snprintf(buf, sizeof(buf), "[%s]: [%s]\n", key, name);
380 props[num_props++] = strdup(buf);
381 }
382 }
383
compare_prop(const void * a,const void * b)384 static int compare_prop(const void *a, const void *b) {
385 return strcmp(*(char * const *) a, *(char * const *) b);
386 }
387
388 /* prints all the system properties */
print_properties()389 void print_properties() {
390 size_t i;
391 num_props = 0;
392 property_list(print_prop, NULL);
393 qsort(&props, num_props, sizeof(props[0]), compare_prop);
394
395 printf("------ SYSTEM PROPERTIES ------\n");
396 for (i = 0; i < num_props; ++i) {
397 fputs(props[i], stdout);
398 free(props[i]);
399 }
400 printf("\n");
401 }
402
403 /* redirect output to a service control socket */
redirect_to_socket(FILE * redirect,const char * service)404 void redirect_to_socket(FILE *redirect, const char *service) {
405 int s = android_get_control_socket(service);
406 if (s < 0) {
407 fprintf(stderr, "android_get_control_socket(%s): %s\n", service, strerror(errno));
408 exit(1);
409 }
410 if (listen(s, 4) < 0) {
411 fprintf(stderr, "listen(control socket): %s\n", strerror(errno));
412 exit(1);
413 }
414
415 struct sockaddr addr;
416 socklen_t alen = sizeof(addr);
417 int fd = accept(s, &addr, &alen);
418 if (fd < 0) {
419 fprintf(stderr, "accept(control socket): %s\n", strerror(errno));
420 exit(1);
421 }
422
423 fflush(redirect);
424 dup2(fd, fileno(redirect));
425 close(fd);
426 }
427
428 /* redirect output to a file, optionally gzipping; returns gzip pid (or -1) */
redirect_to_file(FILE * redirect,char * path,int gzip_level)429 pid_t redirect_to_file(FILE *redirect, char *path, int gzip_level) {
430 char *chp = path;
431
432 /* skip initial slash */
433 if (chp[0] == '/')
434 chp++;
435
436 /* create leading directories, if necessary */
437 while (chp && chp[0]) {
438 chp = strchr(chp, '/');
439 if (chp) {
440 *chp = 0;
441 mkdir(path, 0770); /* drwxrwx--- */
442 *chp++ = '/';
443 }
444 }
445
446 int fd = open(path, O_WRONLY | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
447 if (fd < 0) {
448 fprintf(stderr, "%s: %s\n", path, strerror(errno));
449 exit(1);
450 }
451
452 pid_t gzip_pid = -1;
453 if (gzip_level > 0) {
454 int fds[2];
455 if (pipe(fds)) {
456 fprintf(stderr, "pipe: %s\n", strerror(errno));
457 exit(1);
458 }
459
460 fflush(redirect);
461 fflush(stdout);
462
463 gzip_pid = fork();
464 if (gzip_pid < 0) {
465 fprintf(stderr, "fork: %s\n", strerror(errno));
466 exit(1);
467 }
468
469 if (gzip_pid == 0) {
470 dup2(fds[0], STDIN_FILENO);
471 dup2(fd, STDOUT_FILENO);
472
473 close(fd);
474 close(fds[0]);
475 close(fds[1]);
476
477 char level[10];
478 snprintf(level, sizeof(level), "-%d", gzip_level);
479 execlp("gzip", "gzip", level, NULL);
480 fprintf(stderr, "exec(gzip): %s\n", strerror(errno));
481 _exit(-1);
482 }
483
484 close(fd);
485 close(fds[0]);
486 fd = fds[1];
487 }
488
489 dup2(fd, fileno(redirect));
490 close(fd);
491 return gzip_pid;
492 }
493
should_dump_native_traces(const char * path)494 static bool should_dump_native_traces(const char* path) {
495 for (const char** p = native_processes_to_dump; *p; p++) {
496 if (!strcmp(*p, path)) {
497 return true;
498 }
499 }
500 return false;
501 }
502
503 /* dump Dalvik and native stack traces, return the trace file location (NULL if none) */
dump_traces()504 const char *dump_traces() {
505 const char* result = NULL;
506
507 char traces_path[PROPERTY_VALUE_MAX] = "";
508 property_get("dalvik.vm.stack-trace-file", traces_path, "");
509 if (!traces_path[0]) return NULL;
510
511 /* move the old traces.txt (if any) out of the way temporarily */
512 char anr_traces_path[PATH_MAX];
513 strlcpy(anr_traces_path, traces_path, sizeof(anr_traces_path));
514 strlcat(anr_traces_path, ".anr", sizeof(anr_traces_path));
515 if (rename(traces_path, anr_traces_path) && errno != ENOENT) {
516 fprintf(stderr, "rename(%s, %s): %s\n", traces_path, anr_traces_path, strerror(errno));
517 return NULL; // Can't rename old traces.txt -- no permission? -- leave it alone instead
518 }
519
520 /* make the directory if necessary */
521 char anr_traces_dir[PATH_MAX];
522 strlcpy(anr_traces_dir, traces_path, sizeof(anr_traces_dir));
523 char *slash = strrchr(anr_traces_dir, '/');
524 if (slash != NULL) {
525 *slash = '\0';
526 if (!mkdir(anr_traces_dir, 0775)) {
527 chown(anr_traces_dir, AID_SYSTEM, AID_SYSTEM);
528 chmod(anr_traces_dir, 0775);
529 if (selinux_android_restorecon(anr_traces_dir, 0) == -1) {
530 fprintf(stderr, "restorecon failed for %s: %s\n", anr_traces_dir, strerror(errno));
531 }
532 } else if (errno != EEXIST) {
533 fprintf(stderr, "mkdir(%s): %s\n", anr_traces_dir, strerror(errno));
534 return NULL;
535 }
536 }
537
538 /* create a new, empty traces.txt file to receive stack dumps */
539 int fd = open(traces_path, O_CREAT | O_WRONLY | O_TRUNC | O_NOFOLLOW, 0666); /* -rw-rw-rw- */
540 if (fd < 0) {
541 fprintf(stderr, "%s: %s\n", traces_path, strerror(errno));
542 return NULL;
543 }
544 int chmod_ret = fchmod(fd, 0666);
545 if (chmod_ret < 0) {
546 fprintf(stderr, "fchmod on %s failed: %s\n", traces_path, strerror(errno));
547 close(fd);
548 return NULL;
549 }
550
551 /* walk /proc and kill -QUIT all Dalvik processes */
552 DIR *proc = opendir("/proc");
553 if (proc == NULL) {
554 fprintf(stderr, "/proc: %s\n", strerror(errno));
555 goto error_close_fd;
556 }
557
558 /* use inotify to find when processes are done dumping */
559 int ifd = inotify_init();
560 if (ifd < 0) {
561 fprintf(stderr, "inotify_init: %s\n", strerror(errno));
562 goto error_close_fd;
563 }
564
565 int wfd = inotify_add_watch(ifd, traces_path, IN_CLOSE_WRITE);
566 if (wfd < 0) {
567 fprintf(stderr, "inotify_add_watch(%s): %s\n", traces_path, strerror(errno));
568 goto error_close_ifd;
569 }
570
571 struct dirent *d;
572 int dalvik_found = 0;
573 while ((d = readdir(proc))) {
574 int pid = atoi(d->d_name);
575 if (pid <= 0) continue;
576
577 char path[PATH_MAX];
578 char data[PATH_MAX];
579 snprintf(path, sizeof(path), "/proc/%d/exe", pid);
580 ssize_t len = readlink(path, data, sizeof(data) - 1);
581 if (len <= 0) {
582 continue;
583 }
584 data[len] = '\0';
585
586 if (!strncmp(data, "/system/bin/app_process", strlen("/system/bin/app_process"))) {
587 /* skip zygote -- it won't dump its stack anyway */
588 snprintf(path, sizeof(path), "/proc/%d/cmdline", pid);
589 int cfd = open(path, O_RDONLY);
590 len = read(cfd, data, sizeof(data) - 1);
591 close(cfd);
592 if (len <= 0) {
593 continue;
594 }
595 data[len] = '\0';
596 if (!strncmp(data, "zygote", strlen("zygote"))) {
597 continue;
598 }
599
600 ++dalvik_found;
601 int64_t start = nanotime();
602 if (kill(pid, SIGQUIT)) {
603 fprintf(stderr, "kill(%d, SIGQUIT): %s\n", pid, strerror(errno));
604 continue;
605 }
606
607 /* wait for the writable-close notification from inotify */
608 struct pollfd pfd = { ifd, POLLIN, 0 };
609 int ret = poll(&pfd, 1, 5000); /* 5 sec timeout */
610 if (ret < 0) {
611 fprintf(stderr, "poll: %s\n", strerror(errno));
612 } else if (ret == 0) {
613 fprintf(stderr, "warning: timed out dumping pid %d\n", pid);
614 } else {
615 struct inotify_event ie;
616 read(ifd, &ie, sizeof(ie));
617 }
618
619 if (lseek(fd, 0, SEEK_END) < 0) {
620 fprintf(stderr, "lseek: %s\n", strerror(errno));
621 } else {
622 dprintf(fd, "[dump dalvik stack %d: %.3fs elapsed]\n",
623 pid, (float)(nanotime() - start) / NANOS_PER_SEC);
624 }
625 } else if (should_dump_native_traces(data)) {
626 /* dump native process if appropriate */
627 if (lseek(fd, 0, SEEK_END) < 0) {
628 fprintf(stderr, "lseek: %s\n", strerror(errno));
629 } else {
630 static uint16_t timeout_failures = 0;
631 int64_t start = nanotime();
632
633 /* If 3 backtrace dumps fail in a row, consider debuggerd dead. */
634 if (timeout_failures == 3) {
635 dprintf(fd, "too many stack dump failures, skipping...\n");
636 } else if (dump_backtrace_to_file_timeout(pid, fd, 20) == -1) {
637 dprintf(fd, "dumping failed, likely due to a timeout\n");
638 timeout_failures++;
639 } else {
640 timeout_failures = 0;
641 }
642 dprintf(fd, "[dump native stack %d: %.3fs elapsed]\n",
643 pid, (float)(nanotime() - start) / NANOS_PER_SEC);
644 }
645 }
646 }
647
648 if (dalvik_found == 0) {
649 fprintf(stderr, "Warning: no Dalvik processes found to dump stacks\n");
650 }
651
652 static char dump_traces_path[PATH_MAX];
653 strlcpy(dump_traces_path, traces_path, sizeof(dump_traces_path));
654 strlcat(dump_traces_path, ".bugreport", sizeof(dump_traces_path));
655 if (rename(traces_path, dump_traces_path)) {
656 fprintf(stderr, "rename(%s, %s): %s\n", traces_path, dump_traces_path, strerror(errno));
657 goto error_close_ifd;
658 }
659 result = dump_traces_path;
660
661 /* replace the saved [ANR] traces.txt file */
662 rename(anr_traces_path, traces_path);
663
664 error_close_ifd:
665 close(ifd);
666 error_close_fd:
667 close(fd);
668 return result;
669 }
670
dump_route_tables()671 void dump_route_tables() {
672 const char* const RT_TABLES_PATH = "/data/misc/net/rt_tables";
673 dump_file("RT_TABLES", RT_TABLES_PATH);
674 FILE* fp = fopen(RT_TABLES_PATH, "r");
675 if (!fp) {
676 printf("*** %s: %s\n", RT_TABLES_PATH, strerror(errno));
677 return;
678 }
679 char table[16];
680 // Each line has an integer (the table number), a space, and a string (the table name). We only
681 // need the table number. It's a 32-bit unsigned number, so max 10 chars. Skip the table name.
682 // Add a fixed max limit so this doesn't go awry.
683 for (int i = 0; i < 64 && fscanf(fp, " %10s %*s", table) == 1; ++i) {
684 run_command("ROUTE TABLE IPv4", 10, "ip", "-4", "route", "show", "table", table, NULL);
685 run_command("ROUTE TABLE IPv6", 10, "ip", "-6", "route", "show", "table", table, NULL);
686 }
687 fclose(fp);
688 }
689