SAFE public projects git trees. - safe/jmp/linux-2.6/blob - tools/perf/builtin-annotate.c

   1 /*
   2  * builtin-annotate.c
   3  *
   4  * Builtin annotate command: Analyze the perf.data input file,
   5  * look up and read DSOs and symbol information and display
   6  * a histogram of results, along various sorting keys.
   7  */
   8 #include "builtin.h"
   9
  10 #include "util/util.h"
  11
  12 #include "util/color.h"
  13 #include <linux/list.h>
  14 #include "util/cache.h"
  15 #include <linux/rbtree.h>
  16 #include "util/symbol.h"
  17 #include "util/string.h"
  18
  19 #include "perf.h"
  20 #include "util/debug.h"
  21
  22 #include "util/parse-options.h"
  23 #include "util/parse-events.h"
  24 #include "util/thread.h"
  25 #include "util/sort.h"
  26 #include "util/hist.h"
  27
  28 static char             const *input_name = "perf.data";
  29
  30 static int              force;
  31 static int              input;
  32 static int              show_mask = SHOW_KERNEL | SHOW_USER | SHOW_HV;
  33
  34 static int              full_paths;
  35
  36 static int              print_line;
  37
  38 static unsigned long    page_size;
  39 static unsigned long    mmap_window = 32;
  40
  41 static struct rb_root   threads;
  42 static struct thread    *last_match;
  43
  44
  45 struct sym_ext {
  46         struct rb_node  node;
  47         double          percent;
  48         char            *path;
  49 };
  50
  51
  52 /*
  53  * collect histogram counts
  54  */
  55 static void hist_hit(struct hist_entry *he, u64 ip)
  56 {
  57         unsigned int sym_size, offset;
  58         struct symbol *sym = he->sym;
  59
  60         he->count++;
  61
  62         if (!sym || !sym->hist)
  63                 return;
  64
  65         sym_size = sym->end - sym->start;
  66         ip = he->map->map_ip(he->map, ip);
  67         offset = ip - sym->start;
  68
  69         if (offset >= sym_size)
  70                 return;
  71
  72         sym->hist_sum++;
  73         sym->hist[offset]++;
  74
  75         if (verbose >= 3)
  76                 printf("%p %s: count++ [ip: %p, %08Lx] => %Ld\n",
  77                         (void *)(unsigned long)he->sym->start,
  78                         he->sym->name,
  79                         (void *)(unsigned long)ip, ip - he->sym->start,
  80                         sym->hist[offset]);
  81 }
  82
  83 static int
  84 hist_entry__add(struct thread *thread, struct map *map,
  85                 struct symbol *sym, u64 ip, char level)
  86 {
  87         struct rb_node **p = &hist.rb_node;
  88         struct rb_node *parent = NULL;
  89         struct hist_entry *he;
  90         struct hist_entry entry = {
  91                 .thread = thread,
  92                 .map    = map,
  93                 .sym    = sym,
  94                 .ip     = ip,
  95                 .level  = level,
  96                 .count  = 1,
  97         };
  98         int cmp;
  99
 100         while (*p != NULL) {
 101                 parent = *p;
 102                 he = rb_entry(parent, struct hist_entry, rb_node);
 103
 104                 cmp = hist_entry__cmp(&entry, he);
 105
 106                 if (!cmp) {
 107                         hist_hit(he, ip);
 108
 109                         return 0;
 110                 }
 111
 112                 if (cmp < 0)
 113                         p = &(*p)->rb_left;
 114                 else
 115                         p = &(*p)->rb_right;
 116         }
 117
 118         he = malloc(sizeof(*he));
 119         if (!he)
 120                 return -ENOMEM;
 121         *he = entry;
 122         rb_link_node(&he->rb_node, parent, p);
 123         rb_insert_color(&he->rb_node, &hist);
 124
 125         return 0;
 126 }
 127
 128 static int
 129 process_sample_event(event_t *event, unsigned long offset, unsigned long head)
 130 {
 131         char level;
 132         int show = 0;
 133         struct thread *thread;
 134         u64 ip = event->ip.ip;
 135         struct map *map = NULL;
 136         struct symbol *sym = NULL;
 137
 138         thread = threads__findnew(event->ip.pid, &threads, &last_match);
 139
 140         dump_printf("%p [%p]: PERF_EVENT (IP, %d): %d: %p\n",
 141                 (void *)(offset + head),
 142                 (void *)(long)(event->header.size),
 143                 event->header.misc,
 144                 event->ip.pid,
 145                 (void *)(long)ip);
 146
 147         dump_printf(" ... thread: %s:%d\n", thread->comm, thread->pid);
 148
 149         if (thread == NULL) {
 150                 fprintf(stderr, "problem processing %d event, skipping it.\n",
 151                         event->header.type);
 152                 return -1;
 153         }
 154
 155         if (event->header.misc & PERF_RECORD_MISC_KERNEL) {
 156                 show = SHOW_KERNEL;
 157                 level = 'k';
 158                 sym = kernel_maps__find_symbol(ip, &map);
 159                 dump_printf(" ...... dso: %s\n",
 160                             map ? map->dso->long_name : "<not found>");
 161         } else if (event->header.misc & PERF_RECORD_MISC_USER) {
 162                 show = SHOW_USER;
 163                 level = '.';
 164                 map = thread__find_map(thread, ip);
 165                 if (map != NULL) {
 166 got_map:
 167                         ip = map->map_ip(map, ip);
 168                         sym = map->dso->find_symbol(map->dso, ip);
 169                 } else {
 170                         /*
 171                          * If this is outside of all known maps,
 172                          * and is a negative address, try to look it
 173                          * up in the kernel dso, as it might be a
 174                          * vsyscall or vdso (which executes in user-mode).
 175                          *
 176                          * XXX This is nasty, we should have a symbol list in
 177                          * the "[vdso]" dso, but for now lets use the old
 178                          * trick of looking in the whole kernel symbol list.
 179                          */
 180                         if ((long long)ip < 0) {
 181                                 map = kernel_map;
 182                                 goto got_map;
 183                         }
 184                 }
 185                 dump_printf(" ...... dso: %s\n",
 186                             map ? map->dso->long_name : "<not found>");
 187         } else {
 188                 show = SHOW_HV;
 189                 level = 'H';
 190                 dump_printf(" ...... dso: [hypervisor]\n");
 191         }
 192
 193         if (show & show_mask) {
 194                 if (hist_entry__add(thread, map, sym, ip, level)) {
 195                         fprintf(stderr,
 196                 "problem incrementing symbol count, skipping event\n");
 197                         return -1;
 198                 }
 199         }
 200         total++;
 201
 202         return 0;
 203 }
 204
 205 static int
 206 process_mmap_event(event_t *event, unsigned long offset, unsigned long head)
 207 {
 208         struct thread *thread;
 209         struct map *map = map__new(&event->mmap, NULL, 0);
 210
 211         thread = threads__findnew(event->mmap.pid, &threads, &last_match);
 212
 213         dump_printf("%p [%p]: PERF_RECORD_MMAP %d: [%p(%p) @ %p]: %s\n",
 214                 (void *)(offset + head),
 215                 (void *)(long)(event->header.size),
 216                 event->mmap.pid,
 217                 (void *)(long)event->mmap.start,
 218                 (void *)(long)event->mmap.len,
 219                 (void *)(long)event->mmap.pgoff,
 220                 event->mmap.filename);
 221
 222         if (thread == NULL || map == NULL) {
 223                 dump_printf("problem processing PERF_RECORD_MMAP, skipping event.\n");
 224                 return 0;
 225         }
 226
 227         thread__insert_map(thread, map);
 228         total_mmap++;
 229
 230         return 0;
 231 }
 232
 233 static int
 234 process_comm_event(event_t *event, unsigned long offset, unsigned long head)
 235 {
 236         struct thread *thread;
 237
 238         thread = threads__findnew(event->comm.pid, &threads, &last_match);
 239         dump_printf("%p [%p]: PERF_RECORD_COMM: %s:%d\n",
 240                 (void *)(offset + head),
 241                 (void *)(long)(event->header.size),
 242                 event->comm.comm, event->comm.pid);
 243
 244         if (thread == NULL ||
 245             thread__set_comm(thread, event->comm.comm)) {
 246                 dump_printf("problem processing PERF_RECORD_COMM, skipping event.\n");
 247                 return -1;
 248         }
 249         total_comm++;
 250
 251         return 0;
 252 }
 253
 254 static int
 255 process_fork_event(event_t *event, unsigned long offset, unsigned long head)
 256 {
 257         struct thread *thread;
 258         struct thread *parent;
 259
 260         thread = threads__findnew(event->fork.pid, &threads, &last_match);
 261         parent = threads__findnew(event->fork.ppid, &threads, &last_match);
 262         dump_printf("%p [%p]: PERF_RECORD_FORK: %d:%d\n",
 263                 (void *)(offset + head),
 264                 (void *)(long)(event->header.size),
 265                 event->fork.pid, event->fork.ppid);
 266
 267         /*
 268          * A thread clone will have the same PID for both
 269          * parent and child.
 270          */
 271         if (thread == parent)
 272                 return 0;
 273
 274         if (!thread || !parent || thread__fork(thread, parent)) {
 275                 dump_printf("problem processing PERF_RECORD_FORK, skipping event.\n");
 276                 return -1;
 277         }
 278         total_fork++;
 279
 280         return 0;
 281 }
 282
 283 static int
 284 process_event(event_t *event, unsigned long offset, unsigned long head)
 285 {
 286         switch (event->header.type) {
 287         case PERF_RECORD_SAMPLE:
 288                 return process_sample_event(event, offset, head);
 289
 290         case PERF_RECORD_MMAP:
 291                 return process_mmap_event(event, offset, head);
 292
 293         case PERF_RECORD_COMM:
 294                 return process_comm_event(event, offset, head);
 295
 296         case PERF_RECORD_FORK:
 297                 return process_fork_event(event, offset, head);
 298         /*
 299          * We dont process them right now but they are fine:
 300          */
 301
 302         case PERF_RECORD_THROTTLE:
 303         case PERF_RECORD_UNTHROTTLE:
 304                 return 0;
 305
 306         default:
 307                 return -1;
 308         }
 309
 310         return 0;
 311 }
 312
 313 static int
 314 parse_line(FILE *file, struct symbol *sym, u64 len)
 315 {
 316         char *line = NULL, *tmp, *tmp2;
 317         static const char *prev_line;
 318         static const char *prev_color;
 319         unsigned int offset;
 320         size_t line_len;
 321         s64 line_ip;
 322         int ret;
 323         char *c;
 324
 325         if (getline(&line, &line_len, file) < 0)
 326                 return -1;
 327         if (!line)
 328                 return -1;
 329
 330         c = strchr(line, '\n');
 331         if (c)
 332                 *c = 0;
 333
 334         line_ip = -1;
 335         offset = 0;
 336         ret = -2;
 337
 338         /*
 339          * Strip leading spaces:
 340          */
 341         tmp = line;
 342         while (*tmp) {
 343                 if (*tmp != ' ')
 344                         break;
 345                 tmp++;
 346         }
 347
 348         if (*tmp) {
 349                 /*
 350                  * Parse hexa addresses followed by ':'
 351                  */
 352                 line_ip = strtoull(tmp, &tmp2, 16);
 353                 if (*tmp2 != ':')
 354                         line_ip = -1;
 355         }
 356
 357         if (line_ip != -1) {
 358                 const char *path = NULL;
 359                 unsigned int hits = 0;
 360                 double percent = 0.0;
 361                 const char *color;
 362                 struct sym_ext *sym_ext = sym->priv;
 363
 364                 offset = line_ip - sym->start;
 365                 if (offset < len)
 366                         hits = sym->hist[offset];
 367
 368                 if (offset < len && sym_ext) {
 369                         path = sym_ext[offset].path;
 370                         percent = sym_ext[offset].percent;
 371                 } else if (sym->hist_sum)
 372                         percent = 100.0 * hits / sym->hist_sum;
 373
 374                 color = get_percent_color(percent);
 375
 376                 /*
 377                  * Also color the filename and line if needed, with
 378                  * the same color than the percentage. Don't print it
 379                  * twice for close colored ip with the same filename:line
 380                  */
 381                 if (path) {
 382                         if (!prev_line || strcmp(prev_line, path)
 383                                        || color != prev_color) {
 384                                 color_fprintf(stdout, color, " %s", path);
 385                                 prev_line = path;
 386                                 prev_color = color;
 387                         }
 388                 }
 389
 390                 color_fprintf(stdout, color, " %7.2f", percent);
 391                 printf(" :      ");
 392                 color_fprintf(stdout, PERF_COLOR_BLUE, "%s\n", line);
 393         } else {
 394                 if (!*line)
 395                         printf("         :\n");
 396                 else
 397                         printf("         :      %s\n", line);
 398         }
 399
 400         return 0;
 401 }
 402
 403 static struct rb_root root_sym_ext;
 404
 405 static void insert_source_line(struct sym_ext *sym_ext)
 406 {
 407         struct sym_ext *iter;
 408         struct rb_node **p = &root_sym_ext.rb_node;
 409         struct rb_node *parent = NULL;
 410
 411         while (*p != NULL) {
 412                 parent = *p;
 413                 iter = rb_entry(parent, struct sym_ext, node);
 414
 415                 if (sym_ext->percent > iter->percent)
 416                         p = &(*p)->rb_left;
 417                 else
 418                         p = &(*p)->rb_right;
 419         }
 420
 421         rb_link_node(&sym_ext->node, parent, p);
 422         rb_insert_color(&sym_ext->node, &root_sym_ext);
 423 }
 424
 425 static void free_source_line(struct symbol *sym, int len)
 426 {
 427         struct sym_ext *sym_ext = sym->priv;
 428         int i;
 429
 430         if (!sym_ext)
 431                 return;
 432
 433         for (i = 0; i < len; i++)
 434                 free(sym_ext[i].path);
 435         free(sym_ext);
 436
 437         sym->priv = NULL;
 438         root_sym_ext = RB_ROOT;
 439 }
 440
 441 /* Get the filename:line for the colored entries */
 442 static void
 443 get_source_line(struct symbol *sym, int len, const char *filename)
 444 {
 445         int i;
 446         char cmd[PATH_MAX * 2];
 447         struct sym_ext *sym_ext;
 448
 449         if (!sym->hist_sum)
 450                 return;
 451
 452         sym->priv = calloc(len, sizeof(struct sym_ext));
 453         if (!sym->priv)
 454                 return;
 455
 456         sym_ext = sym->priv;
 457
 458         for (i = 0; i < len; i++) {
 459                 char *path = NULL;
 460                 size_t line_len;
 461                 u64 offset;
 462                 FILE *fp;
 463
 464                 sym_ext[i].percent = 100.0 * sym->hist[i] / sym->hist_sum;
 465                 if (sym_ext[i].percent <= 0.5)
 466                         continue;
 467
 468                 offset = sym->start + i;
 469                 sprintf(cmd, "addr2line -e %s %016llx", filename, offset);
 470                 fp = popen(cmd, "r");
 471                 if (!fp)
 472                         continue;
 473
 474                 if (getline(&path, &line_len, fp) < 0 || !line_len)
 475                         goto next;
 476
 477                 sym_ext[i].path = malloc(sizeof(char) * line_len + 1);
 478                 if (!sym_ext[i].path)
 479                         goto next;
 480
 481                 strcpy(sym_ext[i].path, path);
 482                 insert_source_line(&sym_ext[i]);
 483
 484         next:
 485                 pclose(fp);
 486         }
 487 }
 488
 489 static void print_summary(const char *filename)
 490 {
 491         struct sym_ext *sym_ext;
 492         struct rb_node *node;
 493
 494         printf("\nSorted summary for file %s\n", filename);
 495         printf("----------------------------------------------\n\n");
 496
 497         if (RB_EMPTY_ROOT(&root_sym_ext)) {
 498                 printf(" Nothing higher than %1.1f%%\n", MIN_GREEN);
 499                 return;
 500         }
 501
 502         node = rb_first(&root_sym_ext);
 503         while (node) {
 504                 double percent;
 505                 const char *color;
 506                 char *path;
 507
 508                 sym_ext = rb_entry(node, struct sym_ext, node);
 509                 percent = sym_ext->percent;
 510                 color = get_percent_color(percent);
 511                 path = sym_ext->path;
 512
 513                 color_fprintf(stdout, color, " %7.2f %s", percent, path);
 514                 node = rb_next(node);
 515         }
 516 }
 517
 518 static void annotate_sym(struct dso *dso, struct symbol *sym)
 519 {
 520         const char *filename = dso->long_name, *d_filename;
 521         u64 len;
 522         char command[PATH_MAX*2];
 523         FILE *file;
 524
 525         if (!filename)
 526                 return;
 527
 528         if (full_paths)
 529                 d_filename = filename;
 530         else
 531                 d_filename = basename(filename);
 532
 533         len = sym->end - sym->start;
 534
 535         if (print_line) {
 536                 get_source_line(sym, len, filename);
 537                 print_summary(filename);
 538         }
 539
 540         printf("\n\n------------------------------------------------\n");
 541         printf(" Percent |      Source code & Disassembly of %s\n", d_filename);
 542         printf("------------------------------------------------\n");
 543
 544         if (verbose >= 2)
 545                 printf("annotating [%p] %30s : [%p] %30s\n",
 546                        dso, dso->long_name, sym, sym->name);
 547
 548         sprintf(command, "objdump --start-address=0x%016Lx --stop-address=0x%016Lx -dS %s|grep -v %s",
 549                 sym->start, sym->end, filename, filename);
 550
 551         if (verbose >= 3)
 552                 printf("doing: %s\n", command);
 553
 554         file = popen(command, "r");
 555         if (!file)
 556                 return;
 557
 558         while (!feof(file)) {
 559                 if (parse_line(file, sym, len) < 0)
 560                         break;
 561         }
 562
 563         pclose(file);
 564         if (print_line)
 565                 free_source_line(sym, len);
 566 }
 567
 568 static void find_annotations(void)
 569 {
 570         struct rb_node *nd;
 571         struct dso *dso;
 572         int count = 0;
 573
 574         list_for_each_entry(dso, &dsos, node) {
 575
 576                 for (nd = rb_first(&dso->syms); nd; nd = rb_next(nd)) {
 577                         struct symbol *sym = rb_entry(nd, struct symbol, rb_node);
 578
 579                         if (sym->hist) {
 580                                 annotate_sym(dso, sym);
 581                                 count++;
 582                         }
 583                 }
 584         }
 585
 586         if (!count)
 587                 printf(" Error: symbol '%s' not present amongst the samples.\n", sym_hist_filter);
 588 }
 589
 590 static int __cmd_annotate(void)
 591 {
 592         int ret, rc = EXIT_FAILURE;
 593         unsigned long offset = 0;
 594         unsigned long head = 0;
 595         struct stat input_stat;
 596         event_t *event;
 597         uint32_t size;
 598         char *buf;
 599
 600         register_idle_thread(&threads, &last_match);
 601
 602         input = open(input_name, O_RDONLY);
 603         if (input < 0) {
 604                 perror("failed to open file");
 605                 exit(-1);
 606         }
 607
 608         ret = fstat(input, &input_stat);
 609         if (ret < 0) {
 610                 perror("failed to stat file");
 611                 exit(-1);
 612         }
 613
 614         if (!force && input_stat.st_uid && (input_stat.st_uid != geteuid())) {
 615                 fprintf(stderr, "file: %s not owned by current user or root\n", input_name);
 616                 exit(-1);
 617         }
 618
 619         if (!input_stat.st_size) {
 620                 fprintf(stderr, "zero-sized file, nothing to do!\n");
 621                 exit(0);
 622         }
 623
 624         if (load_kernel() < 0) {
 625                 perror("failed to load kernel symbols");
 626                 return EXIT_FAILURE;
 627         }
 628
 629 remap:
 630         buf = (char *)mmap(NULL, page_size * mmap_window, PROT_READ,
 631                            MAP_SHARED, input, offset);
 632         if (buf == MAP_FAILED) {
 633                 perror("failed to mmap file");
 634                 exit(-1);
 635         }
 636
 637 more:
 638         event = (event_t *)(buf + head);
 639
 640         size = event->header.size;
 641         if (!size)
 642                 size = 8;
 643
 644         if (head + event->header.size >= page_size * mmap_window) {
 645                 unsigned long shift = page_size * (head / page_size);
 646                 int munmap_ret;
 647
 648                 munmap_ret = munmap(buf, page_size * mmap_window);
 649                 assert(munmap_ret == 0);
 650
 651                 offset += shift;
 652                 head -= shift;
 653                 goto remap;
 654         }
 655
 656         size = event->header.size;
 657
 658         dump_printf("%p [%p]: event: %d\n",
 659                         (void *)(offset + head),
 660                         (void *)(long)event->header.size,
 661                         event->header.type);
 662
 663         if (!size || process_event(event, offset, head) < 0) {
 664
 665                 dump_printf("%p [%p]: skipping unknown header type: %d\n",
 666                         (void *)(offset + head),
 667                         (void *)(long)(event->header.size),
 668                         event->header.type);
 669
 670                 total_unknown++;
 671
 672                 /*
 673                  * assume we lost track of the stream, check alignment, and
 674                  * increment a single u64 in the hope to catch on again 'soon'.
 675                  */
 676
 677                 if (unlikely(head & 7))
 678                         head &= ~7ULL;
 679
 680                 size = 8;
 681         }
 682
 683         head += size;
 684
 685         if (offset + head < (unsigned long)input_stat.st_size)
 686                 goto more;
 687
 688         rc = EXIT_SUCCESS;
 689         close(input);
 690
 691         dump_printf("      IP events: %10ld\n", total);
 692         dump_printf("    mmap events: %10ld\n", total_mmap);
 693         dump_printf("    comm events: %10ld\n", total_comm);
 694         dump_printf("    fork events: %10ld\n", total_fork);
 695         dump_printf(" unknown events: %10ld\n", total_unknown);
 696
 697         if (dump_trace)
 698                 return 0;
 699
 700         if (verbose >= 3)
 701                 threads__fprintf(stdout, &threads);
 702
 703         if (verbose >= 2)
 704                 dsos__fprintf(stdout);
 705
 706         collapse__resort();
 707         output__resort(total);
 708
 709         find_annotations();
 710
 711         return rc;
 712 }
 713
 714 static const char * const annotate_usage[] = {
 715         "perf annotate [<options>] <command>",
 716         NULL
 717 };
 718
 719 static const struct option options[] = {
 720         OPT_STRING('i', "input", &input_name, "file",
 721                     "input file name"),
 722         OPT_STRING('s', "symbol", &sym_hist_filter, "symbol",
 723                     "symbol to annotate"),
 724         OPT_BOOLEAN('f', "force", &force, "don't complain, do it"),
 725         OPT_BOOLEAN('v', "verbose", &verbose,
 726                     "be more verbose (show symbol address, etc)"),
 727         OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
 728                     "dump raw trace in ASCII"),
 729         OPT_STRING('k', "vmlinux", &vmlinux_name, "file", "vmlinux pathname"),
 730         OPT_BOOLEAN('m', "modules", &modules,
 731                     "load module symbols - WARNING: use only with -k and LIVE kernel"),
 732         OPT_BOOLEAN('l', "print-line", &print_line,
 733                     "print matching source lines (may be slow)"),
 734         OPT_BOOLEAN('P', "full-paths", &full_paths,
 735                     "Don't shorten the displayed pathnames"),
 736         OPT_END()
 737 };
 738
 739 static void setup_sorting(void)
 740 {
 741         char *tmp, *tok, *str = strdup(sort_order);
 742
 743         for (tok = strtok_r(str, ", ", &tmp);
 744                         tok; tok = strtok_r(NULL, ", ", &tmp)) {
 745                 if (sort_dimension__add(tok) < 0) {
 746                         error("Unknown --sort key: `%s'", tok);
 747                         usage_with_options(annotate_usage, options);
 748                 }
 749         }
 750
 751         free(str);
 752 }
 753
 754 int cmd_annotate(int argc, const char **argv, const char *prefix __used)
 755 {
 756         symbol__init();
 757
 758         page_size = getpagesize();
 759
 760         argc = parse_options(argc, argv, options, annotate_usage, 0);
 761
 762         setup_sorting();
 763
 764         if (argc) {
 765                 /*
 766                  * Special case: if there's an argument left then assume tha
 767                  * it's a symbol filter:
 768                  */
 769                 if (argc > 1)
 770                         usage_with_options(annotate_usage, options);
 771
 772                 sym_hist_filter = argv[0];
 773         }
 774
 775         if (!sym_hist_filter)
 776                 usage_with_options(annotate_usage, options);
 777
 778         setup_pager();
 779
 780         if (field_sep && *field_sep == '.') {
 781                 fputs("'.' is the only non valid --field-separator argument\n",
 782                                 stderr);
 783                 exit(129);
 784         }
 785
 786         return __cmd_annotate();
 787 }