From 9ca6ee74e305cb8be064bab35674dcddde232148 Mon Sep 17 00:00:00 2001 From: Nick Gasson Date: Sun, 23 Jul 2023 17:18:47 +0100 Subject: [PATCH] Add command to export coverage data in Cobertura format --- .github/workflows/build-test.yml | 2 +- NEWS.md | 2 + configure.ac | 7 + nvc.1 | 49 +++++- src/nvc.c | 127 ++++++++++++++-- src/option.c | 1 + src/option.h | 1 + src/rt/cover.c | 249 ++++++++++++++++++++++++++++++- src/rt/cover.h | 3 + src/rt/stdenv.c | 7 +- src/util.c | 13 ++ src/util.h | 1 + test/cobertura.dtd | 60 ++++++++ test/regress/cover17.vhd | 45 ++++++ test/regress/gold/cover17.xml | 46 ++++++ test/regress/testlist.txt | 1 + test/run_regr.c | 115 ++++++++++++-- 17 files changed, 686 insertions(+), 43 deletions(-) create mode 100644 test/cobertura.dtd create mode 100644 test/regress/cover17.vhd create mode 100644 test/regress/gold/cover17.xml diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml index 80184b1f0..fa0dba6bf 100644 --- a/.github/workflows/build-test.yml +++ b/.github/workflows/build-test.yml @@ -20,7 +20,7 @@ jobs: run: | sudo apt-get install automake flex llvm-dev check lcov \ libdw-dev libffi-dev bison libreadline-dev tcl8.6-dev \ - libzstd-dev libmicrohttpd-dev + libzstd-dev libmicrohttpd-dev libxml2-utils - name: Generate configure script run: ./autogen.sh - name: Configure diff --git a/NEWS.md b/NEWS.md index f6fc7c3c1..ce7716392 100644 --- a/NEWS.md +++ b/NEWS.md @@ -3,6 +3,8 @@ (#731). - The format of fractional `time` values returned by the standard `to_string` function was changed to match other simulators. +- New command `--cover-export` exports coverage data in the Cobertura + XML format which is supported by most CI environments such as GitLab. ## Version 1.10.0 - 2023-07-14 - The Zstandard compression library is now a build dependency. Install diff --git a/configure.ac b/configure.ac index b44761b3b..2342ec4e5 100644 --- a/configure.ac +++ b/configure.ac @@ -145,6 +145,13 @@ AC_DEFINE_UNQUOTED([SH_PATH], ["`$pathprog $sh_path`"], [Path to POSIX shell]) AC_PATH_PROG([diff_path], ["diff"], ["/usr/bin/diff"]) AC_DEFINE_UNQUOTED([DIFF_PATH], ["`$pathprog $diff_path`"], [Path to diff program]) +AC_PATH_PROG([xmllint_path], ["xmllint"]) +if test -n "$xmllint_path"; then + # This is only used for regression tests + AC_DEFINE_UNQUOTED([XMLLINT_PATH], ["`$pathprog $xmllint_path`"], + [Path to xmllint program for tests]) +fi + AC_DEFINE_UNQUOTED([DIR_SEP], ["$DIR_SEP"], [Directory separator]) AC_DEFINE_UNQUOTED([EXEEXT], ["$EXEEXT"], [Executable file extension]) diff --git a/nvc.1 b/nvc.1 index 72d175429..f60e4681b 100644 --- a/nvc.1 +++ b/nvc.1 @@ -53,6 +53,10 @@ Execute a previously elaborated top level design unit. Process code coverage data from .Ar file and generate coverage report. +.\" --cover-export +.It Fl \-cover-export Ar unit +Export collected coverage information for a previously executed top +level design unit to an external format such as Cobertura XML. .\" --dump .It Fl \-dump Ar unit Print out a pseudo-VHDL representation of an analysed unit. This is @@ -79,21 +83,30 @@ Check input files for syntax errors only. .El .\" .Pp -Commands can be chained together. For example to analyse a file -.Ar foo.vhd +Commands can be chained together arbitrarily and the top-level unit +name need only be specified once. For example to analyse a file +.Ql source.vhd and then elaborate and run a top-level entity -.Ar bar : +.Ql tb : .Bd -literal -offset indent -$ nvc -a foo.vhd -e bar -r +$ nvc -a source.vhd -e tb -r .Ed .Pp -Note that the +Or to elaborate again with coverage collection enabled, run the +simulation, and export the data to the Cobertura format: +.Bd -literal -offset indent +$ nvc -e --cover tb -r --cover-export --format=cobertura -o out.xml +.Ed +.Pp +Note how the .Ar unit argument for the .Fl r -run command is taken from the earlier +and +.Fl \-cover-export +commands is taken from the earlier .Fl e -elaborate command. +command. .\" ------------------------------------------------------------ .\" Global options .\" ------------------------------------------------------------ @@ -447,6 +460,28 @@ is 5000. Prints detailed hierarchy coverage when generating code coverage report. .El .\" ------------------------------------------------------------ +.\" Coverage export options +.\" ------------------------------------------------------------ +.Ss Coverage export options +.Bl -tag -width Ds +.\" --format +.It Fl \-format= Ns Ar format +Output file format. Currently the only valid value is +.Ql cobertura +which is the Cobertura XML format widely supported by CI systems. +.\" --output +.It Fl o , Fl \-output= Ns Ar file +Write output to +.Ar file . +If this option is not specified the standard output stream is used. +.\" --relative +.It Fl \-relative Ns Op = Ns Ar path +Strip +.Ar path +or the current working directory from the front of any absolute path +names in the output. +.El +.\" ------------------------------------------------------------ .\" Make options .\" ------------------------------------------------------------ .Ss Make options diff --git a/src/nvc.c b/src/nvc.c index 8785a0617..18ea588b8 100644 --- a/src/nvc.c +++ b/src/nvc.c @@ -88,6 +88,7 @@ static int scan_cmd(int start, int argc, char **argv) const char *commands[] = { "-a", "-e", "-r", "-c", "--dump", "--make", "--syntax", "--list", "--init", "--install", "--print-deps", "--aotgen", "--do", "-i", + "--cover-export" }; for (int i = start; i < argc; i++) { @@ -266,6 +267,9 @@ static void set_top_level(char **argv, int next_cmd) if (top_level == NULL) fatal("missing top-level unit name"); } + else if (optind != next_cmd - 1) + fatal("excess positional argument '%s' following top-level unit name", + argv[optind + 1]); else { free(top_level_orig); top_level_orig = xstrdup(argv[optind]); @@ -1292,6 +1296,7 @@ static int coverage(int argc, char **argv) static struct option long_options[] = { { "report", required_argument, 0, 'r' }, { "exclude-file", required_argument, 0, 'e' }, + { "export", required_argument, 0, 'E' }, { "merge", required_argument, 0, 'm' }, { "dont-print", required_argument, 0, 'd' }, { "item-limit", required_argument, 0, 'l' }, @@ -1300,6 +1305,7 @@ static int coverage(int argc, char **argv) }; const char *out_db = NULL, *rpt_file = NULL, *exclude_file = NULL; + const char *export_file = NULL; int c, index; const char *spec = ":V"; cover_mask_t rpt_mask = 0; @@ -1316,6 +1322,9 @@ static int coverage(int argc, char **argv) case 'e': exclude_file = optarg; break; + case 'E': + export_file = optarg; + break; case 'd': rpt_mask = parse_cover_print_spec(optarg); break; @@ -1375,19 +1384,104 @@ static int coverage(int argc, char **argv) cover_report(rpt_file, cover, item_limit); } + if (export_file && cover) { + progress("Exporting XML coverage report"); + + FILE *f = fopen(export_file, "w"); + if (f == NULL) + fatal_errno("cannot open %s", export_file); + + cover_export_cobertura(cover, f, NULL); + fclose(f); + } + return 0; } +static int cover_export_cmd(int argc, char **argv) +{ + static struct option long_options[] = { + { "format", required_argument, 0, 'f' }, + { "output", required_argument, 0, 'o' }, + { "relative", optional_argument, 0, 'r' }, + { 0, 0, 0, 0 } + }; + + const int next_cmd = scan_cmd(2, argc, argv); + + enum { UNSET, COBERTURA } format = UNSET; + const char *output = NULL, *relative = NULL; + int c, index; + const char *spec = ":o"; + while ((c = getopt_long(argc, argv, spec, long_options, &index)) != -1) { + switch (c) { + case 'f': + if (strcasecmp(optarg, "cobertura") == 0) + format = COBERTURA; + else + fatal("unknown format '%s', valid formats are: cobertura", optarg); + break; + case 'o': + output = optarg; + break; + case 'r': + relative = optarg ?: "."; + break; + case '?': + bad_option("coverage", argv); + case ':': + missing_argument("coverage", argv); + default: + abort(); + } + } + + set_top_level(argv, next_cmd); + + if (format == UNSET) { + diag_t *d = diag_new(DIAG_FATAL, NULL); + diag_printf(d, "the $bold$--format$$ option is required"); + diag_hint(d, NULL, "pass $bold$--format=cobertura$$ for Cobertura XML"); + diag_emit(d); + return EXIT_FAILURE; + } + + char *fname LOCAL = xasprintf("_%s.elab.covdb", istr(top_level)); + fbuf_t *f = lib_fbuf_open(lib_work(), fname, FBUF_IN, FBUF_CS_NONE); + + if (f == NULL) + fatal("no coverage database for %s", istr(top_level)); + + int rpt_mask = 0; + cover_tagging_t *cover = cover_read_tags(f, rpt_mask); + fbuf_close(f, NULL); + + FILE *file = stdout; + if (output != NULL && (file = fopen(output, "w")) == NULL) + fatal_errno("cannot create %s", output); + + cover_export_cobertura(cover, file, relative); + + if (file != stdout) + fclose(file); + + argc -= next_cmd - 1; + argv += next_cmd - 1; + + return argc > 1 ? process_command(argc, argv) : 0; +} + static void usage(void) { printf("Usage: %s [OPTION]... COMMAND [OPTION]...\n" "\n" "COMMAND is one of:\n" " -a [OPTION]... FILE...\t\tAnalyse FILEs into work library\n" - " -e [OPTION]... UNIT\t\tElaborate and generate code for UNIT\n" - " -r [OPTION]... UNIT\t\tExecute previously elaborated UNIT\n" + " -e [OPTION]... TOP\t\tElaborate design unit TOP\n" + " -r [OPTION]... TOP\t\tExecute previously elaborated TOP\n" " -c [OPTION]... FILE...\t\tProcess code coverage from FILEs\n" " -i\t\t\t\tLaunch interactive TCL shell\n" + " --cover-export TOP\t\tExport code coverage statistics for TOP\n" " --do SCRIPT\t\t\tEvaluate TCL script\n" " --dump [OPTION]... UNIT\tPrint out previously analysed UNIT\n" " --init\t\t\t\tInitialise work library directory\n" @@ -1414,7 +1508,7 @@ static void usage(void) " --bootstrap\tAllow compilation of STANDARD package\n" " -D, --define NAME=VAL\tSet preprocessor symbol NAME to VAL\n" " --error-limit=NUM\tStop after NUM errors\n" - " -f, --files=LIST\t\tRead files to analyse from LIST\n" + " -f, --files=LIST\tRead files to analyse from LIST\n" " --psl\t\tEnable parsing of PSL directives in comments\n" " --relaxed\t\tDisable certain pedantic rule checks\n" "\n" @@ -1458,6 +1552,7 @@ static void usage(void) " --merge=OUTPUT\tMerge all input coverage databases from FILEs\n" " \tto OUTPUT coverage database\n" " --exclude-file=\tApply exclude file when generating report\n" + " --export=FILE\tEquivalent to `--cover-export -o FILE'\n" " --dont-print=\tDo not include specified tags in generated " "code\n" " \tcoverage report. Argument is a list of:\n" @@ -1467,6 +1562,11 @@ static void usage(void) " --report=DIR\tGenerate HTML report with code coverage results\n" " \tto DIR folder.\n" "\n" + "Coverage export options:\n" + " --format=FMT\tFile format (must be 'cobertura')\n" + " -o, --output=FILE\tOutput file name\n" + " --relative=PATH\tStrip PATH from prefix of absolute paths\n" + "\n" "Dump options:\n" " -e, --elab\t\tDump an elaborated unit\n" " -b, --body\t\tDump package body\n" @@ -1568,15 +1668,16 @@ static void parse_library_map(char *str) static int process_command(int argc, char **argv) { static struct option long_options[] = { - { "dump", no_argument, 0, 'd' }, - { "make", no_argument, 0, 'm' }, - { "syntax", no_argument, 0, 's' }, - { "list", no_argument, 0, 'l' }, - { "init", no_argument, 0, 'n' }, - { "install", no_argument, 0, 'I' }, - { "print-deps", no_argument, 0, 'P' }, - { "aotgen", no_argument, 0, 'A' }, - { "do", no_argument, 0, 'D' }, + { "dump", no_argument, 0, 'd' }, + { "make", no_argument, 0, 'm' }, + { "syntax", no_argument, 0, 's' }, + { "list", no_argument, 0, 'l' }, + { "init", no_argument, 0, 'n' }, + { "install", no_argument, 0, 'I' }, + { "print-deps", no_argument, 0, 'P' }, + { "aotgen", no_argument, 0, 'A' }, + { "do", no_argument, 0, 'D' }, + { "cover-export", no_argument, 0, 'E' }, { 0, 0, 0, 0 } }; @@ -1614,6 +1715,8 @@ static int process_command(int argc, char **argv) return do_cmd(argc, argv); case 'i': return interact_cmd(argc, argv); + case 'E': + return cover_export_cmd(argc, argv); default: fatal("missing command, try %s --help for usage", PACKAGE); return EXIT_FAILURE; diff --git a/src/option.c b/src/option.c index ed89002bd..0c3662c66 100644 --- a/src/option.c +++ b/src/option.c @@ -164,4 +164,5 @@ void set_default_options(void) opt_set_int(OPT_PSL_COMMENTS, 0); opt_set_int(OPT_NO_COLLAPSE, 0); opt_set_int(OPT_COVER_VERBOSE, get_int_env("NVC_COVER_VERBOSE", 0)); + opt_set_int(OPT_COVER_TIMESTAMP, get_int_env("NVC_COVER_TIMESTAMP", -1)); } diff --git a/src/option.h b/src/option.h index 10e6e1d33..ca6bc7858 100644 --- a/src/option.h +++ b/src/option.h @@ -60,6 +60,7 @@ typedef enum { OPT_PSL_COMMENTS, OPT_NO_COLLAPSE, OPT_COVER_VERBOSE, + OPT_COVER_TIMESTAMP, OPT_LAST_NAME } opt_name_t; diff --git a/src/rt/cover.c b/src/rt/cover.c index 09401f690..99ee75eed 100644 --- a/src/rt/cover.c +++ b/src/rt/cover.c @@ -19,6 +19,7 @@ #include "array.h" #include "common.h" #include "cover.h" +#include "hash.h" #include "lib.h" #include "option.h" #include "rt/model.h" @@ -36,6 +37,7 @@ #include #include #include +#include //#define COVER_DEBUG_EMIT //#define COVER_DEBUG_DUMP @@ -458,6 +460,9 @@ static void cover_write_scope(cover_scope_t *s, fbuf_t *f, ident_write(s->hier, ident_ctx); loc_write(&s->loc, loc_ctx); + if (s->type == CSCOPE_INSTANCE) + ident_write(s->block_name, ident_ctx); + fbuf_put_uint(f, s->tags.count); for (int i = 0; i < s->tags.count; i++) { cover_tag_t *tag = &(s->tags.items[i]); @@ -535,9 +540,11 @@ static bool cover_should_emit_scope(cover_tagging_t *tagging, tree_t t) // Block (entity, package instance or block) name if (ts->block_name) { + ident_t ename = ident_until(ts->block_name, '-'); + for (int i = 0; i < spc->block_exclude.count; i++) - if (ident_glob(ts->block_name, AGET(spc->block_exclude, i), -1)) { + if (ident_glob(ename, AGET(spc->block_exclude, i), -1)) { #ifdef COVER_DEBUG_EMIT printf("Cover emit: False, block (Block: %s, Pattern: %s)\n", istr(ts->block_name), AGET(spc->block_exclude, i)); @@ -546,7 +553,7 @@ static bool cover_should_emit_scope(cover_tagging_t *tagging, tree_t t) } for (int i = 0; i < tagging->spec->block_include.count; i++) - if (ident_glob(ts->block_name, AGET(spc->block_include, i), -1)) { + if (ident_glob(ename, AGET(spc->block_include, i), -1)) { #ifdef COVER_DEBUG_EMIT printf("Cover emit: True, block (Block: %s, Pattern: %s)\n", istr(ts->block_name), AGET(spc->block_include, i)); @@ -649,7 +656,7 @@ void cover_push_scope(cover_tagging_t *tagging, tree_t t) tree_t unit = tree_ref(hier); if (tree_kind(unit) == T_ARCH) { - s->block_name = ident_rfrom(tree_ident(tree_primary(unit)), '.'); + s->block_name = ident_rfrom(tree_ident(unit), '.'); s->type = CSCOPE_INSTANCE; } } @@ -773,6 +780,9 @@ static cover_scope_t *cover_read_scope(fbuf_t *f, ident_rd_ctx_t ident_ctx, loc_read(&s->loc, loc_ctx); + if (s->type == CSCOPE_INSTANCE) + s->block_name = ident_read(ident_ctx); + const int ntags = fbuf_get_uint(f); for (int i = 0; i < ntags; i++) { cover_tag_t new; @@ -2516,3 +2526,236 @@ void cover_report(const char *path, cover_tagging_t *tagging, int item_limit) fclose(f); } + +//////////////////////////////////////////////////////////////////////////////// +// Cobertura XML export + +typedef struct _cobertura_class cobertura_class_t; + +typedef struct { + unsigned lineno; + unsigned hits; + bool branch; + unsigned bflags; +} cobertura_line_t; + +typedef struct _cobertura_class { + const char *file; + ident_t name; + cobertura_class_t *next; + unsigned nlines; + unsigned maxlines; + cobertura_line_t *lines; +} cobertura_class_t; + +typedef struct { + hash_t *class_map; + cobertura_class_t *classes; + char *relative; +} cobertura_report_t; + +static cobertura_class_t *cobertura_get_class(cobertura_report_t *report, + ident_t name, const loc_t *loc) +{ + cobertura_class_t *c = hash_get(report->class_map, name); + if (c == NULL) { + c = xcalloc(sizeof(cobertura_class_t)); + c->name = name; + c->file = loc_file_str(loc); + c->next = report->classes; + + if (is_absolute_path(c->file) && report->relative != NULL) { + const size_t rlen = strlen(report->relative); + if (strncmp(report->relative, c->file, rlen) == 0) { + c->file += rlen; + while (c->file[0] == DIR_SEP[0] || c->file[0] == '/') + c->file++; + } + } + + report->classes = c; + hash_put(report->class_map, name, c); + } + + return c; +} + +static cobertura_line_t *cobertura_get_line(cobertura_class_t *class, + const loc_t *loc) +{ + if (class->nlines > 0) { + cobertura_line_t *last = &(class->lines[class->nlines - 1]); + if (last->lineno == loc->first_line) // Most likely + return last; + + for (int i = 0; i < class->nlines - 1; i++) { + cobertura_line_t *line = &(class->lines[i]); + if (line->lineno == loc->first_line) + return line; + } + } + + if (class->nlines == class->maxlines) { + class->maxlines = MAX(class->maxlines * 2, 100); + class->lines = xrealloc_array(class->lines, class->maxlines, + sizeof(cobertura_line_t)); + } + + cobertura_line_t *line = &(class->lines[class->nlines++]); + memset(line, '\0', sizeof(cobertura_line_t)); + line->lineno = loc->first_line; + return line; +} + +static void cobertura_export_scope(cobertura_report_t *report, + cobertura_class_t *class, + cover_scope_t *s) +{ + if (s->block_name != NULL) + class = cobertura_get_class(report, s->block_name, &s->loc); + + for (int i = 0; i < s->tags.count; i++) { + cover_tag_t *t = &(s->tags.items[i]); + switch (t->kind) { + case TAG_STMT: + { + cobertura_line_t *l = cobertura_get_line(class, &(t->loc)); + l->hits += t->data; + } + break; + case TAG_BRANCH: + { + cobertura_line_t *l = cobertura_get_line(class, &(t->loc)); + l->branch = true; + l->bflags |= t->data; + } + break; + default: + break; + } + } + + for (list_iter(cover_scope_t *, it, s->children)) + cobertura_export_scope(report, class, it); +} + +static void cobertura_class_stats(const cobertura_class_t *class, + int *nlines, int *hitlines, + int *nbranches, int *hitbranches) +{ + *nlines += class->nlines; + for (int i = 0; i < class->nlines; i++) { + const cobertura_line_t *line = &(class->lines[i]); + if (line->hits > 0) + (*hitlines)++; + if (line->branch) { + (*nbranches)++; + if ((line->bflags & COV_FLAG_TRUE) && (line->bflags & COV_FLAG_FALSE)) + (*hitbranches)++; + } + } +} + +static void cobertura_print_class(cobertura_class_t *class, FILE *f) +{ + ident_t ename = ident_until(class->name, '-'); + ident_t aname = ident_from(class->name, '-'); + + int nlines = 0, hitlines = 0, nbranches = 0, hitbranches = 0; + cobertura_class_stats(class, &nlines, &hitlines, &nbranches, &hitbranches); + + const double line_rate = (double)hitlines / (double)nlines; + const double branch_rate = (double)hitbranches / (double)nbranches; + + fprintf(f, "\n", + istr(ename), istr(aname), class->file, line_rate, branch_rate); + fprintf(f, "\n"); + + fprintf(f, "\n"); + for (int i = 0; i < class->nlines; i++) { + const cobertura_line_t *line = &(class->lines[i]); + if (line->branch) { + int pct = 0; + if (line->bflags & COV_FLAG_TRUE) pct += 50; + if (line->bflags & COV_FLAG_FALSE) pct += 50; + + fprintf(f, "\n", + line->lineno, line->hits, pct); + fprintf(f, "\n"); + fprintf(f, "\n", pct); + fprintf(f, "\n"); + fprintf(f, "\n"); + } + else + fprintf(f, "\n", + line->lineno, line->hits); + } + fprintf(f, "\n"); + + fprintf(f, "\n"); +} + +void cover_export_cobertura(cover_tagging_t *tagging, FILE *f, + const char *relative) +{ + cobertura_report_t report = { + .class_map = hash_new(64), + .relative = relative ? realpath(relative, NULL) : NULL, + }; + + cobertura_export_scope(&report, NULL, tagging->root_scope); + + fprintf(f, "\n"); + fprintf(f, "\n"); + + int nlines = 0, hitlines = 0, nbranches = 0, hitbranches = 0; + for (cobertura_class_t *it = report.classes; it; it = it->next) + cobertura_class_stats(it, &nlines, &hitlines, &nbranches, &hitbranches); + + const double line_rate = (double)hitlines / (double)nlines; + const double branch_rate = (double)hitbranches / (double)nbranches; + + unsigned long timestamp; + const long override_time = opt_get_int(OPT_COVER_TIMESTAMP); + if (override_time >= 0) + timestamp = override_time; + else + timestamp = time(NULL); + + fprintf(f, "\n", + line_rate, branch_rate, nlines, hitlines, nbranches, hitbranches, + timestamp); + fprintf(f, "\n"); + fprintf(f, ".\n"); + fprintf(f, "\n"); + fprintf(f, "\n"); + fprintf(f, "\n", + istr(tagging->root_scope->name), line_rate, branch_rate); + + fprintf(f, "\n"); + for (cobertura_class_t *it = report.classes; it; it = it->next) + cobertura_print_class(it, f); + fprintf(f, "\n"); + + fprintf(f, "\n"); + fprintf(f, "\n"); + fprintf(f, "\n"); + + for (cobertura_class_t *it = report.classes, *tmp; it; it = tmp) { + tmp = it->next; + free(it->lines); + free(it); + } + + free(report.relative); + hash_free(report.class_map); +} diff --git a/src/rt/cover.h b/src/rt/cover.h index 047c399c5..c4d3967c4 100644 --- a/src/rt/cover.h +++ b/src/rt/cover.h @@ -200,4 +200,7 @@ cover_tagging_t *cover_read_tags(fbuf_t *f, uint32_t pre_mask); void cover_merge_tags(fbuf_t *f, cover_tagging_t *tagging); +void cover_export_cobertura(cover_tagging_t *tagging, FILE *f, + const char *relative); + #endif // _COVER_H diff --git a/src/rt/stdenv.c b/src/rt/stdenv.c index 63448b2f6..df0133604 100644 --- a/src/rt/stdenv.c +++ b/src/rt/stdenv.c @@ -140,14 +140,9 @@ static const char *find_dir_separator(const char *str) static ffi_uarray_t *to_absolute_path(const char *input, size_t len) { - if (input[0] == DIR_SEP[0] || input[0] == '/') + if (is_absolute_path(input)) return to_line_n(input, len); -#ifdef __MINGW32__ - if (isalpha((int)input[0]) && input[1] == ':') - return to_line_n(input, len); -#endif - char buf[PATH_MAX]; if (realpath(input, buf) == NULL) return to_line(input); diff --git a/src/util.c b/src/util.c index 228df5783..811e18f8b 100644 --- a/src/util.c +++ b/src/util.c @@ -2083,6 +2083,19 @@ void get_data_dir(text_buf_t *tb) #endif } +bool is_absolute_path(const char *path) +{ + if (path[0] == DIR_SEP[0] || path[0] == '/') + return true; + +#ifdef __MINGW32__ + if (isalpha((int)path[0]) && path[1] == ':') + return true; +#endif + + return false; +} + void progress(const char *fmt, ...) { if (opt_get_int(OPT_VERBOSE)) { diff --git a/src/util.h b/src/util.h index 5e8548d18..f9eca48d0 100644 --- a/src/util.h +++ b/src/util.h @@ -364,6 +364,7 @@ void get_lib_dir(text_buf_t *tb); void get_data_dir(text_buf_t *tb); bool get_exe_path(text_buf_t *tb); void open_pipe(int *rfd, int *wfd); +bool is_absolute_path(const char *path); struct cpu_state; typedef void (*fault_fn_t)(int, void *, struct cpu_state *, void *); diff --git a/test/cobertura.dtd b/test/cobertura.dtd new file mode 100644 index 000000000..e5a21bb15 --- /dev/null +++ b/test/cobertura.dtd @@ -0,0 +1,60 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/regress/cover17.vhd b/test/regress/cover17.vhd new file mode 100644 index 000000000..6eafd1238 --- /dev/null +++ b/test/regress/cover17.vhd @@ -0,0 +1,45 @@ +entity sub is + port ( i : integer ); +end entity; + +architecture test of sub is +begin + + process (i) is + begin + if i > 0 then -- Taken in U1 + report "> 0"; + else -- Taken in U1 and U2 + report "< 0"; + end if; + end process; + +end architecture; + +------------------------------------------------------------------------------- + +entity cover17 is +end entity; + +architecture test of cover17 is + signal x, y : integer; +begin + + u1: entity work.sub port map ( x ); + u2: entity work.sub port map ( y ); + + tb: process is + begin + x <= 0; + wait for 1 ns; + if x = 0 then + y <= 600; + end if; + wait for 1 ns; + if x < 0 then -- Never taken + assert false; -- Not covered + end if; + wait; + end process; + +end architecture; diff --git a/test/regress/gold/cover17.xml b/test/regress/gold/cover17.xml new file mode 100644 index 000000000..164a52398 --- /dev/null +++ b/test/regress/gold/cover17.xml @@ -0,0 +1,46 @@ + + + + +. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/test/regress/testlist.txt b/test/regress/testlist.txt index 7c7cfbb2b..3d1d6c682 100644 --- a/test/regress/testlist.txt +++ b/test/regress/testlist.txt @@ -824,3 +824,4 @@ tcl2 normal,tcl psl3 fail,gold,2008 issue731 normal,2008 psl4 fail,psl,gold,2008 +cover17 cover,export=cobertura diff --git a/test/run_regr.c b/test/run_regr.c index 27d27a45f..f0494088e 100644 --- a/test/run_regr.c +++ b/test/run_regr.c @@ -93,6 +93,7 @@ #define F_TCL (1 << 20) #define F_GTKW (1 << 21) #define F_NOCOLL (1 << 22) +#define F_EXPORT (1 << 23) typedef struct test test_t; typedef struct param param_t; @@ -122,6 +123,7 @@ struct test { char *heapsz; char *cover; char *define; + char *export; }; struct arglist { @@ -471,6 +473,17 @@ static bool parse_test_list(int argc, char **argv) test->flags |= F_DEFINE; test->define = strdup(value + 1); } + else if (strncmp(opt, "export", 6) == 0) { + char *value = strchr(opt, '='); + if (value == NULL) { + fprintf(stderr, "Error on testlist line %d: missing argument to " + "export option in test %s\n", lineno, name); + goto out_close; + } + + test->flags |= F_EXPORT; + test->export = strdup(value + 1); + } else { fprintf(stderr, "Error on testlist line %d: invalid option %s in " "test %s\n", lineno, opt, name); @@ -697,22 +710,17 @@ static bool enter_test_directory(test_t *test, char *dir) return true; } -__attribute__((format(printf, 1, 2))) -static void failed(const char *fmt, ...) +static void explain(int attr, const char *prefix, const char *fmt, va_list ap) { char *reason = NULL; - if (fmt != NULL) { - va_list ap; - va_start(ap, fmt); + if (fmt != NULL) reason = xvasprintf(fmt, ap); - va_end(ap); - } - set_attr(ANSI_FG_RED); + set_attr(attr); if (reason != NULL) - printf("failed (%s)", reason); + printf("%s (%s)", prefix, reason); else - printf("failed"); + printf("%s", prefix); set_attr(ANSI_RESET); printf("\n"); fflush(stdout); @@ -720,6 +728,24 @@ static void failed(const char *fmt, ...) free(reason); } +__attribute__((format(printf, 1, 2))) +static void failed(const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + explain(ANSI_FG_RED, "failed", fmt, ap); + va_end(ap); +} + +__attribute__((format(printf, 1, 2))) +static void skipped(const char *fmt, ...) +{ + va_list ap; + va_start(ap, fmt); + explain(ANSI_FG_CYAN, "skipped", fmt, ap); + va_end(ap); +} + static bool run_test(test_t *test) { printf("%15s : ", test->name); @@ -742,9 +768,17 @@ static bool run_test(test_t *test) #endif if (skip) { - set_attr(ANSI_FG_CYAN); - printf("skipped\n"); - set_attr(ANSI_RESET); + if (skip & F_SLOW) + skipped("slow with interpreter"); + else if (skip & F_VERILOG) + skipped("verilog not enabled"); + else if (skip & F_TCL) + skipped("tcl not enabled"); + else if (skip & (F_NOTWIN | F_WAVE)) + skipped("disabled on windows"); + else + skipped(NULL); + return true; } @@ -896,7 +930,58 @@ static bool run_test(test_t *test) goto out_print; } - if ((test->flags & F_COVER) && !(test->flags & F_SHELL)) { + if ((test->flags & F_COVER) && (test->flags & F_EXPORT)) { + // Generate and check XML report + push_arg(&args, "%s/nvc%s", bin_dir, EXEEXT); + push_arg(&args, "--cover-export"); + push_arg(&args, "--relative=%s", test_dir); + push_arg(&args, "--out=export.xml"); + push_arg(&args, "--format=%s", test->export); + push_arg(&args, "%s", test->name); + + if (run_cmd(outf, &args) != RUN_OK) { + failed("coverage report"); + result = false; + goto out_print; + } + else if (!file_exists("export.xml")) { + failed("missing exported coverage report"); + result = false; + goto out_print; + } + +#ifdef XMLLINT_PATH + push_arg(&args, "%s", XMLLINT_PATH); + push_arg(&args, "--noout"); + push_arg(&args, "--nonet"); + push_arg(&args, "--dtdvalid"); + push_arg(&args, "%s/%s.dtd", test_dir, test->export); + push_arg(&args, "export.xml"); + + if (run_cmd(outf, &args) != RUN_OK) { + failed("XML lint failed"); + result = false; + goto out_print; + } +#else + skipped("missing xmllint"); + goto out_close; +#endif + +#ifndef __MINGW32__ // Directory separator different on Windows + push_arg(&args, "%s", DIFF_PATH); + push_arg(&args, "-u"); + push_arg(&args, "%s/regress/gold/%s.xml", test_dir, test->name); + push_arg(&args, "export.xml"); + + if (run_cmd(outf, &args) != RUN_OK) { + failed("XML mismatch"); + result = false; + goto out_print; + } +#endif + } + else if ((test->flags & F_COVER) && !(test->flags & F_SHELL)) { // Generate coverage report push_arg(&args, "%s/nvc%s", bin_dir, EXEEXT); push_arg(&args, "-c"); @@ -1103,6 +1188,8 @@ int main(int argc, char **argv) setenv("LANG", "en_US.UTF-8", 1); #endif + setenv("NVC_COVER_TIMESTAMP", "0", 1); + if (getenv("QUICK")) return 0;