diff -r 4a80c6053a66 -r 4c8606eb60c1 test/harness/testcase.c --- a/test/harness/testcase.c Mon Jan 30 23:32:47 2012 +0000 +++ b/test/harness/testcase.c Sun May 26 01:46:33 2013 +0100 @@ -239,6 +239,218 @@ } /* + * Parse a warning of the form: + * [blank line] + * (ignored) + * " Line " [" column " ] " - " "\n" + * If not specified, the column is returned as 0. + * Returns: the number of bytes parsed, or -1 on error. + */ +static ssize_t testcase_parse_warning(Testcase *testcase,const char *output, + guint *line,guint *column,char **text) +{ + ssize_t offset=0; + guint64 tmp; + char *s,*endp; + if (output[offset]!='\n') + { + g_print("%s: FAIL\n",testcase->basename); + g_print("Unexpected output from bookloupe:\n"); + print_unexpected(output,offset); + return -1; + } + offset++; + s=strchr(output+offset,'\n'); + if (!s) + { + g_print("%s: FAIL\n",testcase->basename); + g_print("Missing new-line in output from bookloupe:\n"); + print_unexpected(output,offset); + return -1; + } + offset=s-output+1; + if (!g_str_has_prefix(output+offset," Line ")) + { + g_print("%s: FAIL\n",testcase->basename); + g_print("Unexpected output from bookloupe:\n"); + offset+=common_prefix_length(output+offset," Line "); + print_unexpected(output,offset); + return -1; + } + offset+=9; + tmp=g_ascii_strtoull(output+offset,&endp,10); + if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64) + { + g_print("%s: FAIL\n",testcase->basename); + g_print("Unexpected output from bookloupe:\n"); + print_unexpected(output,offset); + return -1; + } + *line=tmp; + offset=endp-output; + if (g_str_has_prefix(output+offset," column ")) + { + offset+=8; + tmp=g_ascii_strtoull(output+offset,&endp,10); + if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64) + { + g_print("%s: FAIL\n",testcase->basename); + g_print("Unexpected output from bookloupe:\n"); + print_unexpected(output,offset); + return -1; + } + *column=tmp; + offset=endp-output; + } + else + *column=0; + if (!g_str_has_prefix(output+offset," - ")) + { + g_print("%s: FAIL\n",testcase->basename); + g_print("Unexpected output from bookloupe:\n"); + offset+=common_prefix_length(output+offset," - "); + print_unexpected(output,offset); + return -1; + } + offset+=3; + s=strchr(output+offset,'\n'); + if (!s) + { + g_print("%s: FAIL\n",testcase->basename); + g_print("Missing new-line in output from bookloupe:\n"); + print_unexpected(output,offset); + return -1; + } + *text=g_strndup(output+offset,s-(output+offset)); + return s-output+1; +} + +/* + * Check the warnings produced by bookloupe against either the + * unstructured testcase->expected or the structured testcase->warnings + * as appropriate. + */ +static gboolean testcase_check_warnings(Testcase *testcase,const char *output, + char **xfail) +{ + gboolean r=TRUE; + size_t offset; + ssize_t off; + int i,count_false_positive,count_false_negative; + int total_false_positive,total_false_negative; + char *text; + guint *counts,line,column; + GSList *link,*link2; + TestcaseWarning *warning; + TestcaseLocation *location; + *xfail=NULL; + if (testcase->expected) + { + if (strcmp(output,testcase->expected)) + { + g_print("%s: FAIL\n",testcase->basename); + offset=common_prefix_length(output,testcase->expected); + if (!offset && !output[offset]) + g_print("Unexpected zero warnings from bookloupe.\n"); + else + { + g_print("Unexpected output from bookloupe:\n"); + print_unexpected(output,offset); + } + return FALSE; + } + return TRUE; + } + counts=g_new0(guint,g_slist_length(testcase->warnings)); + for(offset=0;output[offset];) + { + off=testcase_parse_warning(testcase,output+offset,&line,&column,&text); + if (off<0) + { + r=FALSE; + break; + } + offset+=off; + for(link=testcase->warnings,i=0;link;link=link->next,i++) + { + warning=link->data; + if (strcmp(warning->text,text)) + continue; + for(link2=warning->locations;link2;link2=link2->next) + { + location=link2->data; + if (location->line!=line || location->column!=column) + continue; + counts[i]++; + break; + } + if (link2) + break; + } + if (!link) + { + g_print("%s: FAIL\n",testcase->basename); + g_print("Unexpected warning from bookloupe:\n"); + if (column) + g_print(" Line %u column %u - %s\n",line,column,text); + else + g_print(" Line %u - %s\n",line,text); + r=FALSE; + g_free(text); + break; + } + g_free(text); + } + count_false_positive=total_false_positive=0; + count_false_negative=total_false_negative=0; + for(link=testcase->warnings,i=0;r && link;link=link->next,i++) + { + warning=link->data; + if (!counts[i] && warning->is_real && !warning->xfail) + { + location=warning->locations->data; + g_print("%s: FAIL\n",testcase->basename); + g_print("Missing warning from bookloupe:\n"); + if (location->column) + g_print(" Line %u column %u - %s\n",location->line, + location->column,warning->text); + else + g_print(" Line %u - %s\n",location->line,warning->text); + r=FALSE; + break; + } + else if (warning->xfail) + { + if (warning->is_real) + { + total_false_negative++; + if (!counts[i]) + count_false_negative++; + } + else if (!warning->is_real) + { + total_false_positive++; + if (counts[i]) + count_false_positive++; + } + } + } + g_free(counts); + if (count_false_positive && count_false_negative) + *xfail=g_strdup_printf( + "with %d of %d false positives and %d of %d false negatives", + count_false_positive,total_false_positive, + count_false_negative,total_false_negative); + else if (count_false_positive) + *xfail=g_strdup_printf("with %d of %d false positives", + count_false_positive,total_false_positive); + else if (count_false_negative) + *xfail=g_strdup_printf("with %d of %d false negatives", + count_false_negative,total_false_negative); + return r; +} + +/* * Run a testcase, returning FALSE on fail or error and * TRUE on pass or expected-fail. * Suitable message(s) will be printed in all cases. @@ -247,8 +459,8 @@ { gboolean r; size_t pos,offset; - GString *header,*expected; - char *output,*filename,*s; + GString *header; + char *output,*filename,*s,*xfail=NULL; GError *error=NULL; if (!testcase_create_input_files(testcase,&error)) { @@ -257,7 +469,7 @@ g_error_free(error); return FALSE; } - if (testcase->expected) + if (testcase->expected || testcase->warnings) r=testcase_spawn_bookloupe(testcase,&output,&error); else { @@ -280,17 +492,16 @@ g_error_free(error); return FALSE; } - if (testcase->expected) + if (testcase->expected || testcase->warnings) { header=g_string_new("\n\nFile: "); g_string_append(header,filename); g_string_append(header,"\n"); - expected=g_string_new(testcase->expected); if (!g_str_has_prefix(output,header->str)) { g_print("%s: FAIL\n",testcase->basename); + g_print("Unexpected header from bookloupe:\n"); offset=common_prefix_length(output,header->str); - g_print("Unexpected header from bookloupe:\n"); print_unexpected(output,offset); r=FALSE; } @@ -304,36 +515,39 @@ else { g_print("%s: FAIL\n",testcase->basename); - offset=common_prefix_length(output,header->str); g_print("Unterminated summary from bookloupe:\n%s\n", output+pos); r=FALSE; } } - if (r && strcmp(output+pos,expected->str)) - { - g_print("%s: FAIL\n",testcase->basename); - offset=common_prefix_length(output+pos,expected->str); - if (!offset && !output[pos+offset]) - g_print("Unexpected zero warnings from bookloupe.\n"); - else - { - g_print("Unexpected output from bookloupe:\n"); - print_unexpected(output+pos,offset); - } - r=FALSE; - } g_string_free(header,TRUE); - g_string_free(expected,TRUE); + r=testcase_check_warnings(testcase,output+pos,&xfail); } g_free(filename); g_free(output); if (r) - g_print("%s: PASS\n",testcase->basename); + { + if (xfail) + g_print("%s: PASS (%s)\n",testcase->basename,xfail); + else + g_print("%s: PASS\n",testcase->basename); + } + g_free(xfail); return r; } /* + * Free a testcase warning. + */ +void testcase_warning_free(TestcaseWarning *warning) +{ + g_slist_foreach(warning->locations,(GFunc)g_free,NULL); + g_slist_free(warning->locations); + g_free(warning->text); + g_free(warning); +} + +/* * Free a testcase. */ void testcase_free(Testcase *testcase) @@ -342,6 +556,8 @@ g_slist_foreach(testcase->inputs,(GFunc)testcase_input_free,NULL); g_slist_free(testcase->inputs); g_free(testcase->expected); + g_slist_foreach(testcase->warnings,(GFunc)testcase_warning_free,NULL); + g_slist_free(testcase->warnings); g_free(testcase->encoding); g_strfreev(testcase->options); g_free(testcase);