1.1 --- a/test/harness/testcase.c Mon Jan 30 23:32:47 2012 +0000
1.2 +++ b/test/harness/testcase.c Sat May 25 19:14:21 2013 +0100
1.3 @@ -239,6 +239,218 @@
1.4 }
1.5
1.6 /*
1.7 + * Parse a warning of the form:
1.8 + * [blank line]
1.9 + * <echoed line> (ignored)
1.10 + * " Line " <number> [" column " <number>] " - " <text> "\n"
1.11 + * If not specified, the column is returned as 0.
1.12 + * Returns: the number of bytes parsed, or -1 on error.
1.13 + */
1.14 +static ssize_t testcase_parse_warning(Testcase *testcase,const char *output,
1.15 + guint *line,guint *column,char **text)
1.16 +{
1.17 + ssize_t offset=0;
1.18 + guint64 tmp;
1.19 + char *s,*endp;
1.20 + if (output[offset]!='\n')
1.21 + {
1.22 + g_print("%s: FAIL\n",testcase->basename);
1.23 + g_print("Unexpected output from bookloupe:\n");
1.24 + print_unexpected(output,offset);
1.25 + return -1;
1.26 + }
1.27 + offset++;
1.28 + s=strchr(output+offset,'\n');
1.29 + if (!s)
1.30 + {
1.31 + g_print("%s: FAIL\n",testcase->basename);
1.32 + g_print("Missing new-line in output from bookloupe:\n");
1.33 + print_unexpected(output,offset);
1.34 + return -1;
1.35 + }
1.36 + offset=s-output+1;
1.37 + if (!g_str_has_prefix(output+offset," Line "))
1.38 + {
1.39 + g_print("%s: FAIL\n",testcase->basename);
1.40 + g_print("Unexpected output from bookloupe:\n");
1.41 + offset+=common_prefix_length(output+offset," Line ");
1.42 + print_unexpected(output,offset);
1.43 + return -1;
1.44 + }
1.45 + offset+=9;
1.46 + tmp=g_ascii_strtoull(output+offset,&endp,10);
1.47 + if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64)
1.48 + {
1.49 + g_print("%s: FAIL\n",testcase->basename);
1.50 + g_print("Unexpected output from bookloupe:\n");
1.51 + print_unexpected(output,offset);
1.52 + return -1;
1.53 + }
1.54 + *line=tmp;
1.55 + offset=endp-output;
1.56 + if (g_str_has_prefix(output+offset," column "))
1.57 + {
1.58 + offset+=8;
1.59 + tmp=g_ascii_strtoull(output+offset,&endp,10);
1.60 + if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64)
1.61 + {
1.62 + g_print("%s: FAIL\n",testcase->basename);
1.63 + g_print("Unexpected output from bookloupe:\n");
1.64 + print_unexpected(output,offset);
1.65 + return -1;
1.66 + }
1.67 + *column=tmp;
1.68 + offset=endp-output;
1.69 + }
1.70 + else
1.71 + *column=0;
1.72 + if (!g_str_has_prefix(output+offset," - "))
1.73 + {
1.74 + g_print("%s: FAIL\n",testcase->basename);
1.75 + g_print("Unexpected output from bookloupe:\n");
1.76 + offset+=common_prefix_length(output+offset," - ");
1.77 + print_unexpected(output,offset);
1.78 + return -1;
1.79 + }
1.80 + offset+=3;
1.81 + s=strchr(output+offset,'\n');
1.82 + if (!s)
1.83 + {
1.84 + g_print("%s: FAIL\n",testcase->basename);
1.85 + g_print("Missing new-line in output from bookloupe:\n");
1.86 + print_unexpected(output,offset);
1.87 + return -1;
1.88 + }
1.89 + *text=g_strndup(output+offset,s-(output+offset));
1.90 + return s-output+1;
1.91 +}
1.92 +
1.93 +/*
1.94 + * Check the warnings produced by bookloupe against either the
1.95 + * unstructured testcase->expected or the structured testcase->warnings
1.96 + * as appropriate.
1.97 + */
1.98 +static gboolean testcase_check_warnings(Testcase *testcase,const char *output,
1.99 + char **xfail)
1.100 +{
1.101 + gboolean r=TRUE;
1.102 + size_t offset;
1.103 + ssize_t off;
1.104 + int i,count_false_positive,count_false_negative;
1.105 + int total_false_positive,total_false_negative;
1.106 + char *text;
1.107 + guint *counts,line,column;
1.108 + GSList *link,*link2;
1.109 + TestcaseWarning *warning;
1.110 + TestcaseLocation *location;
1.111 + *xfail=NULL;
1.112 + if (testcase->expected)
1.113 + {
1.114 + if (strcmp(output,testcase->expected))
1.115 + {
1.116 + g_print("%s: FAIL\n",testcase->basename);
1.117 + offset=common_prefix_length(output,testcase->expected);
1.118 + if (!offset && !output[offset])
1.119 + g_print("Unexpected zero warnings from bookloupe.\n");
1.120 + else
1.121 + {
1.122 + g_print("Unexpected output from bookloupe:\n");
1.123 + print_unexpected(output,offset);
1.124 + }
1.125 + return FALSE;
1.126 + }
1.127 + return TRUE;
1.128 + }
1.129 + counts=g_new0(guint,g_slist_length(testcase->warnings));
1.130 + for(offset=0;output[offset];)
1.131 + {
1.132 + off=testcase_parse_warning(testcase,output+offset,&line,&column,&text);
1.133 + if (off<0)
1.134 + {
1.135 + r=FALSE;
1.136 + break;
1.137 + }
1.138 + offset+=off;
1.139 + for(link=testcase->warnings,i=0;link;link=link->next,i++)
1.140 + {
1.141 + warning=link->data;
1.142 + if (strcmp(warning->text,text))
1.143 + continue;
1.144 + for(link2=warning->locations;link2;link2=link2->next)
1.145 + {
1.146 + location=link2->data;
1.147 + if (location->line!=line || location->column!=column)
1.148 + continue;
1.149 + counts[i]++;
1.150 + break;
1.151 + }
1.152 + if (link2)
1.153 + break;
1.154 + }
1.155 + if (!link)
1.156 + {
1.157 + g_print("%s: FAIL\n",testcase->basename);
1.158 + g_print("Unexpected warning from bookloupe:\n");
1.159 + if (column)
1.160 + g_print(" Line %u column %u - %s\n",line,column,text);
1.161 + else
1.162 + g_print(" Line %u - %s\n",line,text);
1.163 + r=FALSE;
1.164 + g_free(text);
1.165 + break;
1.166 + }
1.167 + g_free(text);
1.168 + }
1.169 + count_false_positive=total_false_positive=0;
1.170 + count_false_negative=total_false_negative=0;
1.171 + for(link=testcase->warnings,i=0;r && link;link=link->next,i++)
1.172 + {
1.173 + warning=link->data;
1.174 + if (!counts[i] && warning->is_real && !warning->xfail)
1.175 + {
1.176 + location=warning->locations->data;
1.177 + g_print("%s: FAIL\n",testcase->basename);
1.178 + g_print("Missing warning from bookloupe:\n");
1.179 + if (location->column)
1.180 + g_print(" Line %u column %u - %s\n",location->line,
1.181 + location->column,warning->text);
1.182 + else
1.183 + g_print(" Line %u - %s\n",location->line,warning->text);
1.184 + r=FALSE;
1.185 + break;
1.186 + }
1.187 + else if (warning->xfail)
1.188 + {
1.189 + if (warning->is_real)
1.190 + {
1.191 + total_false_negative++;
1.192 + if (!counts[i])
1.193 + count_false_negative++;
1.194 + }
1.195 + else if (!warning->is_real)
1.196 + {
1.197 + total_false_positive++;
1.198 + if (counts[i])
1.199 + count_false_positive++;
1.200 + }
1.201 + }
1.202 + }
1.203 + g_free(counts);
1.204 + if (count_false_positive && count_false_negative)
1.205 + *xfail=g_strdup_printf(
1.206 + "with %d of %d false positives and %d of %d false negatives",
1.207 + count_false_positive,total_false_positive,
1.208 + count_false_negative,total_false_negative);
1.209 + else if (count_false_positive)
1.210 + *xfail=g_strdup_printf("with %d of %d false positives",
1.211 + count_false_positive,total_false_positive);
1.212 + else if (count_false_negative)
1.213 + *xfail=g_strdup_printf("with %d of %d false negatives",
1.214 + count_false_negative,total_false_negative);
1.215 + return r;
1.216 +}
1.217 +
1.218 +/*
1.219 * Run a testcase, returning FALSE on fail or error and
1.220 * TRUE on pass or expected-fail.
1.221 * Suitable message(s) will be printed in all cases.
1.222 @@ -247,8 +459,8 @@
1.223 {
1.224 gboolean r;
1.225 size_t pos,offset;
1.226 - GString *header,*expected;
1.227 - char *output,*filename,*s;
1.228 + GString *header;
1.229 + char *output,*filename,*s,*xfail=NULL;
1.230 GError *error=NULL;
1.231 if (!testcase_create_input_files(testcase,&error))
1.232 {
1.233 @@ -257,7 +469,7 @@
1.234 g_error_free(error);
1.235 return FALSE;
1.236 }
1.237 - if (testcase->expected)
1.238 + if (testcase->expected || testcase->warnings)
1.239 r=testcase_spawn_bookloupe(testcase,&output,&error);
1.240 else
1.241 {
1.242 @@ -280,17 +492,16 @@
1.243 g_error_free(error);
1.244 return FALSE;
1.245 }
1.246 - if (testcase->expected)
1.247 + if (testcase->expected || testcase->warnings)
1.248 {
1.249 header=g_string_new("\n\nFile: ");
1.250 g_string_append(header,filename);
1.251 g_string_append(header,"\n");
1.252 - expected=g_string_new(testcase->expected);
1.253 if (!g_str_has_prefix(output,header->str))
1.254 {
1.255 g_print("%s: FAIL\n",testcase->basename);
1.256 + g_print("Unexpected header from bookloupe:\n");
1.257 offset=common_prefix_length(output,header->str);
1.258 - g_print("Unexpected header from bookloupe:\n");
1.259 print_unexpected(output,offset);
1.260 r=FALSE;
1.261 }
1.262 @@ -304,36 +515,39 @@
1.263 else
1.264 {
1.265 g_print("%s: FAIL\n",testcase->basename);
1.266 - offset=common_prefix_length(output,header->str);
1.267 g_print("Unterminated summary from bookloupe:\n%s\n",
1.268 output+pos);
1.269 r=FALSE;
1.270 }
1.271 }
1.272 - if (r && strcmp(output+pos,expected->str))
1.273 - {
1.274 - g_print("%s: FAIL\n",testcase->basename);
1.275 - offset=common_prefix_length(output+pos,expected->str);
1.276 - if (!offset && !output[pos+offset])
1.277 - g_print("Unexpected zero warnings from bookloupe.\n");
1.278 - else
1.279 - {
1.280 - g_print("Unexpected output from bookloupe:\n");
1.281 - print_unexpected(output+pos,offset);
1.282 - }
1.283 - r=FALSE;
1.284 - }
1.285 g_string_free(header,TRUE);
1.286 - g_string_free(expected,TRUE);
1.287 + r=testcase_check_warnings(testcase,output+pos,&xfail);
1.288 }
1.289 g_free(filename);
1.290 g_free(output);
1.291 if (r)
1.292 - g_print("%s: PASS\n",testcase->basename);
1.293 + {
1.294 + if (xfail)
1.295 + g_print("%s: PASS (%s)\n",testcase->basename,xfail);
1.296 + else
1.297 + g_print("%s: PASS\n",testcase->basename);
1.298 + }
1.299 + g_free(xfail);
1.300 return r;
1.301 }
1.302
1.303 /*
1.304 + * Free a testcase warning.
1.305 + */
1.306 +void testcase_warning_free(TestcaseWarning *warning)
1.307 +{
1.308 + g_slist_foreach(warning->locations,(GFunc)g_free,NULL);
1.309 + g_slist_free(warning->locations);
1.310 + g_free(warning->text);
1.311 + g_free(warning);
1.312 +}
1.313 +
1.314 +/*
1.315 * Free a testcase.
1.316 */
1.317 void testcase_free(Testcase *testcase)
1.318 @@ -342,6 +556,8 @@
1.319 g_slist_foreach(testcase->inputs,(GFunc)testcase_input_free,NULL);
1.320 g_slist_free(testcase->inputs);
1.321 g_free(testcase->expected);
1.322 + g_slist_foreach(testcase->warnings,(GFunc)testcase_warning_free,NULL);
1.323 + g_slist_free(testcase->warnings);
1.324 g_free(testcase->encoding);
1.325 g_strfreev(testcase->options);
1.326 g_free(testcase);