test/harness/testcase.c
changeset 17 2c88fd553e5d
parent 11 4a80c6053a66
child 93 9fb13a5dde3b
     1.1 --- a/test/harness/testcase.c	Mon Jan 30 23:32:47 2012 +0000
     1.2 +++ b/test/harness/testcase.c	Mon Feb 06 23:55:27 2012 +0000
     1.3 @@ -239,6 +239,218 @@
     1.4  }
     1.5  
     1.6  /*
     1.7 + * Parse a warning of the form:
     1.8 + *	[blank line]
     1.9 + *	<echoed line> (ignored)
    1.10 + *	"    Line " <number> [" column " <number>] " - " <text> "\n"
    1.11 + * If not specified, the column is returned as 0.
    1.12 + * Returns: the number of bytes parsed, or -1 on error.
    1.13 + */
    1.14 +static ssize_t testcase_parse_warning(Testcase *testcase,const char *output,
    1.15 +  guint *line,guint *column,char **text)
    1.16 +{
    1.17 +    ssize_t offset=0;
    1.18 +    guint64 tmp;
    1.19 +    char *s,*endp;
    1.20 +    if (output[offset]!='\n')
    1.21 +    {
    1.22 +	g_print("%s: FAIL\n",testcase->basename);
    1.23 +	g_print("Unexpected output from bookloupe:\n");
    1.24 +	print_unexpected(output,offset);
    1.25 +	return -1;
    1.26 +    }
    1.27 +    offset++;
    1.28 +    s=strchr(output+offset,'\n');
    1.29 +    if (!s)
    1.30 +    {
    1.31 +	g_print("%s: FAIL\n",testcase->basename);
    1.32 +	g_print("Missing new-line in output from bookloupe:\n");
    1.33 +	print_unexpected(output,offset);
    1.34 +	return -1;
    1.35 +    }
    1.36 +    offset=s-output+1;
    1.37 +    if (!g_str_has_prefix(output+offset,"    Line "))
    1.38 +    {
    1.39 +	g_print("%s: FAIL\n",testcase->basename);
    1.40 +	g_print("Unexpected output from bookloupe:\n");
    1.41 +	offset+=common_prefix_length(output+offset,"    Line ");
    1.42 +	print_unexpected(output,offset);
    1.43 +	return -1;
    1.44 +    }
    1.45 +    offset+=9;
    1.46 +    tmp=g_ascii_strtoull(output+offset,&endp,10);
    1.47 +    if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64)
    1.48 +    {
    1.49 +	g_print("%s: FAIL\n",testcase->basename);
    1.50 +	g_print("Unexpected output from bookloupe:\n");
    1.51 +	print_unexpected(output,offset);
    1.52 +	return -1;
    1.53 +    }
    1.54 +    *line=tmp;
    1.55 +    offset=endp-output;
    1.56 +    if (g_str_has_prefix(output+offset," column "))
    1.57 +    {
    1.58 +	offset+=8;
    1.59 +	tmp=g_ascii_strtoull(output+offset,&endp,10);
    1.60 +	if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64)
    1.61 +	{
    1.62 +	    g_print("%s: FAIL\n",testcase->basename);
    1.63 +	    g_print("Unexpected output from bookloupe:\n");
    1.64 +	    print_unexpected(output,offset);
    1.65 +	    return -1;
    1.66 +	}
    1.67 +	*column=tmp;
    1.68 +	offset=endp-output;
    1.69 +    }
    1.70 +    else
    1.71 +	*column=0;
    1.72 +    if (!g_str_has_prefix(output+offset," - "))
    1.73 +    {
    1.74 +	g_print("%s: FAIL\n",testcase->basename);
    1.75 +	g_print("Unexpected output from bookloupe:\n");
    1.76 +	offset+=common_prefix_length(output+offset," - ");
    1.77 +	print_unexpected(output,offset);
    1.78 +	return -1;
    1.79 +    }
    1.80 +    offset+=3;
    1.81 +    s=strchr(output+offset,'\n');
    1.82 +    if (!s)
    1.83 +    {
    1.84 +	g_print("%s: FAIL\n",testcase->basename);
    1.85 +	g_print("Missing new-line in output from bookloupe:\n");
    1.86 +	print_unexpected(output,offset);
    1.87 +	return -1;
    1.88 +    }
    1.89 +    *text=g_strndup(output+offset,s-(output+offset));
    1.90 +    return s-output+1;
    1.91 +}
    1.92 +
    1.93 +/*
    1.94 + * Check the warnings produced by bookloupe against either the
    1.95 + * unstructured testcase->expected or the structured testcase->warnings
    1.96 + * as appropriate.
    1.97 + */
    1.98 +static gboolean testcase_check_warnings(Testcase *testcase,const char *output,
    1.99 +  char **xfail)
   1.100 +{
   1.101 +    gboolean r=TRUE;
   1.102 +    size_t offset;
   1.103 +    ssize_t off;
   1.104 +    int i,count_false_positive,count_false_negative;
   1.105 +    int total_false_positive,total_false_negative;
   1.106 +    char *text;
   1.107 +    guint *counts,line,column;
   1.108 +    GSList *link,*link2;
   1.109 +    TestcaseWarning *warning;
   1.110 +    TestcaseLocation *location;
   1.111 +    *xfail=NULL;
   1.112 +    if (testcase->expected)
   1.113 +    {
   1.114 +	if (strcmp(output,testcase->expected))
   1.115 +	{
   1.116 +	    g_print("%s: FAIL\n",testcase->basename);
   1.117 +	    offset=common_prefix_length(output,testcase->expected);
   1.118 +	    if (!offset && !output[offset])
   1.119 +		g_print("Unexpected zero warnings from bookloupe.\n");
   1.120 +	    else
   1.121 +	    {
   1.122 +		g_print("Unexpected output from bookloupe:\n");
   1.123 +		print_unexpected(output,offset);
   1.124 +	    }
   1.125 +	    return FALSE;
   1.126 +	}
   1.127 +	return TRUE;
   1.128 +    }
   1.129 +    counts=g_new0(guint,g_slist_length(testcase->warnings));
   1.130 +    for(offset=0;output[offset];)
   1.131 +    {
   1.132 +	off=testcase_parse_warning(testcase,output+offset,&line,&column,&text);
   1.133 +	if (off<0)
   1.134 +	{
   1.135 +	    r=FALSE;
   1.136 +	    break;
   1.137 +	}
   1.138 +	offset+=off;
   1.139 +	for(link=testcase->warnings,i=0;link;link=link->next,i++)
   1.140 +	{
   1.141 +	    warning=link->data;
   1.142 +	    if (strcmp(warning->text,text))
   1.143 +		continue;
   1.144 +	    for(link2=warning->locations;link2;link2=link2->next)
   1.145 +	    {
   1.146 +		location=link2->data;
   1.147 +		if (location->line!=line || location->column!=column)
   1.148 +		    continue;
   1.149 +		counts[i]++;
   1.150 +		break;
   1.151 +	    }
   1.152 +	    if (link2)
   1.153 +		break;
   1.154 +	}
   1.155 +	if (!link)
   1.156 +	{
   1.157 +	    g_print("%s: FAIL\n",testcase->basename);
   1.158 +	    g_print("Unexpected warning from bookloupe:\n");
   1.159 +	    if (column)
   1.160 +		g_print("    Line %u column %u - %s\n",line,column,text);
   1.161 +	    else
   1.162 +		g_print("    Line %u - %s\n",line,text);
   1.163 +	    r=FALSE;
   1.164 +	    g_free(text);
   1.165 +	    break;
   1.166 +	}
   1.167 +	g_free(text);
   1.168 +    }
   1.169 +    count_false_positive=total_false_positive=0;
   1.170 +    count_false_negative=total_false_negative=0;
   1.171 +    for(link=testcase->warnings,i=0;r && link;link=link->next,i++)
   1.172 +    {
   1.173 +	warning=link->data;
   1.174 +	if (!counts[i] && warning->is_real && !warning->xfail)
   1.175 +	{
   1.176 +	    location=warning->locations->data;
   1.177 +	    g_print("%s: FAIL\n",testcase->basename);
   1.178 +	    g_print("Missing warning from bookloupe:\n");
   1.179 +	    if (location->column)
   1.180 +		g_print("    Line %u column %u - %s\n",location->line,
   1.181 +		  location->column,warning->text);
   1.182 +	    else
   1.183 +		g_print("    Line %u - %s\n",location->line,warning->text);
   1.184 +	    r=FALSE;
   1.185 +	    break;
   1.186 +	}
   1.187 +	else if (warning->xfail)
   1.188 +	{
   1.189 +	    if (warning->is_real)
   1.190 +	    {
   1.191 +		total_false_negative++;
   1.192 +		if (!counts[i])
   1.193 +		    count_false_negative++;
   1.194 +	    }
   1.195 +	    else if (!warning->is_real)
   1.196 +	    {
   1.197 +		total_false_positive++;
   1.198 +		if (counts[i])
   1.199 +		    count_false_positive++;
   1.200 +	    }
   1.201 +	}
   1.202 +    }
   1.203 +    g_free(counts);
   1.204 +    if (count_false_positive && count_false_negative)
   1.205 +	*xfail=g_strdup_printf(
   1.206 +	  "with %d of %d false positives and %d of %d false negatives",
   1.207 +	  count_false_positive,total_false_positive,
   1.208 +	  count_false_negative,total_false_negative);
   1.209 +    else if (count_false_positive)
   1.210 +	*xfail=g_strdup_printf("with %d of %d false positives",
   1.211 +	  count_false_positive,total_false_positive);
   1.212 +    else if (count_false_negative)
   1.213 +	*xfail=g_strdup_printf("with %d of %d false negatives",
   1.214 +	  count_false_negative,total_false_negative);
   1.215 +    return r;
   1.216 +}
   1.217 +
   1.218 +/*
   1.219   * Run a testcase, returning FALSE on fail or error and
   1.220   * TRUE on pass or expected-fail.
   1.221   * Suitable message(s) will be printed in all cases.
   1.222 @@ -247,8 +459,8 @@
   1.223  {
   1.224      gboolean r;
   1.225      size_t pos,offset;
   1.226 -    GString *header,*expected;
   1.227 -    char *output,*filename,*s;
   1.228 +    GString *header;
   1.229 +    char *output,*filename,*s,*xfail=NULL;
   1.230      GError *error=NULL;
   1.231      if (!testcase_create_input_files(testcase,&error))
   1.232      {
   1.233 @@ -257,7 +469,7 @@
   1.234  	g_error_free(error);
   1.235  	return FALSE;
   1.236      }
   1.237 -    if (testcase->expected)
   1.238 +    if (testcase->expected || testcase->warnings)
   1.239  	r=testcase_spawn_bookloupe(testcase,&output,&error);
   1.240      else
   1.241      {
   1.242 @@ -280,17 +492,16 @@
   1.243  	g_error_free(error);
   1.244  	return FALSE;
   1.245      }
   1.246 -    if (testcase->expected)
   1.247 +    if (testcase->expected || testcase->warnings)
   1.248      {
   1.249  	header=g_string_new("\n\nFile: ");
   1.250  	g_string_append(header,filename);
   1.251  	g_string_append(header,"\n");
   1.252 -	expected=g_string_new(testcase->expected);
   1.253  	if (!g_str_has_prefix(output,header->str))
   1.254  	{
   1.255  	    g_print("%s: FAIL\n",testcase->basename);
   1.256 +	    g_print("Unexpected header from bookloupe:\n");
   1.257  	    offset=common_prefix_length(output,header->str);
   1.258 -	    g_print("Unexpected header from bookloupe:\n");
   1.259  	    print_unexpected(output,offset);
   1.260  	    r=FALSE;
   1.261  	}
   1.262 @@ -304,36 +515,39 @@
   1.263  	    else
   1.264  	    {
   1.265  		g_print("%s: FAIL\n",testcase->basename);
   1.266 -		offset=common_prefix_length(output,header->str);
   1.267  		g_print("Unterminated summary from bookloupe:\n%s\n",
   1.268  		  output+pos);
   1.269  		r=FALSE;
   1.270  	    }
   1.271  	}
   1.272 -	if (r && strcmp(output+pos,expected->str))
   1.273 -	{
   1.274 -	    g_print("%s: FAIL\n",testcase->basename);
   1.275 -	    offset=common_prefix_length(output+pos,expected->str);
   1.276 -	    if (!offset && !output[pos+offset])
   1.277 -		g_print("Unexpected zero warnings from bookloupe.\n");
   1.278 -	    else
   1.279 -	    {
   1.280 -		g_print("Unexpected output from bookloupe:\n");
   1.281 -		print_unexpected(output+pos,offset);
   1.282 -	    }
   1.283 -	    r=FALSE;
   1.284 -	}
   1.285  	g_string_free(header,TRUE);
   1.286 -	g_string_free(expected,TRUE);
   1.287 +	r=testcase_check_warnings(testcase,output+pos,&xfail);
   1.288      }
   1.289      g_free(filename);
   1.290      g_free(output);
   1.291      if (r)
   1.292 -	g_print("%s: PASS\n",testcase->basename);
   1.293 +    {
   1.294 +	if (xfail)
   1.295 +	    g_print("%s: PASS (%s)\n",testcase->basename,xfail);
   1.296 +	else
   1.297 +	    g_print("%s: PASS\n",testcase->basename);
   1.298 +    }
   1.299 +    g_free(xfail);
   1.300      return r;
   1.301  }
   1.302  
   1.303  /*
   1.304 + * Free a testcase warning.
   1.305 + */
   1.306 +void testcase_warning_free(TestcaseWarning *warning)
   1.307 +{
   1.308 +    g_slist_foreach(warning->locations,(GFunc)g_free,NULL);
   1.309 +    g_slist_free(warning->locations);
   1.310 +    g_free(warning->text);
   1.311 +    g_free(warning);
   1.312 +}
   1.313 +
   1.314 +/*
   1.315   * Free a testcase.
   1.316   */
   1.317  void testcase_free(Testcase *testcase)
   1.318 @@ -342,6 +556,8 @@
   1.319      g_slist_foreach(testcase->inputs,(GFunc)testcase_input_free,NULL);
   1.320      g_slist_free(testcase->inputs);
   1.321      g_free(testcase->expected);
   1.322 +    g_slist_foreach(testcase->warnings,(GFunc)testcase_warning_free,NULL);
   1.323 +    g_slist_free(testcase->warnings);
   1.324      g_free(testcase->encoding);
   1.325      g_strfreev(testcase->options);
   1.326      g_free(testcase);