test/harness/testcase.c
author ali <ali@juiblex.co.uk>
Fri Oct 25 11:15:18 2013 +0100 (2013-10-25)
changeset 102 ff0aa9b1397a
parent 101 f44c530f80da
permissions -rw-r--r--
Fix bug #14: Add a configuration file
     1 #include <stdlib.h>
     2 #include <stdio.h>
     3 #include <string.h>
     4 #include <errno.h>
     5 #include <glib.h>
     6 #include <glib/gstdio.h>
     7 #include <bl/bl.h>
     8 #include "testcase.h"
     9 #include "testcaseinput.h"
    10 #include "testcaseoutput.h"
    11 
    12 GQuark testcase_error_quark(void)
    13 {
    14     return g_quark_from_static_string("testcase-error-quark");
    15 }
    16 
    17 /*
    18  * Return the length (in bytes) of any common prefix between s1 and s2.
    19  * The returned length will always represent an exact number of characters.
    20  */
    21 size_t common_prefix_length(const char *s1,const char *s2)
    22 {
    23     gunichar c1,c2;
    24     const char *s=s1;
    25     while(*s1 && *s2)
    26     {
    27 	c1=g_utf8_get_char(s1);
    28 	c2=g_utf8_get_char(s2);
    29 	if (c1!=c2)
    30 	    break;
    31 	s1=g_utf8_next_char(s1);
    32 	s2=g_utf8_next_char(s2);
    33     }
    34     return s1-s;
    35 }
    36 
    37 void print_unexpected(const char *unexpected,gsize differs_at)
    38 {
    39     int col;
    40     gunichar c;
    41     const char *endp,*bol,*s;
    42     GString *string;
    43     endp=strchr(unexpected+differs_at,'\n');
    44     if (!endp)
    45 	endp=unexpected+strlen(unexpected);
    46     string=g_string_new_len(unexpected,endp-unexpected);
    47     bol=strrchr(string->str,'\n');
    48     if (bol)
    49 	bol++;
    50     else
    51 	bol=string->str;
    52     col=0;
    53     s=bol;
    54     endp=string->str+differs_at;
    55     while(s<endp)
    56     {
    57 	c=g_utf8_get_char(s);
    58 	s=g_utf8_next_char(s);
    59 	if (c=='\t')
    60 	    col=(col&~7)+8;
    61 	else if (g_unichar_iswide(c))
    62 	    col+=2;
    63 	else if (!g_unichar_iszerowidth(c))
    64 	    col++;
    65     }
    66     g_print("%s\n%*s^\n",string->str,col,"");
    67     g_string_free(string,TRUE);
    68 }
    69 
    70 /*
    71  * Create all the input files needed by a testcase and, if required,
    72  * a temporary directory in which to store them.
    73  */
    74 gboolean testcase_create_input_files(Testcase *testcase,GError **error)
    75 {
    76     GSList *link,*link2;
    77     if (testcase->flags&TESTCASE_TMP_DIR)
    78     {
    79 	testcase->tmpdir=g_strdup("TEST-XXXXXX");
    80 	if (!g_mkdtemp(testcase->tmpdir))
    81 	{
    82 	    g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
    83 	      "Failed to create temporary directory: %s",g_strerror(errno));
    84 	    g_free(testcase->tmpdir);
    85 	    testcase->tmpdir=NULL;
    86 	    return FALSE;
    87 	}
    88     }
    89     for(link=testcase->inputs;link;link=link->next)
    90 	if (!testcase_input_create(testcase,link->data,error))
    91 	{
    92 	    for(link2=testcase->inputs;link2!=link;link2=link2->next)
    93 		(void)testcase_input_remove(testcase,link2->data,NULL);
    94 	    if (testcase->tmpdir)
    95 	    {
    96 		(void)g_rmdir(testcase->tmpdir);
    97 		g_free(testcase->tmpdir);
    98 		testcase->tmpdir=NULL;
    99 	    }
   100 	    return FALSE;
   101 	}
   102     return TRUE;
   103 }
   104 
   105 /*
   106  * Remove all the input files used by a testcase and, if created,
   107  * the temporary directory in which they are stored.
   108  */
   109 gboolean testcase_remove_input_files(Testcase *testcase,GError **error)
   110 {
   111     GSList *link;
   112     GError *tmp_err=NULL;
   113     gboolean retval=TRUE;
   114     for(link=testcase->inputs;link;link=link->next)
   115 	if (!testcase_input_remove(testcase,link->data,&tmp_err))
   116 	{
   117 	    if (error && !*error)
   118 		g_propagate_error(error,tmp_err);
   119 	    else
   120 		g_clear_error(&tmp_err);
   121 	    retval=FALSE;
   122 	}
   123     if (testcase->tmpdir)
   124     {
   125 	if (g_rmdir(testcase->tmpdir))
   126 	{
   127 	    if (error && !*error)
   128 		g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
   129 		  "Failed to remove temporary directory: %s",g_strerror(errno));
   130 	    retval=FALSE;
   131 	}
   132 	g_free(testcase->tmpdir);
   133 	testcase->tmpdir=NULL;
   134     }
   135     return retval;
   136 }
   137 
   138 /*
   139  * Replace every occurance of an input file name in <str> with the
   140  * filename which holds that input. For input files with fixed names,
   141  * this is a noop. For input files which use the "XXXXXX" sequence
   142  * to create a unique filename, the XXXXXX will be replaced with the
   143  * 6 characters that were chosen to be unique.
   144  */
   145 char *testcase_resolve_input_files(Testcase *testcase,const char *str)
   146 {
   147     GSList *link;
   148     gsize offset,pos;
   149     char *s;
   150     TestcaseInput *input;
   151     GString *filename=g_string_new(str);
   152     for(link=testcase->inputs;link;link=link->next)
   153     {
   154 	input=link->data;
   155 	if (!input->name_used)
   156 	{
   157 	    g_warning("%s: Input file uninstantiated",input->name);
   158 	    continue;
   159 	}
   160 	offset=0;
   161 	do
   162 	{
   163 	    s=strstr(filename->str+offset,input->name);
   164 	    if (s)
   165 	    {
   166 		pos=s-filename->str;
   167 		g_string_overwrite(filename,pos,input->name_used);
   168 		offset=pos+strlen(input->name);
   169 	    }
   170 	} while(s);
   171     }
   172     return g_string_free(filename,FALSE);
   173 }
   174 
   175 /*
   176  * Verify that all the output files specified by a testcase are present
   177  * with the expected contents. 
   178  */
   179 gboolean testcase_verify_output_files(Testcase *testcase)
   180 {
   181     GSList *link;
   182     GError *tmp_err=NULL;
   183     gboolean retval=TRUE;
   184     ssize_t offset;
   185     gchar *contents;
   186     TestcaseOutput *output;
   187     for(link=testcase->outputs;link;link=link->next)
   188     {
   189 	output=link->data;
   190 	if (!testcase_output_read(testcase,output,&contents,NULL,&tmp_err))
   191 	{
   192 	    g_print("%s: FAIL\n",testcase->basename);
   193 	    g_print("%s\n",tmp_err->message);
   194 	    g_clear_error(&tmp_err);
   195 	    retval=FALSE;
   196 	    break;
   197 	}
   198 	else
   199 	{
   200 	    if (strcmp(contents,output->contents))
   201 	    {
   202 		g_print("%s: FAIL\n",testcase->basename);
   203 		offset=common_prefix_length(contents,output->contents);
   204 		if (!offset && !contents[offset])
   205 		    g_print("%s: Unexpected empty output from bookloupe.\n",
   206 		      output->name);
   207 		else
   208 		{
   209 		    g_print("%s: Unexpected output from bookloupe:\n",
   210 		      output->name);
   211 		    print_unexpected(contents,offset);
   212 		}
   213 		retval=FALSE;
   214 	    }
   215 	    g_free(contents);
   216 	    break;
   217 	}
   218     }
   219     for(link=testcase->outputs;link;link=link->next)
   220 	if (!testcase_output_remove(testcase,link->data,&tmp_err))
   221 	{
   222 	    if (retval)
   223 	    {
   224 		g_print("%s: FAIL\n",testcase->basename);
   225 		g_print("%s\n",tmp_err->message);
   226 		retval=TRUE;
   227 	    }
   228 	    g_clear_error(&tmp_err);
   229 	}
   230     return retval;
   231 }
   232 
   233 gboolean testcase_spawn_bookloupe(Testcase *testcase,char **standard_output,
   234   GError **error)
   235 {
   236     gboolean r;
   237     int i,exit_status;
   238     char **argv;
   239     char *output,*s;
   240     GError *tmp_err=NULL;
   241     if (testcase->options)
   242 	argv=g_new(char *,g_strv_length(testcase->options)+3);
   243     else
   244 	argv=g_new(char *,3);
   245     s=getenv("BOOKLOUPE");
   246     if (!s)
   247 	s="bookloupe";
   248     argv[0]=path_to_absolute(s);
   249     for(i=0;testcase->options && testcase->options[i];i++)
   250 	argv[i+1]=testcase_resolve_input_files(testcase,testcase->options[i]);
   251     argv[i+1]=testcase_resolve_input_files(testcase,"TEST-XXXXXX");
   252     argv[i+2]=NULL;
   253     if (standard_output)
   254     {
   255 	r=spawn_sync(testcase->tmpdir,argv,&s,&exit_status,error);
   256 	if (r)
   257 	{
   258 	    if (testcase->encoding)
   259 	    {
   260 		output=g_convert(s,-1,"UTF-8",testcase->encoding,NULL,NULL,
   261 		  &tmp_err);
   262 		g_free(s);
   263 		if (!output)
   264 		{
   265 		    g_propagate_prefixed_error(error,tmp_err,
   266 		      "Conversion from %s failed: ",testcase->encoding);
   267 		    r=FALSE;
   268 		}
   269 	    }
   270 	    else
   271 	    {
   272 		output=s;
   273 		if (!g_utf8_validate(s,-1,NULL))
   274 		{
   275 		    g_set_error_literal(error,TESTCASE_ERROR,
   276 		      TESTCASE_ERROR_FAILED,
   277 		      "bookloupe output is not valid UTF-8");
   278 		    r=FALSE;
   279 		}
   280 	    }
   281 	}
   282     }
   283     else
   284     {
   285 	r=spawn_sync(testcase->tmpdir,argv,NULL,&exit_status,error);
   286 	output=NULL;
   287     }
   288     g_strfreev(argv);
   289     if (r && exit_status)
   290     {
   291 	g_set_error(error,TESTCASE_ERROR,TESTCASE_ERROR_FAILED,
   292 	  "bookloupe exited with code %d",exit_status);
   293 	r=FALSE;
   294     }
   295     if (r && standard_output)
   296 	*standard_output=output;
   297     return r;
   298 }
   299 
   300 /*
   301  * Parse a warning of the form:
   302  *	[blank line]
   303  *	<echoed line> (ignored)
   304  *	"    Line " <number> [" column " <number>] " - " <text> "\n"
   305  * If not specified, the column is returned as 0.
   306  * Returns: the number of bytes parsed, or -1 on error.
   307  */
   308 static ssize_t testcase_parse_warning(Testcase *testcase,const char *output,
   309   guint *line,guint *column,char **text)
   310 {
   311     ssize_t offset=0;
   312     guint64 tmp;
   313     char *s,*endp;
   314     if (output[offset]!='\n')
   315     {
   316 	g_print("%s: FAIL\n",testcase->basename);
   317 	g_print("Unexpected output from bookloupe:\n");
   318 	print_unexpected(output,offset);
   319 	return -1;
   320     }
   321     offset++;
   322     s=strchr(output+offset,'\n');
   323     if (!s)
   324     {
   325 	g_print("%s: FAIL\n",testcase->basename);
   326 	g_print("Missing new-line in output from bookloupe:\n");
   327 	print_unexpected(output,offset);
   328 	return -1;
   329     }
   330     offset=s-output+1;
   331     if (!g_str_has_prefix(output+offset,"    Line "))
   332     {
   333 	g_print("%s: FAIL\n",testcase->basename);
   334 	g_print("Unexpected output from bookloupe:\n");
   335 	offset+=common_prefix_length(output+offset,"    Line ");
   336 	print_unexpected(output,offset);
   337 	return -1;
   338     }
   339     offset+=9;
   340     tmp=g_ascii_strtoull(output+offset,&endp,10);
   341     if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64)
   342     {
   343 	g_print("%s: FAIL\n",testcase->basename);
   344 	g_print("Unexpected output from bookloupe:\n");
   345 	print_unexpected(output,offset);
   346 	return -1;
   347     }
   348     *line=tmp;
   349     offset=endp-output;
   350     if (g_str_has_prefix(output+offset," column "))
   351     {
   352 	offset+=8;
   353 	tmp=g_ascii_strtoull(output+offset,&endp,10);
   354 	if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64)
   355 	{
   356 	    g_print("%s: FAIL\n",testcase->basename);
   357 	    g_print("Unexpected output from bookloupe:\n");
   358 	    print_unexpected(output,offset);
   359 	    return -1;
   360 	}
   361 	*column=tmp;
   362 	offset=endp-output;
   363     }
   364     else
   365 	*column=0;
   366     if (!g_str_has_prefix(output+offset," - "))
   367     {
   368 	g_print("%s: FAIL\n",testcase->basename);
   369 	g_print("Unexpected output from bookloupe:\n");
   370 	offset+=common_prefix_length(output+offset," - ");
   371 	print_unexpected(output,offset);
   372 	return -1;
   373     }
   374     offset+=3;
   375     s=strchr(output+offset,'\n');
   376     if (!s)
   377     {
   378 	g_print("%s: FAIL\n",testcase->basename);
   379 	g_print("Missing new-line in output from bookloupe:\n");
   380 	print_unexpected(output,offset);
   381 	return -1;
   382     }
   383     *text=g_strndup(output+offset,s-(output+offset));
   384     return s-output+1;
   385 }
   386 
   387 /*
   388  * Check the summary produced by bookloupe against testcase->summary.
   389  */
   390 static gboolean testcase_check_summary(Testcase *testcase,const char *summary)
   391 {
   392     int i;
   393     gboolean r;
   394     gchar **lines;
   395     GSList *texts,*lnk;
   396     if (!testcase->summary.texts)
   397 	return TRUE;
   398     texts=g_slist_copy(testcase->summary.texts);
   399     lines=g_strsplit(summary,"\n",0);
   400     for(i=0;lines[i];i++)
   401     {
   402 	if (!g_str_has_prefix(lines[i],"   --> "))
   403 	    continue;
   404 	for(lnk=texts;lnk;lnk=lnk->next)
   405 	    if (!strcmp(lines[i]+7,lnk->data))
   406 	    {
   407 		texts=g_slist_delete_link(texts,lnk);
   408 		break;
   409 	    }
   410     }
   411     g_strfreev(lines);
   412     r=!texts;
   413     if (texts)
   414     {
   415 	g_print("%s: FAIL\n",testcase->basename);
   416 	g_print("Missing summary text from bookloupe:\n");
   417 	g_print("   --> %s\n",texts->data);
   418     }
   419     g_slist_free(texts);
   420     return r;
   421 }
   422 
   423 /*
   424  * Check the warnings produced by bookloupe against either the
   425  * unstructured testcase->expected or the structured testcase->warnings
   426  * as appropriate.
   427  */
   428 static gboolean testcase_check_warnings(Testcase *testcase,const char *output,
   429   char **xfail)
   430 {
   431     gboolean r=TRUE;
   432     size_t offset;
   433     ssize_t off;
   434     int i,count_false_positive,count_false_negative;
   435     int total_false_positive,total_false_negative;
   436     char *text;
   437     guint *counts,line,column;
   438     GSList *link,*link2;
   439     TestcaseWarning *warning;
   440     TestcaseLocation *location;
   441     *xfail=NULL;
   442     if (testcase->expected)
   443     {
   444 	if (strcmp(output,testcase->expected))
   445 	{
   446 	    g_print("%s: FAIL\n",testcase->basename);
   447 	    offset=common_prefix_length(output,testcase->expected);
   448 	    if (!offset && !output[offset])
   449 		g_print("Unexpected zero warnings from bookloupe.\n");
   450 	    else
   451 	    {
   452 		g_print("Unexpected output from bookloupe:\n");
   453 		print_unexpected(output,offset);
   454 	    }
   455 	    return FALSE;
   456 	}
   457 	return TRUE;
   458     }
   459     counts=g_new0(guint,g_slist_length(testcase->warnings));
   460     for(offset=0;output[offset];)
   461     {
   462 	off=testcase_parse_warning(testcase,output+offset,&line,&column,&text);
   463 	if (off<0)
   464 	{
   465 	    r=FALSE;
   466 	    break;
   467 	}
   468 	offset+=off;
   469 	for(link=testcase->warnings,i=0;link;link=link->next,i++)
   470 	{
   471 	    warning=link->data;
   472 	    if (strcmp(warning->text,text))
   473 		continue;
   474 	    for(link2=warning->locations;link2;link2=link2->next)
   475 	    {
   476 		location=link2->data;
   477 		if (location->line!=line || location->column!=column)
   478 		    continue;
   479 		counts[i]++;
   480 		break;
   481 	    }
   482 	    if (link2)
   483 		break;
   484 	}
   485 	if (!link)
   486 	{
   487 	    g_print("%s: FAIL\n",testcase->basename);
   488 	    g_print("Unexpected warning from bookloupe:\n");
   489 	    if (column)
   490 		g_print("    Line %u column %u - %s\n",line,column,text);
   491 	    else
   492 		g_print("    Line %u - %s\n",line,text);
   493 	    r=FALSE;
   494 	    g_free(text);
   495 	    break;
   496 	}
   497 	g_free(text);
   498     }
   499     count_false_positive=total_false_positive=0;
   500     count_false_negative=total_false_negative=0;
   501     for(link=testcase->warnings,i=0;r && link;link=link->next,i++)
   502     {
   503 	warning=link->data;
   504 	if (!counts[i] && warning->is_real && !warning->xfail)
   505 	{
   506 	    location=warning->locations->data;
   507 	    g_print("%s: FAIL\n",testcase->basename);
   508 	    g_print("Missing warning from bookloupe:\n");
   509 	    if (location->column)
   510 		g_print("    Line %u column %u - %s\n",location->line,
   511 		  location->column,warning->text);
   512 	    else
   513 		g_print("    Line %u - %s\n",location->line,warning->text);
   514 	    r=FALSE;
   515 	    break;
   516 	}
   517 	else if (warning->xfail)
   518 	{
   519 	    if (warning->is_real)
   520 	    {
   521 		total_false_negative++;
   522 		if (!counts[i])
   523 		    count_false_negative++;
   524 	    }
   525 	    else if (!warning->is_real)
   526 	    {
   527 		total_false_positive++;
   528 		if (counts[i])
   529 		    count_false_positive++;
   530 	    }
   531 	}
   532     }
   533     g_free(counts);
   534     if (count_false_positive && count_false_negative)
   535 	*xfail=g_strdup_printf(
   536 	  "with %d of %d false positives and %d of %d false negatives",
   537 	  count_false_positive,total_false_positive,
   538 	  count_false_negative,total_false_negative);
   539     else if (count_false_positive)
   540 	*xfail=g_strdup_printf("with %d of %d false positives",
   541 	  count_false_positive,total_false_positive);
   542     else if (count_false_negative)
   543 	*xfail=g_strdup_printf("with %d of %d false negatives",
   544 	  count_false_negative,total_false_negative);
   545     return r;
   546 }
   547 
   548 /*
   549  * Run a testcase, returning FALSE on fail or error and
   550  * TRUE on pass or expected-fail.
   551  * Suitable message(s) will be printed in all cases.
   552  */
   553 gboolean testcase_run(Testcase *testcase)
   554 {
   555     gboolean r;
   556     size_t pos,offset;
   557     GString *header;
   558     char *filename,*s,*summary,*xfail=NULL;
   559     GError *error=NULL;
   560     if (!testcase_create_input_files(testcase,&error))
   561     {
   562 	g_print("%s: FAIL\n",testcase->basename);
   563 	g_print("%s\n",error->message);
   564 	g_error_free(error);
   565 	return FALSE;
   566     }
   567     r=testcase_spawn_bookloupe(testcase,&testcase->test_output,&error);
   568     if (!r)
   569     {
   570 	g_print("%s: FAIL\n",testcase->basename);
   571 	g_print("%s\n",error->message);
   572 	g_error_free(error);
   573 	(void)testcase_remove_input_files(testcase,NULL);
   574 	return FALSE;
   575     }
   576     filename=testcase_resolve_input_files(testcase,"TEST-XXXXXX");
   577     if (!testcase_remove_input_files(testcase,&error))
   578     {
   579 	g_print("%s: FAIL\n",testcase->basename);
   580 	g_print("%s\n",error->message);
   581 	g_error_free(error);
   582 	return FALSE;
   583     }
   584     if (testcase->expected || testcase->warnings)
   585     {
   586 	header=g_string_new("\n\nFile: ");
   587 	g_string_append(header,filename);
   588 	g_string_append(header,"\n");
   589 	if (!g_str_has_prefix(testcase->test_output,header->str))
   590 	{
   591 	    g_print("%s: FAIL\n",testcase->basename);
   592 	    g_print("Unexpected header from bookloupe:\n");
   593 	    offset=common_prefix_length(testcase->test_output,header->str);
   594 	    print_unexpected(testcase->test_output,offset);
   595 	    r=FALSE;
   596 	}
   597 	summary=testcase->test_output+header->len;
   598 	pos=header->len;
   599 	if (r)
   600 	{
   601 	    /* Find the end of the summary */
   602 	    s=strstr(summary,"\n\n");
   603 	    if (s)
   604 	    {
   605 		summary=g_strndup(summary,s-summary);
   606 		r=testcase_check_summary(testcase,summary);
   607 		g_free(summary);
   608 		pos=s-testcase->test_output+2;
   609 	    }
   610 	    else
   611 	    {
   612 		g_print("%s: FAIL\n",testcase->basename);
   613 		g_print("Unterminated summary from bookloupe:\n%s\n",summary);
   614 		r=FALSE;
   615 	    }
   616 	}
   617 	g_string_free(header,TRUE);
   618 	if (r)
   619 	    r=testcase_check_warnings(testcase,testcase->test_output+pos,
   620 	      &xfail);
   621     }
   622     if (!testcase_verify_output_files(testcase))
   623 	r=FALSE;
   624     g_free(filename);
   625     if (r)
   626     {
   627 	if (xfail)
   628 	    g_print("%s: PASS (%s)\n",testcase->basename,xfail);
   629 	else
   630 	    g_print("%s: PASS\n",testcase->basename);
   631     }
   632     g_free(xfail);
   633     return r;
   634 }
   635 
   636 /*
   637  * Run a testcase, returning FALSE on error.
   638  * Bookloupe's output or a suitable error message will be shown.
   639  */
   640 gboolean testcase_show_output(Testcase *testcase)
   641 {
   642     gboolean r;
   643     gchar *output;
   644     GError *error=NULL;
   645     r=testcase_create_input_files(testcase,&error);
   646     if (r)
   647     {
   648 	r&=testcase_spawn_bookloupe(testcase,&output,&error);
   649 	r&=testcase_remove_input_files(testcase,&error);
   650     }
   651     if (r)
   652 	g_print("%s",output);
   653     else
   654     {
   655 	g_print("%s\n",error->message);
   656 	g_error_free(error);
   657     }
   658     return r;
   659 }
   660 
   661 /*
   662  * Free a testcase warning.
   663  */
   664 void testcase_warning_free(TestcaseWarning *warning)
   665 {
   666     g_slist_foreach(warning->locations,(GFunc)g_free,NULL);
   667     g_slist_free(warning->locations);
   668     g_free(warning->text);
   669     g_free(warning);
   670 }
   671 
   672 /*
   673  * Free a testcase.
   674  */
   675 void testcase_free(Testcase *testcase)
   676 {
   677     g_free(testcase->basename);
   678     g_slist_foreach(testcase->inputs,(GFunc)testcase_input_free,NULL);
   679     g_slist_free(testcase->inputs);
   680     g_free(testcase->expected);
   681     g_slist_foreach(testcase->warnings,(GFunc)testcase_warning_free,NULL);
   682     g_slist_free(testcase->warnings);
   683     g_free(testcase->encoding);
   684     g_strfreev(testcase->options);
   685     g_free(testcase->test_output);
   686     g_free(testcase);
   687 }