test/harness/testcase.c
author ali <ali@juiblex.co.uk>
Sat Oct 26 18:47:33 2013 +0100 (2013-10-26)
changeset 101 f44c530f80da
parent 96 8c2d6a0cf717
child 102 ff0aa9b1397a
permissions -rw-r--r--
Fix bug #24: Accept alternate form of newline
     1 #include <stdlib.h>
     2 #include <stdio.h>
     3 #include <string.h>
     4 #include <errno.h>
     5 #include <glib.h>
     6 #include <glib/gstdio.h>
     7 #include <bl/bl.h>
     8 #include "testcase.h"
     9 #include "testcaseinput.h"
    10 
    11 GQuark testcase_error_quark(void)
    12 {
    13     return g_quark_from_static_string("testcase-error-quark");
    14 }
    15 
    16 /*
    17  * Return the length (in bytes) of any common prefix between s1 and s2.
    18  * The returned length will always represent an exact number of characters.
    19  */
    20 size_t common_prefix_length(const char *s1,const char *s2)
    21 {
    22     gunichar c1,c2;
    23     const char *s=s1;
    24     while(*s1 && *s2)
    25     {
    26 	c1=g_utf8_get_char(s1);
    27 	c2=g_utf8_get_char(s2);
    28 	if (c1!=c2)
    29 	    break;
    30 	s1=g_utf8_next_char(s1);
    31 	s2=g_utf8_next_char(s2);
    32     }
    33     return s1-s;
    34 }
    35 
    36 void print_unexpected(const char *unexpected,gsize differs_at)
    37 {
    38     int col;
    39     gunichar c;
    40     const char *endp,*bol,*s;
    41     GString *string;
    42     endp=strchr(unexpected+differs_at,'\n');
    43     if (!endp)
    44 	endp=unexpected+strlen(unexpected);
    45     string=g_string_new_len(unexpected,endp-unexpected);
    46     bol=strrchr(string->str,'\n');
    47     if (bol)
    48 	bol++;
    49     else
    50 	bol=string->str;
    51     col=0;
    52     s=bol;
    53     endp=string->str+differs_at;
    54     while(s<endp)
    55     {
    56 	c=g_utf8_get_char(s);
    57 	s=g_utf8_next_char(s);
    58 	if (c=='\t')
    59 	    col=(col&~7)+8;
    60 	else if (g_unichar_iswide(c))
    61 	    col+=2;
    62 	else if (!g_unichar_iszerowidth(c))
    63 	    col++;
    64     }
    65     g_print("%s\n%*s^\n",string->str,col,"");
    66     g_string_free(string,TRUE);
    67 }
    68 
    69 /*
    70  * Create all the input files needed by a testcase and, if required,
    71  * a temporary directory in which to store them.
    72  */
    73 gboolean testcase_create_input_files(Testcase *testcase,GError **error)
    74 {
    75     GSList *link,*link2;
    76     if (testcase->flags&TESTCASE_TMP_DIR)
    77     {
    78 	testcase->tmpdir=g_strdup("TEST-XXXXXX");
    79 	if (!g_mkdtemp(testcase->tmpdir))
    80 	{
    81 	    g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
    82 	      "Failed to create temporary directory: %s",g_strerror(errno));
    83 	    g_free(testcase->tmpdir);
    84 	    testcase->tmpdir=NULL;
    85 	    return FALSE;
    86 	}
    87     }
    88     for(link=testcase->inputs;link;link=link->next)
    89 	if (!testcase_input_create(testcase,link->data,error))
    90 	{
    91 	    for(link2=testcase->inputs;link2!=link;link2=link2->next)
    92 		(void)testcase_input_remove(testcase,link2->data,NULL);
    93 	    if (testcase->tmpdir)
    94 	    {
    95 		(void)g_rmdir(testcase->tmpdir);
    96 		g_free(testcase->tmpdir);
    97 		testcase->tmpdir=NULL;
    98 	    }
    99 	    return FALSE;
   100 	}
   101     return TRUE;
   102 }
   103 
   104 /*
   105  * Remove all the input files used by a testcase and, if created,
   106  * the temporary directory in which they are stored.
   107  */
   108 gboolean testcase_remove_input_files(Testcase *testcase,GError **error)
   109 {
   110     GSList *link;
   111     GError *tmp_err=NULL;
   112     gboolean retval=TRUE;
   113     for(link=testcase->inputs;link;link=link->next)
   114 	if (!testcase_input_remove(testcase,link->data,&tmp_err))
   115 	{
   116 	    if (error && !*error)
   117 		g_propagate_error(error,tmp_err);
   118 	    else
   119 		g_clear_error(&tmp_err);
   120 	    retval=FALSE;
   121 	}
   122     if (testcase->tmpdir)
   123     {
   124 	if (g_rmdir(testcase->tmpdir))
   125 	{
   126 	    if (error && !*error)
   127 		g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
   128 		  "Failed to remove temporary directory: %s",g_strerror(errno));
   129 	    retval=FALSE;
   130 	}
   131 	g_free(testcase->tmpdir);
   132 	testcase->tmpdir=NULL;
   133     }
   134     return retval;
   135 }
   136 
   137 /*
   138  * Replace every occurance of an input file name in <str> with the
   139  * filename which holds that input. For input files with fixed names,
   140  * this is a noop. For input files which use the "XXXXXX" sequence
   141  * to create a unique filename, the XXXXXX will be replaced with the
   142  * 6 characters that were chosen to be unique.
   143  */
   144 char *testcase_resolve_input_files(Testcase *testcase,const char *str)
   145 {
   146     GSList *link;
   147     gsize offset,pos;
   148     char *s;
   149     TestcaseInput *input;
   150     GString *filename=g_string_new(str);
   151     for(link=testcase->inputs;link;link=link->next)
   152     {
   153 	input=link->data;
   154 	if (!input->name_used)
   155 	{
   156 	    g_warning("%s: Input file uninstantiated",input->name);
   157 	    continue;
   158 	}
   159 	offset=0;
   160 	do
   161 	{
   162 	    s=strstr(filename->str+offset,input->name);
   163 	    if (s)
   164 	    {
   165 		pos=s-filename->str;
   166 		g_string_overwrite(filename,pos,input->name_used);
   167 		offset=pos+strlen(input->name);
   168 	    }
   169 	} while(s);
   170     }
   171     return g_string_free(filename,FALSE);
   172 }
   173 
   174 gboolean testcase_spawn_bookloupe(Testcase *testcase,char **standard_output,
   175   GError **error)
   176 {
   177     gboolean r;
   178     int i,exit_status;
   179     char **argv;
   180     char *output,*s;
   181     GError *tmp_err=NULL;
   182     if (testcase->options)
   183 	argv=g_new(char *,g_strv_length(testcase->options)+3);
   184     else
   185 	argv=g_new(char *,3);
   186     s=getenv("BOOKLOUPE");
   187     if (!s)
   188 	s="bookloupe";
   189     argv[0]=path_to_absolute(s);
   190     for(i=0;testcase->options && testcase->options[i];i++)
   191 	argv[i+1]=testcase_resolve_input_files(testcase,testcase->options[i]);
   192     argv[i+1]=testcase_resolve_input_files(testcase,"TEST-XXXXXX");
   193     argv[i+2]=NULL;
   194     if (standard_output)
   195     {
   196 	r=spawn_sync(testcase->tmpdir,argv,&s,&exit_status,error);
   197 	if (r)
   198 	{
   199 	    if (testcase->encoding)
   200 	    {
   201 		output=g_convert(s,-1,"UTF-8",testcase->encoding,NULL,NULL,
   202 		  &tmp_err);
   203 		g_free(s);
   204 		if (!output)
   205 		{
   206 		    g_propagate_prefixed_error(error,tmp_err,
   207 		      "Conversion from %s failed: ",testcase->encoding);
   208 		    r=FALSE;
   209 		}
   210 	    }
   211 	    else
   212 	    {
   213 		output=s;
   214 		if (!g_utf8_validate(s,-1,NULL))
   215 		{
   216 		    g_set_error_literal(error,TESTCASE_ERROR,
   217 		      TESTCASE_ERROR_FAILED,
   218 		      "bookloupe output is not valid UTF-8");
   219 		    r=FALSE;
   220 		}
   221 	    }
   222 	}
   223     }
   224     else
   225     {
   226 	r=spawn_sync(testcase->tmpdir,argv,NULL,&exit_status,error);
   227 	output=NULL;
   228     }
   229     g_strfreev(argv);
   230     if (r && exit_status)
   231     {
   232 	g_set_error(error,TESTCASE_ERROR,TESTCASE_ERROR_FAILED,
   233 	  "bookloupe exited with code %d",exit_status);
   234 	r=FALSE;
   235     }
   236     if (r && standard_output)
   237 	*standard_output=output;
   238     return r;
   239 }
   240 
   241 /*
   242  * Parse a warning of the form:
   243  *	[blank line]
   244  *	<echoed line> (ignored)
   245  *	"    Line " <number> [" column " <number>] " - " <text> "\n"
   246  * If not specified, the column is returned as 0.
   247  * Returns: the number of bytes parsed, or -1 on error.
   248  */
   249 static ssize_t testcase_parse_warning(Testcase *testcase,const char *output,
   250   guint *line,guint *column,char **text)
   251 {
   252     ssize_t offset=0;
   253     guint64 tmp;
   254     char *s,*endp;
   255     if (output[offset]!='\n')
   256     {
   257 	g_print("%s: FAIL\n",testcase->basename);
   258 	g_print("Unexpected output from bookloupe:\n");
   259 	print_unexpected(output,offset);
   260 	return -1;
   261     }
   262     offset++;
   263     s=strchr(output+offset,'\n');
   264     if (!s)
   265     {
   266 	g_print("%s: FAIL\n",testcase->basename);
   267 	g_print("Missing new-line in output from bookloupe:\n");
   268 	print_unexpected(output,offset);
   269 	return -1;
   270     }
   271     offset=s-output+1;
   272     if (!g_str_has_prefix(output+offset,"    Line "))
   273     {
   274 	g_print("%s: FAIL\n",testcase->basename);
   275 	g_print("Unexpected output from bookloupe:\n");
   276 	offset+=common_prefix_length(output+offset,"    Line ");
   277 	print_unexpected(output,offset);
   278 	return -1;
   279     }
   280     offset+=9;
   281     tmp=g_ascii_strtoull(output+offset,&endp,10);
   282     if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64)
   283     {
   284 	g_print("%s: FAIL\n",testcase->basename);
   285 	g_print("Unexpected output from bookloupe:\n");
   286 	print_unexpected(output,offset);
   287 	return -1;
   288     }
   289     *line=tmp;
   290     offset=endp-output;
   291     if (g_str_has_prefix(output+offset," column "))
   292     {
   293 	offset+=8;
   294 	tmp=g_ascii_strtoull(output+offset,&endp,10);
   295 	if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64)
   296 	{
   297 	    g_print("%s: FAIL\n",testcase->basename);
   298 	    g_print("Unexpected output from bookloupe:\n");
   299 	    print_unexpected(output,offset);
   300 	    return -1;
   301 	}
   302 	*column=tmp;
   303 	offset=endp-output;
   304     }
   305     else
   306 	*column=0;
   307     if (!g_str_has_prefix(output+offset," - "))
   308     {
   309 	g_print("%s: FAIL\n",testcase->basename);
   310 	g_print("Unexpected output from bookloupe:\n");
   311 	offset+=common_prefix_length(output+offset," - ");
   312 	print_unexpected(output,offset);
   313 	return -1;
   314     }
   315     offset+=3;
   316     s=strchr(output+offset,'\n');
   317     if (!s)
   318     {
   319 	g_print("%s: FAIL\n",testcase->basename);
   320 	g_print("Missing new-line in output from bookloupe:\n");
   321 	print_unexpected(output,offset);
   322 	return -1;
   323     }
   324     *text=g_strndup(output+offset,s-(output+offset));
   325     return s-output+1;
   326 }
   327 
   328 /*
   329  * Check the summary produced by bookloupe against testcase->summary.
   330  */
   331 static gboolean testcase_check_summary(Testcase *testcase,const char *summary)
   332 {
   333     int i;
   334     gboolean r;
   335     gchar **lines;
   336     GSList *texts,*lnk;
   337     if (!testcase->summary.texts)
   338 	return TRUE;
   339     texts=g_slist_copy(testcase->summary.texts);
   340     lines=g_strsplit(summary,"\n",0);
   341     for(i=0;lines[i];i++)
   342     {
   343 	if (!g_str_has_prefix(lines[i],"   --> "))
   344 	    continue;
   345 	for(lnk=texts;lnk;lnk=lnk->next)
   346 	    if (!strcmp(lines[i]+7,lnk->data))
   347 	    {
   348 		texts=g_slist_delete_link(texts,lnk);
   349 		break;
   350 	    }
   351     }
   352     g_strfreev(lines);
   353     r=!texts;
   354     if (texts)
   355     {
   356 	g_print("%s: FAIL\n",testcase->basename);
   357 	g_print("Missing summary text from bookloupe:\n");
   358 	g_print("   --> %s\n",texts->data);
   359     }
   360     g_slist_free(texts);
   361     return r;
   362 }
   363 
   364 /*
   365  * Check the warnings produced by bookloupe against either the
   366  * unstructured testcase->expected or the structured testcase->warnings
   367  * as appropriate.
   368  */
   369 static gboolean testcase_check_warnings(Testcase *testcase,const char *output,
   370   char **xfail)
   371 {
   372     gboolean r=TRUE;
   373     size_t offset;
   374     ssize_t off;
   375     int i,count_false_positive,count_false_negative;
   376     int total_false_positive,total_false_negative;
   377     char *text;
   378     guint *counts,line,column;
   379     GSList *link,*link2;
   380     TestcaseWarning *warning;
   381     TestcaseLocation *location;
   382     *xfail=NULL;
   383     if (testcase->expected)
   384     {
   385 	if (strcmp(output,testcase->expected))
   386 	{
   387 	    g_print("%s: FAIL\n",testcase->basename);
   388 	    offset=common_prefix_length(output,testcase->expected);
   389 	    if (!offset && !output[offset])
   390 		g_print("Unexpected zero warnings from bookloupe.\n");
   391 	    else
   392 	    {
   393 		g_print("Unexpected output from bookloupe:\n");
   394 		print_unexpected(output,offset);
   395 	    }
   396 	    return FALSE;
   397 	}
   398 	return TRUE;
   399     }
   400     counts=g_new0(guint,g_slist_length(testcase->warnings));
   401     for(offset=0;output[offset];)
   402     {
   403 	off=testcase_parse_warning(testcase,output+offset,&line,&column,&text);
   404 	if (off<0)
   405 	{
   406 	    r=FALSE;
   407 	    break;
   408 	}
   409 	offset+=off;
   410 	for(link=testcase->warnings,i=0;link;link=link->next,i++)
   411 	{
   412 	    warning=link->data;
   413 	    if (strcmp(warning->text,text))
   414 		continue;
   415 	    for(link2=warning->locations;link2;link2=link2->next)
   416 	    {
   417 		location=link2->data;
   418 		if (location->line!=line || location->column!=column)
   419 		    continue;
   420 		counts[i]++;
   421 		break;
   422 	    }
   423 	    if (link2)
   424 		break;
   425 	}
   426 	if (!link)
   427 	{
   428 	    g_print("%s: FAIL\n",testcase->basename);
   429 	    g_print("Unexpected warning from bookloupe:\n");
   430 	    if (column)
   431 		g_print("    Line %u column %u - %s\n",line,column,text);
   432 	    else
   433 		g_print("    Line %u - %s\n",line,text);
   434 	    r=FALSE;
   435 	    g_free(text);
   436 	    break;
   437 	}
   438 	g_free(text);
   439     }
   440     count_false_positive=total_false_positive=0;
   441     count_false_negative=total_false_negative=0;
   442     for(link=testcase->warnings,i=0;r && link;link=link->next,i++)
   443     {
   444 	warning=link->data;
   445 	if (!counts[i] && warning->is_real && !warning->xfail)
   446 	{
   447 	    location=warning->locations->data;
   448 	    g_print("%s: FAIL\n",testcase->basename);
   449 	    g_print("Missing warning from bookloupe:\n");
   450 	    if (location->column)
   451 		g_print("    Line %u column %u - %s\n",location->line,
   452 		  location->column,warning->text);
   453 	    else
   454 		g_print("    Line %u - %s\n",location->line,warning->text);
   455 	    r=FALSE;
   456 	    break;
   457 	}
   458 	else if (warning->xfail)
   459 	{
   460 	    if (warning->is_real)
   461 	    {
   462 		total_false_negative++;
   463 		if (!counts[i])
   464 		    count_false_negative++;
   465 	    }
   466 	    else if (!warning->is_real)
   467 	    {
   468 		total_false_positive++;
   469 		if (counts[i])
   470 		    count_false_positive++;
   471 	    }
   472 	}
   473     }
   474     g_free(counts);
   475     if (count_false_positive && count_false_negative)
   476 	*xfail=g_strdup_printf(
   477 	  "with %d of %d false positives and %d of %d false negatives",
   478 	  count_false_positive,total_false_positive,
   479 	  count_false_negative,total_false_negative);
   480     else if (count_false_positive)
   481 	*xfail=g_strdup_printf("with %d of %d false positives",
   482 	  count_false_positive,total_false_positive);
   483     else if (count_false_negative)
   484 	*xfail=g_strdup_printf("with %d of %d false negatives",
   485 	  count_false_negative,total_false_negative);
   486     return r;
   487 }
   488 
   489 /*
   490  * Run a testcase, returning FALSE on fail or error and
   491  * TRUE on pass or expected-fail.
   492  * Suitable message(s) will be printed in all cases.
   493  */
   494 gboolean testcase_run(Testcase *testcase)
   495 {
   496     gboolean r;
   497     size_t pos,offset;
   498     GString *header;
   499     char *output,*filename,*s,*summary,*xfail=NULL;
   500     GError *error=NULL;
   501     if (!testcase_create_input_files(testcase,&error))
   502     {
   503 	g_print("%s: FAIL\n",testcase->basename);
   504 	g_print("%s\n",error->message);
   505 	g_error_free(error);
   506 	return FALSE;
   507     }
   508     r=testcase_spawn_bookloupe(testcase,&output,&error);
   509     if (!r)
   510     {
   511 	g_print("%s: FAIL\n",testcase->basename);
   512 	g_print("%s\n",error->message);
   513 	g_error_free(error);
   514 	(void)testcase_remove_input_files(testcase,NULL);
   515 	return FALSE;
   516     }
   517     filename=testcase_resolve_input_files(testcase,"TEST-XXXXXX");
   518     if (!testcase_remove_input_files(testcase,&error))
   519     {
   520 	g_print("%s: FAIL\n",testcase->basename);
   521 	g_print("%s\n",error->message);
   522 	g_error_free(error);
   523 	return FALSE;
   524     }
   525     header=g_string_new("\n\nFile: ");
   526     g_string_append(header,filename);
   527     g_string_append(header,"\n");
   528     if (!g_str_has_prefix(output,header->str))
   529     {
   530 	g_print("%s: FAIL\n",testcase->basename);
   531 	g_print("Unexpected header from bookloupe:\n");
   532 	offset=common_prefix_length(output,header->str);
   533 	print_unexpected(output,offset);
   534 	r=FALSE;
   535     }
   536     pos=header->len;
   537     if (r)
   538     {
   539 	/* Find the end of the summary */
   540 	s=strstr(output+pos,"\n\n");
   541 	if (s)
   542 	{
   543 	    summary=g_strndup(output+pos,s-(output+pos));
   544 	    r=testcase_check_summary(testcase,summary);
   545 	    g_free(summary);
   546 	    pos=s-output+2;
   547 	}
   548 	else
   549 	{
   550 	    g_print("%s: FAIL\n",testcase->basename);
   551 	    g_print("Unterminated summary from bookloupe:\n%s\n",output+pos);
   552 	    r=FALSE;
   553 	}
   554     }
   555     g_string_free(header,TRUE);
   556     if (r)
   557 	r=testcase_check_warnings(testcase,output+pos,&xfail);
   558     g_free(filename);
   559     g_free(output);
   560     if (r)
   561     {
   562 	if (xfail)
   563 	    g_print("%s: PASS (%s)\n",testcase->basename,xfail);
   564 	else
   565 	    g_print("%s: PASS\n",testcase->basename);
   566     }
   567     g_free(xfail);
   568     return r;
   569 }
   570 
   571 /*
   572  * Run a testcase, returning FALSE on error.
   573  * Bookloupe's output or a suitable error message will be shown.
   574  */
   575 gboolean testcase_show_output(Testcase *testcase)
   576 {
   577     gboolean r;
   578     gchar *output;
   579     GError *error=NULL;
   580     r=testcase_create_input_files(testcase,&error);
   581     if (r)
   582     {
   583 	r&=testcase_spawn_bookloupe(testcase,&output,&error);
   584 	r&=testcase_remove_input_files(testcase,&error);
   585     }
   586     if (r)
   587 	g_print("%s",output);
   588     else
   589     {
   590 	g_print("%s\n",error->message);
   591 	g_error_free(error);
   592     }
   593     return r;
   594 }
   595 
   596 /*
   597  * Free a testcase warning.
   598  */
   599 void testcase_warning_free(TestcaseWarning *warning)
   600 {
   601     g_slist_foreach(warning->locations,(GFunc)g_free,NULL);
   602     g_slist_free(warning->locations);
   603     g_free(warning->text);
   604     g_free(warning);
   605 }
   606 
   607 /*
   608  * Free a testcase.
   609  */
   610 void testcase_free(Testcase *testcase)
   611 {
   612     g_free(testcase->basename);
   613     g_slist_foreach(testcase->inputs,(GFunc)testcase_input_free,NULL);
   614     g_slist_free(testcase->inputs);
   615     g_free(testcase->expected);
   616     g_slist_foreach(testcase->warnings,(GFunc)testcase_warning_free,NULL);
   617     g_slist_free(testcase->warnings);
   618     g_free(testcase->encoding);
   619     g_strfreev(testcase->options);
   620     g_free(testcase);
   621 }