ali@0: #include <stdlib.h>
ali@0: #include <stdio.h>
ali@0: #include <string.h>
ali@0: #include <errno.h>
ali@9: #include <glib.h>
ali@9: #include <glib/gstdio.h>
ali@5: #include <bl/bl.h>
ali@0: #include "testcase.h"
ali@9: #include "testcaseinput.h"
ali@0: 
ali@7: GQuark testcase_error_quark(void)
ali@7: {
ali@7:     return g_quark_from_static_string("testcase-error-quark");
ali@7: }
ali@7: 
ali@0: /*
ali@8:  * Return the length (in bytes) of any common prefix between s1 and s2.
ali@8:  * The returned length will always represent an exact number of characters.
ali@0:  */
ali@0: size_t common_prefix_length(const char *s1,const char *s2)
ali@0: {
ali@8:     gunichar c1,c2;
ali@8:     const char *s=s1;
ali@8:     while(*s1 && *s2)
ali@8:     {
ali@8: 	c1=g_utf8_get_char(s1);
ali@8: 	c2=g_utf8_get_char(s2);
ali@8: 	if (c1!=c2)
ali@8: 	    break;
ali@8: 	s1=g_utf8_next_char(s1);
ali@8: 	s2=g_utf8_next_char(s2);
ali@8:     }
ali@8:     return s1-s;
ali@0: }
ali@0: 
ali@7: void print_unexpected(const char *unexpected,gsize differs_at)
ali@7: {
ali@7:     int col;
ali@8:     gunichar c;
ali@8:     const char *endp,*bol,*s;
ali@7:     GString *string;
ali@7:     endp=strchr(unexpected+differs_at,'\n');
ali@7:     if (!endp)
ali@7: 	endp=unexpected+strlen(unexpected);
ali@7:     string=g_string_new_len(unexpected,endp-unexpected);
ali@7:     bol=strrchr(string->str,'\n');
ali@7:     if (bol)
ali@7: 	bol++;
ali@7:     else
ali@7: 	bol=string->str;
ali@8:     col=0;
ali@8:     s=bol;
ali@8:     endp=string->str+differs_at;
ali@8:     while(s<endp)
ali@8:     {
ali@8: 	c=g_utf8_get_char(s);
ali@8: 	s=g_utf8_next_char(s);
ali@8: 	if (c=='\t')
ali@8: 	    col=(col&~7)+8;
ali@8: 	else if (g_unichar_iswide(c))
ali@8: 	    col+=2;
ali@8: 	else if (!g_unichar_iszerowidth(c))
ali@8: 	    col++;
ali@8:     }
ali@11:     g_print("%s\n%*s^\n",string->str,col,"");
ali@7:     g_string_free(string,TRUE);
ali@7: }
ali@7: 
ali@8: /*
ali@9:  * Create all the input files needed by a testcase and, if required,
ali@9:  * a temporary directory in which to store them.
ali@8:  */
ali@9: gboolean testcase_create_input_files(Testcase *testcase,GError **error)
ali@8: {
ali@9:     GSList *link,*link2;
ali@9:     if (testcase->flags&TESTCASE_TMP_DIR)
ali@8:     {
ali@9: 	testcase->tmpdir=g_strdup("TEST-XXXXXX");
ali@9: 	if (!g_mkdtemp(testcase->tmpdir))
ali@9: 	{
ali@9: 	    g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
ali@9: 	      "Failed to create temporary directory: %s",g_strerror(errno));
ali@9: 	    g_free(testcase->tmpdir);
ali@9: 	    testcase->tmpdir=NULL;
ali@9: 	    return FALSE;
ali@9: 	}
ali@8:     }
ali@9:     for(link=testcase->inputs;link;link=link->next)
ali@9: 	if (!testcase_input_create(testcase,link->data,error))
ali@9: 	{
ali@9: 	    for(link2=testcase->inputs;link2!=link;link2=link2->next)
ali@9: 		(void)testcase_input_remove(testcase,link2->data,NULL);
ali@9: 	    if (testcase->tmpdir)
ali@9: 	    {
ali@9: 		(void)g_rmdir(testcase->tmpdir);
ali@9: 		g_free(testcase->tmpdir);
ali@9: 		testcase->tmpdir=NULL;
ali@9: 	    }
ali@9: 	    return FALSE;
ali@9: 	}
ali@9:     return TRUE;
ali@8: }
ali@8: 
ali@9: /*
ali@9:  * Remove all the input files used by a testcase and, if created,
ali@9:  * the temporary directory in which they are stored.
ali@9:  */
ali@9: gboolean testcase_remove_input_files(Testcase *testcase,GError **error)
ali@9: {
ali@9:     GSList *link;
ali@9:     GError *tmp_err=NULL;
ali@9:     gboolean retval=TRUE;
ali@9:     for(link=testcase->inputs;link;link=link->next)
ali@9: 	if (!testcase_input_remove(testcase,link->data,&tmp_err))
ali@9: 	{
ali@9: 	    if (error && !*error)
ali@9: 		g_propagate_error(error,tmp_err);
ali@9: 	    else
ali@9: 		g_clear_error(&tmp_err);
ali@9: 	    retval=FALSE;
ali@9: 	}
ali@9:     if (testcase->tmpdir)
ali@9:     {
ali@9: 	if (g_rmdir(testcase->tmpdir))
ali@9: 	{
ali@9: 	    if (error && !*error)
ali@9: 		g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
ali@9: 		  "Failed to remove temporary directory: %s",g_strerror(errno));
ali@9: 	    retval=FALSE;
ali@9: 	}
ali@9: 	g_free(testcase->tmpdir);
ali@9: 	testcase->tmpdir=NULL;
ali@9:     }
ali@9:     return retval;
ali@9: }
ali@9: 
ali@9: /*
ali@9:  * Replace every occurance of an input file name in <str> with the
ali@9:  * filename which holds that input. For input files with fixed names,
ali@9:  * this is a noop. For input files which use the "XXXXXX" sequence
ali@9:  * to create a unique filename, the XXXXXX will be replaced with the
ali@9:  * 6 characters that were chosen to be unique.
ali@9:  */
ali@9: char *testcase_resolve_input_files(Testcase *testcase,const char *str)
ali@9: {
ali@9:     GSList *link;
ali@9:     gsize offset,pos;
ali@9:     char *s;
ali@9:     TestcaseInput *input;
ali@9:     GString *filename=g_string_new(str);
ali@9:     for(link=testcase->inputs;link;link=link->next)
ali@9:     {
ali@9: 	input=link->data;
ali@9: 	if (!input->name_used)
ali@9: 	{
ali@9: 	    g_warning("%s: Input file uninstantiated",input->name);
ali@9: 	    continue;
ali@9: 	}
ali@9: 	offset=0;
ali@9: 	do
ali@9: 	{
ali@9: 	    s=strstr(filename->str+offset,input->name);
ali@9: 	    if (s)
ali@9: 	    {
ali@9: 		pos=s-filename->str;
ali@9: 		g_string_overwrite(filename,pos,input->name_used);
ali@9: 		offset=pos+strlen(input->name);
ali@9: 	    }
ali@9: 	} while(s);
ali@9:     }
ali@9:     return g_string_free(filename,FALSE);
ali@9: }
ali@9: 
ali@9: gboolean testcase_spawn_bookloupe(Testcase *testcase,char **standard_output,
ali@9:   GError **error)
ali@7: {
ali@7:     gboolean r;
ali@9:     int i,exit_status;
ali@9:     char **argv;
ali@9:     char *output,*s;
ali@7:     GError *tmp_err=NULL;
ali@9:     if (testcase->options)
ali@9: 	argv=g_new(char *,g_strv_length(testcase->options)+3);
ali@7:     else
ali@9: 	argv=g_new(char *,3);
ali@9:     s=getenv("BOOKLOUPE");
ali@9:     if (!s)
ali@9: 	s="bookloupe";
ali@9:     argv[0]=path_to_absolute(s);
ali@9:     for(i=0;testcase->options && testcase->options[i];i++)
ali@9: 	argv[i+1]=testcase_resolve_input_files(testcase,testcase->options[i]);
ali@9:     argv[i+1]=testcase_resolve_input_files(testcase,"TEST-XXXXXX");
ali@9:     argv[i+2]=NULL;
ali@7:     if (standard_output)
ali@7:     {
ali@9: 	r=spawn_sync(testcase->tmpdir,argv,&s,&exit_status,error);
ali@7: 	if (r)
ali@7: 	{
ali@9: 	    if (testcase->encoding)
ali@7: 	    {
ali@9: 		output=g_convert(s,-1,"UTF-8",testcase->encoding,NULL,NULL,
ali@9: 		  &tmp_err);
ali@7: 		g_free(s);
ali@7: 		if (!output)
ali@7: 		{
ali@7: 		    g_propagate_prefixed_error(error,tmp_err,
ali@9: 		      "Conversion from %s failed: ",testcase->encoding);
ali@7: 		    r=FALSE;
ali@7: 		}
ali@7: 	    }
ali@7: 	    else
ali@7: 	    {
ali@7: 		output=s;
ali@7: 		if (!g_utf8_validate(s,-1,NULL))
ali@7: 		{
ali@7: 		    g_set_error_literal(error,TESTCASE_ERROR,
ali@7: 		      TESTCASE_ERROR_FAILED,
ali@7: 		      "bookloupe output is not valid UTF-8");
ali@7: 		    r=FALSE;
ali@7: 		}
ali@7: 	    }
ali@7: 	}
ali@7:     }
ali@7:     else
ali@7:     {
ali@9: 	r=spawn_sync(testcase->tmpdir,argv,NULL,&exit_status,error);
ali@7: 	output=NULL;
ali@7:     }
ali@9:     g_strfreev(argv);
ali@7:     if (r && exit_status)
ali@7:     {
ali@7: 	g_set_error(error,TESTCASE_ERROR,TESTCASE_ERROR_FAILED,
ali@7: 	  "bookloupe exited with code %d",exit_status);
ali@7: 	r=FALSE;
ali@7:     }
ali@7:     if (r && standard_output)
ali@7: 	*standard_output=output;
ali@7:     return r;
ali@7: }
ali@7: 
ali@0: /*
ali@17:  * Parse a warning of the form:
ali@17:  *	[blank line]
ali@17:  *	<echoed line> (ignored)
ali@17:  *	"    Line " <number> [" column " <number>] " - " <text> "\n"
ali@17:  * If not specified, the column is returned as 0.
ali@17:  * Returns: the number of bytes parsed, or -1 on error.
ali@17:  */
ali@17: static ssize_t testcase_parse_warning(Testcase *testcase,const char *output,
ali@17:   guint *line,guint *column,char **text)
ali@17: {
ali@17:     ssize_t offset=0;
ali@17:     guint64 tmp;
ali@17:     char *s,*endp;
ali@17:     if (output[offset]!='\n')
ali@17:     {
ali@17: 	g_print("%s: FAIL\n",testcase->basename);
ali@17: 	g_print("Unexpected output from bookloupe:\n");
ali@17: 	print_unexpected(output,offset);
ali@17: 	return -1;
ali@17:     }
ali@17:     offset++;
ali@17:     s=strchr(output+offset,'\n');
ali@17:     if (!s)
ali@17:     {
ali@17: 	g_print("%s: FAIL\n",testcase->basename);
ali@17: 	g_print("Missing new-line in output from bookloupe:\n");
ali@17: 	print_unexpected(output,offset);
ali@17: 	return -1;
ali@17:     }
ali@17:     offset=s-output+1;
ali@17:     if (!g_str_has_prefix(output+offset,"    Line "))
ali@17:     {
ali@17: 	g_print("%s: FAIL\n",testcase->basename);
ali@17: 	g_print("Unexpected output from bookloupe:\n");
ali@17: 	offset+=common_prefix_length(output+offset,"    Line ");
ali@17: 	print_unexpected(output,offset);
ali@17: 	return -1;
ali@17:     }
ali@17:     offset+=9;
ali@17:     tmp=g_ascii_strtoull(output+offset,&endp,10);
ali@17:     if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64)
ali@17:     {
ali@17: 	g_print("%s: FAIL\n",testcase->basename);
ali@17: 	g_print("Unexpected output from bookloupe:\n");
ali@17: 	print_unexpected(output,offset);
ali@17: 	return -1;
ali@17:     }
ali@17:     *line=tmp;
ali@17:     offset=endp-output;
ali@17:     if (g_str_has_prefix(output+offset," column "))
ali@17:     {
ali@17: 	offset+=8;
ali@17: 	tmp=g_ascii_strtoull(output+offset,&endp,10);
ali@17: 	if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64)
ali@17: 	{
ali@17: 	    g_print("%s: FAIL\n",testcase->basename);
ali@17: 	    g_print("Unexpected output from bookloupe:\n");
ali@17: 	    print_unexpected(output,offset);
ali@17: 	    return -1;
ali@17: 	}
ali@17: 	*column=tmp;
ali@17: 	offset=endp-output;
ali@17:     }
ali@17:     else
ali@17: 	*column=0;
ali@17:     if (!g_str_has_prefix(output+offset," - "))
ali@17:     {
ali@17: 	g_print("%s: FAIL\n",testcase->basename);
ali@17: 	g_print("Unexpected output from bookloupe:\n");
ali@17: 	offset+=common_prefix_length(output+offset," - ");
ali@17: 	print_unexpected(output,offset);
ali@17: 	return -1;
ali@17:     }
ali@17:     offset+=3;
ali@17:     s=strchr(output+offset,'\n');
ali@17:     if (!s)
ali@17:     {
ali@17: 	g_print("%s: FAIL\n",testcase->basename);
ali@17: 	g_print("Missing new-line in output from bookloupe:\n");
ali@17: 	print_unexpected(output,offset);
ali@17: 	return -1;
ali@17:     }
ali@17:     *text=g_strndup(output+offset,s-(output+offset));
ali@17:     return s-output+1;
ali@17: }
ali@17: 
ali@17: /*
ali@17:  * Check the warnings produced by bookloupe against either the
ali@17:  * unstructured testcase->expected or the structured testcase->warnings
ali@17:  * as appropriate.
ali@17:  */
ali@17: static gboolean testcase_check_warnings(Testcase *testcase,const char *output,
ali@17:   char **xfail)
ali@17: {
ali@17:     gboolean r=TRUE;
ali@17:     size_t offset;
ali@17:     ssize_t off;
ali@17:     int i,count_false_positive,count_false_negative;
ali@17:     int total_false_positive,total_false_negative;
ali@17:     char *text;
ali@17:     guint *counts,line,column;
ali@17:     GSList *link,*link2;
ali@17:     TestcaseWarning *warning;
ali@17:     TestcaseLocation *location;
ali@17:     *xfail=NULL;
ali@17:     if (testcase->expected)
ali@17:     {
ali@17: 	if (strcmp(output,testcase->expected))
ali@17: 	{
ali@17: 	    g_print("%s: FAIL\n",testcase->basename);
ali@17: 	    offset=common_prefix_length(output,testcase->expected);
ali@17: 	    if (!offset && !output[offset])
ali@17: 		g_print("Unexpected zero warnings from bookloupe.\n");
ali@17: 	    else
ali@17: 	    {
ali@17: 		g_print("Unexpected output from bookloupe:\n");
ali@17: 		print_unexpected(output,offset);
ali@17: 	    }
ali@17: 	    return FALSE;
ali@17: 	}
ali@17: 	return TRUE;
ali@17:     }
ali@17:     counts=g_new0(guint,g_slist_length(testcase->warnings));
ali@17:     for(offset=0;output[offset];)
ali@17:     {
ali@17: 	off=testcase_parse_warning(testcase,output+offset,&line,&column,&text);
ali@17: 	if (off<0)
ali@17: 	{
ali@17: 	    r=FALSE;
ali@17: 	    break;
ali@17: 	}
ali@17: 	offset+=off;
ali@17: 	for(link=testcase->warnings,i=0;link;link=link->next,i++)
ali@17: 	{
ali@17: 	    warning=link->data;
ali@17: 	    if (strcmp(warning->text,text))
ali@17: 		continue;
ali@17: 	    for(link2=warning->locations;link2;link2=link2->next)
ali@17: 	    {
ali@17: 		location=link2->data;
ali@17: 		if (location->line!=line || location->column!=column)
ali@17: 		    continue;
ali@17: 		counts[i]++;
ali@17: 		break;
ali@17: 	    }
ali@17: 	    if (link2)
ali@17: 		break;
ali@17: 	}
ali@17: 	if (!link)
ali@17: 	{
ali@17: 	    g_print("%s: FAIL\n",testcase->basename);
ali@17: 	    g_print("Unexpected warning from bookloupe:\n");
ali@17: 	    if (column)
ali@17: 		g_print("    Line %u column %u - %s\n",line,column,text);
ali@17: 	    else
ali@17: 		g_print("    Line %u - %s\n",line,text);
ali@17: 	    r=FALSE;
ali@17: 	    g_free(text);
ali@17: 	    break;
ali@17: 	}
ali@17: 	g_free(text);
ali@17:     }
ali@17:     count_false_positive=total_false_positive=0;
ali@17:     count_false_negative=total_false_negative=0;
ali@17:     for(link=testcase->warnings,i=0;r && link;link=link->next,i++)
ali@17:     {
ali@17: 	warning=link->data;
ali@17: 	if (!counts[i] && warning->is_real && !warning->xfail)
ali@17: 	{
ali@17: 	    location=warning->locations->data;
ali@17: 	    g_print("%s: FAIL\n",testcase->basename);
ali@17: 	    g_print("Missing warning from bookloupe:\n");
ali@17: 	    if (location->column)
ali@17: 		g_print("    Line %u column %u - %s\n",location->line,
ali@17: 		  location->column,warning->text);
ali@17: 	    else
ali@17: 		g_print("    Line %u - %s\n",location->line,warning->text);
ali@17: 	    r=FALSE;
ali@17: 	    break;
ali@17: 	}
ali@17: 	else if (warning->xfail)
ali@17: 	{
ali@17: 	    if (warning->is_real)
ali@17: 	    {
ali@17: 		total_false_negative++;
ali@17: 		if (!counts[i])
ali@17: 		    count_false_negative++;
ali@17: 	    }
ali@17: 	    else if (!warning->is_real)
ali@17: 	    {
ali@17: 		total_false_positive++;
ali@17: 		if (counts[i])
ali@17: 		    count_false_positive++;
ali@17: 	    }
ali@17: 	}
ali@17:     }
ali@17:     g_free(counts);
ali@17:     if (count_false_positive && count_false_negative)
ali@17: 	*xfail=g_strdup_printf(
ali@17: 	  "with %d of %d false positives and %d of %d false negatives",
ali@17: 	  count_false_positive,total_false_positive,
ali@17: 	  count_false_negative,total_false_negative);
ali@17:     else if (count_false_positive)
ali@17: 	*xfail=g_strdup_printf("with %d of %d false positives",
ali@17: 	  count_false_positive,total_false_positive);
ali@17:     else if (count_false_negative)
ali@17: 	*xfail=g_strdup_printf("with %d of %d false negatives",
ali@17: 	  count_false_negative,total_false_negative);
ali@17:     return r;
ali@17: }
ali@17: 
ali@17: /*
ali@0:  * Run a testcase, returning FALSE on fail or error and
ali@0:  * TRUE on pass or expected-fail.
ali@0:  * Suitable message(s) will be printed in all cases.
ali@0:  */
ali@6: gboolean testcase_run(Testcase *testcase)
ali@0: {
ali@6:     gboolean r;
ali@7:     size_t pos,offset;
ali@17:     GString *header;
ali@17:     char *output,*filename,*s,*xfail=NULL;
ali@7:     GError *error=NULL;
ali@9:     if (!testcase_create_input_files(testcase,&error))
ali@9:     {
ali@11: 	g_print("%s: FAIL\n",testcase->basename);
ali@11: 	g_print("%s\n",error->message);
ali@9: 	g_error_free(error);
ali@9: 	return FALSE;
ali@9:     }
ali@93:     r=testcase_spawn_bookloupe(testcase,&output,&error);
ali@7:     if (!r)
ali@7:     {
ali@11: 	g_print("%s: FAIL\n",testcase->basename);
ali@11: 	g_print("%s\n",error->message);
ali@7: 	g_error_free(error);
ali@9: 	(void)testcase_remove_input_files(testcase,NULL);
ali@9: 	return FALSE;
ali@9:     }
ali@9:     filename=testcase_resolve_input_files(testcase,"TEST-XXXXXX");
ali@9:     if (!testcase_remove_input_files(testcase,&error))
ali@9:     {
ali@11: 	g_print("%s: FAIL\n",testcase->basename);
ali@11: 	g_print("%s\n",error->message);
ali@9: 	g_error_free(error);
ali@0: 	return FALSE;
ali@0:     }
ali@93:     header=g_string_new("\n\nFile: ");
ali@93:     g_string_append(header,filename);
ali@93:     g_string_append(header,"\n");
ali@93:     if (!g_str_has_prefix(output,header->str))
ali@0:     {
ali@93: 	g_print("%s: FAIL\n",testcase->basename);
ali@93: 	g_print("Unexpected header from bookloupe:\n");
ali@93: 	offset=common_prefix_length(output,header->str);
ali@93: 	print_unexpected(output,offset);
ali@93: 	r=FALSE;
ali@93:     }
ali@93:     pos=header->len;
ali@93:     if (r)
ali@93:     {
ali@93: 	/* Skip the summary */
ali@93: 	s=strstr(output+pos,"\n\n");
ali@93: 	if (s)
ali@93: 	    pos=s-output+2;
ali@93: 	else
ali@7: 	{
ali@11: 	    g_print("%s: FAIL\n",testcase->basename);
ali@93: 	    g_print("Unterminated summary from bookloupe:\n%s\n",output+pos);
ali@7: 	    r=FALSE;
ali@7: 	}
ali@0:     }
ali@93:     g_string_free(header,TRUE);
ali@93:     r=testcase_check_warnings(testcase,output+pos,&xfail);
ali@9:     g_free(filename);
ali@6:     g_free(output);
ali@7:     if (r)
ali@17:     {
ali@17: 	if (xfail)
ali@17: 	    g_print("%s: PASS (%s)\n",testcase->basename,xfail);
ali@17: 	else
ali@17: 	    g_print("%s: PASS\n",testcase->basename);
ali@17:     }
ali@17:     g_free(xfail);
ali@7:     return r;
ali@0: }
ali@0: 
ali@0: /*
ali@17:  * Free a testcase warning.
ali@17:  */
ali@17: void testcase_warning_free(TestcaseWarning *warning)
ali@17: {
ali@17:     g_slist_foreach(warning->locations,(GFunc)g_free,NULL);
ali@17:     g_slist_free(warning->locations);
ali@17:     g_free(warning->text);
ali@17:     g_free(warning);
ali@17: }
ali@17: 
ali@17: /*
ali@0:  * Free a testcase.
ali@0:  */
ali@0: void testcase_free(Testcase *testcase)
ali@0: {
ali@6:     g_free(testcase->basename);
ali@9:     g_slist_foreach(testcase->inputs,(GFunc)testcase_input_free,NULL);
ali@9:     g_slist_free(testcase->inputs);
ali@6:     g_free(testcase->expected);
ali@17:     g_slist_foreach(testcase->warnings,(GFunc)testcase_warning_free,NULL);
ali@17:     g_slist_free(testcase->warnings);
ali@7:     g_free(testcase->encoding);
ali@9:     g_strfreev(testcase->options);
ali@6:     g_free(testcase);
ali@0: }