# HG changeset patch
# User ali <ali@juiblex.co.uk>
# Date 1328572527 0
# Node ID 2c88fd553e5d0bea53df93e6a2cc8eb3835f0cba
# Parent  7294f2bea92da4e1de77b296ec92201dc311eb9e
Add support for false-positives, etc.

diff -r 7294f2bea92d -r 2c88fd553e5d test/harness/Makefile.am
--- a/test/harness/Makefile.am	Thu Feb 02 23:32:12 2012 +0000
+++ b/test/harness/Makefile.am	Mon Feb 06 23:55:27 2012 +0000
@@ -5,5 +5,5 @@
 
 loupe_test_SOURCES=loupe-test.c testcase.c testcase.h testcaseio.c \
 	testcaseio.h testcaseparser.c testcaseparser.h testcaseinput.c \
-	testcaseinput.h
+	testcaseinput.h warningsparser.c warningsparser.h
 loupe_test_LDADD=../../bl/libbl.la
diff -r 7294f2bea92d -r 2c88fd553e5d test/harness/testcase.c
--- a/test/harness/testcase.c	Thu Feb 02 23:32:12 2012 +0000
+++ b/test/harness/testcase.c	Mon Feb 06 23:55:27 2012 +0000
@@ -239,6 +239,218 @@
 }
 
 /*
+ * Parse a warning of the form:
+ *	[blank line]
+ *	<echoed line> (ignored)
+ *	"    Line " <number> [" column " <number>] " - " <text> "\n"
+ * If not specified, the column is returned as 0.
+ * Returns: the number of bytes parsed, or -1 on error.
+ */
+static ssize_t testcase_parse_warning(Testcase *testcase,const char *output,
+  guint *line,guint *column,char **text)
+{
+    ssize_t offset=0;
+    guint64 tmp;
+    char *s,*endp;
+    if (output[offset]!='\n')
+    {
+	g_print("%s: FAIL\n",testcase->basename);
+	g_print("Unexpected output from bookloupe:\n");
+	print_unexpected(output,offset);
+	return -1;
+    }
+    offset++;
+    s=strchr(output+offset,'\n');
+    if (!s)
+    {
+	g_print("%s: FAIL\n",testcase->basename);
+	g_print("Missing new-line in output from bookloupe:\n");
+	print_unexpected(output,offset);
+	return -1;
+    }
+    offset=s-output+1;
+    if (!g_str_has_prefix(output+offset,"    Line "))
+    {
+	g_print("%s: FAIL\n",testcase->basename);
+	g_print("Unexpected output from bookloupe:\n");
+	offset+=common_prefix_length(output+offset,"    Line ");
+	print_unexpected(output,offset);
+	return -1;
+    }
+    offset+=9;
+    tmp=g_ascii_strtoull(output+offset,&endp,10);
+    if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64)
+    {
+	g_print("%s: FAIL\n",testcase->basename);
+	g_print("Unexpected output from bookloupe:\n");
+	print_unexpected(output,offset);
+	return -1;
+    }
+    *line=tmp;
+    offset=endp-output;
+    if (g_str_has_prefix(output+offset," column "))
+    {
+	offset+=8;
+	tmp=g_ascii_strtoull(output+offset,&endp,10);
+	if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64)
+	{
+	    g_print("%s: FAIL\n",testcase->basename);
+	    g_print("Unexpected output from bookloupe:\n");
+	    print_unexpected(output,offset);
+	    return -1;
+	}
+	*column=tmp;
+	offset=endp-output;
+    }
+    else
+	*column=0;
+    if (!g_str_has_prefix(output+offset," - "))
+    {
+	g_print("%s: FAIL\n",testcase->basename);
+	g_print("Unexpected output from bookloupe:\n");
+	offset+=common_prefix_length(output+offset," - ");
+	print_unexpected(output,offset);
+	return -1;
+    }
+    offset+=3;
+    s=strchr(output+offset,'\n');
+    if (!s)
+    {
+	g_print("%s: FAIL\n",testcase->basename);
+	g_print("Missing new-line in output from bookloupe:\n");
+	print_unexpected(output,offset);
+	return -1;
+    }
+    *text=g_strndup(output+offset,s-(output+offset));
+    return s-output+1;
+}
+
+/*
+ * Check the warnings produced by bookloupe against either the
+ * unstructured testcase->expected or the structured testcase->warnings
+ * as appropriate.
+ */
+static gboolean testcase_check_warnings(Testcase *testcase,const char *output,
+  char **xfail)
+{
+    gboolean r=TRUE;
+    size_t offset;
+    ssize_t off;
+    int i,count_false_positive,count_false_negative;
+    int total_false_positive,total_false_negative;
+    char *text;
+    guint *counts,line,column;
+    GSList *link,*link2;
+    TestcaseWarning *warning;
+    TestcaseLocation *location;
+    *xfail=NULL;
+    if (testcase->expected)
+    {
+	if (strcmp(output,testcase->expected))
+	{
+	    g_print("%s: FAIL\n",testcase->basename);
+	    offset=common_prefix_length(output,testcase->expected);
+	    if (!offset && !output[offset])
+		g_print("Unexpected zero warnings from bookloupe.\n");
+	    else
+	    {
+		g_print("Unexpected output from bookloupe:\n");
+		print_unexpected(output,offset);
+	    }
+	    return FALSE;
+	}
+	return TRUE;
+    }
+    counts=g_new0(guint,g_slist_length(testcase->warnings));
+    for(offset=0;output[offset];)
+    {
+	off=testcase_parse_warning(testcase,output+offset,&line,&column,&text);
+	if (off<0)
+	{
+	    r=FALSE;
+	    break;
+	}
+	offset+=off;
+	for(link=testcase->warnings,i=0;link;link=link->next,i++)
+	{
+	    warning=link->data;
+	    if (strcmp(warning->text,text))
+		continue;
+	    for(link2=warning->locations;link2;link2=link2->next)
+	    {
+		location=link2->data;
+		if (location->line!=line || location->column!=column)
+		    continue;
+		counts[i]++;
+		break;
+	    }
+	    if (link2)
+		break;
+	}
+	if (!link)
+	{
+	    g_print("%s: FAIL\n",testcase->basename);
+	    g_print("Unexpected warning from bookloupe:\n");
+	    if (column)
+		g_print("    Line %u column %u - %s\n",line,column,text);
+	    else
+		g_print("    Line %u - %s\n",line,text);
+	    r=FALSE;
+	    g_free(text);
+	    break;
+	}
+	g_free(text);
+    }
+    count_false_positive=total_false_positive=0;
+    count_false_negative=total_false_negative=0;
+    for(link=testcase->warnings,i=0;r && link;link=link->next,i++)
+    {
+	warning=link->data;
+	if (!counts[i] && warning->is_real && !warning->xfail)
+	{
+	    location=warning->locations->data;
+	    g_print("%s: FAIL\n",testcase->basename);
+	    g_print("Missing warning from bookloupe:\n");
+	    if (location->column)
+		g_print("    Line %u column %u - %s\n",location->line,
+		  location->column,warning->text);
+	    else
+		g_print("    Line %u - %s\n",location->line,warning->text);
+	    r=FALSE;
+	    break;
+	}
+	else if (warning->xfail)
+	{
+	    if (warning->is_real)
+	    {
+		total_false_negative++;
+		if (!counts[i])
+		    count_false_negative++;
+	    }
+	    else if (!warning->is_real)
+	    {
+		total_false_positive++;
+		if (counts[i])
+		    count_false_positive++;
+	    }
+	}
+    }
+    g_free(counts);
+    if (count_false_positive && count_false_negative)
+	*xfail=g_strdup_printf(
+	  "with %d of %d false positives and %d of %d false negatives",
+	  count_false_positive,total_false_positive,
+	  count_false_negative,total_false_negative);
+    else if (count_false_positive)
+	*xfail=g_strdup_printf("with %d of %d false positives",
+	  count_false_positive,total_false_positive);
+    else if (count_false_negative)
+	*xfail=g_strdup_printf("with %d of %d false negatives",
+	  count_false_negative,total_false_negative);
+    return r;
+}
+
+/*
  * Run a testcase, returning FALSE on fail or error and
  * TRUE on pass or expected-fail.
  * Suitable message(s) will be printed in all cases.
@@ -247,8 +459,8 @@
 {
     gboolean r;
     size_t pos,offset;
-    GString *header,*expected;
-    char *output,*filename,*s;
+    GString *header;
+    char *output,*filename,*s,*xfail=NULL;
     GError *error=NULL;
     if (!testcase_create_input_files(testcase,&error))
     {
@@ -257,7 +469,7 @@
 	g_error_free(error);
 	return FALSE;
     }
-    if (testcase->expected)
+    if (testcase->expected || testcase->warnings)
 	r=testcase_spawn_bookloupe(testcase,&output,&error);
     else
     {
@@ -280,17 +492,16 @@
 	g_error_free(error);
 	return FALSE;
     }
-    if (testcase->expected)
+    if (testcase->expected || testcase->warnings)
     {
 	header=g_string_new("\n\nFile: ");
 	g_string_append(header,filename);
 	g_string_append(header,"\n");
-	expected=g_string_new(testcase->expected);
 	if (!g_str_has_prefix(output,header->str))
 	{
 	    g_print("%s: FAIL\n",testcase->basename);
+	    g_print("Unexpected header from bookloupe:\n");
 	    offset=common_prefix_length(output,header->str);
-	    g_print("Unexpected header from bookloupe:\n");
 	    print_unexpected(output,offset);
 	    r=FALSE;
 	}
@@ -304,36 +515,39 @@
 	    else
 	    {
 		g_print("%s: FAIL\n",testcase->basename);
-		offset=common_prefix_length(output,header->str);
 		g_print("Unterminated summary from bookloupe:\n%s\n",
 		  output+pos);
 		r=FALSE;
 	    }
 	}
-	if (r && strcmp(output+pos,expected->str))
-	{
-	    g_print("%s: FAIL\n",testcase->basename);
-	    offset=common_prefix_length(output+pos,expected->str);
-	    if (!offset && !output[pos+offset])
-		g_print("Unexpected zero warnings from bookloupe.\n");
-	    else
-	    {
-		g_print("Unexpected output from bookloupe:\n");
-		print_unexpected(output+pos,offset);
-	    }
-	    r=FALSE;
-	}
 	g_string_free(header,TRUE);
-	g_string_free(expected,TRUE);
+	r=testcase_check_warnings(testcase,output+pos,&xfail);
     }
     g_free(filename);
     g_free(output);
     if (r)
-	g_print("%s: PASS\n",testcase->basename);
+    {
+	if (xfail)
+	    g_print("%s: PASS (%s)\n",testcase->basename,xfail);
+	else
+	    g_print("%s: PASS\n",testcase->basename);
+    }
+    g_free(xfail);
     return r;
 }
 
 /*
+ * Free a testcase warning.
+ */
+void testcase_warning_free(TestcaseWarning *warning)
+{
+    g_slist_foreach(warning->locations,(GFunc)g_free,NULL);
+    g_slist_free(warning->locations);
+    g_free(warning->text);
+    g_free(warning);
+}
+
+/*
  * Free a testcase.
  */
 void testcase_free(Testcase *testcase)
@@ -342,6 +556,8 @@
     g_slist_foreach(testcase->inputs,(GFunc)testcase_input_free,NULL);
     g_slist_free(testcase->inputs);
     g_free(testcase->expected);
+    g_slist_foreach(testcase->warnings,(GFunc)testcase_warning_free,NULL);
+    g_slist_free(testcase->warnings);
     g_free(testcase->encoding);
     g_strfreev(testcase->options);
     g_free(testcase);
diff -r 7294f2bea92d -r 2c88fd553e5d test/harness/testcase.h
--- a/test/harness/testcase.h	Thu Feb 02 23:32:12 2012 +0000
+++ b/test/harness/testcase.h	Mon Feb 06 23:55:27 2012 +0000
@@ -10,10 +10,35 @@
 } TestcaseError;
 
 typedef struct {
+    guint line;
+    guint column;		/* or 0 for unspecified */
+} TestcaseLocation;
+
+typedef struct {
+    /*
+     * Does this warning relate to a real problem in the etext
+     * (eg., error and false-negative).
+     */
+    gboolean is_real;
+    /*
+     * Do we "expect" BOOKLOUPE to get this wrong
+     * (eg., false-negative and false-positive)
+     */
+    gboolean xfail;
+    /*
+     * For real problems, the first location should be the
+     * actual location of the problem.
+     */
+    GSList *locations;
+    char *text;
+} TestcaseWarning;
+
+typedef struct {
     char *basename;
     char *tmpdir;
     GSList *inputs;
     char *expected;
+    GSList *warnings;
     char *encoding;	/* The character encoding to talk to BOOKLOUPE in */
     char **options;
     enum {
diff -r 7294f2bea92d -r 2c88fd553e5d test/harness/testcaseio.c
--- a/test/harness/testcaseio.c	Thu Feb 02 23:32:12 2012 +0000
+++ b/test/harness/testcaseio.c	Mon Feb 06 23:55:27 2012 +0000
@@ -6,6 +6,7 @@
 #include "testcaseparser.h"
 #include "testcaseinput.h"
 #include "testcaseio.h"
+#include "warningsparser.h"
 
 /*
  * Read a testcase in from a file.
@@ -17,6 +18,8 @@
     Testcase *testcase;
     TestcaseParser *parser;
     TestcaseInput *input=NULL;
+    GMarkupParseContext *context;
+    GError *err=NULL;
     char *s,*arg;
     const char *tag,*text;
     gboolean found_tag=FALSE;
@@ -64,8 +67,25 @@
 		testcase->flags|=TESTCASE_TMP_DIR;
 	    g_free(arg);
 	}
-	else if (!testcase->expected && !strcmp(tag,"EXPECTED"))
+	else if (!testcase->expected && !testcase->warnings &&
+	  !strcmp(tag,"EXPECTED"))
 	    testcase->expected=g_strdup(text);
+	else if (!testcase->expected && !testcase->warnings &&
+	  !strcmp(tag,"WARNINGS"))
+	{
+	    context=warnings_parse_context_new(testcase);
+	    if (!g_markup_parse_context_parse(context,text,-1,&err) ||
+	      !g_markup_parse_context_end_parse(context,&err))
+	    {
+		g_markup_parse_context_free(context);
+		g_printerr("%s\n",err->message);
+		g_clear_error(&err);
+		testcase_free(testcase);
+		testcase_parser_free(parser);
+		return NULL;
+	    }
+	    g_markup_parse_context_free(context);
+	}
 	else if (!testcase->encoding && !strcmp(tag,"ENCODING"))
 	    testcase->encoding=g_strchomp(g_strdup(text));
 	else if (!testcase->encoding && !strcmp(tag,"OPTIONS"))
diff -r 7294f2bea92d -r 2c88fd553e5d test/harness/warningsparser.c
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/harness/warningsparser.c	Mon Feb 06 23:55:27 2012 +0000
@@ -0,0 +1,244 @@
+#include <stdlib.h>
+#include <string.h>
+#include <glib.h>
+#include "testcase.h"
+#include "warningsparser.h"
+
+/*
+ * A GMarkupParser for the contents of a WARNINGS tag.
+ */
+
+typedef struct {
+    Testcase *testcase;
+    TestcaseWarning *warning;
+    TestcaseLocation *location;
+    enum {
+	WARNINGS_INIT,
+	WARNINGS_IN_EXPECTED,
+	WARNINGS_IN_WARNING,
+	WARNINGS_IN_AT,
+	WARNINGS_IN_TEXT,
+	WARNINGS_DONE,
+    } state;
+} WarningsBaton;
+
+static void warnings_parser_start_element(GMarkupParseContext *context,
+  const char *element_name,const char **attribute_names,
+  const char **attribute_values,void *user_data,GError **error)
+{
+    int i;
+    guint64 tmp;
+    char *endp;
+    WarningsBaton *baton=user_data;
+    switch(baton->state)
+    {
+	case WARNINGS_INIT:
+	    if (strcmp(element_name,"expected"))
+		g_set_error(error,G_MARKUP_ERROR,G_MARKUP_ERROR_UNKNOWN_ELEMENT,
+		  "Unknown root element: '%s'",element_name);
+	    else if (attribute_names[0])
+		g_set_error(error,G_MARKUP_ERROR,
+		  G_MARKUP_ERROR_UNKNOWN_ATTRIBUTE,
+		  "Unknown attribute on element 'expected': '%s'",
+		  attribute_names[0]);
+	    else
+		baton->state=WARNINGS_IN_EXPECTED;
+	    break;
+	case WARNINGS_IN_EXPECTED:
+	    baton->warning=g_new0(TestcaseWarning,1);
+	    if (!strcmp(element_name,"error"))
+		baton->warning->is_real=TRUE;
+	    else if (!strcmp(element_name,"false-positive"))
+		baton->warning->xfail=TRUE;
+	    else if (!strcmp(element_name,"false-negative"))
+		baton->warning->is_real=baton->warning->xfail=TRUE;
+	    else
+	    {
+		g_set_error(error,G_MARKUP_ERROR,G_MARKUP_ERROR_UNKNOWN_ELEMENT,
+		  "Unknown element in 'expected': '%s'",element_name);
+		g_free(baton->warning);
+		baton->warning=NULL;
+		return;
+	    }
+	    if (attribute_names[0])
+	    {
+		g_set_error(error,G_MARKUP_ERROR,
+		  G_MARKUP_ERROR_UNKNOWN_ATTRIBUTE,
+		  "Unknown attribute on element '%s': '%s'",element_name,
+		  attribute_names[0]);
+		g_free(baton->warning);
+		baton->warning=NULL;
+		return;
+	    }
+	    else
+		baton->state=WARNINGS_IN_WARNING;
+	    break;
+	case WARNINGS_IN_WARNING:
+	    if (!strcmp(element_name,"at"))
+	    {
+		baton->location=g_new0(TestcaseLocation,1);
+		for(i=0;attribute_names[i];i++)
+		{
+		    if (!strcmp(attribute_names[i],"line"))
+		    {
+			tmp=g_ascii_strtoull(attribute_values[i],&endp,0);
+			if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64)
+			{
+			    g_set_error(error,G_MARKUP_ERROR,
+			      G_MARKUP_ERROR_INVALID_CONTENT,"Invalid value "
+			      "for attribute 'line' on element '%s': '%s'",
+			      element_name,attribute_values[i]);
+			    return;
+			}
+			baton->location->line=(guint)tmp;
+		    }
+		    else if (!strcmp(attribute_names[i],"column"))
+		    {
+			tmp=g_ascii_strtoull(attribute_values[i],&endp,0);
+			if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64)
+			{
+			    g_set_error(error,G_MARKUP_ERROR,
+			      G_MARKUP_ERROR_INVALID_CONTENT,"Invalid value "
+			      "for attribute 'column' on element '%s': '%s'",
+			      element_name,attribute_values[i]);
+			    return;
+			}
+			baton->location->column=(guint)tmp;
+		    }
+		    else
+		    {
+			g_set_error(error,G_MARKUP_ERROR,
+			  G_MARKUP_ERROR_UNKNOWN_ATTRIBUTE,
+			  "Unknown attribute on element '%s': '%s'",
+			  element_name,attribute_names[i]);
+			return;
+		    }
+		}
+		if (!baton->location->line)
+		{
+		    g_set_error(error,G_MARKUP_ERROR,
+		      G_MARKUP_ERROR_MISSING_ATTRIBUTE,
+		      "Missing attribute on element '%s': 'line'",element_name);
+		    return;
+		}
+		baton->state=WARNINGS_IN_AT;
+	    }
+	    else if (!strcmp(element_name,"text"))
+	    {
+		if (attribute_names[0])
+		{
+		    g_set_error(error,G_MARKUP_ERROR,
+		      G_MARKUP_ERROR_UNKNOWN_ATTRIBUTE,
+		      "Unknown attribute on element 'text': '%s'",
+		      attribute_names[0]);
+		    return;
+		}
+		baton->state=WARNINGS_IN_TEXT;
+	    }
+	    break;
+	case WARNINGS_IN_AT:
+	    g_set_error(error,G_MARKUP_ERROR,G_MARKUP_ERROR_UNKNOWN_ELEMENT,
+	      "Unknown element in 'at': '%s'",element_name);
+	    return;
+	case WARNINGS_IN_TEXT:
+	    g_set_error(error,G_MARKUP_ERROR,G_MARKUP_ERROR_UNKNOWN_ELEMENT,
+	      "Unknown element in 'text': '%s'",element_name);
+	    return;
+	default:
+	    g_set_error(error,G_MARKUP_ERROR,G_MARKUP_ERROR_UNKNOWN_ELEMENT,
+	      "Unexpected element: '%s'",element_name);
+	    return;
+    }
+}
+
+static void warnings_parser_end_element(GMarkupParseContext *context,
+  const char *element_name,void *user_data,GError **error)
+{
+    WarningsBaton *baton=user_data;
+    switch(baton->state)
+    {
+	case WARNINGS_IN_EXPECTED:
+	    baton->testcase->warnings=
+	      g_slist_reverse(baton->testcase->warnings);
+	    baton->state=WARNINGS_DONE;
+	    break;
+	case WARNINGS_IN_WARNING:
+	    baton->warning->locations=
+	      g_slist_reverse(baton->warning->locations);
+	    baton->testcase->warnings=g_slist_prepend(baton->testcase->warnings,
+	      baton->warning);
+	    baton->warning=NULL;
+	    baton->state=WARNINGS_IN_EXPECTED;
+	    break;
+	case WARNINGS_IN_AT:
+	    baton->warning->locations=g_slist_prepend(baton->warning->locations,
+	      baton->location);
+	    baton->location=NULL;
+	    baton->state=WARNINGS_IN_WARNING;
+	    break;
+	case WARNINGS_IN_TEXT:
+	    baton->state=WARNINGS_IN_WARNING;
+	    break;
+	default:
+	    g_set_error(error,G_MARKUP_ERROR,G_MARKUP_ERROR_UNKNOWN_ELEMENT,
+	      "Unexpected element ending: '%s'",element_name);
+	    return;
+    }
+}
+
+static void warnings_parser_text(GMarkupParseContext *context,
+  const char *text,gsize text_len,void *user_data,GError **error)
+{
+    char *s,*t;
+    WarningsBaton *baton=user_data;
+    switch(baton->state)
+    {
+	case WARNINGS_IN_EXPECTED:
+	    if (strspn(text," \t\n")!=text_len)
+		g_set_error(error,G_MARKUP_ERROR,G_MARKUP_ERROR_INVALID_CONTENT,
+		  "The 'expected' tag does not take any content");
+	    break;
+	case WARNINGS_IN_WARNING:
+	    if (strspn(text," \t\n")!=text_len)
+		g_set_error(error,G_MARKUP_ERROR,G_MARKUP_ERROR_INVALID_CONTENT,
+		  "The warning tags do not take any content");
+	    break;
+	case WARNINGS_IN_AT:
+	    if (strspn(text," \t\n")!=text_len)
+		g_set_error(error,G_MARKUP_ERROR,G_MARKUP_ERROR_INVALID_CONTENT,
+		  "The 'at' tag does not take any content");
+	    break;
+	case WARNINGS_IN_TEXT:
+	    s=g_strdup(text+strspn(text," \t\n"));
+	    g_strchomp(s);
+	    if (baton->warning->text)
+	    {
+		t=g_strconcat(baton->warning->text,s,NULL);
+		g_free(baton->warning->text);
+		g_free(s);
+		baton->warning->text=t;
+	    }
+	    else
+		baton->warning->text=s;
+	    break;
+	default:
+	    g_set_error(error,G_MARKUP_ERROR,G_MARKUP_ERROR_INVALID_CONTENT,
+	      "Unexpected content: '%s'",text);
+	    return;
+    }
+}
+
+GMarkupParseContext *warnings_parse_context_new(Testcase *testcase)
+{
+    static GMarkupParser parser={0};
+    WarningsBaton *baton;
+    parser.start_element=warnings_parser_start_element;
+    parser.end_element=warnings_parser_end_element;
+    parser.text=warnings_parser_text;
+    baton=g_new0(WarningsBaton,1);
+    baton->testcase=testcase;
+    baton->state=WARNINGS_INIT;
+    return g_markup_parse_context_new(&parser,
+      G_MARKUP_TREAT_CDATA_AS_TEXT|G_MARKUP_PREFIX_ERROR_POSITION,
+      baton,g_free);
+}