# HG changeset patch
# User ali <ali@juiblex.co.uk>
# Date 1327700435 0
# Node ID 721e468c10f34914f9b79eb346e13dc95ddfdd27
# Parent  faab25d520dd1f771f1291e14da78f6c318de459
Add support for non-ASCII testcases

diff -r faab25d520dd -r 721e468c10f3 bl/spawn.c
--- a/bl/spawn.c	Fri Jan 27 16:18:02 2012 +0000
+++ b/bl/spawn.c	Fri Jan 27 21:40:35 2012 +0000
@@ -7,26 +7,22 @@
 
 #define SPAWN_BUFSIZE	128
 
-gboolean spawn_sync(char **argv,char **standard_output,int *exit_status)
+gboolean spawn_sync(char **argv,char **standard_output,int *exit_status,
+  GError **error)
 {
 /* Don't use g_spawn_sync on WIN32 for now to avoid needing the helper */
 #ifndef WIN32
-    char *standard_error;
-    GError *error=NULL;
+    char *standard_error=NULL;
     gboolean retval;
     GSpawnFlags flags=G_SPAWN_SEARCH_PATH;
     if (!standard_output)
 	flags=G_SPAWN_STDOUT_TO_DEV_NULL;
     retval=g_spawn_sync(NULL,argv,NULL,flags,NULL,NULL,standard_output,
-      &standard_error,exit_status,&error);
-    fputs(standard_error,stderr);
+      &standard_error,exit_status,error);
+    if (standard_error)
+	fputs(standard_error,stderr);
     g_free(standard_error);
-    if (!retval)
-    {
-	fprintf(stderr,"%s\n",error->message);
-	g_error_free(error);
-    }
-    else if (exit_status)
+    if (retval && exit_status)
 	*exit_status=WEXITSTATUS(*exit_status);
     return retval;
 #else
@@ -45,7 +41,8 @@
     g_string_free(command_line,TRUE);
     if (!fp)
     {
-	perror(command_line->str);
+	g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
+	  "%s: %s",command_line->str,g_strerror(errno));
 	return FALSE;
     }
     string=g_string_new(NULL);
@@ -56,7 +53,8 @@
 	n=fread(string->str+len,1,SPAWN_BUFSIZE,fp);
 	if (n<0)
 	{
-	    perror("fread");
+	    g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
+	      "Error reading from bookloupe: %s",g_strerror(errno));
 	    (void)pclose(fp);
 	    g_string_free(string,TRUE);
 	    return FALSE;
@@ -66,7 +64,8 @@
     r=pclose(fp);
     if (r<0)
     {
-	perror("pclose");
+	g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
+	  "Error reading from bookloupe: %s",g_strerror(errno));
 	g_string_free(string,TRUE);
 	return FALSE;
     }
diff -r faab25d520dd -r 721e468c10f3 bl/spawn.h
--- a/bl/spawn.h	Fri Jan 27 16:18:02 2012 +0000
+++ b/bl/spawn.h	Fri Jan 27 21:40:35 2012 +0000
@@ -3,6 +3,7 @@
 
 #include <glib.h>
 
-gboolean spawn_sync(char **argv,char **standard_output,int *exit_status);
+gboolean spawn_sync(char **argv,char **standard_output,int *exit_status,
+  GError **error);
 
 #endif /* BL_SPAWN_H */
diff -r faab25d520dd -r 721e468c10f3 test/compatibility/Makefile.am
--- a/test/compatibility/Makefile.am	Fri Jan 27 16:18:02 2012 +0000
+++ b/test/compatibility/Makefile.am	Fri Jan 27 21:40:35 2012 +0000
@@ -2,6 +2,6 @@
 TESTS=missing-space.tst spaced-punctuation.tst html-tag.tst html-symbol.tst \
 	spaced-doublequote.tst mismatched-quotes.tst he-be.tst digits.tst \
 	extra-period.tst ellipsis.tst short-line.tst abbreviation.tst \
-	example.tst
+	example.tst non-ascii.tst
 
 dist_pkgdata_DATA=$(TESTS)
diff -r faab25d520dd -r 721e468c10f3 test/compatibility/non-ascii.tst
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/test/compatibility/non-ascii.tst	Fri Jan 27 21:40:35 2012 +0000
@@ -0,0 +1,8 @@
+**************** ENCODING ****************
+ISO-8859-1
+**************** INPUT ****************
+"Hello," he said, "I wanted to bave a tête-à-tête with you."
+**************** EXPECTED ****************
+
+"Hello," he said, "I wanted to bave a tête-à-tête with you."
+    Line 1 column 31 - Query word bave - not reporting duplicates
diff -r faab25d520dd -r 721e468c10f3 test/harness/testcase.c
--- a/test/harness/testcase.c	Fri Jan 27 16:18:02 2012 +0000
+++ b/test/harness/testcase.c	Fri Jan 27 21:40:35 2012 +0000
@@ -10,6 +10,11 @@
 #include <bl/bl.h>
 #include "testcase.h"
 
+GQuark testcase_error_quark(void)
+{
+    return g_quark_from_static_string("testcase-error-quark");
+}
+
 #if !HAVE_MKSTEMP
 /*
  * An insecure implementation of mkstemp(), for those platforms that
@@ -45,7 +50,7 @@
 /*
  * As write(), but always convert NL to CR NL.
  */
-static size_t write_text(int fd,const char *buf,size_t count)
+static size_t write_text(int fd,const char *buf,size_t count,GError **error)
 {
     size_t i;
     FILE *fp;
@@ -56,6 +61,9 @@
     if (_setmode(fd,_O_BINARY)<0)
     {
 	close(fd);
+	g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
+	  "Failed to set mode of bookloupe input file to binary: %s",
+	  g_strerror(errno));
 	return -1;
     }
 #endif
@@ -63,6 +71,9 @@
     if (!fp)
     {
 	close(fd);
+	g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
+	  "Failed to open stream to bookloupe input file: %s",
+	  g_strerror(errno));
 	return -1;
     }
     for(i=0;i<count;i++)
@@ -70,17 +81,25 @@
 	if (buf[i]=='\n')
 	    if (putc('\r',fp)==EOF)
 	    {
+		g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
+		  "Error writing bookloupe input file: %s",g_strerror(errno));
 		(void)fclose(fp);
 		return -1;
 	    }
 	if (putc(buf[i],fp)==EOF)
 	{
+	    g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
+	      "Error writing bookloupe input file: %s",g_strerror(errno));
 	    (void)fclose(fp);
 	    return -1;
 	}
     }
     if (fclose(fp))
+    {
+	g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
+	  "Error writing bookloupe input file: %s",g_strerror(errno));
 	return -1;
+    }
     return count;
 }
 
@@ -95,6 +114,122 @@
     return i;
 }
 
+void print_unexpected(const char *unexpected,gsize differs_at)
+{
+    int col;
+    const char *endp,*bol;
+    GString *string;
+    endp=strchr(unexpected+differs_at,'\n');
+    if (!endp)
+	endp=unexpected+strlen(unexpected);
+    string=g_string_new_len(unexpected,endp-unexpected);
+    bol=strrchr(string->str,'\n');
+    if (bol)
+	bol++;
+    else
+	bol=string->str;
+    col=differs_at-(bol-string->str);
+    fprintf(stderr,"%s\n%*s^\n",string->str,col,"");
+    g_string_free(string,TRUE);
+}
+
+gboolean spawn_bootloupe(const char *encoding,const char *standard_input,
+  char **standard_output,char **filename,GError **error)
+{
+    gboolean r;
+    int fd,exit_status;
+    size_t n,pos,offset;
+    FILE *fp;
+    char input[]="TEST-XXXXXX";
+    char *command[3];
+    char *output,*s;
+    GError *tmp_err=NULL;
+    if (standard_input)
+    {
+	if (encoding)
+	{
+	    s=g_convert(standard_input,-1,encoding,"UTF-8",NULL,&n,&tmp_err);
+	    if (!s)
+	    {
+		g_propagate_prefixed_error(error,tmp_err,
+		  "Conversion to %s failed: ",encoding);
+		return FALSE;
+	    }
+	}
+	else
+	{
+	    s=g_strdup(standard_input);
+	    n=strlen(s);
+	}
+    }
+    else
+    {
+	n=0;
+	s=NULL;
+    }
+    fd=mkstemp(input);
+    if (n && write_text(fd,s,n,error)!=n)
+    {
+	g_free(s);
+	close(fd);
+	(void)remove(input);
+	return FALSE;
+    }
+    g_free(s);
+    close(fd);
+    command[0]=getenv("BOOKLOUPE");
+    if (!command[0])
+	command[0]="." G_DIR_SEPARATOR_S "bookloupe";
+    command[1]=input;
+    command[2]=NULL;
+    if (standard_output)
+    {
+	r=spawn_sync(command,&s,&exit_status,error);
+	if (r)
+	{
+	    if (encoding)
+	    {
+		output=g_convert(s,-1,"UTF-8",encoding,NULL,NULL,&tmp_err);
+		g_free(s);
+		if (!output)
+		{
+		    g_propagate_prefixed_error(error,tmp_err,
+		      "Conversion from %s failed: ",encoding);
+		    r=FALSE;
+		}
+	    }
+	    else
+	    {
+		output=s;
+		if (!g_utf8_validate(s,-1,NULL))
+		{
+		    g_set_error_literal(error,TESTCASE_ERROR,
+		      TESTCASE_ERROR_FAILED,
+		      "bookloupe output is not valid UTF-8");
+		    r=FALSE;
+		}
+	    }
+	}
+    }
+    else
+    {
+	r=spawn_sync(command,NULL,&exit_status,error);
+	output=NULL;
+    }
+    (void)remove(input);
+    if (r && exit_status)
+    {
+	g_set_error(error,TESTCASE_ERROR,TESTCASE_ERROR_FAILED,
+	  "bookloupe exited with code %d",exit_status);
+	r=FALSE;
+    }
+    if (r && filename)
+	*filename=g_strdup(input);
+    if (r && standard_output)
+	*standard_output=output;
+    return r;
+}
+
 /*
  * Run a testcase, returning FALSE on fail or error and
  * TRUE on pass or expected-fail.
@@ -103,92 +238,76 @@
 gboolean testcase_run(Testcase *testcase)
 {
     gboolean r;
-    int fd,exit_status,col;
-    size_t n,pos,offset,header_len;
-    FILE *fp;
-    char input[]="TEST-XXXXXX";
-    char *endp,*bol;
-    char *command[3];
-    GString *expected,*report;
-    char *output;
-    fd=mkstemp(input);
-    if (testcase->input)
-	n=strlen(testcase->input);
+    size_t pos,offset;
+    GString *header,*expected;
+    char *output,*filename,*s;
+    GError *error=NULL;
+    if (testcase->expected)
+	r=spawn_bootloupe(testcase->encoding,testcase->input,&output,&filename,
+	  &error);
     else
-	n=0;
-    if (n && write_text(fd,testcase->input,n)!=n)
     {
-	perror(input);
-	close(fd);
-	(void)remove(input);
+	r=spawn_bootloupe(testcase->encoding,testcase->input,NULL,NULL,&error);
+        output=filename=NULL;
+    }
+    if (!r)
+    {
+	fprintf(stderr,"%s: FAIL\n",testcase->basename);
+	fprintf(stderr,"%s\n",error->message);
+	g_error_free(error);
 	return FALSE;
     }
-    close(fd);
-    command[0]=getenv("BOOKLOUPE");
-    if (!command[0])
-	command[0]="." G_DIR_SEPARATOR_S "bookloupe";
-    command[1]=input;
-    command[2]=NULL;
-    if (testcase->expected)
-	r=spawn_sync(command,&output,&exit_status);
-    else
-    {
-	r=spawn_sync(command,NULL,&exit_status);
-	output=NULL;
-    }
-    (void)remove(input);
-    if (!r)
-	return FALSE;
     if (testcase->expected)
     {
-	expected=g_string_new("\n\nFile: ");
-	g_string_append(expected,input);
-	g_string_append(expected,"\n\n\n");
-	header_len=expected->len;
-	g_string_append(expected,testcase->expected);
+	header=g_string_new("\n\nFile: ");
+	g_string_append(header,filename);
+	g_string_append(header,"\n");
+	expected=g_string_new(testcase->expected);
+	if (!g_str_has_prefix(output,header->str))
+	{
+	    fprintf(stderr,"%s: FAIL\n",testcase->basename);
+	    offset=common_prefix_length(output,header->str);
+	    fprintf(stderr,"Unexpected header from bookloupe:\n");
+	    print_unexpected(output,offset);
+	    r=FALSE;
+	}
+	pos=header->len;
+	if (r)
+	{
+	    /* Skip the summary */
+	    s=strstr(output+pos,"\n\n");
+	    if (s)
+		pos=s-output+2;
+	    else
+	    {
+		fprintf(stderr,"%s: FAIL\n",testcase->basename);
+		offset=common_prefix_length(output,header->str);
+		fprintf(stderr,"Unterminated summary from bookloupe:\n%s\n",
+		  output+pos);
+		r=FALSE;
+	    }
+	}
+	if (r && strcmp(output+pos,expected->str))
+	{
+	    fprintf(stderr,"%s: FAIL\n",testcase->basename);
+	    offset=common_prefix_length(output+pos,expected->str);
+	    if (!offset && !output[pos+offset])
+		fprintf(stderr,"Unexpected zero warnings from bookloupe.\n");
+	    else
+	    {
+		fprintf(stderr,"Unexpected output from bookloupe:\n");
+		print_unexpected(output+pos,offset);
+	    }
+	    r=FALSE;
+	}
+	g_string_free(header,TRUE);
+	g_string_free(expected,TRUE);
     }
-    else
-    {
-	expected=NULL;
-	header_len=0;
-    }
-    if (expected && strcmp(output,expected->str))
-    {
-	fprintf(stderr,"%s: FAIL\n",testcase->basename);
-	offset=common_prefix_length(output,expected->str);
-	if (offset==header_len && !output[offset])
-	    fprintf(stderr,"Unexpected zero warnings from bookloupe.\n");
-	else
-	{
-	    endp=strchr(output+offset,'\n');
-	    if (!endp)
-		endp=output+strlen(output);
-	    report=g_string_new(NULL);
-	    g_string_append_len(report,output,endp-output);
-	    bol=strrchr(report->str,'\n');
-	    if (bol)
-		bol++;
-	    else
-		bol=report->str;
-	    col=offset-(bol-report->str);
-	    fprintf(stderr,"Unexpected output from bookloupe:\n");
-	    if (report->len>=header_len)
-		fprintf(stderr,"%s\n%*s^\n",report->str+header_len,col,"");
-	    else
-		fprintf(stderr,"%s\n%*s^\n",report->str,col,"");
-	    g_string_free(report,TRUE);
-	}
-	g_string_free(expected,TRUE);
-	g_free(output);
-	return FALSE;
-    }
-    g_string_free(expected,TRUE);
     g_free(output);
-    if (exit_status)
-	fprintf(stderr,"bookloupe exited with code %d\n",r);
-    if (!exit_status)
+    g_free(filename);
+    if (r)
 	fprintf(stderr,"%s: PASS\n",testcase->basename);
-    return !exit_status;
+    return r;
 }
 
 /*
@@ -199,5 +318,6 @@
     g_free(testcase->basename);
     g_free(testcase->input);
     g_free(testcase->expected);
+    g_free(testcase->encoding);
     g_free(testcase);
 }
diff -r faab25d520dd -r 721e468c10f3 test/harness/testcase.h
--- a/test/harness/testcase.h	Fri Jan 27 16:18:02 2012 +0000
+++ b/test/harness/testcase.h	Fri Jan 27 21:40:35 2012 +0000
@@ -3,15 +3,23 @@
 
 #include <glib.h>
 
+#define TESTCASE_ERROR testcase_error_quark()
+
+typedef enum {
+    TESTCASE_ERROR_FAILED
+} TestcaseError;
+
 typedef struct {
     char *basename;
     char *input;
     char *expected;
+    char *encoding;	/* The character encoding to talk to BOOKLOUPE in */
     enum {
 	TESTCASE_XFAIL=1<<0,
     } flags;
 } Testcase;
 
+GQuark testcase_error_quark(void);
 gboolean testcase_run(Testcase *testcase);
 void testcase_free(Testcase *testcase);
 
diff -r faab25d520dd -r 721e468c10f3 test/harness/testcaseio.c
--- a/test/harness/testcaseio.c	Fri Jan 27 16:18:02 2012 +0000
+++ b/test/harness/testcaseio.c	Fri Jan 27 21:40:35 2012 +0000
@@ -38,6 +38,8 @@
 	    testcase->input=g_strdup(text);
 	else if (!testcase->expected && !strcmp(tag,"EXPECTED"))
 	    testcase->expected=g_strdup(text);
+	else if (!testcase->encoding && !strcmp(tag,"ENCODING"))
+	    testcase->encoding=g_strdup(text);
 	else
 	{
 	    fprintf(stderr,"%s: Not a valid testcase (%s)\n",filename,tag);
diff -r faab25d520dd -r 721e468c10f3 test/harness/testcaseparser.c
--- a/test/harness/testcaseparser.c	Fri Jan 27 16:18:02 2012 +0000
+++ b/test/harness/testcaseparser.c	Fri Jan 27 21:40:35 2012 +0000
@@ -90,12 +90,20 @@
 TestcaseParser *testcase_parser_new_from_file(const char *filename)
 {
     TestcaseParser *parser;
+    gsize len;
     parser=g_new0(TestcaseParser,1);
-    if (!file_get_contents_text(filename,&parser->contents,NULL))
+    if (!file_get_contents_text(filename,&parser->contents,&len))
     {
 	g_free(parser);
 	return NULL;
     }
+    if (!g_utf8_validate(parser->contents,len,NULL))
+    {
+	fprintf(stderr,"%s: Does not contain valid UTF-8\n",filename);
+	g_free(parser->contents);
+	g_free(parser);
+	return NULL;
+    }
     parser->filename=g_strdup(filename);
     return parser;
 }