1.1 --- a/bl/spawn.c	Fri Jan 27 16:18:02 2012 +0000
     1.2 +++ b/bl/spawn.c	Fri Jan 27 21:40:35 2012 +0000
     1.3 @@ -7,26 +7,22 @@
     1.4  
     1.5  #define SPAWN_BUFSIZE	128
     1.6  
     1.7 -gboolean spawn_sync(char **argv,char **standard_output,int *exit_status)
     1.8 +gboolean spawn_sync(char **argv,char **standard_output,int *exit_status,
     1.9 +  GError **error)
    1.10  {
    1.11  /* Don't use g_spawn_sync on WIN32 for now to avoid needing the helper */
    1.12  #ifndef WIN32
    1.13 -    char *standard_error;
    1.14 -    GError *error=NULL;
    1.15 +    char *standard_error=NULL;
    1.16      gboolean retval;
    1.17      GSpawnFlags flags=G_SPAWN_SEARCH_PATH;
    1.18      if (!standard_output)
    1.19  	flags=G_SPAWN_STDOUT_TO_DEV_NULL;
    1.20      retval=g_spawn_sync(NULL,argv,NULL,flags,NULL,NULL,standard_output,
    1.21 -      &standard_error,exit_status,&error);
    1.22 -    fputs(standard_error,stderr);
    1.23 +      &standard_error,exit_status,error);
    1.24 +    if (standard_error)
    1.25 +	fputs(standard_error,stderr);
    1.26      g_free(standard_error);
    1.27 -    if (!retval)
    1.28 -    {
    1.29 -	fprintf(stderr,"%s\n",error->message);
    1.30 -	g_error_free(error);
    1.31 -    }
    1.32 -    else if (exit_status)
    1.33 +    if (retval && exit_status)
    1.34  	*exit_status=WEXITSTATUS(*exit_status);
    1.35      return retval;
    1.36  #else
    1.37 @@ -45,7 +41,8 @@
    1.38      g_string_free(command_line,TRUE);
    1.39      if (!fp)
    1.40      {
    1.41 -	perror(command_line->str);
    1.42 +	g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
    1.43 +	  "%s: %s",command_line->str,g_strerror(errno));
    1.44  	return FALSE;
    1.45      }
    1.46      string=g_string_new(NULL);
    1.47 @@ -56,7 +53,8 @@
    1.48  	n=fread(string->str+len,1,SPAWN_BUFSIZE,fp);
    1.49  	if (n<0)
    1.50  	{
    1.51 -	    perror("fread");
    1.52 +	    g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
    1.53 +	      "Error reading from bookloupe: %s",g_strerror(errno));
    1.54  	    (void)pclose(fp);
    1.55  	    g_string_free(string,TRUE);
    1.56  	    return FALSE;
    1.57 @@ -66,7 +64,8 @@
    1.58      r=pclose(fp);
    1.59      if (r<0)
    1.60      {
    1.61 -	perror("pclose");
    1.62 +	g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
    1.63 +	  "Error reading from bookloupe: %s",g_strerror(errno));
    1.64  	g_string_free(string,TRUE);
    1.65  	return FALSE;
    1.66      }

     2.1 --- a/bl/spawn.h	Fri Jan 27 16:18:02 2012 +0000
     2.2 +++ b/bl/spawn.h	Fri Jan 27 21:40:35 2012 +0000
     2.3 @@ -3,6 +3,7 @@
     2.4  
     2.5  #include <glib.h>
     2.6  
     2.7 -gboolean spawn_sync(char **argv,char **standard_output,int *exit_status);
     2.8 +gboolean spawn_sync(char **argv,char **standard_output,int *exit_status,
     2.9 +  GError **error);
    2.10  
    2.11  #endif /* BL_SPAWN_H */

     3.1 --- a/test/compatibility/Makefile.am	Fri Jan 27 16:18:02 2012 +0000
     3.2 +++ b/test/compatibility/Makefile.am	Fri Jan 27 21:40:35 2012 +0000
     3.3 @@ -2,6 +2,6 @@
     3.4  TESTS=missing-space.tst spaced-punctuation.tst html-tag.tst html-symbol.tst \
     3.5  	spaced-doublequote.tst mismatched-quotes.tst he-be.tst digits.tst \
     3.6  	extra-period.tst ellipsis.tst short-line.tst abbreviation.tst \
     3.7 -	example.tst
     3.8 +	example.tst non-ascii.tst
     3.9  
    3.10  dist_pkgdata_DATA=$(TESTS)

     4.1 --- /dev/null	Thu Jan 01 00:00:00 1970 +0000
     4.2 +++ b/test/compatibility/non-ascii.tst	Fri Jan 27 21:40:35 2012 +0000
     4.3 @@ -0,0 +1,8 @@
     4.4 +**************** ENCODING ****************
     4.5 +ISO-8859-1
     4.6 +**************** INPUT ****************
     4.7 +"Hello," he said, "I wanted to bave a tête-à-tête with you."
     4.8 +**************** EXPECTED ****************
     4.9 +
    4.10 +"Hello," he said, "I wanted to bave a tête-à-tête with you."
    4.11 +    Line 1 column 31 - Query word bave - not reporting duplicates

     5.1 --- a/test/harness/testcase.c	Fri Jan 27 16:18:02 2012 +0000
     5.2 +++ b/test/harness/testcase.c	Fri Jan 27 21:40:35 2012 +0000
     5.3 @@ -10,6 +10,11 @@
     5.4  #include <bl/bl.h>
     5.5  #include "testcase.h"
     5.6  
     5.7 +GQuark testcase_error_quark(void)
     5.8 +{
     5.9 +    return g_quark_from_static_string("testcase-error-quark");
    5.10 +}
    5.11 +
    5.12  #if !HAVE_MKSTEMP
    5.13  /*
    5.14   * An insecure implementation of mkstemp(), for those platforms that
    5.15 @@ -45,7 +50,7 @@
    5.16  /*
    5.17   * As write(), but always convert NL to CR NL.
    5.18   */
    5.19 -static size_t write_text(int fd,const char *buf,size_t count)
    5.20 +static size_t write_text(int fd,const char *buf,size_t count,GError **error)
    5.21  {
    5.22      size_t i;
    5.23      FILE *fp;
    5.24 @@ -56,6 +61,9 @@
    5.25      if (_setmode(fd,_O_BINARY)<0)
    5.26      {
    5.27  	close(fd);
    5.28 +	g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
    5.29 +	  "Failed to set mode of bookloupe input file to binary: %s",
    5.30 +	  g_strerror(errno));
    5.31  	return -1;
    5.32      }
    5.33  #endif
    5.34 @@ -63,6 +71,9 @@
    5.35      if (!fp)
    5.36      {
    5.37  	close(fd);
    5.38 +	g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
    5.39 +	  "Failed to open stream to bookloupe input file: %s",
    5.40 +	  g_strerror(errno));
    5.41  	return -1;
    5.42      }
    5.43      for(i=0;i<count;i++)
    5.44 @@ -70,17 +81,25 @@
    5.45  	if (buf[i]=='\n')
    5.46  	    if (putc('\r',fp)==EOF)
    5.47  	    {
    5.48 +		g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
    5.49 +		  "Error writing bookloupe input file: %s",g_strerror(errno));
    5.50  		(void)fclose(fp);
    5.51  		return -1;
    5.52  	    }
    5.53  	if (putc(buf[i],fp)==EOF)
    5.54  	{
    5.55 +	    g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
    5.56 +	      "Error writing bookloupe input file: %s",g_strerror(errno));
    5.57  	    (void)fclose(fp);
    5.58  	    return -1;
    5.59  	}
    5.60      }
    5.61      if (fclose(fp))
    5.62 +    {
    5.63 +	g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
    5.64 +	  "Error writing bookloupe input file: %s",g_strerror(errno));
    5.65  	return -1;
    5.66 +    }
    5.67      return count;
    5.68  }
    5.69  
    5.70 @@ -95,6 +114,122 @@
    5.71      return i;
    5.72  }
    5.73  
    5.74 +void print_unexpected(const char *unexpected,gsize differs_at)
    5.75 +{
    5.76 +    int col;
    5.77 +    const char *endp,*bol;
    5.78 +    GString *string;
    5.79 +    endp=strchr(unexpected+differs_at,'\n');
    5.80 +    if (!endp)
    5.81 +	endp=unexpected+strlen(unexpected);
    5.82 +    string=g_string_new_len(unexpected,endp-unexpected);
    5.83 +    bol=strrchr(string->str,'\n');
    5.84 +    if (bol)
    5.85 +	bol++;
    5.86 +    else
    5.87 +	bol=string->str;
    5.88 +    col=differs_at-(bol-string->str);
    5.89 +    fprintf(stderr,"%s\n%*s^\n",string->str,col,"");
    5.90 +    g_string_free(string,TRUE);
    5.91 +}
    5.92 +
    5.93 +gboolean spawn_bootloupe(const char *encoding,const char *standard_input,
    5.94 +  char **standard_output,char **filename,GError **error)
    5.95 +{
    5.96 +    gboolean r;
    5.97 +    int fd,exit_status;
    5.98 +    size_t n,pos,offset;
    5.99 +    FILE *fp;
   5.100 +    char input[]="TEST-XXXXXX";
   5.101 +    char *command[3];
   5.102 +    char *output,*s;
   5.103 +    GError *tmp_err=NULL;
   5.104 +    if (standard_input)
   5.105 +    {
   5.106 +	if (encoding)
   5.107 +	{
   5.108 +	    s=g_convert(standard_input,-1,encoding,"UTF-8",NULL,&n,&tmp_err);
   5.109 +	    if (!s)
   5.110 +	    {
   5.111 +		g_propagate_prefixed_error(error,tmp_err,
   5.112 +		  "Conversion to %s failed: ",encoding);
   5.113 +		return FALSE;
   5.114 +	    }
   5.115 +	}
   5.116 +	else
   5.117 +	{
   5.118 +	    s=g_strdup(standard_input);
   5.119 +	    n=strlen(s);
   5.120 +	}
   5.121 +    }
   5.122 +    else
   5.123 +    {
   5.124 +	n=0;
   5.125 +	s=NULL;
   5.126 +    }
   5.127 +    fd=mkstemp(input);
   5.128 +    if (n && write_text(fd,s,n,error)!=n)
   5.129 +    {
   5.130 +	g_free(s);
   5.131 +	close(fd);
   5.132 +	(void)remove(input);
   5.133 +	return FALSE;
   5.134 +    }
   5.135 +    g_free(s);
   5.136 +    close(fd);
   5.137 +    command[0]=getenv("BOOKLOUPE");
   5.138 +    if (!command[0])
   5.139 +	command[0]="." G_DIR_SEPARATOR_S "bookloupe";
   5.140 +    command[1]=input;
   5.141 +    command[2]=NULL;
   5.142 +    if (standard_output)
   5.143 +    {
   5.144 +	r=spawn_sync(command,&s,&exit_status,error);
   5.145 +	if (r)
   5.146 +	{
   5.147 +	    if (encoding)
   5.148 +	    {
   5.149 +		output=g_convert(s,-1,"UTF-8",encoding,NULL,NULL,&tmp_err);
   5.150 +		g_free(s);
   5.151 +		if (!output)
   5.152 +		{
   5.153 +		    g_propagate_prefixed_error(error,tmp_err,
   5.154 +		      "Conversion from %s failed: ",encoding);
   5.155 +		    r=FALSE;
   5.156 +		}
   5.157 +	    }
   5.158 +	    else
   5.159 +	    {
   5.160 +		output=s;
   5.161 +		if (!g_utf8_validate(s,-1,NULL))
   5.162 +		{
   5.163 +		    g_set_error_literal(error,TESTCASE_ERROR,
   5.164 +		      TESTCASE_ERROR_FAILED,
   5.165 +		      "bookloupe output is not valid UTF-8");
   5.166 +		    r=FALSE;
   5.167 +		}
   5.168 +	    }
   5.169 +	}
   5.170 +    }
   5.171 +    else
   5.172 +    {
   5.173 +	r=spawn_sync(command,NULL,&exit_status,error);
   5.174 +	output=NULL;
   5.175 +    }
   5.176 +    (void)remove(input);
   5.177 +    if (r && exit_status)
   5.178 +    {
   5.179 +	g_set_error(error,TESTCASE_ERROR,TESTCASE_ERROR_FAILED,
   5.180 +	  "bookloupe exited with code %d",exit_status);
   5.181 +	r=FALSE;
   5.182 +    }
   5.183 +    if (r && filename)
   5.184 +	*filename=g_strdup(input);
   5.185 +    if (r && standard_output)
   5.186 +	*standard_output=output;
   5.187 +    return r;
   5.188 +}
   5.189 +
   5.190  /*
   5.191   * Run a testcase, returning FALSE on fail or error and
   5.192   * TRUE on pass or expected-fail.
   5.193 @@ -103,92 +238,76 @@
   5.194  gboolean testcase_run(Testcase *testcase)
   5.195  {
   5.196      gboolean r;
   5.197 -    int fd,exit_status,col;
   5.198 -    size_t n,pos,offset,header_len;
   5.199 -    FILE *fp;
   5.200 -    char input[]="TEST-XXXXXX";
   5.201 -    char *endp,*bol;
   5.202 -    char *command[3];
   5.203 -    GString *expected,*report;
   5.204 -    char *output;
   5.205 -    fd=mkstemp(input);
   5.206 -    if (testcase->input)
   5.207 -	n=strlen(testcase->input);
   5.208 +    size_t pos,offset;
   5.209 +    GString *header,*expected;
   5.210 +    char *output,*filename,*s;
   5.211 +    GError *error=NULL;
   5.212 +    if (testcase->expected)
   5.213 +	r=spawn_bootloupe(testcase->encoding,testcase->input,&output,&filename,
   5.214 +	  &error);
   5.215      else
   5.216 -	n=0;
   5.217 -    if (n && write_text(fd,testcase->input,n)!=n)
   5.218      {
   5.219 -	perror(input);
   5.220 -	close(fd);
   5.221 -	(void)remove(input);
   5.222 +	r=spawn_bootloupe(testcase->encoding,testcase->input,NULL,NULL,&error);
   5.223 +        output=filename=NULL;
   5.224 +    }
   5.225 +    if (!r)
   5.226 +    {
   5.227 +	fprintf(stderr,"%s: FAIL\n",testcase->basename);
   5.228 +	fprintf(stderr,"%s\n",error->message);
   5.229 +	g_error_free(error);
   5.230  	return FALSE;
   5.231      }
   5.232 -    close(fd);
   5.233 -    command[0]=getenv("BOOKLOUPE");
   5.234 -    if (!command[0])
   5.235 -	command[0]="." G_DIR_SEPARATOR_S "bookloupe";
   5.236 -    command[1]=input;
   5.237 -    command[2]=NULL;
   5.238 -    if (testcase->expected)
   5.239 -	r=spawn_sync(command,&output,&exit_status);
   5.240 -    else
   5.241 -    {
   5.242 -	r=spawn_sync(command,NULL,&exit_status);
   5.243 -	output=NULL;
   5.244 -    }
   5.245 -    (void)remove(input);
   5.246 -    if (!r)
   5.247 -	return FALSE;
   5.248      if (testcase->expected)
   5.249      {
   5.250 -	expected=g_string_new("\n\nFile: ");
   5.251 -	g_string_append(expected,input);
   5.252 -	g_string_append(expected,"\n\n\n");
   5.253 -	header_len=expected->len;
   5.254 -	g_string_append(expected,testcase->expected);
   5.255 +	header=g_string_new("\n\nFile: ");
   5.256 +	g_string_append(header,filename);
   5.257 +	g_string_append(header,"\n");
   5.258 +	expected=g_string_new(testcase->expected);
   5.259 +	if (!g_str_has_prefix(output,header->str))
   5.260 +	{
   5.261 +	    fprintf(stderr,"%s: FAIL\n",testcase->basename);
   5.262 +	    offset=common_prefix_length(output,header->str);
   5.263 +	    fprintf(stderr,"Unexpected header from bookloupe:\n");
   5.264 +	    print_unexpected(output,offset);
   5.265 +	    r=FALSE;
   5.266 +	}
   5.267 +	pos=header->len;
   5.268 +	if (r)
   5.269 +	{
   5.270 +	    /* Skip the summary */
   5.271 +	    s=strstr(output+pos,"\n\n");
   5.272 +	    if (s)
   5.273 +		pos=s-output+2;
   5.274 +	    else
   5.275 +	    {
   5.276 +		fprintf(stderr,"%s: FAIL\n",testcase->basename);
   5.277 +		offset=common_prefix_length(output,header->str);
   5.278 +		fprintf(stderr,"Unterminated summary from bookloupe:\n%s\n",
   5.279 +		  output+pos);
   5.280 +		r=FALSE;
   5.281 +	    }
   5.282 +	}
   5.283 +	if (r && strcmp(output+pos,expected->str))
   5.284 +	{
   5.285 +	    fprintf(stderr,"%s: FAIL\n",testcase->basename);
   5.286 +	    offset=common_prefix_length(output+pos,expected->str);
   5.287 +	    if (!offset && !output[pos+offset])
   5.288 +		fprintf(stderr,"Unexpected zero warnings from bookloupe.\n");
   5.289 +	    else
   5.290 +	    {
   5.291 +		fprintf(stderr,"Unexpected output from bookloupe:\n");
   5.292 +		print_unexpected(output+pos,offset);
   5.293 +	    }
   5.294 +	    r=FALSE;
   5.295 +	}
   5.296 +	g_string_free(header,TRUE);
   5.297 +	g_string_free(expected,TRUE);
   5.298      }
   5.299 -    else
   5.300 -    {
   5.301 -	expected=NULL;
   5.302 -	header_len=0;
   5.303 -    }
   5.304 -    if (expected && strcmp(output,expected->str))
   5.305 -    {
   5.306 -	fprintf(stderr,"%s: FAIL\n",testcase->basename);
   5.307 -	offset=common_prefix_length(output,expected->str);
   5.308 -	if (offset==header_len && !output[offset])
   5.309 -	    fprintf(stderr,"Unexpected zero warnings from bookloupe.\n");
   5.310 -	else
   5.311 -	{
   5.312 -	    endp=strchr(output+offset,'\n');
   5.313 -	    if (!endp)
   5.314 -		endp=output+strlen(output);
   5.315 -	    report=g_string_new(NULL);
   5.316 -	    g_string_append_len(report,output,endp-output);
   5.317 -	    bol=strrchr(report->str,'\n');
   5.318 -	    if (bol)
   5.319 -		bol++;
   5.320 -	    else
   5.321 -		bol=report->str;
   5.322 -	    col=offset-(bol-report->str);
   5.323 -	    fprintf(stderr,"Unexpected output from bookloupe:\n");
   5.324 -	    if (report->len>=header_len)
   5.325 -		fprintf(stderr,"%s\n%*s^\n",report->str+header_len,col,"");
   5.326 -	    else
   5.327 -		fprintf(stderr,"%s\n%*s^\n",report->str,col,"");
   5.328 -	    g_string_free(report,TRUE);
   5.329 -	}
   5.330 -	g_string_free(expected,TRUE);
   5.331 -	g_free(output);
   5.332 -	return FALSE;
   5.333 -    }
   5.334 -    g_string_free(expected,TRUE);
   5.335      g_free(output);
   5.336 -    if (exit_status)
   5.337 -	fprintf(stderr,"bookloupe exited with code %d\n",r);
   5.338 -    if (!exit_status)
   5.339 +    g_free(filename);
   5.340 +    if (r)
   5.341  	fprintf(stderr,"%s: PASS\n",testcase->basename);
   5.342 -    return !exit_status;
   5.343 +    return r;
   5.344  }
   5.345  
   5.346  /*
   5.347 @@ -199,5 +318,6 @@
   5.348      g_free(testcase->basename);
   5.349      g_free(testcase->input);
   5.350      g_free(testcase->expected);
   5.351 +    g_free(testcase->encoding);
   5.352      g_free(testcase);
   5.353  }

     6.1 --- a/test/harness/testcase.h	Fri Jan 27 16:18:02 2012 +0000
     6.2 +++ b/test/harness/testcase.h	Fri Jan 27 21:40:35 2012 +0000
     6.3 @@ -3,15 +3,23 @@
     6.4  
     6.5  #include <glib.h>
     6.6  
     6.7 +#define TESTCASE_ERROR testcase_error_quark()
     6.8 +
     6.9 +typedef enum {
    6.10 +    TESTCASE_ERROR_FAILED
    6.11 +} TestcaseError;
    6.12 +
    6.13  typedef struct {
    6.14      char *basename;
    6.15      char *input;
    6.16      char *expected;
    6.17 +    char *encoding;	/* The character encoding to talk to BOOKLOUPE in */
    6.18      enum {
    6.19  	TESTCASE_XFAIL=1<<0,
    6.20      } flags;
    6.21  } Testcase;
    6.22  
    6.23 +GQuark testcase_error_quark(void);
    6.24  gboolean testcase_run(Testcase *testcase);
    6.25  void testcase_free(Testcase *testcase);
    6.26  

     7.1 --- a/test/harness/testcaseio.c	Fri Jan 27 16:18:02 2012 +0000
     7.2 +++ b/test/harness/testcaseio.c	Fri Jan 27 21:40:35 2012 +0000
     7.3 @@ -38,6 +38,8 @@
     7.4  	    testcase->input=g_strdup(text);
     7.5  	else if (!testcase->expected && !strcmp(tag,"EXPECTED"))
     7.6  	    testcase->expected=g_strdup(text);
     7.7 +	else if (!testcase->encoding && !strcmp(tag,"ENCODING"))
     7.8 +	    testcase->encoding=g_strdup(text);
     7.9  	else
    7.10  	{
    7.11  	    fprintf(stderr,"%s: Not a valid testcase (%s)\n",filename,tag);

     8.1 --- a/test/harness/testcaseparser.c	Fri Jan 27 16:18:02 2012 +0000
     8.2 +++ b/test/harness/testcaseparser.c	Fri Jan 27 21:40:35 2012 +0000
     8.3 @@ -90,12 +90,20 @@
     8.4  TestcaseParser *testcase_parser_new_from_file(const char *filename)
     8.5  {
     8.6      TestcaseParser *parser;
     8.7 +    gsize len;
     8.8      parser=g_new0(TestcaseParser,1);
     8.9 -    if (!file_get_contents_text(filename,&parser->contents,NULL))
    8.10 +    if (!file_get_contents_text(filename,&parser->contents,&len))
    8.11      {
    8.12  	g_free(parser);
    8.13  	return NULL;
    8.14      }
    8.15 +    if (!g_utf8_validate(parser->contents,len,NULL))
    8.16 +    {
    8.17 +	fprintf(stderr,"%s: Does not contain valid UTF-8\n",filename);
    8.18 +	g_free(parser->contents);
    8.19 +	g_free(parser);
    8.20 +	return NULL;
    8.21 +    }
    8.22      parser->filename=g_strdup(filename);
    8.23      return parser;
    8.24  }