test/harness/testcase.c
author ali <ali@juiblex.co.uk>
Wed Oct 16 22:51:29 2013 +0100 (2013-10-16)
changeset 104 70cc629ec1e0
parent 101 f44c530f80da
permissions -rw-r--r--
Fix bug #28: Don't report ., as double punctuation after "etc" or "&c"
ali@0
     1
#include <stdlib.h>
ali@0
     2
#include <stdio.h>
ali@0
     3
#include <string.h>
ali@0
     4
#include <errno.h>
ali@9
     5
#include <glib.h>
ali@9
     6
#include <glib/gstdio.h>
ali@5
     7
#include <bl/bl.h>
ali@0
     8
#include "testcase.h"
ali@9
     9
#include "testcaseinput.h"
ali@102
    10
#include "testcaseoutput.h"
ali@0
    11
ali@7
    12
GQuark testcase_error_quark(void)
ali@7
    13
{
ali@7
    14
    return g_quark_from_static_string("testcase-error-quark");
ali@7
    15
}
ali@7
    16
ali@0
    17
/*
ali@8
    18
 * Return the length (in bytes) of any common prefix between s1 and s2.
ali@8
    19
 * The returned length will always represent an exact number of characters.
ali@0
    20
 */
ali@0
    21
size_t common_prefix_length(const char *s1,const char *s2)
ali@0
    22
{
ali@8
    23
    gunichar c1,c2;
ali@8
    24
    const char *s=s1;
ali@8
    25
    while(*s1 && *s2)
ali@8
    26
    {
ali@8
    27
	c1=g_utf8_get_char(s1);
ali@8
    28
	c2=g_utf8_get_char(s2);
ali@8
    29
	if (c1!=c2)
ali@8
    30
	    break;
ali@8
    31
	s1=g_utf8_next_char(s1);
ali@8
    32
	s2=g_utf8_next_char(s2);
ali@8
    33
    }
ali@8
    34
    return s1-s;
ali@0
    35
}
ali@0
    36
ali@7
    37
void print_unexpected(const char *unexpected,gsize differs_at)
ali@7
    38
{
ali@7
    39
    int col;
ali@8
    40
    gunichar c;
ali@8
    41
    const char *endp,*bol,*s;
ali@7
    42
    GString *string;
ali@7
    43
    endp=strchr(unexpected+differs_at,'\n');
ali@7
    44
    if (!endp)
ali@7
    45
	endp=unexpected+strlen(unexpected);
ali@7
    46
    string=g_string_new_len(unexpected,endp-unexpected);
ali@7
    47
    bol=strrchr(string->str,'\n');
ali@7
    48
    if (bol)
ali@7
    49
	bol++;
ali@7
    50
    else
ali@7
    51
	bol=string->str;
ali@8
    52
    col=0;
ali@8
    53
    s=bol;
ali@8
    54
    endp=string->str+differs_at;
ali@8
    55
    while(s<endp)
ali@8
    56
    {
ali@8
    57
	c=g_utf8_get_char(s);
ali@8
    58
	s=g_utf8_next_char(s);
ali@8
    59
	if (c=='\t')
ali@8
    60
	    col=(col&~7)+8;
ali@8
    61
	else if (g_unichar_iswide(c))
ali@8
    62
	    col+=2;
ali@8
    63
	else if (!g_unichar_iszerowidth(c))
ali@8
    64
	    col++;
ali@8
    65
    }
ali@11
    66
    g_print("%s\n%*s^\n",string->str,col,"");
ali@7
    67
    g_string_free(string,TRUE);
ali@7
    68
}
ali@7
    69
ali@8
    70
/*
ali@9
    71
 * Create all the input files needed by a testcase and, if required,
ali@9
    72
 * a temporary directory in which to store them.
ali@8
    73
 */
ali@9
    74
gboolean testcase_create_input_files(Testcase *testcase,GError **error)
ali@8
    75
{
ali@9
    76
    GSList *link,*link2;
ali@9
    77
    if (testcase->flags&TESTCASE_TMP_DIR)
ali@8
    78
    {
ali@9
    79
	testcase->tmpdir=g_strdup("TEST-XXXXXX");
ali@9
    80
	if (!g_mkdtemp(testcase->tmpdir))
ali@9
    81
	{
ali@9
    82
	    g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
ali@9
    83
	      "Failed to create temporary directory: %s",g_strerror(errno));
ali@9
    84
	    g_free(testcase->tmpdir);
ali@9
    85
	    testcase->tmpdir=NULL;
ali@9
    86
	    return FALSE;
ali@9
    87
	}
ali@8
    88
    }
ali@9
    89
    for(link=testcase->inputs;link;link=link->next)
ali@9
    90
	if (!testcase_input_create(testcase,link->data,error))
ali@9
    91
	{
ali@9
    92
	    for(link2=testcase->inputs;link2!=link;link2=link2->next)
ali@9
    93
		(void)testcase_input_remove(testcase,link2->data,NULL);
ali@9
    94
	    if (testcase->tmpdir)
ali@9
    95
	    {
ali@9
    96
		(void)g_rmdir(testcase->tmpdir);
ali@9
    97
		g_free(testcase->tmpdir);
ali@9
    98
		testcase->tmpdir=NULL;
ali@9
    99
	    }
ali@9
   100
	    return FALSE;
ali@9
   101
	}
ali@9
   102
    return TRUE;
ali@8
   103
}
ali@8
   104
ali@9
   105
/*
ali@9
   106
 * Remove all the input files used by a testcase and, if created,
ali@9
   107
 * the temporary directory in which they are stored.
ali@9
   108
 */
ali@9
   109
gboolean testcase_remove_input_files(Testcase *testcase,GError **error)
ali@9
   110
{
ali@9
   111
    GSList *link;
ali@9
   112
    GError *tmp_err=NULL;
ali@9
   113
    gboolean retval=TRUE;
ali@9
   114
    for(link=testcase->inputs;link;link=link->next)
ali@9
   115
	if (!testcase_input_remove(testcase,link->data,&tmp_err))
ali@9
   116
	{
ali@9
   117
	    if (error && !*error)
ali@9
   118
		g_propagate_error(error,tmp_err);
ali@9
   119
	    else
ali@9
   120
		g_clear_error(&tmp_err);
ali@9
   121
	    retval=FALSE;
ali@9
   122
	}
ali@9
   123
    if (testcase->tmpdir)
ali@9
   124
    {
ali@9
   125
	if (g_rmdir(testcase->tmpdir))
ali@9
   126
	{
ali@9
   127
	    if (error && !*error)
ali@9
   128
		g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
ali@9
   129
		  "Failed to remove temporary directory: %s",g_strerror(errno));
ali@9
   130
	    retval=FALSE;
ali@9
   131
	}
ali@9
   132
	g_free(testcase->tmpdir);
ali@9
   133
	testcase->tmpdir=NULL;
ali@9
   134
    }
ali@9
   135
    return retval;
ali@9
   136
}
ali@9
   137
ali@9
   138
/*
ali@9
   139
 * Replace every occurance of an input file name in <str> with the
ali@9
   140
 * filename which holds that input. For input files with fixed names,
ali@9
   141
 * this is a noop. For input files which use the "XXXXXX" sequence
ali@9
   142
 * to create a unique filename, the XXXXXX will be replaced with the
ali@9
   143
 * 6 characters that were chosen to be unique.
ali@9
   144
 */
ali@9
   145
char *testcase_resolve_input_files(Testcase *testcase,const char *str)
ali@9
   146
{
ali@9
   147
    GSList *link;
ali@9
   148
    gsize offset,pos;
ali@9
   149
    char *s;
ali@9
   150
    TestcaseInput *input;
ali@9
   151
    GString *filename=g_string_new(str);
ali@9
   152
    for(link=testcase->inputs;link;link=link->next)
ali@9
   153
    {
ali@9
   154
	input=link->data;
ali@9
   155
	if (!input->name_used)
ali@9
   156
	{
ali@9
   157
	    g_warning("%s: Input file uninstantiated",input->name);
ali@9
   158
	    continue;
ali@9
   159
	}
ali@9
   160
	offset=0;
ali@9
   161
	do
ali@9
   162
	{
ali@9
   163
	    s=strstr(filename->str+offset,input->name);
ali@9
   164
	    if (s)
ali@9
   165
	    {
ali@9
   166
		pos=s-filename->str;
ali@9
   167
		g_string_overwrite(filename,pos,input->name_used);
ali@9
   168
		offset=pos+strlen(input->name);
ali@9
   169
	    }
ali@9
   170
	} while(s);
ali@9
   171
    }
ali@9
   172
    return g_string_free(filename,FALSE);
ali@9
   173
}
ali@9
   174
ali@102
   175
/*
ali@102
   176
 * Verify that all the output files specified by a testcase are present
ali@102
   177
 * with the expected contents. 
ali@102
   178
 */
ali@102
   179
gboolean testcase_verify_output_files(Testcase *testcase)
ali@102
   180
{
ali@102
   181
    GSList *link;
ali@102
   182
    GError *tmp_err=NULL;
ali@102
   183
    gboolean retval=TRUE;
ali@102
   184
    ssize_t offset;
ali@102
   185
    gchar *contents;
ali@102
   186
    TestcaseOutput *output;
ali@102
   187
    for(link=testcase->outputs;link;link=link->next)
ali@102
   188
    {
ali@102
   189
	output=link->data;
ali@102
   190
	if (!testcase_output_read(testcase,output,&contents,NULL,&tmp_err))
ali@102
   191
	{
ali@102
   192
	    g_print("%s: FAIL\n",testcase->basename);
ali@102
   193
	    g_print("%s\n",tmp_err->message);
ali@102
   194
	    g_clear_error(&tmp_err);
ali@102
   195
	    retval=FALSE;
ali@102
   196
	    break;
ali@102
   197
	}
ali@102
   198
	else
ali@102
   199
	{
ali@102
   200
	    if (strcmp(contents,output->contents))
ali@102
   201
	    {
ali@102
   202
		g_print("%s: FAIL\n",testcase->basename);
ali@102
   203
		offset=common_prefix_length(contents,output->contents);
ali@102
   204
		if (!offset && !contents[offset])
ali@102
   205
		    g_print("%s: Unexpected empty output from bookloupe.\n",
ali@102
   206
		      output->name);
ali@102
   207
		else
ali@102
   208
		{
ali@102
   209
		    g_print("%s: Unexpected output from bookloupe:\n",
ali@102
   210
		      output->name);
ali@102
   211
		    print_unexpected(contents,offset);
ali@102
   212
		}
ali@102
   213
		retval=FALSE;
ali@102
   214
	    }
ali@102
   215
	    g_free(contents);
ali@102
   216
	    break;
ali@102
   217
	}
ali@102
   218
    }
ali@102
   219
    for(link=testcase->outputs;link;link=link->next)
ali@102
   220
	if (!testcase_output_remove(testcase,link->data,&tmp_err))
ali@102
   221
	{
ali@102
   222
	    if (retval)
ali@102
   223
	    {
ali@102
   224
		g_print("%s: FAIL\n",testcase->basename);
ali@102
   225
		g_print("%s\n",tmp_err->message);
ali@102
   226
		retval=TRUE;
ali@102
   227
	    }
ali@102
   228
	    g_clear_error(&tmp_err);
ali@102
   229
	}
ali@102
   230
    return retval;
ali@102
   231
}
ali@102
   232
ali@9
   233
gboolean testcase_spawn_bookloupe(Testcase *testcase,char **standard_output,
ali@9
   234
  GError **error)
ali@7
   235
{
ali@7
   236
    gboolean r;
ali@9
   237
    int i,exit_status;
ali@9
   238
    char **argv;
ali@9
   239
    char *output,*s;
ali@7
   240
    GError *tmp_err=NULL;
ali@9
   241
    if (testcase->options)
ali@9
   242
	argv=g_new(char *,g_strv_length(testcase->options)+3);
ali@7
   243
    else
ali@9
   244
	argv=g_new(char *,3);
ali@9
   245
    s=getenv("BOOKLOUPE");
ali@9
   246
    if (!s)
ali@9
   247
	s="bookloupe";
ali@9
   248
    argv[0]=path_to_absolute(s);
ali@9
   249
    for(i=0;testcase->options && testcase->options[i];i++)
ali@9
   250
	argv[i+1]=testcase_resolve_input_files(testcase,testcase->options[i]);
ali@9
   251
    argv[i+1]=testcase_resolve_input_files(testcase,"TEST-XXXXXX");
ali@9
   252
    argv[i+2]=NULL;
ali@7
   253
    if (standard_output)
ali@7
   254
    {
ali@9
   255
	r=spawn_sync(testcase->tmpdir,argv,&s,&exit_status,error);
ali@7
   256
	if (r)
ali@7
   257
	{
ali@9
   258
	    if (testcase->encoding)
ali@7
   259
	    {
ali@9
   260
		output=g_convert(s,-1,"UTF-8",testcase->encoding,NULL,NULL,
ali@9
   261
		  &tmp_err);
ali@7
   262
		g_free(s);
ali@7
   263
		if (!output)
ali@7
   264
		{
ali@7
   265
		    g_propagate_prefixed_error(error,tmp_err,
ali@9
   266
		      "Conversion from %s failed: ",testcase->encoding);
ali@7
   267
		    r=FALSE;
ali@7
   268
		}
ali@7
   269
	    }
ali@7
   270
	    else
ali@7
   271
	    {
ali@7
   272
		output=s;
ali@7
   273
		if (!g_utf8_validate(s,-1,NULL))
ali@7
   274
		{
ali@7
   275
		    g_set_error_literal(error,TESTCASE_ERROR,
ali@7
   276
		      TESTCASE_ERROR_FAILED,
ali@7
   277
		      "bookloupe output is not valid UTF-8");
ali@7
   278
		    r=FALSE;
ali@7
   279
		}
ali@7
   280
	    }
ali@7
   281
	}
ali@7
   282
    }
ali@7
   283
    else
ali@7
   284
    {
ali@9
   285
	r=spawn_sync(testcase->tmpdir,argv,NULL,&exit_status,error);
ali@7
   286
	output=NULL;
ali@7
   287
    }
ali@9
   288
    g_strfreev(argv);
ali@7
   289
    if (r && exit_status)
ali@7
   290
    {
ali@7
   291
	g_set_error(error,TESTCASE_ERROR,TESTCASE_ERROR_FAILED,
ali@7
   292
	  "bookloupe exited with code %d",exit_status);
ali@7
   293
	r=FALSE;
ali@7
   294
    }
ali@7
   295
    if (r && standard_output)
ali@7
   296
	*standard_output=output;
ali@7
   297
    return r;
ali@7
   298
}
ali@7
   299
ali@0
   300
/*
ali@17
   301
 * Parse a warning of the form:
ali@17
   302
 *	[blank line]
ali@17
   303
 *	<echoed line> (ignored)
ali@17
   304
 *	"    Line " <number> [" column " <number>] " - " <text> "\n"
ali@17
   305
 * If not specified, the column is returned as 0.
ali@17
   306
 * Returns: the number of bytes parsed, or -1 on error.
ali@17
   307
 */
ali@17
   308
static ssize_t testcase_parse_warning(Testcase *testcase,const char *output,
ali@17
   309
  guint *line,guint *column,char **text)
ali@17
   310
{
ali@17
   311
    ssize_t offset=0;
ali@17
   312
    guint64 tmp;
ali@17
   313
    char *s,*endp;
ali@17
   314
    if (output[offset]!='\n')
ali@17
   315
    {
ali@17
   316
	g_print("%s: FAIL\n",testcase->basename);
ali@17
   317
	g_print("Unexpected output from bookloupe:\n");
ali@17
   318
	print_unexpected(output,offset);
ali@17
   319
	return -1;
ali@17
   320
    }
ali@17
   321
    offset++;
ali@17
   322
    s=strchr(output+offset,'\n');
ali@17
   323
    if (!s)
ali@17
   324
    {
ali@17
   325
	g_print("%s: FAIL\n",testcase->basename);
ali@17
   326
	g_print("Missing new-line in output from bookloupe:\n");
ali@17
   327
	print_unexpected(output,offset);
ali@17
   328
	return -1;
ali@17
   329
    }
ali@17
   330
    offset=s-output+1;
ali@17
   331
    if (!g_str_has_prefix(output+offset,"    Line "))
ali@17
   332
    {
ali@17
   333
	g_print("%s: FAIL\n",testcase->basename);
ali@17
   334
	g_print("Unexpected output from bookloupe:\n");
ali@17
   335
	offset+=common_prefix_length(output+offset,"    Line ");
ali@17
   336
	print_unexpected(output,offset);
ali@17
   337
	return -1;
ali@17
   338
    }
ali@17
   339
    offset+=9;
ali@17
   340
    tmp=g_ascii_strtoull(output+offset,&endp,10);
ali@17
   341
    if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64)
ali@17
   342
    {
ali@17
   343
	g_print("%s: FAIL\n",testcase->basename);
ali@17
   344
	g_print("Unexpected output from bookloupe:\n");
ali@17
   345
	print_unexpected(output,offset);
ali@17
   346
	return -1;
ali@17
   347
    }
ali@17
   348
    *line=tmp;
ali@17
   349
    offset=endp-output;
ali@17
   350
    if (g_str_has_prefix(output+offset," column "))
ali@17
   351
    {
ali@17
   352
	offset+=8;
ali@17
   353
	tmp=g_ascii_strtoull(output+offset,&endp,10);
ali@17
   354
	if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64)
ali@17
   355
	{
ali@17
   356
	    g_print("%s: FAIL\n",testcase->basename);
ali@17
   357
	    g_print("Unexpected output from bookloupe:\n");
ali@17
   358
	    print_unexpected(output,offset);
ali@17
   359
	    return -1;
ali@17
   360
	}
ali@17
   361
	*column=tmp;
ali@17
   362
	offset=endp-output;
ali@17
   363
    }
ali@17
   364
    else
ali@17
   365
	*column=0;
ali@17
   366
    if (!g_str_has_prefix(output+offset," - "))
ali@17
   367
    {
ali@17
   368
	g_print("%s: FAIL\n",testcase->basename);
ali@17
   369
	g_print("Unexpected output from bookloupe:\n");
ali@17
   370
	offset+=common_prefix_length(output+offset," - ");
ali@17
   371
	print_unexpected(output,offset);
ali@17
   372
	return -1;
ali@17
   373
    }
ali@17
   374
    offset+=3;
ali@17
   375
    s=strchr(output+offset,'\n');
ali@17
   376
    if (!s)
ali@17
   377
    {
ali@17
   378
	g_print("%s: FAIL\n",testcase->basename);
ali@17
   379
	g_print("Missing new-line in output from bookloupe:\n");
ali@17
   380
	print_unexpected(output,offset);
ali@17
   381
	return -1;
ali@17
   382
    }
ali@17
   383
    *text=g_strndup(output+offset,s-(output+offset));
ali@17
   384
    return s-output+1;
ali@17
   385
}
ali@17
   386
ali@17
   387
/*
ali@101
   388
 * Check the summary produced by bookloupe against testcase->summary.
ali@101
   389
 */
ali@101
   390
static gboolean testcase_check_summary(Testcase *testcase,const char *summary)
ali@101
   391
{
ali@101
   392
    int i;
ali@101
   393
    gboolean r;
ali@101
   394
    gchar **lines;
ali@101
   395
    GSList *texts,*lnk;
ali@101
   396
    if (!testcase->summary.texts)
ali@101
   397
	return TRUE;
ali@101
   398
    texts=g_slist_copy(testcase->summary.texts);
ali@101
   399
    lines=g_strsplit(summary,"\n",0);
ali@101
   400
    for(i=0;lines[i];i++)
ali@101
   401
    {
ali@101
   402
	if (!g_str_has_prefix(lines[i],"   --> "))
ali@101
   403
	    continue;
ali@101
   404
	for(lnk=texts;lnk;lnk=lnk->next)
ali@101
   405
	    if (!strcmp(lines[i]+7,lnk->data))
ali@101
   406
	    {
ali@101
   407
		texts=g_slist_delete_link(texts,lnk);
ali@101
   408
		break;
ali@101
   409
	    }
ali@101
   410
    }
ali@101
   411
    g_strfreev(lines);
ali@101
   412
    r=!texts;
ali@101
   413
    if (texts)
ali@101
   414
    {
ali@101
   415
	g_print("%s: FAIL\n",testcase->basename);
ali@101
   416
	g_print("Missing summary text from bookloupe:\n");
ali@101
   417
	g_print("   --> %s\n",texts->data);
ali@101
   418
    }
ali@101
   419
    g_slist_free(texts);
ali@101
   420
    return r;
ali@101
   421
}
ali@101
   422
ali@101
   423
/*
ali@17
   424
 * Check the warnings produced by bookloupe against either the
ali@17
   425
 * unstructured testcase->expected or the structured testcase->warnings
ali@17
   426
 * as appropriate.
ali@17
   427
 */
ali@17
   428
static gboolean testcase_check_warnings(Testcase *testcase,const char *output,
ali@17
   429
  char **xfail)
ali@17
   430
{
ali@17
   431
    gboolean r=TRUE;
ali@17
   432
    size_t offset;
ali@17
   433
    ssize_t off;
ali@17
   434
    int i,count_false_positive,count_false_negative;
ali@17
   435
    int total_false_positive,total_false_negative;
ali@17
   436
    char *text;
ali@17
   437
    guint *counts,line,column;
ali@17
   438
    GSList *link,*link2;
ali@17
   439
    TestcaseWarning *warning;
ali@17
   440
    TestcaseLocation *location;
ali@17
   441
    *xfail=NULL;
ali@17
   442
    if (testcase->expected)
ali@17
   443
    {
ali@17
   444
	if (strcmp(output,testcase->expected))
ali@17
   445
	{
ali@17
   446
	    g_print("%s: FAIL\n",testcase->basename);
ali@17
   447
	    offset=common_prefix_length(output,testcase->expected);
ali@17
   448
	    if (!offset && !output[offset])
ali@17
   449
		g_print("Unexpected zero warnings from bookloupe.\n");
ali@17
   450
	    else
ali@17
   451
	    {
ali@17
   452
		g_print("Unexpected output from bookloupe:\n");
ali@17
   453
		print_unexpected(output,offset);
ali@17
   454
	    }
ali@17
   455
	    return FALSE;
ali@17
   456
	}
ali@17
   457
	return TRUE;
ali@17
   458
    }
ali@17
   459
    counts=g_new0(guint,g_slist_length(testcase->warnings));
ali@17
   460
    for(offset=0;output[offset];)
ali@17
   461
    {
ali@17
   462
	off=testcase_parse_warning(testcase,output+offset,&line,&column,&text);
ali@17
   463
	if (off<0)
ali@17
   464
	{
ali@17
   465
	    r=FALSE;
ali@17
   466
	    break;
ali@17
   467
	}
ali@17
   468
	offset+=off;
ali@17
   469
	for(link=testcase->warnings,i=0;link;link=link->next,i++)
ali@17
   470
	{
ali@17
   471
	    warning=link->data;
ali@17
   472
	    if (strcmp(warning->text,text))
ali@17
   473
		continue;
ali@17
   474
	    for(link2=warning->locations;link2;link2=link2->next)
ali@17
   475
	    {
ali@17
   476
		location=link2->data;
ali@17
   477
		if (location->line!=line || location->column!=column)
ali@17
   478
		    continue;
ali@17
   479
		counts[i]++;
ali@17
   480
		break;
ali@17
   481
	    }
ali@17
   482
	    if (link2)
ali@17
   483
		break;
ali@17
   484
	}
ali@17
   485
	if (!link)
ali@17
   486
	{
ali@17
   487
	    g_print("%s: FAIL\n",testcase->basename);
ali@17
   488
	    g_print("Unexpected warning from bookloupe:\n");
ali@17
   489
	    if (column)
ali@17
   490
		g_print("    Line %u column %u - %s\n",line,column,text);
ali@17
   491
	    else
ali@17
   492
		g_print("    Line %u - %s\n",line,text);
ali@17
   493
	    r=FALSE;
ali@17
   494
	    g_free(text);
ali@17
   495
	    break;
ali@17
   496
	}
ali@17
   497
	g_free(text);
ali@17
   498
    }
ali@17
   499
    count_false_positive=total_false_positive=0;
ali@17
   500
    count_false_negative=total_false_negative=0;
ali@17
   501
    for(link=testcase->warnings,i=0;r && link;link=link->next,i++)
ali@17
   502
    {
ali@17
   503
	warning=link->data;
ali@17
   504
	if (!counts[i] && warning->is_real && !warning->xfail)
ali@17
   505
	{
ali@17
   506
	    location=warning->locations->data;
ali@17
   507
	    g_print("%s: FAIL\n",testcase->basename);
ali@17
   508
	    g_print("Missing warning from bookloupe:\n");
ali@17
   509
	    if (location->column)
ali@17
   510
		g_print("    Line %u column %u - %s\n",location->line,
ali@17
   511
		  location->column,warning->text);
ali@17
   512
	    else
ali@17
   513
		g_print("    Line %u - %s\n",location->line,warning->text);
ali@17
   514
	    r=FALSE;
ali@17
   515
	    break;
ali@17
   516
	}
ali@17
   517
	else if (warning->xfail)
ali@17
   518
	{
ali@17
   519
	    if (warning->is_real)
ali@17
   520
	    {
ali@17
   521
		total_false_negative++;
ali@17
   522
		if (!counts[i])
ali@17
   523
		    count_false_negative++;
ali@17
   524
	    }
ali@17
   525
	    else if (!warning->is_real)
ali@17
   526
	    {
ali@17
   527
		total_false_positive++;
ali@17
   528
		if (counts[i])
ali@17
   529
		    count_false_positive++;
ali@17
   530
	    }
ali@17
   531
	}
ali@17
   532
    }
ali@17
   533
    g_free(counts);
ali@17
   534
    if (count_false_positive && count_false_negative)
ali@17
   535
	*xfail=g_strdup_printf(
ali@17
   536
	  "with %d of %d false positives and %d of %d false negatives",
ali@17
   537
	  count_false_positive,total_false_positive,
ali@17
   538
	  count_false_negative,total_false_negative);
ali@17
   539
    else if (count_false_positive)
ali@17
   540
	*xfail=g_strdup_printf("with %d of %d false positives",
ali@17
   541
	  count_false_positive,total_false_positive);
ali@17
   542
    else if (count_false_negative)
ali@17
   543
	*xfail=g_strdup_printf("with %d of %d false negatives",
ali@17
   544
	  count_false_negative,total_false_negative);
ali@17
   545
    return r;
ali@17
   546
}
ali@17
   547
ali@17
   548
/*
ali@0
   549
 * Run a testcase, returning FALSE on fail or error and
ali@0
   550
 * TRUE on pass or expected-fail.
ali@0
   551
 * Suitable message(s) will be printed in all cases.
ali@0
   552
 */
ali@6
   553
gboolean testcase_run(Testcase *testcase)
ali@0
   554
{
ali@6
   555
    gboolean r;
ali@7
   556
    size_t pos,offset;
ali@17
   557
    GString *header;
ali@102
   558
    char *filename,*s,*summary,*xfail=NULL;
ali@7
   559
    GError *error=NULL;
ali@9
   560
    if (!testcase_create_input_files(testcase,&error))
ali@9
   561
    {
ali@11
   562
	g_print("%s: FAIL\n",testcase->basename);
ali@11
   563
	g_print("%s\n",error->message);
ali@9
   564
	g_error_free(error);
ali@9
   565
	return FALSE;
ali@9
   566
    }
ali@102
   567
    r=testcase_spawn_bookloupe(testcase,&testcase->test_output,&error);
ali@7
   568
    if (!r)
ali@7
   569
    {
ali@11
   570
	g_print("%s: FAIL\n",testcase->basename);
ali@11
   571
	g_print("%s\n",error->message);
ali@7
   572
	g_error_free(error);
ali@9
   573
	(void)testcase_remove_input_files(testcase,NULL);
ali@9
   574
	return FALSE;
ali@9
   575
    }
ali@9
   576
    filename=testcase_resolve_input_files(testcase,"TEST-XXXXXX");
ali@9
   577
    if (!testcase_remove_input_files(testcase,&error))
ali@9
   578
    {
ali@11
   579
	g_print("%s: FAIL\n",testcase->basename);
ali@11
   580
	g_print("%s\n",error->message);
ali@9
   581
	g_error_free(error);
ali@0
   582
	return FALSE;
ali@0
   583
    }
ali@102
   584
    if (testcase->expected || testcase->warnings)
ali@0
   585
    {
ali@102
   586
	header=g_string_new("\n\nFile: ");
ali@102
   587
	g_string_append(header,filename);
ali@102
   588
	g_string_append(header,"\n");
ali@102
   589
	if (!g_str_has_prefix(testcase->test_output,header->str))
ali@7
   590
	{
ali@11
   591
	    g_print("%s: FAIL\n",testcase->basename);
ali@102
   592
	    g_print("Unexpected header from bookloupe:\n");
ali@102
   593
	    offset=common_prefix_length(testcase->test_output,header->str);
ali@102
   594
	    print_unexpected(testcase->test_output,offset);
ali@7
   595
	    r=FALSE;
ali@7
   596
	}
ali@102
   597
	summary=testcase->test_output+header->len;
ali@102
   598
	pos=header->len;
ali@102
   599
	if (r)
ali@102
   600
	{
ali@102
   601
	    /* Find the end of the summary */
ali@102
   602
	    s=strstr(summary,"\n\n");
ali@102
   603
	    if (s)
ali@102
   604
	    {
ali@102
   605
		summary=g_strndup(summary,s-summary);
ali@102
   606
		r=testcase_check_summary(testcase,summary);
ali@102
   607
		g_free(summary);
ali@102
   608
		pos=s-testcase->test_output+2;
ali@102
   609
	    }
ali@102
   610
	    else
ali@102
   611
	    {
ali@102
   612
		g_print("%s: FAIL\n",testcase->basename);
ali@102
   613
		g_print("Unterminated summary from bookloupe:\n%s\n",summary);
ali@102
   614
		r=FALSE;
ali@102
   615
	    }
ali@102
   616
	}
ali@102
   617
	g_string_free(header,TRUE);
ali@102
   618
	if (r)
ali@102
   619
	    r=testcase_check_warnings(testcase,testcase->test_output+pos,
ali@102
   620
	      &xfail);
ali@0
   621
    }
ali@102
   622
    if (!testcase_verify_output_files(testcase))
ali@102
   623
	r=FALSE;
ali@9
   624
    g_free(filename);
ali@7
   625
    if (r)
ali@17
   626
    {
ali@17
   627
	if (xfail)
ali@17
   628
	    g_print("%s: PASS (%s)\n",testcase->basename,xfail);
ali@17
   629
	else
ali@17
   630
	    g_print("%s: PASS\n",testcase->basename);
ali@17
   631
    }
ali@17
   632
    g_free(xfail);
ali@7
   633
    return r;
ali@0
   634
}
ali@0
   635
ali@0
   636
/*
ali@96
   637
 * Run a testcase, returning FALSE on error.
ali@96
   638
 * Bookloupe's output or a suitable error message will be shown.
ali@96
   639
 */
ali@96
   640
gboolean testcase_show_output(Testcase *testcase)
ali@96
   641
{
ali@96
   642
    gboolean r;
ali@96
   643
    gchar *output;
ali@96
   644
    GError *error=NULL;
ali@96
   645
    r=testcase_create_input_files(testcase,&error);
ali@96
   646
    if (r)
ali@96
   647
    {
ali@96
   648
	r&=testcase_spawn_bookloupe(testcase,&output,&error);
ali@96
   649
	r&=testcase_remove_input_files(testcase,&error);
ali@96
   650
    }
ali@96
   651
    if (r)
ali@96
   652
	g_print("%s",output);
ali@96
   653
    else
ali@96
   654
    {
ali@96
   655
	g_print("%s\n",error->message);
ali@96
   656
	g_error_free(error);
ali@96
   657
    }
ali@96
   658
    return r;
ali@96
   659
}
ali@96
   660
ali@96
   661
/*
ali@17
   662
 * Free a testcase warning.
ali@17
   663
 */
ali@17
   664
void testcase_warning_free(TestcaseWarning *warning)
ali@17
   665
{
ali@17
   666
    g_slist_foreach(warning->locations,(GFunc)g_free,NULL);
ali@17
   667
    g_slist_free(warning->locations);
ali@17
   668
    g_free(warning->text);
ali@17
   669
    g_free(warning);
ali@17
   670
}
ali@17
   671
ali@17
   672
/*
ali@0
   673
 * Free a testcase.
ali@0
   674
 */
ali@0
   675
void testcase_free(Testcase *testcase)
ali@0
   676
{
ali@6
   677
    g_free(testcase->basename);
ali@9
   678
    g_slist_foreach(testcase->inputs,(GFunc)testcase_input_free,NULL);
ali@9
   679
    g_slist_free(testcase->inputs);
ali@6
   680
    g_free(testcase->expected);
ali@17
   681
    g_slist_foreach(testcase->warnings,(GFunc)testcase_warning_free,NULL);
ali@17
   682
    g_slist_free(testcase->warnings);
ali@7
   683
    g_free(testcase->encoding);
ali@9
   684
    g_strfreev(testcase->options);
ali@102
   685
    g_free(testcase->test_output);
ali@6
   686
    g_free(testcase);
ali@0
   687
}