6 #include <glib/gstdio.h>
9 #include "testcaseinput.h"
10 #include "testcaseoutput.h"
12 GQuark testcase_error_quark(void)
14 return g_quark_from_static_string("testcase-error-quark");
18 * Return the length (in bytes) of any common prefix between s1 and s2.
19 * The returned length will always represent an exact number of characters.
21 size_t common_prefix_length(const char *s1,const char *s2)
27 c1=g_utf8_get_char(s1);
28 c2=g_utf8_get_char(s2);
31 s1=g_utf8_next_char(s1);
32 s2=g_utf8_next_char(s2);
37 void print_unexpected(const char *unexpected,gsize differs_at)
41 const char *endp,*bol,*s;
43 endp=strchr(unexpected+differs_at,'\n');
45 endp=unexpected+strlen(unexpected);
46 string=g_string_new_len(unexpected,endp-unexpected);
47 bol=strrchr(string->str,'\n');
54 endp=string->str+differs_at;
58 s=g_utf8_next_char(s);
61 else if (g_unichar_iswide(c))
63 else if (!g_unichar_iszerowidth(c))
66 g_print("%s\n%*s^\n",string->str,col,"");
67 g_string_free(string,TRUE);
71 * Create all the input files needed by a testcase and, if required,
72 * a temporary directory in which to store them.
74 gboolean testcase_create_input_files(Testcase *testcase,GError **error)
77 if (testcase->flags&TESTCASE_TMP_DIR)
79 testcase->tmpdir=g_strdup("TEST-XXXXXX");
80 if (!g_mkdtemp(testcase->tmpdir))
82 g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
83 "Failed to create temporary directory: %s",g_strerror(errno));
84 g_free(testcase->tmpdir);
85 testcase->tmpdir=NULL;
89 for(link=testcase->inputs;link;link=link->next)
90 if (!testcase_input_create(testcase,link->data,error))
92 for(link2=testcase->inputs;link2!=link;link2=link2->next)
93 (void)testcase_input_remove(testcase,link2->data,NULL);
96 (void)g_rmdir(testcase->tmpdir);
97 g_free(testcase->tmpdir);
98 testcase->tmpdir=NULL;
106 * Remove all the input files used by a testcase and, if created,
107 * the temporary directory in which they are stored.
109 gboolean testcase_remove_input_files(Testcase *testcase,GError **error)
112 GError *tmp_err=NULL;
113 gboolean retval=TRUE;
114 for(link=testcase->inputs;link;link=link->next)
115 if (!testcase_input_remove(testcase,link->data,&tmp_err))
117 if (error && !*error)
118 g_propagate_error(error,tmp_err);
120 g_clear_error(&tmp_err);
123 if (testcase->tmpdir)
125 if (g_rmdir(testcase->tmpdir))
127 if (error && !*error)
128 g_set_error(error,G_FILE_ERROR,g_file_error_from_errno(errno),
129 "Failed to remove temporary directory: %s",g_strerror(errno));
132 g_free(testcase->tmpdir);
133 testcase->tmpdir=NULL;
139 * Replace every occurance of an input file name in <str> with the
140 * filename which holds that input. For input files with fixed names,
141 * this is a noop. For input files which use the "XXXXXX" sequence
142 * to create a unique filename, the XXXXXX will be replaced with the
143 * 6 characters that were chosen to be unique.
145 char *testcase_resolve_input_files(Testcase *testcase,const char *str)
150 TestcaseInput *input;
151 GString *filename=g_string_new(str);
152 for(link=testcase->inputs;link;link=link->next)
155 if (!input->name_used)
157 g_warning("%s: Input file uninstantiated",input->name);
163 s=strstr(filename->str+offset,input->name);
167 g_string_overwrite(filename,pos,input->name_used);
168 offset=pos+strlen(input->name);
172 return g_string_free(filename,FALSE);
176 * Verify that all the output files specified by a testcase are present
177 * with the expected contents.
179 gboolean testcase_verify_output_files(Testcase *testcase)
182 GError *tmp_err=NULL;
183 gboolean retval=TRUE;
186 TestcaseOutput *output;
187 for(link=testcase->outputs;link;link=link->next)
190 if (!testcase_output_read(testcase,output,&contents,NULL,&tmp_err))
192 g_print("%s: FAIL\n",testcase->basename);
193 g_print("%s\n",tmp_err->message);
194 g_clear_error(&tmp_err);
200 if (strcmp(contents,output->contents))
202 g_print("%s: FAIL\n",testcase->basename);
203 offset=common_prefix_length(contents,output->contents);
204 if (!offset && !contents[offset])
205 g_print("%s: Unexpected empty output from bookloupe.\n",
209 g_print("%s: Unexpected output from bookloupe:\n",
211 print_unexpected(contents,offset);
219 for(link=testcase->outputs;link;link=link->next)
220 if (!testcase_output_remove(testcase,link->data,&tmp_err))
224 g_print("%s: FAIL\n",testcase->basename);
225 g_print("%s\n",tmp_err->message);
228 g_clear_error(&tmp_err);
233 gboolean testcase_spawn_bookloupe(Testcase *testcase,char **standard_output,
240 GError *tmp_err=NULL;
241 if (testcase->options)
242 argv=g_new(char *,g_strv_length(testcase->options)+3);
244 argv=g_new(char *,3);
245 s=getenv("BOOKLOUPE");
248 argv[0]=path_to_absolute(s);
249 for(i=0;testcase->options && testcase->options[i];i++)
250 argv[i+1]=testcase_resolve_input_files(testcase,testcase->options[i]);
251 argv[i+1]=testcase_resolve_input_files(testcase,"TEST-XXXXXX");
255 r=spawn_sync(testcase->tmpdir,argv,&s,&exit_status,error);
258 if (testcase->encoding)
260 output=g_convert(s,-1,"UTF-8",testcase->encoding,NULL,NULL,
265 g_propagate_prefixed_error(error,tmp_err,
266 "Conversion from %s failed: ",testcase->encoding);
273 if (!g_utf8_validate(s,-1,NULL))
275 g_set_error_literal(error,TESTCASE_ERROR,
276 TESTCASE_ERROR_FAILED,
277 "bookloupe output is not valid UTF-8");
285 r=spawn_sync(testcase->tmpdir,argv,NULL,&exit_status,error);
289 if (r && exit_status)
291 g_set_error(error,TESTCASE_ERROR,TESTCASE_ERROR_FAILED,
292 "bookloupe exited with code %d",exit_status);
295 if (r && standard_output)
296 *standard_output=output;
301 * Parse a warning of the form:
303 * <echoed line> (ignored)
304 * " Line " <number> [" column " <number>] " - " <text> "\n"
305 * If not specified, the column is returned as 0.
306 * Returns: the number of bytes parsed, or -1 on error.
308 static ssize_t testcase_parse_warning(Testcase *testcase,const char *output,
309 guint *line,guint *column,char **text)
314 if (output[offset]!='\n')
316 g_print("%s: FAIL\n",testcase->basename);
317 g_print("Unexpected output from bookloupe:\n");
318 print_unexpected(output,offset);
322 s=strchr(output+offset,'\n');
325 g_print("%s: FAIL\n",testcase->basename);
326 g_print("Missing new-line in output from bookloupe:\n");
327 print_unexpected(output,offset);
331 if (!g_str_has_prefix(output+offset," Line "))
333 g_print("%s: FAIL\n",testcase->basename);
334 g_print("Unexpected output from bookloupe:\n");
335 offset+=common_prefix_length(output+offset," Line ");
336 print_unexpected(output,offset);
340 tmp=g_ascii_strtoull(output+offset,&endp,10);
341 if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64)
343 g_print("%s: FAIL\n",testcase->basename);
344 g_print("Unexpected output from bookloupe:\n");
345 print_unexpected(output,offset);
350 if (g_str_has_prefix(output+offset," column "))
353 tmp=g_ascii_strtoull(output+offset,&endp,10);
354 if (tmp<1 || tmp>G_MAXUINT || tmp==G_MAXUINT64)
356 g_print("%s: FAIL\n",testcase->basename);
357 g_print("Unexpected output from bookloupe:\n");
358 print_unexpected(output,offset);
366 if (!g_str_has_prefix(output+offset," - "))
368 g_print("%s: FAIL\n",testcase->basename);
369 g_print("Unexpected output from bookloupe:\n");
370 offset+=common_prefix_length(output+offset," - ");
371 print_unexpected(output,offset);
375 s=strchr(output+offset,'\n');
378 g_print("%s: FAIL\n",testcase->basename);
379 g_print("Missing new-line in output from bookloupe:\n");
380 print_unexpected(output,offset);
383 *text=g_strndup(output+offset,s-(output+offset));
388 * Check the summary produced by bookloupe against testcase->summary.
390 static gboolean testcase_check_summary(Testcase *testcase,const char *summary)
396 if (!testcase->summary.texts)
398 texts=g_slist_copy(testcase->summary.texts);
399 lines=g_strsplit(summary,"\n",0);
400 for(i=0;lines[i];i++)
402 if (!g_str_has_prefix(lines[i]," --> "))
404 for(lnk=texts;lnk;lnk=lnk->next)
405 if (!strcmp(lines[i]+7,lnk->data))
407 texts=g_slist_delete_link(texts,lnk);
415 g_print("%s: FAIL\n",testcase->basename);
416 g_print("Missing summary text from bookloupe:\n");
417 g_print(" --> %s\n",texts->data);
424 * Check the warnings produced by bookloupe against either the
425 * unstructured testcase->expected or the structured testcase->warnings
428 static gboolean testcase_check_warnings(Testcase *testcase,const char *output,
434 int i,count_false_positive,count_false_negative;
435 int total_false_positive,total_false_negative;
437 guint *counts,line,column;
439 TestcaseWarning *warning;
440 TestcaseLocation *location;
442 if (testcase->expected)
444 if (strcmp(output,testcase->expected))
446 g_print("%s: FAIL\n",testcase->basename);
447 offset=common_prefix_length(output,testcase->expected);
448 if (!offset && !output[offset])
449 g_print("Unexpected zero warnings from bookloupe.\n");
452 g_print("Unexpected output from bookloupe:\n");
453 print_unexpected(output,offset);
459 counts=g_new0(guint,g_slist_length(testcase->warnings));
460 for(offset=0;output[offset];)
462 off=testcase_parse_warning(testcase,output+offset,&line,&column,&text);
469 for(link=testcase->warnings,i=0;link;link=link->next,i++)
472 if (strcmp(warning->text,text))
474 for(link2=warning->locations;link2;link2=link2->next)
476 location=link2->data;
477 if (location->line!=line || location->column!=column)
487 g_print("%s: FAIL\n",testcase->basename);
488 g_print("Unexpected warning from bookloupe:\n");
490 g_print(" Line %u column %u - %s\n",line,column,text);
492 g_print(" Line %u - %s\n",line,text);
499 count_false_positive=total_false_positive=0;
500 count_false_negative=total_false_negative=0;
501 for(link=testcase->warnings,i=0;r && link;link=link->next,i++)
504 if (!counts[i] && warning->is_real && !warning->xfail)
506 location=warning->locations->data;
507 g_print("%s: FAIL\n",testcase->basename);
508 g_print("Missing warning from bookloupe:\n");
509 if (location->column)
510 g_print(" Line %u column %u - %s\n",location->line,
511 location->column,warning->text);
513 g_print(" Line %u - %s\n",location->line,warning->text);
517 else if (warning->xfail)
519 if (warning->is_real)
521 total_false_negative++;
523 count_false_negative++;
525 else if (!warning->is_real)
527 total_false_positive++;
529 count_false_positive++;
534 if (count_false_positive && count_false_negative)
535 *xfail=g_strdup_printf(
536 "with %d of %d false positives and %d of %d false negatives",
537 count_false_positive,total_false_positive,
538 count_false_negative,total_false_negative);
539 else if (count_false_positive)
540 *xfail=g_strdup_printf("with %d of %d false positives",
541 count_false_positive,total_false_positive);
542 else if (count_false_negative)
543 *xfail=g_strdup_printf("with %d of %d false negatives",
544 count_false_negative,total_false_negative);
549 * Run a testcase, returning FALSE on fail or error and
550 * TRUE on pass or expected-fail.
551 * Suitable message(s) will be printed in all cases.
553 gboolean testcase_run(Testcase *testcase)
558 char *filename,*s,*summary,*xfail=NULL;
560 if (!testcase_create_input_files(testcase,&error))
562 g_print("%s: FAIL\n",testcase->basename);
563 g_print("%s\n",error->message);
567 r=testcase_spawn_bookloupe(testcase,&testcase->test_output,&error);
570 g_print("%s: FAIL\n",testcase->basename);
571 g_print("%s\n",error->message);
573 (void)testcase_remove_input_files(testcase,NULL);
576 filename=testcase_resolve_input_files(testcase,"TEST-XXXXXX");
577 if (!testcase_remove_input_files(testcase,&error))
579 g_print("%s: FAIL\n",testcase->basename);
580 g_print("%s\n",error->message);
584 if (testcase->expected || testcase->warnings)
586 header=g_string_new("\n\nFile: ");
587 g_string_append(header,filename);
588 g_string_append(header,"\n");
589 if (!g_str_has_prefix(testcase->test_output,header->str))
591 g_print("%s: FAIL\n",testcase->basename);
592 g_print("Unexpected header from bookloupe:\n");
593 offset=common_prefix_length(testcase->test_output,header->str);
594 print_unexpected(testcase->test_output,offset);
597 summary=testcase->test_output+header->len;
601 /* Find the end of the summary */
602 s=strstr(summary,"\n\n");
605 summary=g_strndup(summary,s-summary);
606 r=testcase_check_summary(testcase,summary);
608 pos=s-testcase->test_output+2;
612 g_print("%s: FAIL\n",testcase->basename);
613 g_print("Unterminated summary from bookloupe:\n%s\n",summary);
617 g_string_free(header,TRUE);
619 r=testcase_check_warnings(testcase,testcase->test_output+pos,
622 if (!testcase_verify_output_files(testcase))
628 g_print("%s: PASS (%s)\n",testcase->basename,xfail);
630 g_print("%s: PASS\n",testcase->basename);
637 * Run a testcase, returning FALSE on error.
638 * Bookloupe's output or a suitable error message will be shown.
640 gboolean testcase_show_output(Testcase *testcase)
645 r=testcase_create_input_files(testcase,&error);
648 r&=testcase_spawn_bookloupe(testcase,&output,&error);
649 r&=testcase_remove_input_files(testcase,&error);
652 g_print("%s",output);
655 g_print("%s\n",error->message);
662 * Free a testcase warning.
664 void testcase_warning_free(TestcaseWarning *warning)
666 g_slist_foreach(warning->locations,(GFunc)g_free,NULL);
667 g_slist_free(warning->locations);
668 g_free(warning->text);
675 void testcase_free(Testcase *testcase)
677 g_free(testcase->basename);
678 g_slist_foreach(testcase->inputs,(GFunc)testcase_input_free,NULL);
679 g_slist_free(testcase->inputs);
680 g_free(testcase->expected);
681 g_slist_foreach(testcase->warnings,(GFunc)testcase_warning_free,NULL);
682 g_slist_free(testcase->warnings);
683 g_free(testcase->encoding);
684 g_strfreev(testcase->options);
685 g_free(testcase->test_output);