/*************************************************************************/

/* bookloupe--check for assorted weirdnesses in a PG candidate text file */

/*									 */

/* Copyright 2000-2005 Jim Tinsley <jtinsley@pobox.com>			 */

/* Copyright 2012- J. Ali Harlow <ali@juiblex.co.uk>			 */

/*									 */

/* This program is free software; you can redistribute it and/or modify  */

/* it under the terms of the GNU General Public License as published by  */

/* the Free Software Foundation; either version 2 of the License, or     */

/* (at your option) any later version.					 */

/*									 */

/* This program is distributed in the hope that it will be useful,       */

/* but WITHOUT ANY WARRANTY; without even the implied warranty of	 */

/* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the		 */

/* GNU General Public License for more details.				 */

/*									 */

/* You should have received a copy of the GNU General Public License	 */

/* along with this program. If not, see <http://www.gnu.org/licenses/>.	 */

/*************************************************************************/

#include <stdio.h>

#include <stdlib.h>

#include <string.h>

#include <ctype.h>

#ifdef __WIN32__

#include <windows.h>

#endif

#include <glib.h>

#include <bl/bl.h>

#include "bookloupe.h"

#include "counters.h"

#include "pending.h"

#include "HTMLentities.h"

gchar *prevline;

/* Common typos. */

char *typo[] = {

    "teh", "th", "og", "fi", "ro", "adn", "yuo", "ot", "fo", "thet", "ane",

    "nad", "te", "ig", "acn",  "ahve", "alot", "anbd", "andt", "awya", "aywa",

    "bakc", "om", "btu", "byt", "cna", "cxan", "coudl", "dont", "didnt",

    "couldnt", "wouldnt", "doesnt", "shouldnt", "doign", "ehr", "hmi", "hse",

    "esle", "eyt", "fitrs", "firts", "foudn", "frmo", "fromt", "fwe", "gaurd",

    "gerat", "goign", "gruop", "haev", "hda", "hearign", "seeign", "sayign",

    "herat", "hge", "hsa", "hsi", "hte", "htere", "htese", "htey", "htis",

    "hvae", "hwich", "idae", "ihs", "iits", "int", "iwll", "iwth", "jsut",

    "loev", "sefl", "myu", "nkow", "nver", "nwe", "nwo", "ocur", "ohter",

    "omre", "onyl", "otehr", "otu", "owrk", "owuld", "peice", "peices",

    "peolpe", "peopel", "perhasp", "perhpas", "pleasent", "poeple", "porblem",

    "porblems", "rwite", "saidt", "saidh", "saids", "seh", "smae", "smoe",

    "sohw", "stnad", "stopry", "stoyr", "stpo", "tahn", "taht", "tath",

    "tehy", "tghe", "tghis", "theri", "theyll", "thgat", "thge", "thier",

    "thna", "thne", "thnig", "thnigs", "thsi", "thsoe", "thta", "timne",

    "tirne", "tkae", "tthe", "tyhat", "tyhe", "veyr", "vou", "vour", "vrey",

    "waht", "wasnt", "awtn", "watn", "wehn", "whic", "whcih", "whihc", "whta",

    "wihch", "wief", "wiht", "witha", "wiull", "wnat", "wnated", "wnats",

    "woh", "wohle", "wokr", "woudl", "wriet", "wrod", "wroet", "wroking",

    "wtih", "wuould", "wya", "yera", "yeras", "yersa", "yoiu", "youve",

    "ytou", "yuor", "abead", "ahle", "ahout", "ahove", "altbough", "balf",

    "bardly", "bas", "bave", "baving", "bebind", "beld", "belp", "belped",

    "ber", "bere", "bim", "bis", "bome", "bouse", "bowever", "buge",

    "dehates", "deht", "han", "hecause", "hecome", "heen", "hefore", "hegan",

    "hegin", "heing", "helieve", "henefit", "hetter", "hetween", "heyond",

    "hig", "higber", "huild", "huy", "hy", "jobn", "joh", "meanwbile",

    "memher", "memhers", "numher", "numhers", "perbaps", "prohlem", "puhlic",

    "witbout", "arn", "hin", "hirn", "wrok", "wroked", "amd", "aud",

    "prornise", "prornised", "modem", "bo", "heside", "chapteb", "chaptee",

    "se", ""

};

GTree *usertypo;

/* Common abbreviations and other OK words not to query as typos. */

char *okword[] = {

    "mr", "mrs", "mss", "mssrs", "ft", "pm", "st", "dr", "hmm", "h'm", "hmmm",

    "rd", "sh", "br", "pp", "hm", "cf", "jr", "sr", "vs", "lb", "lbs", "ltd",

    "pompeii","hawaii","hawaiian", "hotbed", "heartbeat", "heartbeats",

    "outbid", "outbids", "frostbite", "frostbitten", ""

};

/* Common abbreviations that cause otherwise unexplained periods. */

char *abbrev[] = {

    "cent", "cents", "viz", "vol", "vols", "vid", "ed", "al", "etc", "op",

    "cit", "deg", "min", "chap", "oz", "mme", "mlle", "mssrs", ""

};

/*

 * Two-Letter combinations that rarely if ever start words,

 * but are common scannos or otherwise common letter combinations.

 */

char *nostart[] = {

    "hr", "hl", "cb", "sb", "tb", "wb", "tl", "tn", "rn", "lt", "tj", ""

};

/*

 * Two-Letter combinations that rarely if ever end words,

 * but are common scannos or otherwise common letter combinations.

 */

char *noend[] = {

    "cb", "gb", "pb", "sb", "tb", "wh", "fr", "br", "qu", "tw", "gl", "fl",

    "sw", "gr", "sl", "cl", "iy", ""

};

char *markup[] = {

    "a", "b", "big", "blockquote", "body", "br", "center", "col", "div", "em",

    "font", "h1", "h2", "h3", "h4", "h5", "h6", "head", "hr", "html", "i",

    "img", "li", "meta", "ol", "p", "pre", "small", "span", "strong", "sub",

    "sup", "table", "td", "tfoot", "thead", "title", "tr", "tt", "u", "ul", ""

};

char *DPmarkup[] = {

    "<sc>", "</sc>", "/*", "*/", "/#", "#/", "/$", "$/", "<tb>", ""

};

char *nocomma[] = {

    "the", "it's", "their", "an", "mrs", "a", "our", "that's", "its", "whose",

    "every", "i'll", "your", "my", "mr", "mrs", "mss", "mssrs", "ft", "pm",

    "st", "dr", "rd", "pp", "cf", "jr", "sr", "vs", "lb", "lbs", "ltd", "i'm",

    "during", "let", "toward", "among", ""

};

char *noperiod[] = {

    "every", "i'm", "during", "that's", "their", "your", "our", "my", "or",

    "and", "but", "as", "if", "the", "its", "it's", "until", "than", "whether",

    "i'll", "whose", "who", "because", "when", "let", "till", "very", "an",

    "among", "those", "into", "whom", "having", "thence", ""

}; 

gboolean pswit[SWITNO];  /* program switches */

static GOptionEntry options[]={

    { "dp", 'd', 0, G_OPTION_ARG_NONE, pswit+DP_SWITCH,

      "Ignore DP-specific markup", NULL },

    { "noecho", 'e', 0, G_OPTION_ARG_NONE, pswit+ECHO_SWITCH,

      "Don't echo queried line", NULL },

    { "squote", 's', 0, G_OPTION_ARG_NONE, pswit+SQUOTE_SWITCH,

      "Check single quotes", NULL },

    { "typo", 't', 0, G_OPTION_ARG_NONE, pswit+TYPO_SWITCH,

      "Check common typos", NULL },

    { "qpara", 'p', 0, G_OPTION_ARG_NONE, pswit+QPARA_SWITCH,

      "Require closure of quotes on every paragraph", NULL },

    { "relaxed", 'x', 0, G_OPTION_ARG_NONE, pswit+PARANOID_SWITCH,

      "Disable paranoid querying of everything", NULL },

    { "line-end", 'l', 0, G_OPTION_ARG_NONE, pswit+LINE_END_SWITCH,

      "Disable line end checking", NULL },

    { "overview", 'o', 0, G_OPTION_ARG_NONE, pswit+OVERVIEW_SWITCH,

      "Overview: just show counts", NULL },

    { "stdout", 'y', 0, G_OPTION_ARG_NONE, pswit+STDOUT_SWITCH,

      "Output errors to stdout instead of stderr", NULL },

    { "header", 'h', 0, G_OPTION_ARG_NONE, pswit+HEADER_SWITCH,

      "Echo header fields", NULL },

    { "markup", 'm', 0, G_OPTION_ARG_NONE, pswit+MARKUP_SWITCH,

      "Ignore markup in < >", NULL },

    { "usertypo", 'u', 0, G_OPTION_ARG_NONE, pswit+USERTYPO_SWITCH,

      "Use file of user-defined typos", NULL },

    { "web", 'w', 0, G_OPTION_ARG_NONE, pswit+WEB_SWITCH,

      "Defaults for use on www upload", NULL },

    { "verbose", 'v', 0, G_OPTION_ARG_NONE, pswit+VERBOSE_SWITCH,

      "Verbose - list everything", NULL },

    { NULL }

};

long cnt_dquot;		/* for overview mode, count of doublequote queries */

long cnt_squot;		/* for overview mode, count of singlequote queries */

long cnt_brack;		/* for overview mode, count of brackets queries */

long cnt_bin;		/* for overview mode, count of non-ASCII queries */

long cnt_odd;		/* for overview mode, count of odd character queries */

long cnt_long;		/* for overview mode, count of long line errors */

long cnt_short;		/* for overview mode, count of short line queries */

long cnt_punct;		/* for overview mode,

			   count of punctuation and spacing queries */

long cnt_dash;		/* for overview mode, count of dash-related queries */

long cnt_word;		/* for overview mode, count of word queries */

long cnt_html;		/* for overview mode, count of html queries */

long cnt_lineend;	/* for overview mode, count of line-end queries */

long cnt_spacend;	/* count of lines with space at end */

long linecnt;		/* count of total lines in the file */

long checked_linecnt;	/* count of lines actually checked */

void proghelp(GOptionContext *context);

void procfile(const char *);

gchar *running_from;

gboolean mixdigit(const char *);

gchar *getaword(const char **);

char *flgets(char **,long);

void postprocess_for_HTML(char *);

char *linehasmarkup(char *);

char *losemarkup(char *);

gboolean tagcomp(const char *,const char *);

void loseentities(char *);

gboolean isroman(const char *);

void postprocess_for_DP(char *);

void print_as_windows_1252(const char *string);

void print_as_utf_8(const char *string);

GTree *qword,*qperiod;

#ifdef __WIN32__

UINT saved_cp;

#endif

void parse_options(int *argc,char ***argv)

{

    GError *err=NULL;

    GOptionContext *context;

    context=g_option_context_new(

      "file - looks for errors in Project Gutenberg(TM) etexts");

    g_option_context_add_main_entries(context,options,NULL);

    if (!g_option_context_parse(context,argc,argv,&err))

    {

	g_printerr("Bookloupe: %s\n",err->message);

	g_printerr("Use \"%s --help\" for help\n",(*argv)[0]);

	exit(1);

    }

    /* Paranoid checking is turned OFF, not on, by its switch */

    pswit[PARANOID_SWITCH]=!pswit[PARANOID_SWITCH];

    if (pswit[PARANOID_SWITCH])

	/* if running in paranoid mode, typo checks default to enabled */

	pswit[TYPO_SWITCH]=!pswit[TYPO_SWITCH];

    /* Line-end checking is turned OFF, not on, by its switch */

    pswit[LINE_END_SWITCH]=!pswit[LINE_END_SWITCH];

    /* Echoing is turned OFF, not on, by its switch */

    pswit[ECHO_SWITCH]=!pswit[ECHO_SWITCH];

    if (pswit[OVERVIEW_SWITCH])

	/* just print summary; don't echo */

	pswit[ECHO_SWITCH]=FALSE;

    /*

     * Web uploads - for the moment, this is really just a placeholder

     * until we decide what processing we really want to do on web uploads

     */

    if (pswit[WEB_SWITCH])

    {

	/* specific override for web uploads */

	pswit[ECHO_SWITCH]=TRUE;

	pswit[SQUOTE_SWITCH]=FALSE;

	pswit[TYPO_SWITCH]=TRUE;

	pswit[QPARA_SWITCH]=FALSE;

	pswit[PARANOID_SWITCH]=TRUE;

	pswit[LINE_END_SWITCH]=FALSE;

	pswit[OVERVIEW_SWITCH]=FALSE;

	pswit[STDOUT_SWITCH]=FALSE;

	pswit[HEADER_SWITCH]=TRUE;

	pswit[VERBOSE_SWITCH]=FALSE;

	pswit[MARKUP_SWITCH]=FALSE;

	pswit[USERTYPO_SWITCH]=FALSE;

	pswit[DP_SWITCH]=FALSE;

    }

    if (*argc<2)

    {

	proghelp(context);

	exit(1);

    }

    g_option_context_free(context);

}

/*

 * read_user_scannos:

 *

 * Read in the user-defined stealth scanno list.

 */

void read_user_scannos(void)

{

    GError *err=NULL;

    gchar *usertypo_file;

    gboolean okay;

    int i;

    gsize len,nb;

    gchar *contents,*utf8,**lines;

    usertypo_file=g_strdup("bookloupe.typ");

    okay=file_get_contents_text(usertypo_file,&contents,&len,&err);

    if (g_error_matches(err,G_FILE_ERROR,G_FILE_ERROR_NOENT))

    {

	g_clear_error(&err);

	g_free(usertypo_file);

	usertypo_file=g_build_filename(running_from,"bookloupe.typ",NULL);

	okay=file_get_contents_text(usertypo_file,&contents,&len,&err);

    }

    if (g_error_matches(err,G_FILE_ERROR,G_FILE_ERROR_NOENT))

    {

	g_clear_error(&err);

	g_free(usertypo_file);

	usertypo_file=g_strdup("gutcheck.typ");

	okay=file_get_contents_text(usertypo_file,&contents,&len,&err);

    }

    if (g_error_matches(err,G_FILE_ERROR,G_FILE_ERROR_NOENT))

    {

	g_clear_error(&err);

	g_free(usertypo_file);

	usertypo_file=g_build_filename(running_from,"gutcheck.typ",NULL);

	okay=file_get_contents_text(usertypo_file,&contents,&len,&err);

    }

    if (g_error_matches(err,G_FILE_ERROR,G_FILE_ERROR_NOENT))

    {

	g_free(usertypo_file);

	g_print("   --> I couldn't find bookloupe.typ "

	  "-- proceeding without user typos.\n");

	return;

    }

    else if (!okay)

    {

	fprintf(stderr,"%s: %s\n",usertypo_file,err->message);

	g_free(usertypo_file);

	g_clear_error(&err);

	exit(1);

    }

    if (g_utf8_validate(contents,len,NULL))

	utf8=g_utf8_normalize(contents,len,G_NORMALIZE_DEFAULT_COMPOSE);

    else

	utf8=g_convert(contents,len,"UTF-8","WINDOWS-1252",NULL,&nb,NULL);

    g_free(contents);

    lines=g_strsplit_set(utf8,"\r\n",0);

    g_free(utf8);

    usertypo=g_tree_new_full((GCompareDataFunc)strcmp,NULL,g_free,NULL);

    for (i=0;lines[i];i++)

	if (*(unsigned char *)lines[i]>'!')

	    g_tree_insert(usertypo,lines[i],GINT_TO_POINTER(1));

	else

	    g_free(lines[i]);

    g_free(lines);

}

/*

 * read_etext:

 *

 * Read an etext returning a newly allocated string containing the file

 * contents or NULL on error.

 */

gchar *read_etext(const char *filename,GError **err)

{

    GError *tmp_err=NULL;

    gchar *contents,*utf8;

    gsize len,bytes_read,bytes_written;

    int i,line,col;

    if (!g_file_get_contents(filename,&contents,&len,err))

	return NULL;

    if (g_utf8_validate(contents,len,NULL))

    {

	utf8=g_utf8_normalize(contents,len,G_NORMALIZE_DEFAULT_COMPOSE);

	g_set_print_handler(print_as_utf_8);

#ifdef __WIN32__

	SetConsoleOutputCP(CP_UTF8);

#endif

    }

    else

    {

	utf8=g_convert(contents,len,"UTF-8","WINDOWS-1252",&bytes_read,

	  &bytes_written,&tmp_err);

	if (g_error_matches(tmp_err,G_CONVERT_ERROR,

	  G_CONVERT_ERROR_ILLEGAL_SEQUENCE))

	{

	    line=col=1;

	    for(i=0;i<bytes_read;i++)

		if (contents[i]=='\n')

		{

		    line++;

		    col=1;

		}

		else if (contents[i]!='\r')

		    col++;

	    g_set_error(err,G_CONVERT_ERROR,G_CONVERT_ERROR_ILLEGAL_SEQUENCE,

	      "Input conversion failed. Byte %d at line %d, column %d is not a "

	      "valid Windows-1252 character",

	      ((unsigned char *)contents)[bytes_read],line,col);

	}

	else if (tmp_err)

	    g_propagate_error(err,tmp_err);

	g_set_print_handler(print_as_windows_1252);

#ifdef __WIN32__

	SetConsoleOutputCP(1252);

#endif

    }

    g_free(contents);

    return utf8;

}

void cleanup_on_exit(void)

{

#ifdef __WIN32__

    SetConsoleOutputCP(saved_cp);

#endif

}

int main(int argc,char **argv)

{

#ifdef __WIN32__

    atexit(cleanup_on_exit);

    saved_cp=GetConsoleOutputCP();

#endif

    running_from=g_path_get_dirname(argv[0]);

    parse_options(&argc,&argv);

    if (pswit[USERTYPO_SWITCH])

	read_user_scannos();

    fprintf(stderr,"bookloupe: Check and report on an e-text\n");

    procfile(argv[1]);

    if (pswit[OVERVIEW_SWITCH])

    {

	g_print("    Checked %ld lines of %ld (head+foot = %ld)\n\n",

	  checked_linecnt,linecnt,linecnt-checked_linecnt);

	g_print("    --------------- Queries found --------------\n");

	if (cnt_long)

	    g_print("    Long lines:		    %14ld\n",cnt_long);

	if (cnt_short)

	    g_print("    Short lines:		   %14ld\n",cnt_short);

	if (cnt_lineend)

	    g_print("    Line-end problems:	     %14ld\n",cnt_lineend);

	if (cnt_word)

	    g_print("    Common typos:		  %14ld\n",cnt_word);

	if (cnt_dquot)

	    g_print("    Unmatched quotes:	      %14ld\n",cnt_dquot);

	if (cnt_squot)

	    g_print("    Unmatched SingleQuotes:	%14ld\n",cnt_squot);

	if (cnt_brack)

	    g_print("    Unmatched brackets:	    %14ld\n",cnt_brack);

	if (cnt_bin)

	    g_print("    Non-ASCII characters:	  %14ld\n",cnt_bin);

	if (cnt_odd)

	    g_print("    Proofing characters:	   %14ld\n",cnt_odd);

	if (cnt_punct)

	    g_print("    Punctuation & spacing queries: %14ld\n",cnt_punct);

	if (cnt_dash)

	    g_print("    Non-standard dashes:	   %14ld\n",cnt_dash);

	if (cnt_html)

	    g_print("    Possible HTML tags:	    %14ld\n",cnt_html);

	g_print("\n");

	g_print("    TOTAL QUERIES		  %14ld\n",

	  cnt_dquot+cnt_squot+cnt_brack+cnt_bin+cnt_odd+cnt_long+

	  cnt_short+cnt_punct+cnt_dash+cnt_word+cnt_html+cnt_lineend);

    }

    g_free(running_from);

    if (usertypo)

	g_tree_unref(usertypo);

    return 0;

}

/*

 * first_pass:

 *

 * Run a first pass - verify that it's a valid PG

 * file, decide whether to report some things that

 * occur many times in the text like long or short

 * lines, non-standard dashes, etc.

 */

struct first_pass_results *first_pass(const char *etext)

{

    gunichar laststart=CHAR_SPACE;

    const char *s;

    gchar *lc_line;

    int i,j,lbytes,llen;

    gchar **lines;

    unsigned int lastlen=0,lastblen=0;

    long spline=0,nspline=0;

    static struct first_pass_results results={0};

    gchar *inword;

    lines=g_strsplit(etext,"\n",0);

    for (j=0;lines[j];j++)

    {

	lbytes=strlen(lines[j]);

	while (lbytes>0 && lines[j][lbytes-1]=='\r')

	    lines[j][--lbytes]='\0';

	llen=g_utf8_strlen(lines[j],lbytes);

	linecnt++;

	if (strstr(lines[j],"*END") && strstr(lines[j],"SMALL PRINT") &&

	  (strstr(lines[j],"PUBLIC DOMAIN") || strstr(lines[j],"COPYRIGHT")))

	{

	    if (spline)

		g_print("   --> Duplicate header?\n");

	    spline=linecnt+1;   /* first line of non-header text, that is */

	}

	if (!strncmp(lines[j],"*** START",9) &&

	  strstr(lines[j],"PROJECT GUTENBERG"))

	{

	    if (nspline)

		g_print("   --> Duplicate header?\n");

	    nspline=linecnt+1;   /* first line of non-header text, that is */

	}

	if (spline || nspline)

	{

	    lc_line=g_utf8_strdown(lines[j],lbytes);

	    if (strstr(lc_line,"end") && strstr(lc_line,"project gutenberg"))

	    {

		if (strstr(lc_line,"end")<strstr(lc_line,"project gutenberg"))

		{

		    if (results.footerline)

		    {

			/* it's an old-form header - we can detect duplicates */

			if (!nspline)

			    g_print("   --> Duplicate footer?\n");

		    }

		    else

			results.footerline=linecnt;

		}

	    }

	    g_free(lc_line);

	}

	if (spline)

	    results.firstline=spline;

	if (nspline)

	    results.firstline=nspline;  /* override with new */

	if (results.footerline)

	    continue;    /* don't count the boilerplate in the footer */

	results.totlen+=llen;

	for (s=lines[j];*s;s=g_utf8_next_char(s))

	{

	    if (g_utf8_get_char(s)>127)

		results.binlen++;

	    if (g_unichar_isalpha(g_utf8_get_char(s)))

		results.alphalen++;

	    if (s>lines[j] && g_utf8_get_char(s)==CHAR_DQUOTE &&

	      isalpha(g_utf8_get_char(g_utf8_prev_char(s))))

		results.endquote_count++;

	}

	if (llen>2 && lastlen>2 && lastlen<SHORTEST_PG_LINE && lastblen>2 &&

	  lastblen>SHORTEST_PG_LINE && laststart!=CHAR_SPACE)

	    results.shortline++;

	if (lbytes>0 &&

	  g_utf8_get_char(g_utf8_prev_char(lines[j]+lbytes))<=CHAR_SPACE)

	    cnt_spacend++;

	if (strstr(lines[j],".,"))

	    results.dotcomma++;

	/* only count ast lines for ignoring purposes where there is */

	/* locase text on the line */

	if (strchr(lines[j],'*'))

	{

	    for (s=lines[j];*s;s=g_utf8_next_char(s))

		if (g_unichar_islower(g_utf8_get_char(s)))

		    break;

	    if (*s)

		results.astline++;

	}

	if (strchr(lines[j],'/'))

	    results.fslashline++;

	if (lbytes>0)

	{

	    for (s=g_utf8_prev_char(lines[j]+lbytes);

	      s>lines[j] && g_utf8_get_char(s)<=CHAR_SPACE;

	      s=g_utf8_prev_char(s))

		;

	    if (s>g_utf8_next_char(lines[j]) && g_utf8_get_char(s)=='-' &&

	      g_utf8_get_char(g_utf8_prev_char(s))!='-')

		results.hyphens++;

	}

	if (llen>LONGEST_PG_LINE)

	    results.longline++;

	if (llen>WAY_TOO_LONG)

	    results.verylongline++;

	if (strchr(lines[j],'<') && strchr(lines[j],'>'))

	{

	    i=(int)(strchr(lines[j],'>')-strchr(lines[j],'<')+1);

	    if (i>0)

		results.htmcount++;

	    if (strstr(lines[j],"<i>"))

		results.htmcount+=4; /* bonus marks! */

	}

	/* Check for spaced em-dashes */

	if (lines[j][0] && (s=strstr(g_utf8_next_char(lines[j]),"--")))

	{

	    results.emdash++;

	    if (s[-1]==CHAR_SPACE || s[2]==CHAR_SPACE)

		results.space_emdash++;

	    if (s[-1]==CHAR_SPACE && s[2]==CHAR_SPACE)

		/* count of em-dashes with spaces both sides */

		results.non_PG_space_emdash++;

	    if (s[-1]!=CHAR_SPACE && s[2]!=CHAR_SPACE)

		/* count of PG-type em-dashes with no spaces */

		results.PG_space_emdash++;

	}

	for (s=lines[j];*s;)

	{

	    inword=getaword(&s);

	    if (!strcmp(inword,"hij") || !strcmp(inword,"niet")) 

		results.Dutchcount++;

	    if (!strcmp(inword,"dans") || !strcmp(inword,"avec")) 

		results.Frenchcount++;

	    if (!strcmp(inword,"0") || !strcmp(inword,"1")) 

		results.standalone_digit++;

	    g_free(inword);

	}

	/* Check for spaced dashes */

	if (strstr(lines[j]," -") && *(strstr(lines[j]," -")+2)!='-')

	    results.spacedash++;

	lastblen=lastlen;

	lastlen=llen;

	laststart=lines[j][0];

    }

    g_strfreev(lines);

    return &results;

}

/*

 * report_first_pass:

 *

 * Make some snap decisions based on the first pass results.

 */

struct warnings *report_first_pass(struct first_pass_results *results)

{

    static struct warnings warnings={0};

    if (cnt_spacend>0)

	g_print("   --> %ld lines in this file have white space at end\n",

	  cnt_spacend);

    warnings.dotcomma=1;

    if (results->dotcomma>5)

    {

	warnings.dotcomma=0;

	g_print("   --> %ld lines in this file contain '.,'. "

	  "Not reporting them.\n",results->dotcomma);

    }

    /*

     * If more than 50 lines, or one-tenth, are short,

     * don't bother reporting them.

     */

    warnings.shortline=1;

    if (results->shortline>50 || results->shortline*10>linecnt)

    {

	warnings.shortline=0;

	g_print("   --> %ld lines in this file are short. "

	  "Not reporting short lines.\n",results->shortline);

    }

    /*

     * If more than 50 lines, or one-tenth, are long,

     * don't bother reporting them.

     */

    warnings.longline=1;

    if (results->longline>50 || results->longline*10>linecnt)

    {

	warnings.longline=0;

	g_print("   --> %ld lines in this file are long. "

	  "Not reporting long lines.\n",results->longline);

    }

    /* If more than 10 lines contain asterisks, don't bother reporting them. */

    warnings.ast=1;

    if (results->astline>10)

    {

	warnings.ast=0;

	g_print("   --> %ld lines in this file contain asterisks. "

	  "Not reporting them.\n",results->astline);

    }

    /*

     * If more than 10 lines contain forward slashes,

     * don't bother reporting them.

     */

    warnings.fslash=1;

    if (results->fslashline>10)

    {

	warnings.fslash=0;

	g_print("   --> %ld lines in this file contain forward slashes. "

	  "Not reporting them.\n",results->fslashline);

    }

    /*

     * If more than 20 lines contain unpunctuated endquotes,

     * don't bother reporting them.

     */

    warnings.endquote=1;

    if (results->endquote_count>20)

    {

	warnings.endquote=0;

	g_print("   --> %ld lines in this file contain unpunctuated endquotes. "

	  "Not reporting them.\n",results->endquote_count);

    }

    /*

     * If more than 15 lines contain standalone digits,

     * don't bother reporting them.

     */

    warnings.digit=1;

    if (results->standalone_digit>10)

    {

	warnings.digit=0;

	g_print("   --> %ld lines in this file contain standalone 0s and 1s. "

	  "Not reporting them.\n",results->standalone_digit);

    }

    /*

     * If more than 20 lines contain hyphens at end,

     * don't bother reporting them.

     */

    warnings.hyphen=1;

    if (results->hyphens>20)

    {

	warnings.hyphen=0;

	g_print("   --> %ld lines in this file have hyphens at end. "

	  "Not reporting them.\n",results->hyphens);

    }

    if (results->htmcount>20 && !pswit[MARKUP_SWITCH])

    {

	g_print("   --> Looks like this is HTML. Switching HTML mode ON.\n");

	pswit[MARKUP_SWITCH]=1;

    }

    if (results->verylongline>0)

	g_print("   --> %ld lines in this file are VERY long!\n",

	  results->verylongline);

    /*

     * If there are more non-PG spaced dashes than PG em-dashes,

     * assume it's deliberate.

     * Current PG guidelines say don't use them, but older texts do,

     * and some people insist on them whatever the guidelines say.

     */

    warnings.dash=1;

    if (results->spacedash+results->non_PG_space_emdash>

      results->PG_space_emdash)

    {

	warnings.dash=0;

	g_print("   --> There are %ld spaced dashes and em-dashes. "

	  "Not reporting them.\n",

	  results->spacedash+results->non_PG_space_emdash);

    }

    /* If more than a quarter of characters are hi-bit, bug out. */

    warnings.bin=1;

    if (results->binlen*4>results->totlen)

    {

	g_print("   --> This file does not appear to be ASCII. "

	  "Terminating. Best of luck with it!\n");

	exit(1);

    }

    if (results->alphalen*4<results->totlen)

    {

	g_print("   --> This file does not appear to be text. "

	  "Terminating. Best of luck with it!\n");

	exit(1);

    }

    if (results->binlen*100>results->totlen || results->binlen>100)

    {

	g_print("   --> There are a lot of foreign letters here. "

	  "Not reporting them.\n");

	warnings.bin=0;

    }

    warnings.isDutch=FALSE;

    if (results->Dutchcount>50)

    {

	warnings.isDutch=TRUE;

	g_print("   --> This looks like Dutch - "

	  "switching off dashes and warnings for 's Middags case.\n");

    }

    warnings.isFrench=FALSE;

    if (results->Frenchcount>50)

    {

	warnings.isFrench=TRUE;

	g_print("   --> This looks like French - "

	  "switching off some doublepunct.\n");

    }

    if (results->firstline && results->footerline)

	g_print("    The PG header and footer appear to be already on.\n");

    else

    {

	if (results->firstline)

	    g_print("    The PG header is on - no footer.\n");

	if (results->footerline)

	    g_print("    The PG footer is on - no header.\n");

    }

    g_print("\n");

    if (pswit[VERBOSE_SWITCH])

    {

	warnings.bin=1;

	warnings.shortline=1;

	warnings.dotcomma=1;

	warnings.longline=1;

	warnings.dash=1;

	warnings.digit=1;

	warnings.ast=1;

	warnings.fslash=1;

	warnings.hyphen=1;

	warnings.endquote=1;

	g_print("   *** Verbose output is ON -- you asked for it! ***\n");

    }

    if (warnings.isDutch)

	warnings.dash=0;

    if (results->footerline>0 && results->firstline>0 &&

      results->footerline>results->firstline &&

      results->footerline-results->firstline<100)

    {

	g_print("   --> I don't really know where this text starts. \n");

	g_print("       There are no reference points.\n");

	g_print("       I'm going to have to report the header and footer "

	  "as well.\n");

	results->firstline=0;

    }

    return &warnings;

}

/*

 * analyse_quotes:

 *

 * Look along the line, accumulate the count of quotes, and see

 * if this is an empty line - i.e. a line with nothing on it

 * but spaces.

 * If line has just spaces, period, * and/or - on it, don't

 * count it, since empty lines with asterisks or dashes to

 * separate sections are common.

 *

 * Returns: TRUE if the line is empty.

 */

gboolean analyse_quotes(const char *aline,struct counters *counters)

{

    int guessquote=0;

    /* assume the line is empty until proven otherwise */

    gboolean isemptyline=TRUE;

    const char *s=aline,*sprev,*snext;

    gunichar c;

    sprev=NULL;

    while (*s)

    {

	snext=g_utf8_next_char(s);

	c=g_utf8_get_char(s);

	if (c==CHAR_DQUOTE)

	    counters->quot++;

	if (CHAR_IS_SQUOTE(c))

	{

	    if (s==aline)

	    {

		/*

		 * At start of line, it can only be an openquote.

		 * Hardcode a very common exception!

		 */

		if (!g_str_has_prefix(snext,"tis") &&

		  !g_str_has_prefix(snext,"Tis"))

		    increment_matching(counters,c,TRUE);

	    }

	    else if (g_unichar_isalpha(g_utf8_get_char(sprev)) &&

	      g_unichar_isalpha(g_utf8_get_char(snext)))

		/* Do nothing! it's definitely an apostrophe, not a quote */

		;

	    /* it's outside a word - let's check it out */

	    else if (c==CHAR_OPEN_SQUOTE || c==CHAR_LS_QUOTE ||

	      g_unichar_isalpha(g_utf8_get_char(snext)))

	    {

		/* it damwell better BE an openquote */

		if (!g_str_has_prefix(snext,"tis") &&

		  !g_str_has_prefix(snext,"Tis"))

		    /* hardcode a very common exception! */

		    increment_matching(counters,c,TRUE);

	    }

	    else

	    {

		/* now - is it a closequote? */

		guessquote=0;   /* accumulate clues */

		if (g_unichar_isalpha(g_utf8_get_char(sprev)))

		{

		    /* it follows a letter - could be either */

		    guessquote++;

		    if (g_utf8_get_char(sprev)=='s')

		    {

			/* looks like a plural apostrophe */

			guessquote-=3;

			if (g_utf8_get_char(snext)==CHAR_SPACE)

			    /* bonus marks! */

			    guessquote-=2;

		    }

		}

		/* it doesn't have a letter either side */

		else if (strchr(".?!,;:",g_utf8_get_char(sprev)) &&

		  strchr(".?!,;: ",g_utf8_get_char(snext)))

		    guessquote+=8; /* looks like a closequote */

		else

		    guessquote++;

		if (matching_difference(counters,CHAR_SQUOTE)>0)

		    /*

		     * Give it the benefit of some doubt,

		     * if a squote is already open.

		     */

		    guessquote++;

		else

		    guessquote--;

		if (guessquote>=0)

		    increment_matching(counters,c,FALSE);

	    }

	}

	if (c!=CHAR_SPACE && c!='-' && c!='.' && c!=CHAR_ASTERISK &&

	  c!='\r' && c!='\n')

	    isemptyline=FALSE;  /* ignore lines like  *  *  *  as spacers */

	if (c==CHAR_UNDERSCORE)

	    counters->c_unders++;

	if (c==CHAR_OPEN_SBRACK)

	{

	    if (!matching_difference(counters,COUNTER_ILLUSTRATION) &&

	      !matching_difference(counters,c) && s==aline &&

	      g_str_has_prefix(s,"[Illustration:"))

		increment_matching(counters,COUNTER_ILLUSTRATION,TRUE);

	    else

		increment_matching(counters,c,TRUE);

	}

	else if (c==CHAR_OPEN_CBRACK || c==CHAR_OPEN_RBRACK)

	    increment_matching(counters,c,TRUE);

	if (c==CHAR_CLOSE_SBRACK)

	{

	    if (!matching_count(counters,COUNTER_ILLUSTRATION,FALSE) &&

	      !matching_difference(counters,c) && !*snext)

		increment_matching(counters,COUNTER_ILLUSTRATION,FALSE);

	    else

		increment_matching(counters,c,FALSE);

	}

	else if (c==CHAR_CLOSE_CBRACK || c==CHAR_CLOSE_RBRACK)

	    increment_matching(counters,c,FALSE);

	sprev=s;

	s=snext;

    }

    return isemptyline;

}

/*

 * check_for_control_characters:

 *

 * Check for invalid or questionable characters in the line

 * Anything above 127 is invalid for plain ASCII, and

 * non-printable control characters should also be flagged.

 * Tabs should generally not be there.

 */

void check_for_control_characters(const char *aline)

{

    gunichar c;

    const char *s;

    for (s=aline;*s;s=g_utf8_next_char(s))

    {

	c=g_utf8_get_char(s);

	if (c<CHAR_SPACE && c!=CHAR_LF && c!=CHAR_CR && c!=CHAR_TAB)

	{

	    if (pswit[ECHO_SWITCH])

		g_print("\n%s\n",aline);

	    if (!pswit[OVERVIEW_SWITCH])

		g_print("    Line %ld column %ld - Control character %u\n",

		  linecnt,g_utf8_pointer_to_offset(s,aline)+1,c);

	    else

		cnt_bin++;

	}

    }

}

/*

 * check_for_odd_characters:

 *

 * Check for binary and other odd characters.

 */

void check_for_odd_characters(const char *aline,const struct warnings *warnings,

  gboolean isemptyline)

{

    /* Don't repeat multiple warnings on one line. */

    gboolean eNon_A=FALSE,eTab=FALSE,eTilde=FALSE;

    gboolean eCarat=FALSE,eFSlash=FALSE,eAst=FALSE;

    const char *s;

    gunichar c;

    for (s=aline;*s;s=g_utf8_next_char(s))

    {

	c=g_utf8_get_char(s);

	if (!eNon_A && (c<CHAR_SPACE && c!='\t' && c!='\n' || c>127))

	{

	    if (pswit[ECHO_SWITCH])

		g_print("\n%s\n",aline);

	    if (!pswit[OVERVIEW_SWITCH])

		if (c>127 && c<160 || c>255)

		    g_print("    Line %ld column %ld - "

		      "Non-ISO-8859 character %u\n",

		      linecnt,g_utf8_pointer_to_offset(aline,s)+1,c);

		else

		    g_print("    Line %ld column %ld - "

		      "Non-ASCII character %u\n",

		      linecnt,g_utf8_pointer_to_offset(aline,s)+1,c);

	    else

		cnt_bin++;

	    eNon_A=TRUE;

	}

	if (!eTab && c==CHAR_TAB)

	{

	    if (pswit[ECHO_SWITCH])

		g_print("\n%s\n",aline);

	    if (!pswit[OVERVIEW_SWITCH])

		g_print("    Line %ld column %ld - Tab character?\n",

		  linecnt,g_utf8_pointer_to_offset(aline,s)+1);

	    else

		cnt_odd++;

	    eTab=TRUE;

	}

	if (!eTilde && c==CHAR_TILDE)

	{

	    /*

	     * Often used by OCR software to indicate an

	     * unrecognizable character.

	     */

	    if (pswit[ECHO_SWITCH])

		g_print("\n%s\n",aline);

	    if (!pswit[OVERVIEW_SWITCH])

		g_print("    Line %ld column %ld - Tilde character?\n",

		  linecnt,g_utf8_pointer_to_offset(aline,s)+1);

	    else

		cnt_odd++;

	    eTilde=TRUE;

	}

	if (!eCarat && c==CHAR_CARAT)

	{  

	    if (pswit[ECHO_SWITCH])

		g_print("\n%s\n",aline);

	    if (!pswit[OVERVIEW_SWITCH])

		g_print("    Line %ld column %ld - Carat character?\n",

		  linecnt,g_utf8_pointer_to_offset(aline,s)+1);