/*************************************************************************/

/* bookloupe--check for assorted weirdnesses in a PG candidate text file */

/*									 */

/* Copyright 2000-2005 Jim Tinsley <jtinsley@pobox.com>			 */

/* Copyright 2012- J. Ali Harlow <ali@juiblex.co.uk>			 */

/*									 */

/* This program is free software; you can redistribute it and/or modify  */

/* it under the terms of the GNU General Public License as published by  */

/* the Free Software Foundation; either version 2 of the License, or     */

/* (at your option) any later version.					 */

/*									 */

/* This program is distributed in the hope that it will be useful,       */

/* but WITHOUT ANY WARRANTY; without even the implied warranty of	 */

/* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the		 */

/* GNU General Public License for more details.				 */

/*									 */

/* You should have received a copy of the GNU General Public License	 */

/* along with this program. If not, see <http://www.gnu.org/licenses/>.	 */

/*************************************************************************/

#include <stdio.h>

#include <stdlib.h>

#include <string.h>

#include <ctype.h>

#ifdef __WIN32__

#include <windows.h>

#endif

#include <glib.h>

#include <bl/bl.h>

#include "bookloupe.h"

#include "counters.h"

#include "pending.h"

#include "HTMLentities.h"

gchar *prevline;

/* Common typos. */

char *typo[] = {

    "teh", "th", "og", "fi", "ro", "adn", "yuo", "ot", "fo", "thet", "ane",

    "nad", "te", "ig", "acn",  "ahve", "alot", "anbd", "andt", "awya", "aywa",

    "bakc", "om", "btu", "byt", "cna", "cxan", "coudl", "dont", "didnt",

    "couldnt", "wouldnt", "doesnt", "shouldnt", "doign", "ehr", "hmi", "hse",

    "esle", "eyt", "fitrs", "firts", "foudn", "frmo", "fromt", "fwe", "gaurd",

    "gerat", "goign", "gruop", "haev", "hda", "hearign", "seeign", "sayign",

    "herat", "hge", "hsa", "hsi", "hte", "htere", "htese", "htey", "htis",

    "hvae", "hwich", "idae", "ihs", "iits", "int", "iwll", "iwth", "jsut",

    "loev", "sefl", "myu", "nkow", "nver", "nwe", "nwo", "ocur", "ohter",

    "omre", "onyl", "otehr", "otu", "owrk", "owuld", "peice", "peices",

    "peolpe", "peopel", "perhasp", "perhpas", "pleasent", "poeple", "porblem",

    "porblems", "rwite", "saidt", "saidh", "saids", "seh", "smae", "smoe",

    "sohw", "stnad", "stopry", "stoyr", "stpo", "tahn", "taht", "tath",

    "tehy", "tghe", "tghis", "theri", "theyll", "thgat", "thge", "thier",

    "thna", "thne", "thnig", "thnigs", "thsi", "thsoe", "thta", "timne",

    "tirne", "tkae", "tthe", "tyhat", "tyhe", "veyr", "vou", "vour", "vrey",

    "waht", "wasnt", "awtn", "watn", "wehn", "whic", "whcih", "whihc", "whta",

    "wihch", "wief", "wiht", "witha", "wiull", "wnat", "wnated", "wnats",

    "woh", "wohle", "wokr", "woudl", "wriet", "wrod", "wroet", "wroking",

    "wtih", "wuould", "wya", "yera", "yeras", "yersa", "yoiu", "youve",

    "ytou", "yuor", "abead", "ahle", "ahout", "ahove", "altbough", "balf",

    "bardly", "bas", "bave", "baving", "bebind", "beld", "belp", "belped",

    "ber", "bere", "bim", "bis", "bome", "bouse", "bowever", "buge",

    "dehates", "deht", "han", "hecause", "hecome", "heen", "hefore", "hegan",

    "hegin", "heing", "helieve", "henefit", "hetter", "hetween", "heyond",

    "hig", "higber", "huild", "huy", "hy", "jobn", "joh", "meanwbile",

    "memher", "memhers", "numher", "numhers", "perbaps", "prohlem", "puhlic",

    "witbout", "arn", "hin", "hirn", "wrok", "wroked", "amd", "aud",

    "prornise", "prornised", "modem", "bo", "heside", "chapteb", "chaptee",

    "se", ""

};

GTree *usertypo;

/* Common abbreviations and other OK words not to query as typos. */

char *okword[] = {

    "mr", "mrs", "mss", "mssrs", "ft", "pm", "st", "dr", "hmm", "h'm", "hmmm",

    "rd", "sh", "br", "pp", "hm", "cf", "jr", "sr", "vs", "lb", "lbs", "ltd",

    "pompeii","hawaii","hawaiian", "hotbed", "heartbeat", "heartbeats",

    "outbid", "outbids", "frostbite", "frostbitten", ""

};

/* Common abbreviations that cause otherwise unexplained periods. */

char *abbrev[] = {

    "cent", "cents", "viz", "vol", "vols", "vid", "ed", "al", "etc", "op",

    "cit", "deg", "min", "chap", "oz", "mme", "mlle", "mssrs", ""

};

/*

 * Two-Letter combinations that rarely if ever start words,

 * but are common scannos or otherwise common letter combinations.

 */

char *nostart[] = {

    "hr", "hl", "cb", "sb", "tb", "wb", "tl", "tn", "rn", "lt", "tj", ""

};

/*

 * Two-Letter combinations that rarely if ever end words,

 * but are common scannos or otherwise common letter combinations.

 */

char *noend[] = {

    "cb", "gb", "pb", "sb", "tb", "wh", "fr", "br", "qu", "tw", "gl", "fl",

    "sw", "gr", "sl", "cl", "iy", ""

};

char *markup[] = {

    "a", "b", "big", "blockquote", "body", "br", "center", "col", "div", "em",

    "font", "h1", "h2", "h3", "h4", "h5", "h6", "head", "hr", "html", "i",

    "img", "li", "meta", "ol", "p", "pre", "small", "span", "strong", "sub",

    "sup", "table", "td", "tfoot", "thead", "title", "tr", "tt", "u", "ul", ""

};

char *DPmarkup[] = {

    "<sc>", "</sc>", "/*", "*/", "/#", "#/", "/$", "$/", "<tb>", ""

};

char *nocomma[] = {

    "the", "it's", "their", "an", "mrs", "a", "our", "that's", "its", "whose",

    "every", "i'll", "your", "my", "mr", "mrs", "mss", "mssrs", "ft", "pm",

    "st", "dr", "rd", "pp", "cf", "jr", "sr", "vs", "lb", "lbs", "ltd", "i'm",

    "during", "let", "toward", "among", ""

};

char *noperiod[] = {

    "every", "i'm", "during", "that's", "their", "your", "our", "my", "or",

    "and", "but", "as", "if", "the", "its", "it's", "until", "than", "whether",

    "i'll", "whose", "who", "because", "when", "let", "till", "very", "an",

    "among", "those", "into", "whom", "having", "thence", ""

}; 

gboolean pswit[SWITNO];  /* program switches */

gboolean typo_compat,paranoid_compat;

static GOptionEntry options[]={

    { "dp", 'd', 0, G_OPTION_ARG_NONE, pswit+DP_SWITCH,

      "Ignore DP-specific markup", NULL },

    { "no-dp", 0, G_OPTION_FLAG_HIDDEN|G_OPTION_FLAG_REVERSE,

      G_OPTION_ARG_NONE, pswit+DP_SWITCH,

      "Don't ignore DP-specific markup", NULL },

    { "echo", 0, G_OPTION_FLAG_HIDDEN, G_OPTION_ARG_NONE, pswit+ECHO_SWITCH,

      "Echo queried line", NULL },

    { "no-echo", 'e', G_OPTION_FLAG_REVERSE,

      G_OPTION_ARG_NONE, pswit+ECHO_SWITCH,

      "Don't echo queried line", NULL },

    { "squote", 's', 0, G_OPTION_ARG_NONE, pswit+SQUOTE_SWITCH,

      "Check single quotes", NULL },

    { "no-squote", 0, G_OPTION_FLAG_HIDDEN|G_OPTION_FLAG_REVERSE,

      G_OPTION_ARG_NONE, pswit+SQUOTE_SWITCH,

      "Don't check single quotes", NULL },

    { "typo", 0, 0, G_OPTION_ARG_NONE, pswit+TYPO_SWITCH,

      "Check common typos", NULL },

    { "no-typo", 0, G_OPTION_FLAG_HIDDEN|G_OPTION_FLAG_REVERSE,

      G_OPTION_ARG_NONE, pswit+TYPO_SWITCH,

      "Don't check common typos", NULL },

    { "qpara", 'p', 0, G_OPTION_ARG_NONE, pswit+QPARA_SWITCH,

      "Require closure of quotes on every paragraph", NULL },

    { "no-qpara", 0, G_OPTION_FLAG_HIDDEN|G_OPTION_FLAG_REVERSE,

      G_OPTION_ARG_NONE, pswit+QPARA_SWITCH,

      "Don't require closure of quotes on every paragraph", NULL },

    { "paranoid", 0, G_OPTION_FLAG_HIDDEN,

      G_OPTION_ARG_NONE, pswit+PARANOID_SWITCH,

      "Enable paranoid querying of everything", NULL },

    { "no-paranoid", 0, G_OPTION_FLAG_REVERSE,

      G_OPTION_ARG_NONE, pswit+PARANOID_SWITCH,

      "Disable paranoid querying of everything", NULL },

    { "line-end", 0, G_OPTION_FLAG_HIDDEN,

      G_OPTION_ARG_NONE, pswit+LINE_END_SWITCH,

      "Enable line end checking", NULL },

    { "no-line-end", 'l', G_OPTION_FLAG_REVERSE,

      G_OPTION_ARG_NONE, pswit+LINE_END_SWITCH,

      "Diable line end checking", NULL },

    { "overview", 'o', 0, G_OPTION_ARG_NONE, pswit+OVERVIEW_SWITCH,

      "Overview: just show counts", NULL },

    { "no-overview", 0, G_OPTION_FLAG_HIDDEN|G_OPTION_FLAG_REVERSE,

      G_OPTION_ARG_NONE, pswit+OVERVIEW_SWITCH,

      "Show individual warnings", NULL },

    { "stdout", 'y', 0, G_OPTION_ARG_NONE, pswit+STDOUT_SWITCH,

      "Output errors to stdout instead of stderr", NULL },

    { "no-stdout", 0, G_OPTION_FLAG_HIDDEN|G_OPTION_FLAG_REVERSE,

      G_OPTION_ARG_NONE, pswit+STDOUT_SWITCH,

      "Output errors to stderr instead of stdout", NULL },

    { "header", 'h', 0, G_OPTION_ARG_NONE, pswit+HEADER_SWITCH,

      "Echo header fields", NULL },

    { "no-header", 0, G_OPTION_FLAG_HIDDEN|G_OPTION_FLAG_REVERSE,

      G_OPTION_ARG_NONE, pswit+HEADER_SWITCH,

      "Don't echo header fields", NULL },

    { "markup", 'm', 0, G_OPTION_ARG_NONE, pswit+MARKUP_SWITCH,

      "Ignore markup in < >", NULL },

    { "no-markup", 0, G_OPTION_FLAG_HIDDEN|G_OPTION_FLAG_REVERSE,

      G_OPTION_ARG_NONE, pswit+MARKUP_SWITCH,

      "No special handling for markup in < >", NULL },

    { "usertypo", 'u', 0, G_OPTION_ARG_NONE, pswit+USERTYPO_SWITCH,

      "Use file of user-defined typos", NULL },

    { "no-usertypo", 0, G_OPTION_FLAG_HIDDEN|G_OPTION_FLAG_REVERSE,

      G_OPTION_ARG_NONE, pswit+USERTYPO_SWITCH,

      "Ignore file of user-defined typos", NULL },

    { "verbose", 'v', 0, G_OPTION_ARG_NONE, pswit+VERBOSE_SWITCH,

      "Verbose - list everything", NULL },

    { "no-verbose", 0, G_OPTION_FLAG_HIDDEN|G_OPTION_FLAG_REVERSE,

      G_OPTION_ARG_NONE, pswit+VERBOSE_SWITCH,

      "Switch off verbose mode", NULL },

    { NULL }

};

/*

 * Options relating to configuration which make no sense from inside

 * a configuration file.

 */

static GOptionEntry config_options[]={

    { "web", 'w', 0, G_OPTION_ARG_NONE, pswit+WEB_SWITCH,

      "Defaults for use on www upload", NULL },

    { "dump-config", 0, 0, G_OPTION_ARG_NONE, pswit+DUMP_CONFIG_SWITCH,

      "Dump current config settings", NULL },

    { NULL }

};

static GOptionEntry compatibility_options[]={

    { "toggle-typo", 't', 0, G_OPTION_ARG_NONE, &typo_compat,

      "Toggle checking for common typos", NULL },

    { "toggle-relaxed", 'x', 0, G_OPTION_ARG_NONE, &paranoid_compat,

      "Toggle both paranoid mode and common typos", NULL },

    { NULL }

};

long cnt_dquot;		/* for overview mode, count of doublequote queries */

long cnt_squot;		/* for overview mode, count of singlequote queries */

long cnt_brack;		/* for overview mode, count of brackets queries */

long cnt_bin;		/* for overview mode, count of non-ASCII queries */

long cnt_odd;		/* for overview mode, count of odd character queries */

long cnt_long;		/* for overview mode, count of long line errors */

long cnt_short;		/* for overview mode, count of short line queries */

long cnt_punct;		/* for overview mode,

			   count of punctuation and spacing queries */

long cnt_dash;		/* for overview mode, count of dash-related queries */

long cnt_word;		/* for overview mode, count of word queries */

long cnt_html;		/* for overview mode, count of html queries */

long cnt_lineend;	/* for overview mode, count of line-end queries */

long cnt_spacend;	/* count of lines with space at end */

long linecnt;		/* count of total lines in the file */

long checked_linecnt;	/* count of lines actually checked */

void proghelp(GOptionContext *context);

void procfile(const char *);

gchar *running_from;

gboolean mixdigit(const char *);

gchar *getaword(const char **);

char *flgets(char **,long);

void postprocess_for_HTML(char *);

char *linehasmarkup(char *);

char *losemarkup(char *);

gboolean tagcomp(const char *,const char *);

void loseentities(char *);

gboolean isroman(const char *);

void postprocess_for_DP(char *);

void print_as_windows_1252(const char *string);

void print_as_utf_8(const char *string);

GTree *qword,*qperiod;

#ifdef __WIN32__

UINT saved_cp;

#endif

GKeyFile *config;

void config_file_update(GKeyFile *kf)

{

    int i;

    gboolean sw;

    for(i=0;options[i].long_name;i++)

    {

	if (g_str_has_prefix(options[i].long_name,"no-"))

	    continue;

	if (options[i].arg==G_OPTION_ARG_NONE)

	{

	    sw=*(gboolean *)options[i].arg_data;

	    if (options[i].flags&G_OPTION_FLAG_REVERSE)

		sw=!sw;

	    g_key_file_set_boolean(kf,"options",options[i].long_name,sw);

	}

	else

	    g_assert_not_reached();

    }

}

void config_file_add_comments(GKeyFile *kf)

{

    int i;

    gchar *comment;

    g_key_file_set_comment(kf,NULL,NULL," Default configuration for bookloupe",

      NULL);

    for(i=0;options[i].long_name;i++)

    {

	if (g_str_has_prefix(options[i].long_name,"no-"))

	    continue;

	comment=g_strconcat(" ",options[i].description,NULL);

	g_key_file_set_comment(kf,"options",options[i].long_name,comment,NULL);

	g_free(comment);

    }

}

void dump_config(void)

{

    gchar *s;

    if (config)

	config_file_update(config);

    else

    {

	config=g_key_file_new();

	config_file_update(config);

	config_file_add_comments(config);

    }

    s=g_key_file_to_data(config,NULL,NULL);

    if (s)

	g_print("%s",s);

    g_free(s);

}

GKeyFile *read_config_file(gchar **full_path)

{

    int i;

    GError *err=NULL;

    gchar **search_dirs;

    gchar *path;

    const char *search_path;

    GKeyFile *kf;

    kf=g_key_file_new();

    search_path=g_getenv("BOOKLOUPE_CONFIG_PATH");

    if (search_path)

    {

#ifdef __WIN32__

	search_dirs=g_strsplit(search_path,";",0);

#else

	search_dirs=g_strsplit(search_path,":",0);

#endif

    }

    else

    {

	search_dirs=g_new(gchar *,4);

	search_dirs[0]=g_get_current_dir();

	search_dirs[1]=g_strdup(running_from);

	search_dirs[2]=g_strdup(g_get_user_config_dir());

	search_dirs[3]=NULL;

    }

    for(i=0;search_dirs[i];i++)

    {

	path=g_build_filename(search_dirs[i],"bookloupe.ini",NULL);

	if (g_key_file_load_from_file(kf,path,

	  G_KEY_FILE_KEEP_COMMENTS|G_KEY_FILE_KEEP_TRANSLATIONS,&err))

	    break;

	if (!g_error_matches(err,G_FILE_ERROR,G_FILE_ERROR_NOENT))

	{

	    g_printerr("Bookloupe: Error reading %s\n",path);

	    g_printerr("%s\n",err->message);

	    exit(1);

	}

	g_clear_error(&err);

	g_free(path);

	path=NULL;

    }

    if (!search_dirs[i])

    {

	g_key_file_free(kf);

	kf=NULL;

    }

    g_strfreev(search_dirs);

    if (full_path && kf)

	*full_path=path;

    else

	g_free(path);

    return kf;

}

void parse_config_file(void)

{

    int i,j;

    gchar *path;

    gchar **keys;

    gboolean sw;

    GError *err=NULL;

    config=read_config_file(&path);

    if (config)

	keys=g_key_file_get_keys(config,"options",NULL,NULL);

    else

	keys=NULL;

    if (keys)

    {

	for(i=0;keys[i];i++)

	{

	    for(j=0;options[j].long_name;j++)

	    {

		if (g_str_has_prefix(options[j].long_name,"no-"))

		    continue;

		else if (!strcmp(keys[i],options[j].long_name))

		{

		    if (options[j].arg==G_OPTION_ARG_NONE)

		    {

			sw=g_key_file_get_boolean(config,"options",keys[i],

			  &err);

			if (err)

			{

			    g_printerr("Bookloupe: %s: options.%s: %s\n",

			      path,keys[i],err->message);

			    g_clear_error(&err);

			}

			if (options[j].flags&G_OPTION_FLAG_REVERSE)

			    sw=!sw;

			*(gboolean *)options[j].arg_data=sw;

			break;

		    }

		    else

			g_assert_not_reached();

		}

	    }

	    if (!options[j].long_name)

		g_printerr("Bookloupe: %s: Unknown option \"%s\" ignored\n",

		  path,keys[i]);

	}

	g_strfreev(keys);

    }

    if (config)

	g_free(path);

}

void parse_options(int *argc,char ***argv)

{

    GError *err=NULL;

    GOptionContext *context;

    GOptionGroup *compatibility;

    context=g_option_context_new(

      "file - look for errors in Project Gutenberg(TM) etexts");

    g_option_context_add_main_entries(context,options,NULL);

    g_option_context_add_main_entries(context,config_options,NULL);

    compatibility=g_option_group_new("compatibility",

      "Options for Compatibility with Gutcheck:",

      "Show compatibility options",NULL,NULL);

    g_option_group_add_entries(compatibility,compatibility_options);

    g_option_context_add_group(context,compatibility);

    g_option_context_set_description(context,

      "For simplicity, only the switch options which reverse the\n"

      "default configuration are listed. In most cases, both vanilla\n"

      "and \"no-\" prefixed versions are available for use.");

    if (!g_option_context_parse(context,argc,argv,&err))

    {

	g_printerr("Bookloupe: %s\n",err->message);

	g_printerr("Use \"%s --help\" for help\n",(*argv)[0]);

	exit(1);

    }

    if (typo_compat)

	pswit[TYPO_SWITCH]=!pswit[TYPO_SWITCH];

    if (paranoid_compat)

    {

	pswit[PARANOID_SWITCH]=!pswit[PARANOID_SWITCH];

	pswit[TYPO_SWITCH]=!pswit[TYPO_SWITCH];

    }

    /*

     * Web uploads - for the moment, this is really just a placeholder

     * until we decide what processing we really want to do on web uploads

     */

    if (pswit[WEB_SWITCH])

    {

	/* specific override for web uploads */

	pswit[ECHO_SWITCH]=TRUE;

	pswit[SQUOTE_SWITCH]=FALSE;

	pswit[TYPO_SWITCH]=TRUE;

	pswit[QPARA_SWITCH]=FALSE;

	pswit[PARANOID_SWITCH]=TRUE;

	pswit[LINE_END_SWITCH]=FALSE;

	pswit[OVERVIEW_SWITCH]=FALSE;

	pswit[STDOUT_SWITCH]=FALSE;

	pswit[HEADER_SWITCH]=TRUE;

	pswit[VERBOSE_SWITCH]=FALSE;

	pswit[MARKUP_SWITCH]=FALSE;

	pswit[USERTYPO_SWITCH]=FALSE;

	pswit[DP_SWITCH]=FALSE;

    }

    if (pswit[DUMP_CONFIG_SWITCH])

    {

	dump_config();

	exit(0);

    }

    if (pswit[OVERVIEW_SWITCH])

	/* just print summary; don't echo */

	pswit[ECHO_SWITCH]=FALSE;

    if (*argc<2)

    {

	proghelp(context);

	exit(1);

    }

    g_option_context_free(context);

}

/*

 * read_user_scannos:

 *

 * Read in the user-defined stealth scanno list.

 */

void read_user_scannos(void)

{

    GError *err=NULL;

    gchar *usertypo_file;

    gboolean okay;

    int i;

    gsize len,nb;

    gchar *contents,*utf8,**lines;

    usertypo_file=g_strdup("bookloupe.typ");

    okay=file_get_contents_text(usertypo_file,&contents,&len,&err);

    if (g_error_matches(err,G_FILE_ERROR,G_FILE_ERROR_NOENT))

    {

	g_clear_error(&err);

	g_free(usertypo_file);

	usertypo_file=g_build_filename(running_from,"bookloupe.typ",NULL);

	okay=file_get_contents_text(usertypo_file,&contents,&len,&err);

    }

    if (g_error_matches(err,G_FILE_ERROR,G_FILE_ERROR_NOENT))

    {

	g_clear_error(&err);

	g_free(usertypo_file);

	usertypo_file=g_strdup("gutcheck.typ");

	okay=file_get_contents_text(usertypo_file,&contents,&len,&err);

    }

    if (g_error_matches(err,G_FILE_ERROR,G_FILE_ERROR_NOENT))

    {

	g_clear_error(&err);

	g_free(usertypo_file);

	usertypo_file=g_build_filename(running_from,"gutcheck.typ",NULL);

	okay=file_get_contents_text(usertypo_file,&contents,&len,&err);

    }

    if (g_error_matches(err,G_FILE_ERROR,G_FILE_ERROR_NOENT))

    {

	g_free(usertypo_file);

	g_print("   --> I couldn't find bookloupe.typ "

	  "-- proceeding without user typos.\n");

	return;

    }

    else if (!okay)

    {

	fprintf(stderr,"%s: %s\n",usertypo_file,err->message);

	g_free(usertypo_file);

	g_clear_error(&err);

	exit(1);

    }

    if (g_utf8_validate(contents,len,NULL))

	utf8=g_utf8_normalize(contents,len,G_NORMALIZE_DEFAULT_COMPOSE);

    else

	utf8=g_convert(contents,len,"UTF-8","WINDOWS-1252",NULL,&nb,NULL);

    g_free(contents);

    lines=g_strsplit_set(utf8,"\r\n",0);

    g_free(utf8);

    usertypo=g_tree_new_full((GCompareDataFunc)strcmp,NULL,g_free,NULL);

    for (i=0;lines[i];i++)

	if (*(unsigned char *)lines[i]>'!')

	    g_tree_insert(usertypo,lines[i],GINT_TO_POINTER(1));

	else

	    g_free(lines[i]);

    g_free(lines);

}

/*

 * read_etext:

 *

 * Read an etext returning a newly allocated string containing the file

 * contents or NULL on error.

 */

gchar *read_etext(const char *filename,GError **err)

{

    GError *tmp_err=NULL;

    gchar *contents,*utf8;

    gsize len,bytes_read,bytes_written;

    int i,line,col;

    if (!g_file_get_contents(filename,&contents,&len,err))

	return NULL;

    if (g_utf8_validate(contents,len,NULL))

    {

	utf8=g_utf8_normalize(contents,len,G_NORMALIZE_DEFAULT_COMPOSE);

	g_set_print_handler(print_as_utf_8);

#ifdef __WIN32__

	SetConsoleOutputCP(CP_UTF8);

#endif

    }

    else

    {

	utf8=g_convert(contents,len,"UTF-8","WINDOWS-1252",&bytes_read,

	  &bytes_written,&tmp_err);

	if (g_error_matches(tmp_err,G_CONVERT_ERROR,

	  G_CONVERT_ERROR_ILLEGAL_SEQUENCE))

	{

	    line=col=1;

	    for(i=0;i<bytes_read;i++)

		if (contents[i]=='\n')

		{

		    line++;

		    col=1;

		}

		else if (contents[i]!='\r')

		    col++;

	    g_set_error(err,G_CONVERT_ERROR,G_CONVERT_ERROR_ILLEGAL_SEQUENCE,

	      "Input conversion failed. Byte %d at line %d, column %d is not a "

	      "valid Windows-1252 character",

	      ((unsigned char *)contents)[bytes_read],line,col);

	}

	else if (tmp_err)

	    g_propagate_error(err,tmp_err);

	g_set_print_handler(print_as_windows_1252);

#ifdef __WIN32__

	SetConsoleOutputCP(1252);

#endif

    }

    g_free(contents);

    return utf8;

}

void cleanup_on_exit(void)

{

#ifdef __WIN32__

    SetConsoleOutputCP(saved_cp);

#endif

}

int main(int argc,char **argv)

{

#ifdef __WIN32__

    atexit(cleanup_on_exit);

    saved_cp=GetConsoleOutputCP();

#endif

    running_from=g_path_get_dirname(argv[0]);

    /* Paranoid checking is turned OFF, not on, by its switch */

    pswit[PARANOID_SWITCH]=TRUE;

    /* if running in paranoid mode, typo checks default to enabled */

    pswit[TYPO_SWITCH]=TRUE;

    /* Line-end checking is turned OFF, not on, by its switch */

    pswit[LINE_END_SWITCH]=TRUE;

    /* Echoing is turned OFF, not on, by its switch */

    pswit[ECHO_SWITCH]=TRUE;

    parse_config_file();

    parse_options(&argc,&argv);

    if (pswit[USERTYPO_SWITCH])

	read_user_scannos();

    fprintf(stderr,"bookloupe: Check and report on an e-text\n");

    procfile(argv[1]);

    if (pswit[OVERVIEW_SWITCH])

    {

	g_print("    Checked %ld lines of %ld (head+foot = %ld)\n\n",

	  checked_linecnt,linecnt,linecnt-checked_linecnt);

	g_print("    --------------- Queries found --------------\n");

	if (cnt_long)

	    g_print("    Long lines:		    %14ld\n",cnt_long);

	if (cnt_short)

	    g_print("    Short lines:		   %14ld\n",cnt_short);

	if (cnt_lineend)

	    g_print("    Line-end problems:	     %14ld\n",cnt_lineend);

	if (cnt_word)

	    g_print("    Common typos:		  %14ld\n",cnt_word);

	if (cnt_dquot)

	    g_print("    Unmatched quotes:	      %14ld\n",cnt_dquot);

	if (cnt_squot)

	    g_print("    Unmatched SingleQuotes:	%14ld\n",cnt_squot);

	if (cnt_brack)

	    g_print("    Unmatched brackets:	    %14ld\n",cnt_brack);

	if (cnt_bin)

	    g_print("    Non-ASCII characters:	  %14ld\n",cnt_bin);

	if (cnt_odd)

	    g_print("    Proofing characters:	   %14ld\n",cnt_odd);

	if (cnt_punct)

	    g_print("    Punctuation & spacing queries: %14ld\n",cnt_punct);

	if (cnt_dash)

	    g_print("    Non-standard dashes:	   %14ld\n",cnt_dash);

	if (cnt_html)

	    g_print("    Possible HTML tags:	    %14ld\n",cnt_html);

	g_print("\n");

	g_print("    TOTAL QUERIES		  %14ld\n",

	  cnt_dquot+cnt_squot+cnt_brack+cnt_bin+cnt_odd+cnt_long+

	  cnt_short+cnt_punct+cnt_dash+cnt_word+cnt_html+cnt_lineend);

    }

    g_free(running_from);

    if (usertypo)

	g_tree_unref(usertypo);

    if (config)

	g_key_file_free(config);

    return 0;

}

/*

 * first_pass:

 *

 * Run a first pass - verify that it's a valid PG

 * file, decide whether to report some things that

 * occur many times in the text like long or short

 * lines, non-standard dashes, etc.

 */

struct first_pass_results *first_pass(const char *etext)

{

    gunichar laststart=CHAR_SPACE;

    const char *s;

    gchar *lc_line;

    int i,j,lbytes,llen;

    gchar **lines;

    unsigned int lastlen=0,lastblen=0;

    long spline=0,nspline=0;

    static struct first_pass_results results={0};

    gchar *inword;

    lines=g_strsplit(etext,"\n",0);

    for (j=0;lines[j];j++)

    {

	lbytes=strlen(lines[j]);

	while (lbytes>0 && lines[j][lbytes-1]=='\r')

	    lines[j][--lbytes]='\0';

	llen=g_utf8_strlen(lines[j],lbytes);

	linecnt++;

	if (strstr(lines[j],"*END") && strstr(lines[j],"SMALL PRINT") &&

	  (strstr(lines[j],"PUBLIC DOMAIN") || strstr(lines[j],"COPYRIGHT")))

	{

	    if (spline)

		g_print("   --> Duplicate header?\n");

	    spline=linecnt+1;   /* first line of non-header text, that is */

	}

	if (!strncmp(lines[j],"*** START",9) &&

	  strstr(lines[j],"PROJECT GUTENBERG"))

	{

	    if (nspline)

		g_print("   --> Duplicate header?\n");

	    nspline=linecnt+1;   /* first line of non-header text, that is */

	}

	if (spline || nspline)

	{

	    lc_line=g_utf8_strdown(lines[j],lbytes);

	    if (strstr(lc_line,"end") && strstr(lc_line,"project gutenberg"))

	    {

		if (strstr(lc_line,"end")<strstr(lc_line,"project gutenberg"))

		{

		    if (results.footerline)

		    {

			/* it's an old-form header - we can detect duplicates */

			if (!nspline)

			    g_print("   --> Duplicate footer?\n");

		    }

		    else

			results.footerline=linecnt;

		}

	    }

	    g_free(lc_line);

	}

	if (spline)

	    results.firstline=spline;

	if (nspline)

	    results.firstline=nspline;  /* override with new */

	if (results.footerline)

	    continue;    /* don't count the boilerplate in the footer */

	results.totlen+=llen;

	for (s=lines[j];*s;s=g_utf8_next_char(s))

	{

	    if (g_utf8_get_char(s)>127)

		results.binlen++;

	    if (g_unichar_isalpha(g_utf8_get_char(s)))

		results.alphalen++;

	    if (s>lines[j] && g_utf8_get_char(s)==CHAR_DQUOTE &&

	      isalpha(g_utf8_get_char(g_utf8_prev_char(s))))

		results.endquote_count++;

	}

	if (llen>2 && lastlen>2 && lastlen<SHORTEST_PG_LINE && lastblen>2 &&

	  lastblen>SHORTEST_PG_LINE && laststart!=CHAR_SPACE)

	    results.shortline++;

	if (lbytes>0 &&

	  g_utf8_get_char(g_utf8_prev_char(lines[j]+lbytes))<=CHAR_SPACE)

	    cnt_spacend++;

	if (strstr(lines[j],".,"))

	    results.dotcomma++;

	/* only count ast lines for ignoring purposes where there is */

	/* locase text on the line */

	if (strchr(lines[j],'*'))

	{

	    for (s=lines[j];*s;s=g_utf8_next_char(s))

		if (g_unichar_islower(g_utf8_get_char(s)))

		    break;

	    if (*s)

		results.astline++;

	}

	if (strchr(lines[j],'/'))

	    results.fslashline++;

	if (lbytes>0)

	{

	    for (s=g_utf8_prev_char(lines[j]+lbytes);

	      s>lines[j] && g_utf8_get_char(s)<=CHAR_SPACE;

	      s=g_utf8_prev_char(s))

		;

	    if (s>g_utf8_next_char(lines[j]) && g_utf8_get_char(s)=='-' &&

	      g_utf8_get_char(g_utf8_prev_char(s))!='-')

		results.hyphens++;

	}

	if (llen>LONGEST_PG_LINE)

	    results.longline++;

	if (llen>WAY_TOO_LONG)

	    results.verylongline++;

	if (strchr(lines[j],'<') && strchr(lines[j],'>'))

	{

	    i=(int)(strchr(lines[j],'>')-strchr(lines[j],'<')+1);

	    if (i>0)

		results.htmcount++;

	    if (strstr(lines[j],"<i>"))

		results.htmcount+=4; /* bonus marks! */

	}

	/* Check for spaced em-dashes */

	if (lines[j][0] && (s=strstr(g_utf8_next_char(lines[j]),"--")))

	{

	    results.emdash++;

	    if (s[-1]==CHAR_SPACE || s[2]==CHAR_SPACE)

		results.space_emdash++;

	    if (s[-1]==CHAR_SPACE && s[2]==CHAR_SPACE)

		/* count of em-dashes with spaces both sides */

		results.non_PG_space_emdash++;

	    if (s[-1]!=CHAR_SPACE && s[2]!=CHAR_SPACE)

		/* count of PG-type em-dashes with no spaces */

		results.PG_space_emdash++;

	}

	for (s=lines[j];*s;)

	{

	    inword=getaword(&s);

	    if (!strcmp(inword,"hij") || !strcmp(inword,"niet")) 

		results.Dutchcount++;

	    if (!strcmp(inword,"dans") || !strcmp(inword,"avec")) 

		results.Frenchcount++;

	    if (!strcmp(inword,"0") || !strcmp(inword,"1")) 

		results.standalone_digit++;

	    g_free(inword);

	}

	/* Check for spaced dashes */

	if (strstr(lines[j]," -") && *(strstr(lines[j]," -")+2)!='-')

	    results.spacedash++;

	lastblen=lastlen;

	lastlen=llen;

	laststart=lines[j][0];

    }

    g_strfreev(lines);

    return &results;

}

/*

 * report_first_pass:

 *

 * Make some snap decisions based on the first pass results.

 */

struct warnings *report_first_pass(struct first_pass_results *results)

{

    static struct warnings warnings={0};

    if (cnt_spacend>0)

	g_print("   --> %ld lines in this file have white space at end\n",

	  cnt_spacend);

    warnings.dotcomma=1;

    if (results->dotcomma>5)

    {

	warnings.dotcomma=0;

	g_print("   --> %ld lines in this file contain '.,'. "

	  "Not reporting them.\n",results->dotcomma);

    }

    /*

     * If more than 50 lines, or one-tenth, are short,

     * don't bother reporting them.

     */

    warnings.shortline=1;

    if (results->shortline>50 || results->shortline*10>linecnt)

    {

	warnings.shortline=0;

	g_print("   --> %ld lines in this file are short. "

	  "Not reporting short lines.\n",results->shortline);

    }

    /*

     * If more than 50 lines, or one-tenth, are long,

     * don't bother reporting them.

     */

    warnings.longline=1;

    if (results->longline>50 || results->longline*10>linecnt)

    {

	warnings.longline=0;

	g_print("   --> %ld lines in this file are long. "

	  "Not reporting long lines.\n",results->longline);

    }

    /* If more than 10 lines contain asterisks, don't bother reporting them. */

    warnings.ast=1;

    if (results->astline>10)

    {

	warnings.ast=0;

	g_print("   --> %ld lines in this file contain asterisks. "

	  "Not reporting them.\n",results->astline);

    }

    /*

     * If more than 10 lines contain forward slashes,

     * don't bother reporting them.

     */

    warnings.fslash=1;

    if (results->fslashline>10)

    {

	warnings.fslash=0;

	g_print("   --> %ld lines in this file contain forward slashes. "

	  "Not reporting them.\n",results->fslashline);

    }

    /*

     * If more than 20 lines contain unpunctuated endquotes,

     * don't bother reporting them.

     */

    warnings.endquote=1;

    if (results->endquote_count>20)

    {

	warnings.endquote=0;

	g_print("   --> %ld lines in this file contain unpunctuated endquotes. "

	  "Not reporting them.\n",results->endquote_count);

    }

    /*

     * If more than 15 lines contain standalone digits,

     * don't bother reporting them.

     */

    warnings.digit=1;

    if (results->standalone_digit>10)

    {

	warnings.digit=0;

	g_print("   --> %ld lines in this file contain standalone 0s and 1s. "

	  "Not reporting them.\n",results->standalone_digit);

    }

    /*

     * If more than 20 lines contain hyphens at end,

     * don't bother reporting them.

     */

    warnings.hyphen=1;

    if (results->hyphens>20)

    {

	warnings.hyphen=0;

	g_print("   --> %ld lines in this file have hyphens at end. "

	  "Not reporting them.\n",results->hyphens);

    }

    if (results->htmcount>20 && !pswit[MARKUP_SWITCH])

    {

	g_print("   --> Looks like this is HTML. Switching HTML mode ON.\n");

	pswit[MARKUP_SWITCH]=1;

    }

    if (results->verylongline>0)

	g_print("   --> %ld lines in this file are VERY long!\n",

	  results->verylongline);

    /*

     * If there are more non-PG spaced dashes than PG em-dashes,

     * assume it's deliberate.

     * Current PG guidelines say don't use them, but older texts do,

     * and some people insist on them whatever the guidelines say.

     */

    warnings.dash=1;

    if (results->spacedash+results->non_PG_space_emdash>

      results->PG_space_emdash)

    {

	warnings.dash=0;

	g_print("   --> There are %ld spaced dashes and em-dashes. "

	  "Not reporting them.\n",

	  results->spacedash+results->non_PG_space_emdash);

    }

    /* If more than a quarter of characters are hi-bit, bug out. */

    warnings.bin=1;

    if (results->binlen*4>results->totlen)

    {

	g_print("   --> This file does not appear to be ASCII. "

	  "Terminating. Best of luck with it!\n");

	exit(1);

    }

    if (results->alphalen*4<results->totlen)

    {

	g_print("   --> This file does not appear to be text. "

	  "Terminating. Best of luck with it!\n");

	exit(1);

    }

    if (results->binlen*100>results->totlen || results->binlen>100)

    {

	g_print("   --> There are a lot of foreign letters here. "

	  "Not reporting them.\n");

	warnings.bin=0;

    }

    warnings.isDutch=FALSE;

    if (results->Dutchcount>50)

    {

	warnings.isDutch=TRUE;

	g_print("   --> This looks like Dutch - "

	  "switching off dashes and warnings for 's Middags case.\n");

    }

    warnings.isFrench=FALSE;

    if (results->Frenchcount>50)

    {

	warnings.isFrench=TRUE;

	g_print("   --> This looks like French - "

	  "switching off some doublepunct.\n");

    }

    if (results->firstline && results->footerline)

	g_print("    The PG header and footer appear to be already on.\n");

    else

    {

	if (results->firstline)

	    g_print("    The PG header is on - no footer.\n");

	if (results->footerline)