/* Copyright (C) 2000-2006 Lavtech.com corp. All rights reserved.

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
*/

#ifndef _UDM_COMMON_H
#define _UDM_COMMON_H

#include "udm_config.h"

#include <stdio.h> /* for FILE etc. */

#include <sys/types.h>
#include <regex.h>

#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_WINSOCK_H
#include <winsock.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
#include <sys/socket.h>
#endif
#ifdef HAVE_NETINET_IN_H
#include <netinet/in.h>
#endif
#ifdef HAVE_ARPA_INET_H
#include <arpa/inet.h>
#endif
#ifdef HAVE_ARPA_NAMESER_H
#include <arpa/nameser.h>
#endif
#ifdef HAVE_RESOLV_H
#include <resolv.h>
#endif
#ifdef HAVE_NETDB_H
#include <netdb.h>
#endif
#ifdef MECAB
#include <mecab.h>
#endif

#ifdef HAVE_DEBUG
#include <assert.h>
#endif

#include "udm_unicode.h"
#include "udm_uniconv.h"

/* Some constants */
#define UDM_LANGPERDOC				16		/* FIXME */
#define UDM_USER_AGENT				"MnoGoSearch/" VERSION
#define UDM_MAXWORDPERQUERY			32

/* Some sizes and others definitions */
#define UDM_MAXDOCSIZE				2*1024*1024	/**< 2 MB  */
#define UDM_DEFAULT_REINDEX_TIME		7*24*60*60	/**< 1week */
#define UDM_MAXWORDSIZE				32
#define UDM_MAXDISCWORDSIZE			64
#define UDM_DEFAULT_MAX_HOPS			256
#define UDM_READ_TIMEOUT			30
#define UDM_DOC_TIMEOUT				90
#define UDM_MAXNETERRORS			16
#define UDM_DEFAULT_NET_ERROR_DELAY_TIME	86400
#define UDM_DEFAULT_BAD_SINCE_TIME              15*24*60*60     /**< 15 days */
#define UDM_FINDURL_CACHE_SIZE                  128
#define UDM_SERVERID_CACHE_SIZE                 128


/* storage types */
#define UDM_DBMODE_SINGLE	0
#define UDM_DBMODE_MULTI	1
#define UDM_DBMODE_BLOB		6
#define UDM_DBMODE_RAWBLOB	7


/* database open modes */
#define UDM_OPEN_MODE_READ	0
#define UDM_OPEN_MODE_WRITE	1


/* search modes */
#define UDM_ORD_RATE		0
#define UDM_ORD_DATE		1
#define UDM_MODE_ALL		0
#define UDM_MODE_ANY		1
#define UDM_MODE_BOOL		2
#define UDM_MODE_PHRASE		3

/* word match type */
#define UDM_MATCH_FULL		0
#define UDM_MATCH_BEGIN		1
#define UDM_MATCH_SUBSTR	2
#define UDM_MATCH_END		3
#define UDM_MATCH_REGEX		4
#define UDM_MATCH_WILD		5
#define UDM_MATCH_SUBNET	6
#define UDM_MATCH_NUMERIC_LT	7
#define UDM_MATCH_NUMERIC_GT	8

/* Indexer return codes */
#define UDM_OK			0
#define UDM_ERROR		1
#define UDM_NOTARGET		2
#define UDM_TERMINATED		3


/* Flags for indexing */
#define UDM_FLAG_REINDEX	1
#define UDM_FLAG_SORT_EXPIRED	2
#define UDM_FLAG_SORT_HOPS	4
#define UDM_FLAG_ADD_SERV	8
#define UDM_FLAG_SPELL		16
#define UDM_FLAG_LOAD_LANGMAP	32
#define UDM_FLAG_DONTSORT_SEED  64
#define UDM_FLAG_ADD_SERVURL	128


/* URLFile actions */
#define UDM_URL_FILE_REINDEX	1
#define UDM_URL_FILE_CLEAR	2
#define UDM_URL_FILE_INSERT	3
#define UDM_URL_FILE_PARSE	4

/* Ispell mode binary flags */
#define UDM_ISPELL_MODE_DB	1
#define UDM_ISPELL_USE_PREFIXES	2
#define UDM_ISPELL_MODE_SERVER  4

/* Action type: HTTP methods */
#define UDM_METHOD_UNKNOWN	0
#define UDM_METHOD_GET		1
#define UDM_METHOD_DISALLOW	2
#define UDM_METHOD_HEAD		3
#define UDM_METHOD_HREFONLY	4
#define UDM_METHOD_CHECKMP3	5
#define UDM_METHOD_CHECKMP3ONLY	6
#define UDM_METHOD_VISITLATER	7
#define UDM_METHOD_INDEX	8
#define UDM_METHOD_NOINDEX	9
#define UDM_METHOD_IMPORTONLY	10

/* Tags/categories defines */
#define UDM_CATSIZE	8
#define UDM_TAGSIZE	4

/* Words origins */
#define UDM_WORD_ORIGIN_QUERY   1
#define UDM_WORD_ORIGIN_SPELL   2
#define UDM_WORD_ORIGIN_SYNONYM 4
#define UDM_WORD_ORIGIN_STOP    8
#define UDM_WORD_ORIGIN_SUGGEST 16
#define UDM_WORD_ORIGIN_COLLATION 32

/* Locking mutex numbers */
#define UDM_LOCK_CONF		0
#define UDM_LOCK_WIN		1
#define UDM_LOCK_THREAD         2
#define UDM_LOCK_SQL            3
#define UDM_LOCK_SEGMENTER      4
#define UDM_LOCK_DB             5
#define UDM_LOCK_LOG		6
#define UDM_LOCK_MAX		7

/************************ Statistics **********************/
typedef struct stat_struct {
	int	status;
	int	expired;
	int	total;
} UDM_STAT;

typedef struct stat_list_struct{
	time_t		time;
	size_t		nstats;
	UDM_STAT	*Stat;
} UDM_STATLIST;
/************************ VARLISTs ************************/

/* Various variable flags */
#define UDM_VARFLAG_NOCLONE	1  /* Ignore for clone detection        */ 
#define UDM_VARFLAG_USERDEF	2  /* User defined section              */
#define UDM_VARFLAG_KEEPVAL	4  /* If value doesn't need to be freed */
#define UDM_VARFLAG_HTMLSOURCE	8  /* If apply HTML parser, for HTDB    */
#define UDM_VARFLAG_WIKI       16  /* If to remove text between [ and ] */
#define UDM_VARFLAG_HL         32  /* If variable has highlight markers */

/* Variable types */
#define UDM_VAR_INT       1
#define UDM_VAR_STR       2
#define UDM_VAR_DOUBLE    3
#define UDM_VAR_ENV       16
#define UDM_VAR_AGENT     17
#define UDM_VAR_RESULT    18
#define UDM_VAR_DOCUMENT  19
#define UDM_VAR_SQLRESULT 20

#define UDM_VARSRC_QSTRING 1
#define UDM_VARSRC_ENV     2

struct udm_var_handler_st;

typedef struct udm_var_st {
	struct udm_var_handler_st *handler; /* Variable handler */
	int		section;	/**< Number 0..255   */
	size_t		maxlen;		/**< Max length      */
	size_t		curlen;		/**< Cur length      */
	char		*val;		/**< Field Value     */
	char		*name;		/**< Field Name      */
	int		flags;
} UDM_VAR;

typedef struct udm_varlist_st {
	int		freeme;
	size_t		nvars;
	size_t		mvars;
	UDM_VAR		*Var;
} UDM_VARLIST;


#define UDM_TEXTLIST_FLAG_SKIP_ADD_SECTION 1
typedef struct {
	char		*str;
	char		*href;
	char		*section_name;
	int		section;
	int		flags;
} UDM_TEXTITEM;

typedef struct {
	size_t		nitems;
	UDM_TEXTITEM	*Item;
} UDM_TEXTLIST;

/*****************************************************/

/** StopList unit */
typedef struct udm_stopword_struct {
	char		*word;
	char		*lang;
} UDM_STOPWORD;

typedef struct {
	size_t		nstopwords;
	UDM_STOPWORD	*StopWord;
} UDM_STOPLIST;

/*****************************************************/

/** Words parameters */
typedef struct {
	size_t		min_word_len;
	size_t		max_word_len;
} UDM_WORDPARAM;

typedef uint4 udm_pos_t;
typedef unsigned char udm_secno_t;
typedef unsigned char udm_wordnum_t;

typedef struct udm_coord2_st
{
  udm_pos_t pos;
} UDM_COORD2;


typedef struct udm_section_st
{
  UDM_COORD2 *Coord;
  urlid_t url_id;
  uint4 ncoords;
  udm_pos_t seclen;
  udm_pos_t minpos;
  udm_pos_t maxpos;
  udm_secno_t secno;
  udm_wordnum_t wordnum;
} UDM_SECTION;


typedef struct udm_sectionlist_st
{
  size_t mcoords;
  size_t ncoords;
  UDM_COORD2 *Coord;
  size_t msections;
  size_t nsections;
  UDM_SECTION *Section;
} UDM_SECTIONLIST;


typedef struct udm_sectionlistlist_st
{
  size_t nitems;
  size_t mitems;
  UDM_SECTIONLIST *Item;
} UDM_SECTIONLISTLIST;



/** Main search structure */
typedef struct {
  urlid_t url_id;
  uint4   score;
} UDM_URL_SCORE;

typedef struct {
  size_t nitems;
  UDM_URL_SCORE	*Item;
} UDM_URLSCORELIST;

typedef struct {
        urlid_t         url_id;
        uint4           coord;
        uint4           per_site;
        urlid_t         site_id;
        time_t          last_mod_time;
        double          pop_rank;
	char            *url;
        char            *section;
} UDM_URLDATA;

#define UDM_COORD2DBNUM(score) (255 - (int) ((score) & 0xFF))

typedef struct {
	size_t		nitems;
	UDM_URLDATA	*Item;
} UDM_URLDATALIST;


/** Word list unit */
typedef struct
{
  char		*word;
  udm_pos_t     pos;
  udm_secno_t   secno;
  unsigned char hash;
  unsigned char seclen_marker;
} UDM_WORD;

typedef struct {
	size_t		mwords;	/**< Number of memory allocated for words     */
	size_t		nwords;	/**< Real number of words in list             */
	size_t		swords;	/**< Number of words in sorted list           */
	size_t		wordpos;/**< For phrases, number of current word      */
#ifdef TRIAL_VER
	UDM_WORD	Word[256];
#else
	UDM_WORD	*Word;	/**< Word list  itself                        */
#endif
} UDM_WORDLIST;




/***************************************************************/

/** Cross-word list unit */
typedef struct {
	short	pos;
	short	weight;
	char	*word;
	char	*url;
	urlid_t	referree_id;
} UDM_CROSSWORD;

typedef struct {
	size_t		ncrosswords;
	size_t		mcrosswords;
	size_t		wordpos[256];
	UDM_CROSSWORD	*CrossWord;
} UDM_CROSSLIST;

/*****************************************************************/

typedef struct {
	int	max_net_errors;
	int	net_error_delay_time;
	int	read_timeout;
	int	doc_timeout;
	int	period;		/**< Reindex period           */
	int	maxhops;	/**< Max way in mouse clicks  */
	int	index;		/**< Whether to index words   */
	int	follow;		/**< Whether to follow links  */
	int	use_robots;	/**< Whether to use robots.txt*/
	int	use_clones;	/**< Whether to detect clones */
        int	doc_per_site;
        int	collect_links;
} UDM_SPIDERPARAM;

/*****************************************************************/
typedef struct {
	int		match_type;
	int		nomatch;
	int		case_sense;
	char		*section;
	char		*pattern;
	regex_t		*reg;
	char		*arg;
	char		*arg1;
} UDM_MATCH;

typedef struct {
	size_t		nmatches;
	UDM_MATCH	*Match;
} UDM_MATCHLIST;

typedef struct {
	int beg;
	int end;
} UDM_MATCH_PART;
/*****************************************************************/

/** Structure to store server parameters */
typedef struct {
	UDM_MATCH	Match;
	urlid_t         site_id;        /**< server.rec_id            */
	char            command;        /**< 'S' - server,realm, 'F' - disallow,allow */
	int             ordre;          /**< order in list to find    */
	urlid_t         parent;         /**< parent rec_id for grouping by site */
	float           weight;         /**< server weight for popularity rank calculation */
	UDM_VARLIST	Vars;		/**< Default lang, charset,etc*/
        uint4           MaxHops;
} UDM_SERVER;

typedef struct {
	size_t		nservers;
	size_t		mservers;
	int		have_subnets;
	UDM_SERVER	*Server;
} UDM_SERVERLIST;


/*******************************************************/
/* All links are stored in the cache of this structure */
/* before actual INSERT into database                  */

typedef struct {
	char	*url;
        urlid_t referrer;
	uint4	hops;
	int	stored;	
	int	method;
	int	collect_links;
        urlid_t site_id;
        urlid_t server_id;
        urlid_t rec_id;
        size_t  max_doc_per_site;
        UDM_VARLIST Vars;
} UDM_HREF;

typedef struct {
	size_t		mhrefs;
	size_t		nhrefs;
	size_t		shrefs;
	size_t		dhrefs;
	UDM_HREF	*Href;
} UDM_HREFLIST;

/*******************************************************/

/** Resolve stuff */
typedef struct udm_host_addr_struct {
	char		*hostname;
	struct in_addr	addr;
	int		net_errors;
	time_t		last_used;
}UDM_HOST_ADDR;

typedef struct {
	size_t		nhost_addr;
	size_t		mhost_addr;
	UDM_HOST_ADDR	*host_addr;
} UDM_HOSTLIST;

/** Used in FTP sessions */
typedef struct udm_conn_struct {
        int	status;
        int	connected;
        int	err;
        int	retry;
        int	conn_fd;
#ifdef WIN32
        unsigned short port;
#else
        int	port;
#endif
        int	timeout;
        char	*hostname;
        char    *user;
        char    *pass;
        struct	sockaddr_in sin;
        int	buf_len;
        size_t	buf_len_total;
        int	len;
        char	*buf;
        UDM_HOST_ADDR *Host;
        struct	udm_conn_struct *connp;
} UDM_CONN;

/** Parsed URL string */
typedef struct udm_url {
	char	*schema;
	char	*specific;
	char	*hostinfo;
	char	*auth;
	char	*hostname;
	char	*path;
	char	*filename;
	char	*anchor;
	int	port;
	int	default_port;
} UDM_URL;


/***************************************************/

typedef struct {
	char	*buf;		/**< Buffer to download document to          */
	char	*content;	/**< Pointer to content, after headers       */
	size_t	size;		/**< Number of bytes loaded                  */
	size_t	maxsize;	/**< Maximum bytes to load into buf          */
	size_t  content_length; /**  For HTDB **/
} UDM_HTTPBUF;

typedef struct {
	int	freeme;		/**< Whether  memory was allocated for doc   */
	int	stored;		/**< If it is already stored, forAddHref()   */
	int	method;		/**< How to download document: GET, HEAD etc */
	
	UDM_HTTPBUF		Buf;		/**< Buffer       */
	
	UDM_HREFLIST		Hrefs;		/**< Link list    */
	UDM_WORDLIST		Words;		/**< Word list    */
	UDM_CROSSLIST		CrossWords;	/**< Crosswords   */
	
	UDM_VARLIST		RequestHeaders;	/**< Extra headers*/
	UDM_VARLIST		Sections;	/**< User sections*/
	
	UDM_TEXTLIST		TextList;	/**< Text list    */
	UDM_URL			CurURL;		/**< Parsed URL   */
	UDM_CHARSET		*lcs;		/**< LocalCharser */
	UDM_SPIDERPARAM		Spider;		/**< Spider prms  */
	UDM_CONN		connp;		/**< For FTP      */
	
} UDM_DOCUMENT;

/********************************************************/

/** External Parsers */
typedef struct udm_parser_struct{
        char		*from_mime;
	char		*to_mime;
	char		*cmd;
	char		*src;
} UDM_PARSER;

typedef struct {
	size_t		nparsers;
	UDM_PARSER	*Parser;
} UDM_PARSERLIST;


/******* Ispell BEGIN ********/

#define UDM_SPELL_NOPREFIX 1

typedef struct udm_spell_st
{
  char *word;
  char *flags;
} UDM_SPELL;

#define UDM_SPELL_LANGLEN 32
#define UDM_SPELL_CSETLEN 32
#define UDM_SPELL_FILELEN 128
#define UDM_SPELL_FMT_TEXT 0
#define UDM_SPELL_FMT_HASH 1

typedef struct udm_dict_st
{
  char  lang[UDM_SPELL_LANGLEN];
  char  cset[UDM_SPELL_CSETLEN];
  char  fname[UDM_SPELL_FILELEN];
  int   fmt;
  int   fd;
  size_t wordlen;
  UDM_CHARSET *cs;
  char   *fbody;
  size_t nitems;
  size_t mitems;
  UDM_SPELL *Item;
} UDM_SPELLLIST;


typedef struct udm_spelllistlist_st
{
  size_t nitems;
  size_t mitems;
  size_t nspell;              /* Backward capability with php-4.3.x */
  UDM_SPELLLIST *Item;
} UDM_SPELLLISTLIST;


typedef struct udm_aff_st
{
  char flag;
  char type;
  regex_t regex;
  char *find;
  char *repl;
  char *mask;
  size_t findlen;
  size_t replen;
} UDM_AFFIX;


typedef struct udm_afflist_st
{
  size_t mitems;
  size_t nitems;
  char  lang[UDM_SPELL_LANGLEN];
  char  cset[UDM_SPELL_CSETLEN];
  char  fname[UDM_SPELL_FILELEN];
  UDM_CHARSET *cs;
  UDM_AFFIX *Item;
} UDM_AFFIXLIST;


typedef struct udm_afflistlist_st
{
  size_t mitems;
  size_t nitems;
  UDM_AFFIXLIST *Item;
} UDM_AFFIXLISTLIST;


/******* Ispell END **********/


typedef struct{
	int		cmd; /**< 'allow' or 'disallow' */
	char		*path;
} UDM_ROBOT_RULE;

typedef struct{
	char		*hostinfo;
	size_t		nrules;
	UDM_ROBOT_RULE	*Rule;
} UDM_ROBOT;

typedef struct{
	size_t		nrobots;
	UDM_ROBOT	*Robot;
} UDM_ROBOTS;

#define MAX_SEARCH_LIMIT 32

typedef struct{
	int		type;
	char		file_name[1024];
	uint4		hi;
	uint4		lo;
        uint4           f_hi;
        uint4           f_lo;
} UDM_SEARCH_LIMIT;


typedef struct {
	size_t		order;
	size_t		count;
	char		*word;
	size_t		len;
        int     	origin;
        int             weight; /* origin-dependent weight */
        int		match;
        size_t		secno;  /* Which section to search in */
        size_t		phrpos; /* 0 means "not in phrase"    */
        size_t		phrlen; /* phase length               */
        size_t          phrwidth; /* How many additional parts in a multi-word */
} UDM_WIDEWORD;

typedef struct {
	int		wm;              /* Search mode: wrd, sub, beg, end */
	int             strip_noaccents; /* If accent insensitive comparison*/
	size_t		nuniq;
	size_t		nwords;
	UDM_WIDEWORD	*Word;
} UDM_WIDEWORDLIST;


typedef struct {
	char	*p;
	char	*s;
} UDM_SYNONYM;

typedef struct {
	size_t		nsynonyms;
	size_t		msynonyms;
	UDM_SYNONYM	*Synonym;
} UDM_SYNONYMLIST;

typedef struct udm_chinaword_struct {
  int *word;
  int  freq;
} UDM_CHINAWORD;

typedef struct {
  size_t        nwords, mwords;
  size_t        total;
  UDM_CHINAWORD *ChiWord;
  size_t        *hash;
} UDM_CHINALIST;


typedef struct udm_category_struct {
	int		rec_id;
	char		path[128];
	char		link[128];
	char		name[128];
} UDM_CATITEM;

typedef struct {
	char		addr[128];
	size_t		ncategories;
	UDM_CATITEM	*Category;
} UDM_CATEGORY;

/* Boolean search constants and types */
#define UDM_MAXSTACK	 128
#define UDM_STACK_LEFT	 0
#define UDM_STACK_RIGHT	 1
#define UDM_STACK_BOT	 2
#define UDM_STACK_OR	 3
#define UDM_STACK_AND	 4
#define UDM_STACK_NOT	 5
#define UDM_STACK_PHRASE 6
#define UDM_STACK_WORD	 200
#define UDM_STACK_STOP   201

typedef struct {
	size_t		ncstack, mcstack;
	int		*cstack;
	size_t		nastack, mastack;
	unsigned long	*astack;
} UDM_BOOLSTACK;

typedef struct {
	int		cmd;
	unsigned long	arg;
} UDM_STACK_ITEM;

typedef struct {
        size_t nitems;
        size_t mitems;
        size_t ncmds;
        UDM_STACK_ITEM *items;
} UDM_STACKITEMLIST;

typedef struct {
	size_t			work_time;
	size_t			first;
	size_t			last;
	size_t			total_found;
	size_t			num_rows;
	size_t			cur_row;
/*	size_t			offset;*/
	size_t			memused;
	int			freeme;
	UDM_DOCUMENT		*Doc;
	
	UDM_WIDEWORDLIST	WWList;
	UDM_URLDATALIST		URLData;
	UDM_STACKITEMLIST	ItemList;
	
} UDM_RESULT;


#include "udm_db_int.h"

typedef struct {
	size_t		nitems;
        size_t          currdbnum;
	UDM_DB		*db;
} UDM_DBLIST;

/** Forward declaration of UDM_AGENT */
struct udm_indexer_struct;

/** Config file */
typedef struct udm_config_struct {
	int		freeme;
	char		errstr[2048];
	UDM_CHARSET	*bcs;
	UDM_CHARSET	*lcs;
	
	int		url_number;	/**< For indexer -nXXX          */
	
	UDM_SERVERLIST	Servers;	/**< List of servers and realms */
        UDM_SERVER      *Cfg_Srv;
	
	UDM_MATCHLIST	Aliases;	/**< Straight aliases           */
	UDM_MATCHLIST	ReverseAliases;	/**< Reverse aliases            */
	UDM_MATCHLIST	MimeTypes;	/**< For AddType commands       */
	UDM_MATCHLIST	Filters;	/**< Allow, Disallow,etc        */
	UDM_MATCHLIST	SectionFilters; /**< IndexIf, NoIndexIf         */
        UDM_MATCHLIST   SectionHdrMatch;/**< User sections after headers*/
        UDM_MATCHLIST   SectionGsrMatch;/**< User sections after quesser*/
        UDM_MATCHLIST   SectionMatch;   /**< User sections after parser */
	
	UDM_HREFLIST	Hrefs;		/**< Links cache                */
	UDM_RESULT	Targets;	/**< Targets cache              */
	
	UDM_VARLIST	Sections;	/**< document section to parse  */
	UDM_VARLIST	Vars;		/**< Config parameters          */
	UDM_VARLIST	Cookies;	/**< Cookie list                */
	UDM_VARLIST	XMLEnterHooks;
	UDM_VARLIST	XMLLeaveHooks;
	UDM_VARLIST	XMLDataHooks;
	
	UDM_LANGMAPLIST	LangMaps;	/**< For lang+charset quesser   */
	UDM_ROBOTS	Robots;		/**< robots.txt information     */
	UDM_SYNONYMLIST	Synonyms;	/**< Synonims list              */
	UDM_STOPLIST	StopWords;	/**< Stopwords list             */
	UDM_PARSERLIST	Parsers;	/**< External  parsers          */
	UDM_DBLIST	dbl;		/**< Searchd addresses	      */
	UDM_HOSTLIST	Hosts;		/**< Resolve cache              */
	UDM_SPELLLISTLIST Spells;	/**< For ispell dictionaries    */
	UDM_AFFIXLISTLIST Affixes;	/**< For ispell affixes         */
	UDM_WORDPARAM	WordParam;	/**< Word limits                */
        UDM_CHINALIST   Chi;            /**< Chinese words list         */
        UDM_CHINALIST   Thai;           /**< Thai words list            */
	
	/* Various file descriptors */
	int		is_log_open;	/**< if UdmOpenLog is already called   */
	FILE		*logFD;		/**< FILE structure, syslog descriptor */
	int		CVS_ignore;	/**< Skip CVS directgories - for tests */
	void (*ThreadInfo)(struct udm_indexer_struct *,const char * state,const char * str);
	void (*LockProc)(struct udm_indexer_struct *,int command,int type,const char *fname,int lineno);
	void (*RefInfo)(int code,const char *url, const char *ref);
	int (*ThreadCreate)(void **thd, void *(*start_routine)(void *), void *arg);
	int (*ThreadJoin)(void *thd);
#ifdef MECAB
        mecab_t         *mecab;
#endif

} UDM_ENV;



/** Indexer */
typedef struct udm_indexer_struct{
	int		freeme;		/**< whenever it was allocated    */
	int		handle;		/**< Handler for threaded version */
	time_t		start_time;	/**< Time of allocation, for stat */
	unsigned long long nbytes;	/**< Number of bytes downloaded   */
	size_t		ndocs;		/**< Number of documents          */
        size_t          nsleepsecs;     /**> Number of sleep seconds      */
	int		flags;		/**< Callback function to request action*/
        int             action;
	int		doccount;	/**< for UdmGetDocCount()         */
	UDM_ENV		*Conf;		/**< Configuration                */
	UDM_LANGMAP	*LangMap;	/**< LangMap for current document */
	UDM_RESULT	Indexed;	/**< Indexed cache              */
	
	/** Cache mode limits */
	UDM_SEARCH_LIMIT	limits[MAX_SEARCH_LIMIT];
	unsigned int		nlimits;

        char    *UdmFindURLCache[UDM_FINDURL_CACHE_SIZE];
        urlid_t UdmFindURLCacheId[UDM_FINDURL_CACHE_SIZE];
        size_t  pURLCache;
        char    *ServerIdCache[UDM_SERVERID_CACHE_SIZE];
        char    ServerIdCacheCommand[UDM_SERVERID_CACHE_SIZE];
        urlid_t ServerIdCacheId[UDM_SERVERID_CACHE_SIZE];
        size_t  pServerIdCache;

#ifdef USE_TRACE
  FILE *TR;
#endif
	
} UDM_AGENT;



typedef struct {
	char	*url;
	int	status;
} UDM_URLSTATE;

typedef int (*udm_qsort_cmp)(const void*, const void*);

typedef struct {
	uint4	hi,lo;
	off_t	pos;
	size_t	len;
} UDM_UINT8_POS_LEN;

typedef struct {
	uint4	val;
	off_t	pos;
	size_t	len;
} UDM_UINT4_POS_LEN;

typedef struct {
	uint4	val;
	urlid_t	url_id;
} UDM_UINT4URLID;

typedef struct {
	size_t		nitems;
	UDM_UINT4URLID	*Item;
} UDM_UINT4URLIDLIST;

typedef struct {
	uint4	hi,lo;
	urlid_t	url_id;
} UDM_UINT8URLID;

typedef struct {
	size_t		nitems;
	UDM_UINT8URLID	*Item;
} UDM_UINT8URLIDLIST;


#define UDM_LOGD_CMD_WORD  0
#define UDM_LOGD_CMD_DATA  1
#define UDM_LOGD_CMD_CHECK 2

#define UDM_MAXTAGVAL	64

typedef struct {
	int	type;
	int	script;
	int	style;
	int	title;
	int	body;
	int	follow;
	int	index;
	int	comment;
	int	nonbreaking;
	char	*lasthref;
        void    (*next_b)(void *t);
        void    (*next_e)(void *t);
	size_t	ntoks;
	struct {
		const char *name;
		const char *val;
		size_t     nlen;
		size_t     vlen;
	} toks[UDM_MAXTAGVAL+1];
} UDM_HTMLTOK;

typedef struct udm_cfg_st {
        UDM_AGENT       *Indexer;
	UDM_SERVER	*Srv;
	int		flags;
	int		level;
	int		ordre;
} UDM_CFG;

#ifndef udm_max
#define udm_max(a,b) (((a) > (b)) ? (a) : (b))
#endif
#ifndef udm_min
#define udm_min(a,b) (((a) < (b)) ? (a) : (b))
#endif

#ifdef DMALLOC
#include <dmalloc.h>
#endif

#define UDM_DT_BACK    1
#define UDM_DT_ER      2
#define UDM_DT_RANGE   3
#define UDM_DT_UNKNOWN 4

enum udm_indcmd
{
  UDM_IND_AMBIGUOUS,
  UDM_IND_UNKNOWN,
  UDM_IND_INDEX,
  UDM_IND_STAT,
  UDM_IND_CREATE,
  UDM_IND_DROP,
  UDM_IND_DELETE,
  UDM_IND_REFERERS,
  UDM_IND_SQLMON,
  UDM_IND_CHECKCONF,
  UDM_IND_CONVERT,
  UDM_IND_MULTI2BLOB,
  UDM_IND_EXPORT,
  UDM_IND_WRDSTAT,
  UDM_IND_REWRITEURL,
  UDM_IND_HASHSPELL,
  UDM_IND_DUMPSPELL,
  UDM_IND_REWRITELIMITS
};

#endif
