/* Copyright (C) 2004 Datapark corp. All rights reserved.
   Copyright (C) 2000-2002 Lavtech.com corp. All rights reserved.

   This program is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published by
   the Free Software Foundation; either version 2 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU General Public License for more details.

   You should have received a copy of the GNU General Public License
   along with this program; if not, write to the Free Software
   Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA 
*/

#ifndef _DPS_COMMON_H
#define _DPS_COMMON_H

#include "dps_config.h"

#include <stdio.h> /* for FILE etc. */
#include <strings.h> /* for Solaris? */
#include <time.h>

#include <sys/types.h>
#ifndef _PCREPOSIX_H
#include <regex.h>
#endif

#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_WINSOCK_H
#include <winsock.h>
#endif
#ifdef HAVE_SYS_SOCKET_H
#include <sys/socket.h>
#endif
#ifdef HAVE_NETINET_IN_H
#include <netinet/in.h>
#endif
#ifdef HAVE_ARPA_INET_H
#include <arpa/inet.h>
#endif
#ifdef HAVE_ARPA_NAMESER_H
#include <arpa/nameser.h>
#endif
#ifdef HAVE_RESOLV_H
#include <resolv.h>
#endif
#ifdef HAVE_NETDB_H
#include <netdb.h>
#endif
#ifdef MECAB
#include <mecab.h>
#endif

#ifdef DEBUG_CONF_LOCK
#include <assert.h>
#endif

#include "dps_unicode.h"
#include "dps_uniconv.h"

/* Some constants */
#define DPS_LANGPERDOC				16		/* FIXME */
#define DPS_USER_AGENT				"DataparkSearch/" VERSION " (+http://www.dataparksearch.org/)"
#define DPS_MAXWORDPERQUERY			64

/* Some sizes and others definitions */
#define DPS_MAXDOCSIZE				2*1024*1024	/**< 2 MB  */
#define DPS_DEFAULT_REINDEX_TIME		7*24*60*60	/**< 1week */
#define DPS_MAXWORDSIZE				32
#define DPS_MAXDISCWORDSIZE			64
#define DPS_DEFAULT_MAX_HOPS			256
#define DPS_READ_TIMEOUT			30
#define DPS_DOC_TIMEOUT				90
#define DPS_MAXNETERRORS			16
#define DPS_DEFAULT_NET_ERROR_DELAY_TIME	86400
#define DPS_DEFAULT_BAD_SINCE_TIME              15*24*60*60     /**< 15 days */
#define DPS_FINDURL_CACHE_SIZE                  128
#define DPS_SERVERID_CACHE_SIZE                 128

/***********************************/

#define DPS_LOGDIR	"raw"
#define DPS_TREEDIR     "tree"
#define DPS_SPLDIR      "splitter"
#define DPS_URLDIR      "url"


/* storage types */
#define DPS_DBMODE_SINGLE	0
#define DPS_DBMODE_MULTI	1
#define DPS_DBMODE_SINGLE_CRC	2
#define DPS_DBMODE_MULTI_CRC	3
#define DPS_DBMODE_CACHE	4


/* database open modes */
#define DPS_OPEN_MODE_READ	0
#define DPS_OPEN_MODE_WRITE	1


/* search modes */
#define DPS_ORD_RATE		0
#define DPS_ORD_DATE		1
#define DPS_MODE_ALL		0
#define DPS_MODE_ANY		1
#define DPS_MODE_BOOL		2
#define DPS_MODE_PHRASE		3

/* word match type */
#define DPS_MATCH_FULL		0
#define DPS_MATCH_BEGIN		1
#define DPS_MATCH_SUBSTR	2
#define DPS_MATCH_END		3
#define DPS_MATCH_REGEX		4
#define DPS_MATCH_WILD		5
#define DPS_MATCH_SUBNET	6

/* Indexer return codes */
#define DPS_OK			0
#define DPS_ERROR		1
#define DPS_NOTARGET		2
#define DPS_TERMINATED		3


/* Flags for indexing */
#define DPS_FLAG_REINDEX	1
#define DPS_FLAG_SORT_EXPIRED	2
#define DPS_FLAG_SORT_HOPS	4
#define DPS_FLAG_ADD_SERV	8
#define DPS_FLAG_SPELL		16
#define DPS_FLAG_LOAD_LANGMAP	32
#define DPS_FLAG_DONTSORT_SEED  64
#define DPS_FLAG_ADD_SERVURL	128
#define DPS_FLAG_UNOCON         256


/* URLFile actions */
#define DPS_URL_FILE_REINDEX	1
#define DPS_URL_FILE_CLEAR	2
#define DPS_URL_FILE_INSERT	3
#define DPS_URL_FILE_PARSE	4

/* Ispell mode binary flags */
#define DPS_ISPELL_MODE_DB	1
#define DPS_ISPELL_USE_PREFIXES	2
#define DPS_ISPELL_MODE_SERVER  4

/* Action type: HTTP methods */
#define DPS_METHOD_UNKNOWN	0
#define DPS_METHOD_GET		1
#define DPS_METHOD_DISALLOW	2
#define DPS_METHOD_HEAD		3
#define DPS_METHOD_HREFONLY	4
#define DPS_METHOD_CHECKMP3	5
#define DPS_METHOD_CHECKMP3ONLY	6
#define DPS_METHOD_VISITLATER	7
#define DPS_METHOD_INDEX        8
#define DPS_METHOD_NOINDEX      9
#define DPS_METHOD_TAG          10
#define DPS_METHOD_CATEGORY     11

/* Tags/categories defines */
#define DPS_CATSIZE	8
#define DPS_TAGSIZE	4

/* Words origins */
#define DPS_WORD_ORIGIN_QUERY   1
#define DPS_WORD_ORIGIN_SPELL   2
#define DPS_WORD_ORIGIN_SYNONYM 4
#define DPS_WORD_ORIGIN_STOP    8
#define DPS_WORD_ORIGIN_ACCENT  16

/* Locking mutex numbers */
#define DPS_LOCK_CONF		0
#define DPS_LOCK_THREAD         1
#define DPS_LOCK_SEGMENTER      2
#define DPS_LOCK_DB             3
#define DPS_LOCK_RESOLV         4
#define DPS_LOCK_ROBOTS         5
#define DPS_LOCK_CACHED         6 /* should be the last */
#define DPS_LOCK_MAX            7

/************************ Statistics **********************/
typedef struct stat_struct {
	int	        status;
        int	        expired;
	int	        total;
        uint8           expired_size;
        uint8           total_size;
} DPS_STAT;

typedef struct stat_list_struct{
	size_t		nstats;
	DPS_STAT	*Stat;
} DPS_STATLIST;
/************************ VARLISTs ************************/

typedef struct dps_var_st {
	int			section;	/**< Number 0..255   */
	size_t			maxlen;		/**< Max length      */
	size_t			curlen;		/**< Cur length      */
	char			*val;		/**< Field Value     */
        char                    *txt_val;       /**< Field Value in plain text */
	char			*name;		/**< Field Name      */
	struct dps_varlist_st	*subvar;	/**< for Nested vars */
} DPS_VAR;

typedef struct dps_varlist_st {
	int		freeme;
        struct {
	  size_t		nvars, mvars;
	  DPS_VAR		*Var;
	} Root[256];
} DPS_VARLIST;



typedef struct {
	char		*str;
	char		*href;
	char		*section_name;
	int		section;
} DPS_TEXTITEM;

typedef struct {
	size_t		nitems, mitems;
	DPS_TEXTITEM	*Items;
} DPS_TEXTLIST;

/*****************************************************/

/** StopList unit */
typedef struct dps_stopword_struct {
	char		*word;
	char		*lang;
	dpsunicode_t	*uword;
	size_t		len, ulen;
} DPS_STOPWORD;

typedef struct {
	size_t		nstopwords;
	DPS_STOPWORD	*StopWord;
} DPS_STOPLIST;

/*****************************************************/

/** Words parameters */
typedef struct {
	size_t		min_word_len;
	size_t		max_word_len;
	size_t		correct_factor;
	size_t		incorrect_factor;
	size_t		number_factor;
	size_t		alnum_factor;
} DPS_WORDPARAM;


/** Main search structure */
typedef struct{
	urlid_t		url_id;
	uint4		coord;
} DPS_URL_CRD;

typedef struct {
        urlid_t         url_id;
        urlid_t         site_id;
        time_t          last_mod_time;
        double          pop_rank;
} DPS_URLDATA;

typedef struct {
  size_t nrec;
/*  time_t mtime;*/
  DPS_URLDATA *URLData;
} DPS_URLDATA_FILE;

typedef struct {
	size_t		ncoords;
	size_t		order;
	char		*word;
	DPS_URL_CRD	*Coords;
        DPS_URLDATA	*Data;
} DPS_URLCRDLIST;

typedef struct {
	int		freeme;
	size_t		nlists;
	DPS_URLCRDLIST	*List;
} DPS_URLCRDLISTLIST;

/** Word list unit */
typedef struct {
	int		coord;
/*	dpshash32_t	crcword;*/
        char            *word;
	dpsunicode_t	*uword;
	size_t		len, ulen;
} DPS_WORD;

typedef struct {
	size_t		mwords;	/**< Number of memory allocated for words     */
	size_t		nwords;	/**< Real number of words in list             */
	size_t		swords;	/**< Number of words in sorted list           */
	size_t		wordpos;/**< For phrases, number of current word      */
	DPS_WORD	*Word;	/**< Word list  itself                        */
} DPS_WORDLIST;


#define DPS_WRDCOORD(p,w)	( (((unsigned int)(p))<<16)+(((unsigned int)(w))<<8) )
#define DPS_WRDSEC(c)		( (((unsigned int)(c)) >> 8) & 0xFF )
#define DPS_WRDPOS(c)		( ((unsigned int)(c))>>16 )
#define DPS_WRDNUM(c)		( ((unsigned int)(c))&0xFF )
#define DPS_WRDMASK(c)		( 1L << (((unsigned int)(c))&0xFF) )


/***************************************************************/

/** Cross-word list unit */
typedef struct {
	short	pos;
	short	weight;
	char	*word;
	char	*url;
	urlid_t	referree_id;
	dpsunicode_t	*uword;
	size_t		len, ulen;
} DPS_CROSSWORD;

typedef struct {
	size_t		ncrosswords;
	size_t		mcrosswords;
	size_t		wordpos;
	DPS_CROSSWORD	*CrossWord;
} DPS_CROSSLIST;

/*****************************************************************/

typedef struct {
	int	max_net_errors;
	int	net_error_delay_time;
	int	read_timeout;
	int	doc_timeout;
	int	period[DPS_DEFAULT_MAX_HOPS];		/**< Reindex period by hops          */
	int	maxhops;	/**< Max way in mouse clicks  */
	int	index;		/**< Whether to index words   */
	int	follow;		/**< Whether to follow links  */
	int	use_robots;	/**< Whether to use robots.txt*/
	int	use_clones;	/**< Whether to detect clones */
} DPS_SPIDERPARAM;

/*****************************************************************/
typedef struct {
	int		match_type;
	int		nomatch;
	int		case_sense;
        char            *section;
        char            *subsection;
	char		*pattern;
	regex_t		*reg;
	char		*arg;
} DPS_MATCH;

typedef struct {
	size_t		nmatches;
	DPS_MATCH	*Match;
} DPS_MATCHLIST;

typedef struct {
	int beg;
	int end;
} DPS_MATCH_PART;
/*****************************************************************/

/** Structure to store server parameters */
typedef struct {
	DPS_MATCH	Match;
	urlid_t         site_id;        /**< server.rec_id            */
	char            command;        /**< 'S' - server,realm, 'F' - disallow,allow */
	int             ordre;          /**< order in list to find    */
	urlid_t         parent;         /**< parent rec_id for grouping by site */
	float           weight;         /**< server weight for popularity rank calculation */
	DPS_VARLIST	Vars;		/**< Default lang, charset,etc*/
        uint4           MaxHops;
        uint4           ndocs;
        uint4           MaxDocsPerServer;/**< Maximum number of document from Server at one indexer run */
} DPS_SERVER;

typedef struct {
	size_t		nservers;
	size_t		mservers;
	int		have_subnets;
	DPS_SERVER	*Server;
} DPS_SERVERLIST;


/*******************************************************/
/* All links are stored in the cache of this structure */
/* before actual INSERT into database                  */

typedef struct {
	char	*url;
        urlid_t referrer;
	uint4	hops;
	int	stored;	
	int	method;
        int     charset_id;
        urlid_t site_id;
        urlid_t server_id;
        float   weight;
} DPS_HREF;

typedef struct {
	size_t		mhrefs;
	size_t		nhrefs;
	size_t		shrefs;
	size_t		dhrefs;
	DPS_HREF	*Href;
} DPS_HREFLIST;

/*******************************************************/

/** Resolve stuff */
typedef struct dps_host_addr_struct {
	char		*hostname;
	struct in_addr	addr;
/*        size_t  naddr;*/
	int		net_errors;
        int             charset_id;
	time_t		last_used;
} DPS_HOST_ADDR;

typedef struct {
	size_t		nhost_addr;
	size_t		mhost_addr;
	DPS_HOST_ADDR	*host_addr;
} DPS_HOSTLIST;

/** Used in FTP sessions */
typedef struct dps_conn_struct {
        int	status;
        int	connected;
        int	err;
        int	retry;
        int	conn_fd;
#ifdef WIN32
        unsigned short port;
#else
        int	port;
#endif
        int	timeout;
        char	*hostname;
        char    *user;
        char    *pass;
        struct	sockaddr_in sin;
/*        size_t  naddr;*/
        size_t	buf_len_total;
        int	buf_len;
        int	len;
        int     charset_id;
        char	*buf;
        DPS_HOST_ADDR *Host;
        struct	dps_conn_struct *connp;
} DPS_CONN;

/** Parsed URL string */
typedef struct dps_url {
	char	*schema;
	char	*specific;
	char	*hostinfo;
	char	*auth;
	char	*hostname;
	char	*path;
	char	*filename;
	char	*anchor;
        char    *query_string;
	int	port;
	int	default_port;
        int     charset_id;
        int     freeme;
} DPS_URL;


/***************************************************/

typedef struct {
	char	*buf;		/**< Buffer to download document to          */
	char	*content;	/**< Pointer to content, after headers       */
	size_t	size;		/**< Number of bytes loaded                  */
	size_t	maxsize;	/**< Maximum bytes to load into buf          */
} DPS_HTTPBUF;

typedef struct {
	int	freeme;		/**< Whether  memory was allocated for doc   */
	int	stored;		/**< If it is already stored, forAddHref()   */
	int	method;		/**< How to download document: GET, HEAD etc */
        int     fetched;        /**< Whether Doc's info was fetched          */
        int     charset_id;     /**< Document's charset ID                   */
	
	DPS_HTTPBUF		Buf;		/**< Buffer       */
	
	DPS_HREFLIST		Hrefs;		/**< Link list    */
	DPS_WORDLIST		Words;		/**< Word list    */
	DPS_CROSSLIST		CrossWords;	/**< Crosswords   */
	
	DPS_VARLIST		RequestHeaders;	/**< Extra headers*/
	DPS_VARLIST		Sections;	/**< User sections*/
	
	DPS_TEXTLIST		TextList;	/**< Text list    */
	DPS_URL			CurURL;		/**< Parsed URL   */
	DPS_CHARSET		*lcs;		/**< LocalCharser */
	DPS_SPIDERPARAM		Spider;		/**< Spider prms  */
	DPS_CONN		connp;		/**< For FTP      */
	
} DPS_DOCUMENT;

/********************************************************/

/** External Parsers */
typedef struct dps_parser_struct{
        char		*from_mime;
	char		*to_mime;
	char		*cmd;
} DPS_PARSER;

typedef struct {
	size_t		nparsers;
	DPS_PARSER	*Parser;
} DPS_PARSERLIST;

/* Unicode regex lite BEGIN */

#define DPS_UNIREG_SUB	1
#define DPS_UNIREG_BEG	2
#define DPS_UNIREG_END	3
#define DPS_UNIREG_INC	4
#define DPS_UNIREG_EXC	5

typedef struct{
	int		type;
	dpsunicode_t	*str;
} DPS_UNIREG_TOK;

typedef struct {
	size_t		ntokens;
	DPS_UNIREG_TOK	*Token;
} DPS_UNIREG_EXP;

/* Unicode regex lite END */


/* Ispell BEGIN */


typedef struct spell_struct {
	dpsunicode_t	*word;
	char		flag[11];
	char		lang[33];
} DPS_SPELL;


typedef struct aff_struct {
	char		flag;
	char		type;
	char		lang[33];
	dpsunicode_t	mask[41];
        dpsunicode_t	find[16];
	dpsunicode_t	repl[16];
	DPS_UNIREG_EXP	reg;
	size_t		replen;
        size_t		findlen;
        char		compile;
} DPS_AFFIX;

typedef struct Tree_struct {
	int		Left[256];
	int		Right[256];
        char		lang[3];
} Tree_struct;

typedef struct {
	size_t		naffixes;
	size_t		maffixes;
	DPS_AFFIX	*Affix;
	Tree_struct	PrefixTree[DPS_LANGPERDOC];
	Tree_struct	SuffixTree[DPS_LANGPERDOC];
} DPS_AFFIXLIST;

typedef struct {
	size_t		nspell;
	size_t		mspell;
        size_t          nLang;
	DPS_SPELL	*Spell;
	Tree_struct	SpellTree[DPS_LANGPERDOC];
} DPS_SPELLLIST;

/* Ispell END */


typedef struct{
	int		cmd; /**< 'allow' or 'disallow' */
        char	        *path;
} DPS_ROBOT_RULE;

typedef struct{
	char		*hostinfo;
	size_t		nrules;
	DPS_ROBOT_RULE	*Rule;
} DPS_ROBOT;

typedef struct{
	size_t		nrobots;
	DPS_ROBOT	*Robot;
} DPS_ROBOTS;

typedef struct{
	int		type;
	char		file_name[PATH_MAX];
	uint4		hi;
	uint4		lo;
        uint4           f_hi;
        uint4           f_lo;
        urlid_t         *data;
        size_t          size;
        size_t          start;
        int             origin;
        int             need_free;
} DPS_SEARCH_LIMIT;


typedef struct {
	size_t		order;
	size_t		count;
	dpshash32_t	crcword;
	char		*word;
	dpsunicode_t	*uword;
	size_t		len, ulen;
        int     	origin;
} DPS_WIDEWORD;

typedef struct {
	size_t		nuniq;
	size_t		nwords;
	DPS_WIDEWORD	*Word;
} DPS_WIDEWORDLIST;


typedef struct {
	DPS_WIDEWORD	p;
	DPS_WIDEWORD	s;
} DPS_SYNONYM;

typedef struct {
	size_t		nsynonyms;
	size_t		msynonyms;
	DPS_SYNONYM	*Synonym;
} DPS_SYNONYMLIST;

typedef struct dps_chinaword_struct {
  dpsunicode_t *word;
  int          freq;
} DPS_CHINAWORD;

typedef struct {
  size_t        nwords, mwords;
  size_t        total;
  DPS_CHINAWORD *ChiWord;
  size_t        *hash;
} DPS_CHINALIST;


typedef struct dps_category_struct {
	int		rec_id;
	char		path[128];
	char		link[128];
	char		name[128];
} DPS_CATITEM;

typedef struct {
	char		addr[128];
	size_t		ncategories;
	DPS_CATITEM	*Category;
} DPS_CATEGORY;

/* Boolean search constants and types */
#define DPS_MAXSTACK	 128
#define DPS_STACK_LEFT	 0
#define DPS_STACK_RIGHT	 1
#define DPS_STACK_BOT	 -2
#define DPS_STACK_OR	 3
#define DPS_STACK_AND	 4
#define DPS_STACK_NOT	 5
#define DPS_STACK_PHRASE 6
#define DPS_STACK_ERR    -7
#define DPS_STACK_WORD	 200

typedef struct {
	size_t		ncstack, mcstack;
	int		*cstack;
	size_t		nastack, mastack;
	unsigned long	*astack;
} DPS_BOOLSTACK;

typedef struct {
	int		cmd;
	unsigned long	arg;
} DPS_STACK_ITEM;

typedef struct {
	size_t			work_time;
	size_t			first;
	size_t			last;
	size_t			total_found;
        size_t                  fetched;
	size_t			num_rows;
	size_t			cur_row;
/*	size_t			offset;*/
	size_t			memused;
        size_t                  *PerSite;
	int			freeme;
	DPS_DOCUMENT		*Doc;
	
	DPS_WIDEWORDLIST	WWList;
	DPS_URLCRDLIST		CoordList;
	
	/* Bool stuff */
  size_t			nitems, mitems, ncmds, orig_nitems;
  int                           phrase;
	DPS_STACK_ITEM		*items;
	
} DPS_RESULT;


#include "dps_db_int.h"

typedef struct {
	size_t		nitems;
        size_t          currdbnum;
	DPS_DB		*db;
} DPS_DBLIST;


typedef struct {
        int             do_store;         /**< Compressed copies storage flag    */
	int		CVS_ignore;	  /**< Skip CVS directgories - for tests */
        int             collect_links;    /**< Collect links flag                */
        int             use_crc32_url_id; /**< UseCRC32URLId                     */
        int             use_crosswords;
        int             use_newsext;
        int             use_accentext;
        int             use_meta;
        int             update_lm;
        int             OptimizeAtUpdate;
        int             PreloadURLData;
        int             PopRankNeoIterations;
        int             skip_unreferred;
        int             track_hops;
        time_t          hold_cache;       /**> How time in secs hold search cache */
  struct sockaddr_in    bind_addr;        /**> address for outbound connections   */
} DPS_FLAGS;

/** Forward declaration of DPS_AGENT */
struct dps_indexer_struct;

/** Config file */
typedef struct dps_config_struct {
	int		freeme;
	char		errstr[2048];
	DPS_CHARSET	*bcs;
	DPS_CHARSET	*lcs;
	
	int		url_number;	/**< For indexer -nXXX          */
	
	DPS_SERVERLIST	Servers;	/**< List of servers and realms */
        DPS_SERVER      *Cfg_Srv;
	
	DPS_MATCHLIST	Aliases;	/**< Straight aliases           */
	DPS_MATCHLIST	ReverseAliases;	/**< Reverse aliases            */
	DPS_MATCHLIST	MimeTypes;	/**< For AddType commands       */
	DPS_MATCHLIST	Filters;	/**< Allow, Disallow,etc        */
	DPS_MATCHLIST	SectionFilters;	/**< IndexIf, NoIndexIf, etc    */
        DPS_MATCHLIST   SectionMatch;   /**< Section's patterns         */
        DPS_MATCHLIST   HrefSectionMatch;   /**< HrefSection's patterns         */
	DPS_MATCHLIST	SubSectionMatch;/**< TagIf, CategoryIf          */
	
	DPS_RESULT	Targets;	/**< Targets cache              */
	
	DPS_VARLIST	Sections;	/**< document sections to parse */
	DPS_VARLIST	HrefSections;	/**< document href sections     */
	DPS_VARLIST	Vars;		/**< Config parameters          */
	
	DPS_LANGMAPLIST	LangMaps;	/**< For lang+charset quesser   */
	DPS_ROBOTS	Robots;		/**< robots.txt information     */
	DPS_SYNONYMLIST	Synonyms;	/**< Synonims list              */
	DPS_STOPLIST	StopWords;	/**< Stopwords list             */
	DPS_PARSERLIST	Parsers;	/**< External  parsers          */
	DPS_DBLIST	dbl;		/**< DB addresses	      */
	DPS_SPELLLIST	Spells;		/**< For ispell dictionaries    */
	DPS_AFFIXLIST	Affixes;	/**< For ispell affixes         */
	DPS_WORDPARAM	WordParam;	/**< Word limits                */
        DPS_CHINALIST   Chi;            /**< Chinese words list         */
        DPS_CHINALIST   Thai;           /**< Thai words list            */
        DPS_CHINALIST   Korean;         /**< Korean words list          */
        DPS_FLAGS       Flags;
        uint8           flags;
        DPS_URLDATA_FILE *URLDataFile;  /**< url data preloaded         */
        const char      *HlBeg, *HlEnd; /**< template highlighting      */
	
	/* Various file descriptors */
	int		is_log_open;	/**< if DpsOpenLog is already called   */
	FILE		*logFD;		/**< FILE structure, syslog descriptor */
        int             logs_only;      /**< Cache mode writes mode            */

	void (*ThreadInfo)(struct dps_indexer_struct *,const char * state,const char * str);
	void (*LockProc)(struct dps_indexer_struct *, int command, size_t type, const char *fname, int lineno);
	void (*RefInfo)(int code,const char *url, const char *ref);

#ifdef MECAB
        mecab_t         *mecab;
#endif

} DPS_ENV;


typedef struct {
        int             stored_sd;      /* stored connection socket descriptor  */
        int             cached_sd;      /* cached connection socket descriptors */
        int             cached_rv; 
} DPS_DEMONCONN;

typedef struct {
  size_t nitems;
  DPS_DEMONCONN *Demon;
} DPS_DEMONCONNLIST;


/** Indexer */
typedef struct dps_indexer_struct{
	int		freeme;		/**< whenever it was allocated    */
	int		handle;		/**< Handler for threaded version */
	time_t		start_time;	/**< Time of allocation, for stat */
	size_t		nbytes;		/**< Number of bytes downloaded   */
	size_t		ndocs;		/**< Number of documents          */
        size_t          nsleepsecs;     /**> Number of sleep seconds      */
	uint8		flags;		/**< Running flags                */
        int             action;
	int		doccount;	/**< for DpsGetDocCount()         */
	DPS_ENV		*Conf;		/**< Configuration                */
	DPS_LANGMAP	*LangMap;	/**< LangMap for current document */
	DPS_RESULT	Indexed;	/**< Indexed cache              */
	DPS_HREFLIST	Hrefs;		/**< Links cache                */
  DPS_DEMONCONNLIST     Demons;         /**< Daemons connections        */
	DPS_HOSTLIST	Hosts;		/**< Resolve cache              */
	DPS_DBLIST	dbl;		/**< DB addresses	      */
        DPS_VARLIST     tmpl;           /**< parsed template */
        DPS_VARLIST     st_tmpl;        /**< storedoc parsed template */
	DPS_ROBOTS	Robots;		/**< robots.txt information     */

        DPS_FLAGS       Flags;
	
	/** Cache mode limits */
	DPS_SEARCH_LIMIT	*limits;
        size_t		nlimits, loaded_limits;

        char    *DpsFindURLCache[DPS_FINDURL_CACHE_SIZE];
        urlid_t DpsFindURLCacheId[DPS_FINDURL_CACHE_SIZE];
        size_t  pURLCache;
        char    *ServerIdCache[DPS_SERVERID_CACHE_SIZE];
        char    ServerIdCacheCommand[DPS_SERVERID_CACHE_SIZE];
        urlid_t ServerIdCacheId[DPS_SERVERID_CACHE_SIZE];
        size_t  pServerIdCache;

        int     *Locked;  /**< is locked, how many times */

#ifdef WITH_TRACE
  FILE *TR;
  int level;
  char timebuf[32];
#endif
	
} DPS_AGENT;



typedef struct {
	char	*url;
	int	status;
} DPS_URLSTATE;

typedef int (*qsort_cmp)(const void*, const void*);

typedef struct {
	uint4	hi,lo;
	off_t	pos;
        size_t	len;
/*        size_t  orig_len;*/
} DPS_UINT8_POS_LEN;

typedef struct {
	uint4	val;
	off_t	pos;
        size_t	len;
/*        size_t  orig_len;*/
} DPS_UINT4_POS_LEN;

typedef struct {
	uint4	val;
	urlid_t	url_id;
} DPS_UINT4URLID;

typedef struct {
	size_t		nitems;
	DPS_UINT4URLID	*Item;
} DPS_UINT4URLIDLIST;

typedef struct {
	uint4	hi,lo;
	urlid_t	url_id;
} DPS_UINT8URLID;

typedef struct {
	size_t		nitems;
	DPS_UINT8URLID	*Item;
} DPS_UINT8URLIDLIST;


#define DPS_SEARCHD_CMD_ERROR	   1
#define DPS_SEARCHD_CMD_MESSAGE	   2
#define DPS_SEARCHD_CMD_WORDS	   3
#define DPS_SEARCHD_CMD_GOODBYE	   4
#define DPS_SEARCHD_CMD_DOCINFO	   5
#define DPS_SEARCHD_CMD_WITHOFFSET 7
#define DPS_SEARCHD_CMD_WWL        8
#define DPS_SEARCHD_CMD_CATINFO    9
#define DPS_SEARCHD_CMD_URLACTION  10
#define DPS_SEARCHD_CMD_DOCCOUNT   11
#define DPS_SEARCHD_CMD_PERSITE    12
#define DPS_SEARCHD_CMD_DATA       13
#define DPS_SEARCHD_CMD_CLONES     14

#define DPS_LOGD_CMD_WORD    0
#define DPS_LOGD_CMD_DATA    1
#define DPS_LOGD_CMD_CHECK   2
#define DPS_LOGD_CMD_URLINFO 3
#define DPS_LOGD_CMD_FLUSH   4

typedef struct {
	size_t	cmd;
	size_t	len;
} DPS_SEARCHD_PACKET_HEADER;


#define DPS_MAXTAGVAL	64

typedef struct {
	int	type;
	int	script;
	int	style;
	int	title;
	int	body;
	int	follow;
	int	index;
	int	comment;
        int     select;
	char	*lasthref;
        void    (*next_b)(void *t);
        void    (*next_e)(void *t);
        const char *e;
        const char *b;
        const char **lt;
        const char *s;
        int socket;
        int chunks;
        char *Content;
        int finished;
	size_t	ntoks;
	struct {
		const char *name;
		const char *val;
		size_t     nlen;
		size_t     vlen;
	} toks[DPS_MAXTAGVAL+1];
} DPS_HTMLTOK;

typedef struct dps_cfg_st {
        DPS_AGENT       *Indexer;
	DPS_SERVER	*Srv;
	uint8		flags;
	int		level;
	int		ordre;
} DPS_CFG;

typedef int (*DPS_OUTPUTFUNCTION)(void*, const char *fmt, ...);


#define DPS_DT_BACK    1
#define DPS_DT_ER      2
#define DPS_DT_RANGE   3
#define DPS_DT_UNKNOWN 4

enum dps_indcmd
{
  DPS_IND_INDEX,
  DPS_IND_STAT,
  DPS_IND_CREATE,
  DPS_IND_DROP,
  DPS_IND_DELETE,
  DPS_IND_REFERERS,
  DPS_IND_SQLMON,
  DPS_IND_CHECKCONF,
  DPS_IND_CONVERT,
  DPS_IND_DOCINFO
};

#ifdef WITH_TRACE

#define TRACE_IN(A, fn)  {						\
  register int trace_i;							\
  time_t tloc = time(NULL);						\
  ctime_r(&tloc, A->timebuf);						\
  A->timebuf[20] = '\0';						\
  fprintf(A->TR, "%s[%d] in ", A->timebuf+4, A->handle);		\
  for (trace_i = 0; trace_i < A->level; trace_i++) fprintf(A->TR, "-"); \
  A->level++;								\
  fprintf(A->TR, "%s\n", fn);						\
  fflush(A->TR);							\
}
#define TRACE_OUT(A)   {						\
    register int trace_i;						\
    time_t tloc = time(NULL);						\
    ctime_r(&tloc, A->timebuf);						\
    A->timebuf[20] = '\0';						\
    fprintf(A->TR, "%s[%d] out", A->timebuf+4, A->handle);		\
    if (A->level) A->level--;						\
    for (trace_i = 0; trace_i < A->level; trace_i++) fprintf(A->TR, "-"); \
    fprintf(A->TR, "at %s line %d\n", __FILE__, __LINE__);		\
    fflush(A->TR);							\
  }

#else

#define TRACE_IN(A, fn)
#define TRACE_OUT(A)

#endif






#endif /* _DPS_COMMON_H */
