// vim: cindent foldmethod=marker

//
// BEGIN FLOCK GPL
// 
// Copyright Flock Inc. 2005-2006
// http://flock.com
// 
// This file may be used under the terms of of the
// GNU General Public License Version 2 or later (the "GPL"),
// http://www.gnu.org/licenses/gpl.html
// 
// Software distributed under the License is distributed on an "AS IS" basis,
// WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
// for the specific language governing rights and limitations under the
// License.
// 
// END FLOCK GPL
//

//function DEBUG(X) {}
function DEBUG(X) { debug ('flockLuceneIndexer.js: '+X+'\n'); }

// now - the XPCOM service
const flockIFavoriteProperties              = Components.interfaces.flockIFavoriteProperties;
const FIELD_WSID                            = flockIFavoriteProperties.FIELD_WSID;
const FIELD_TAGS                            = flockIFavoriteProperties.FIELD_TAGS;
const FIELD_PERMISSIONS                     = flockIFavoriteProperties.FIELD_PERMISSIONS;
const FIELD_NAME                            = flockIFavoriteProperties.FIELD_NAME;
const FIELD_SHORTCUTURL                     = flockIFavoriteProperties.FIELD_SHORTCUTURL;
const FIELD_DESCRIPTION                     = flockIFavoriteProperties.FIELD_DESCRIPTION;
const FIELD_NSYNC                           = flockIFavoriteProperties.FIELD_NSYNC;
const FIELD_LASTVISITDATE                   = flockIFavoriteProperties.FIELD_LASTVISITDATE;
const FIELD_LASTMODIFIEDDATE                = flockIFavoriteProperties.FIELD_LASTMODIFIEDDATE;
const FIELD_ADDEDDATE                       = flockIFavoriteProperties.FIELD_ADDEDDATE;

const LUCENE_INDEXER_CONTRACTID             = '@flock.com/lucene/indexer;1';
const LUCENE_INDEXER_CID                    = '{15a27de0-59ef-4774-8c76-e61211e55345}';
const flockILuceneIndexer                   = Components.interfaces.flockILuceneIndexer;
const nsIRDFObserver                        = Components.interfaces.nsIRDFObserver;
const nsIRDFContainer                       = Components.interfaces.nsIRDFContainer;
const nsIRDFNode                            = Components.interfaces.nsIRDFNode;
const nsIRDFResource                        = Components.interfaces.nsIRDFResource;
const nsIRDFLiteral                         = Components.interfaces.nsIRDFLiteral;
const nsIRDFInt                             = Components.interfaces.nsIRDFInt;
const nsIRDFDate                            = Components.interfaces.nsIRDFDate;
const nsIRDFBlob                            = Components.interfaces.nsIRDFBlob;/*}}}*/
const nsIClassInfo                          = Components.interfaces.nsIClassInfo;
const nsITimer                              = Components.interfaces.nsITimer;
const nsICache                              = Components.interfaces.nsICache;

const NAME_PROPERTY                         = 'http://home.netscape.com/NC-rdf#Name';
const CHILD_PROPERTY                        = 'http://home.netscape.com/NC-rdf#child';

// a toString method for RDF nodes/*{{{*/
function rdf_repr (node) {
        try { 
            var value = node.QueryInterface (nsIRDFResource).ValueUTF8;
            value = value.replace ('http://www.w3.org/1999/02/22-rdf-syntax-ns#', 'W3:');
            value = value.replace ('http://home.netscape.com/NC-rdf#', 'NS:');
            value = value.replace ('http://developer.mozilla.org/rdf/vocabulary/forward-proxy#', 'FP:');
            value = value.replace ('http://flock.com/rdf#', 'FL:');
            return value;
        }
        catch (e) { }

        try { 
            var date = node.QueryInterface (nsIRDFDate);
            return 'date:'+date.Value;
        }
        catch (e) { }

        try { 
            var blob = node.QueryInterface (nsIRDFBlob);
            return 'blob: length'+blob.Length;
        }
        catch (e) { }

        try { 
            var int = node.QueryInterface (nsIRDFInt);
            return 'int:'+int.Value;
        }
        catch (e) { }

        try { 
            var literal = node.QueryInterface (nsIRDFLiteral);
            return '"'+literal.Value+'"';
        }
        catch (e) { }
        
        return 'unknown:'+node;
}/*}}}*/

function flockLuceneIndexer () {
    this.historyRdf =
        Components.classes['@mozilla.org/rdf/datasource;1?name=history']
            .getService (Components.interfaces.nsIRDFDataSource);
    this.rdfService = Components.classes['@mozilla.org/rdf/rdf-service;1']
            .getService (Components.interfaces.nsIRDFService);

    this.historyRoot = this.rdfService.GetResource ('NC:HistoryRoot');
    this.childResource = this.rdfService.GetResource (CHILD_PROPERTY);
    
    this.historyRdf.AddObserver (this);

    this._initialized = false;
}

flockLuceneIndexer.prototype._init = function() {
    this._searchService = Components.classes['@flock.com/historysearch/service;1'].getService(Components.interfaces.flockIHistorySearchService);
    this._favoritesService = Components.classes['@mozilla.org/rdf/datasource;1?name=flock-favorites'].getService(Components.interfaces.flockIFavoritesService);
    this._cacheService = Components.classes["@mozilla.org/network/cache-service;1"].getService(Components.interfaces.nsICacheService);

    this._unicodeConverter = Components.classes["@mozilla.org/intl/scriptableunicodeconverter"].createInstance(Components.interfaces.nsIScriptableUnicodeConverter);
    this._sstream = Components.classes['@mozilla.org/scriptableinputstream;1'].createInstance(Components.interfaces.nsIScriptableInputStream);
    
    this._initialized = true;
}

flockLuceneIndexer.prototype.delayAddDocument = function (aURL) {
    // add it in five seconds
    var callback = new Object ();
    callback.indexer = this;
    callback.url = aURL;
    callback.notify = function (aTimer) {
        DEBUG ('timer callback!');
        this.indexer.AddDocument (this.url);
    }
    var timer = Components.classes["@mozilla.org/timer;1"]
            .createInstance(nsITimer);
    timer.initWithCallback (callback, 5000, nsITimer.TYPE_ONE_SHOT);
    DEBUG ('set up timer!');
}

// the nsIRDFObserver interface
flockLuceneIndexer.prototype.onAssert = function (ds, src, prop, tar) {/*{{{*/
    DEBUG ('HISTORY: onAssert ('+rdf_repr (src)+','+rdf_repr (prop)+','+rdf_repr (tar)+')');
    if (prop.ValueUTF8 == NAME_PROPERTY 
            && ds.HasAssertion (this.historyRoot, this.childResource, src, 
                true)) {
        this.delayAddDocument (src.ValueUTF8);
    }
}/*}}}*/
flockLuceneIndexer.prototype.onUnassert = function (ds, src, prop, tar) {/*{{{*/
    DEBUG ('HISTORY: onUnassert ('+rdf_repr (src)+','+rdf_repr (prop)+','+rdf_repr (tar)+')');
    if (src.ValueUTF8 == 'NC:HistoryRoot' &&
            prop.ValueUTF8 == 'http://home.netscape.com/NC-rdf#child') {
        // were removing a history entry!
        this.TryToDelete (tar.ValueUTF8);
    }

}/*}}}*/
flockLuceneIndexer.prototype.onChange = function (ds, src, prop, tar0, tar1) {/*{{{*/
    DEBUG ('HISTORY: onChange ('+rdf_repr (src)+','+rdf_repr (prop)+','+rdf_repr (tar0)+','+rdf_repr (tar1)+')');
    if (prop.ValueUTF8 == NAME_PROPERTY 
            && ds.HasAssertion (this.historyRoot, this.childResource, src, 
                true)) {
        this.delayAddDocument (src.ValueUTF8);
    }
}/*}}}*/
flockLuceneIndexer.prototype.onMove = function (ds, src0, src1, prop, tar) {/*{{{*/
    DEBUG ('HISTORY: onMove ('+rdf_repr (src0)+','+rdf_repr (src1)+','+rdf_repr (prop)+','+rdf_repr (tar)+')');
}/*}}}*/
flockLuceneIndexer.prototype.onBeginUpdateBatch = function (ds) {/*{{{*/
    DEBUG ('HISTORY: onBeginUpdateBatch()');
}/*}}}*/
flockLuceneIndexer.prototype.onEndUpdateBatch = function (ds) {/*{{{*/
    DEBUG ('HISTORY: onEndUpdateBatch()');
}/*}}}*/


// the flockILuceneIndexer implementation
flockLuceneIndexer.prototype.AddDocument = function (aURL) {
    if(!this._initialized) {
      this._init();
    }

    DEBUG ('AddDocument ("'+aURL+'")');
    // XXX
    //return;
    
    // FIXME: we need to throttle the frequency with which we re-index pages.

    DEBUG ('flockLuceneIndexer.AddDocument ('+aURL+')');

    // get metadata from the favorites service
    var properties = this._favoritesService.getFavoriteProperties (aURL);

    // title could be url if nothing better comes along
    // FIXME: instead we should handle empty titles in our templates
    this._uri = aURL;
    this._title = '';
    this._tags = '';
    this._description = '';
    this._favorite = false;
    this._data = "";

    if (!this._favoritesService.isFavorite(aURL)) {
        // this is not favoriteed
        // lets try to get the title from history
        DEBUG ('flockLuceneIndexer.AddDocument: no favorite properties - lets get what we can from history');
        var resource = this.historyRdf.GetSource (
            this.rdfService.GetResource ('http://home.netscape.com/NC-rdf#URL'),
            this.rdfService.GetResource (aURL), true);
        if (resource) {
            var title_node = this.historyRdf.GetTarget (resource,
                    this.rdfService.GetResource ('http://home.netscape.com/NC-rdf#Name'), true);
            if (title_node && title_node.QueryInterface (nsIRDFLiteral)) {
                this._title = title_node.Value;
            }
        }
    } else {
        // lets get title & tags from favorites
        if (properties.fields&Components.interfaces.flockIFavoriteProperties.FIELD_TAGS && properties.tags != null) {
            this._tags = properties.tags;
            DEBUG ('flockLuceneIndexer.AddDocument: favorite tags: '+ this._tags);
        }

        if (properties.fields & Components.interfaces.flockIFavoriteProperties.FIELD_NAME) {
            // try to get the title from favorites
            this._title = properties.name;
            DEBUG ('flockLuceneIndexer.AddDocument: favorite title: '+ this._title);
        }

        if (properties.fields & Components.interfaces.flockIFavoriteProperties.FIELD_DESCRIPTION) {
            // try to get the description from favorites
            this._description = properties.description;
            DEBUG ('flockLuceneIndexer.AddDocument: favorite description: '+ this._description);
        }

        this._favorite = true;
    }

    // get the page contents from the cache service
    var cacheSession = this._cacheService.createSession ('HTTP', 0, true);
    cacheSession.doomEntriesIfExpired = false;
    try {
        cacheSession.asyncOpenCacheEntry (aURL, nsICache.ACCESS_READ, this);
    } catch (e) {
        DEBUG ('failed to get page text: '+e);
        this._insertHistoryDocument();
    }
}

flockLuceneIndexer.prototype.onCacheEntryAvailable = function(descriptor, accessGranted, status) {
  try {
      this._charset = descriptor.getMetaDataElement ('charset');
  } catch (e) {
      this._charset = 'ISO-8859-1';
  }
  // FIXME: better header parsing
  // FIXME: handle other transfer encodings (compress?)
  var gzip = (descriptor.getMetaDataElement ('response-head').indexOf ('Content-Encoding: gzip') >=0);
  if (gzip) {
      var decompressor = Components.classes["@flock.com/cachedecompress;1"].createInstance(Components.interfaces.flockICacheDecompress);
      decompressor.decompressEntry(descriptor, this);
  } else {
      var istream = descriptor.openInputStream (0);
      this._sstream.init (istream);
      var data = this._sstream.read (this._sstream.available ());
      try {
          this._unicodeConverter.charset = this._charset;
          data = this._unicodeConverter.ConvertToUnicode(data);
      } catch (e) {
          /* do nothing, data is still set */
      }
      this._data = data;
      this._insertHistoryDocument();
      this._sstream.close();
      descriptor.close();
  }
}

flockLuceneIndexer.prototype.onDecompressed = function(aData) {
  var data;
  try {
    this._unicodeConverter.charset = this._charset;
    data = this._unicodeConverter.ConvertToUnicode(aData);
  } catch (e) {
    data = aData;
  }
  this._data = data;
  this._insertHistoryDocument();
}

flockLuceneIndexer.prototype._insertHistoryDocument = function() {
  var data = this._data;

  // strip tags
  data = data.replace(/<!--[^\0]*?-->|<script[^><]*>[^\0]*?<\/script>|<style[^><]*>[^\0]*?<\/style>|<title[^><]*>[^\0]*?<\/title>|<[^><]*>/gi, " ");
  
  // decode numeric html entities
  data = data.replace(/&#(\d+);/g, function (str, num, offset, s) { return String.fromCharCode(num) });

  // decode named html entities
  var nameNumericMap = {
    "nbsp": 160,
    "copy": 169,
    "reg": 174,
    "sup2": 178,
    "sup3": 179,
    "quot": 34,
    "amp": 38,
    "lt": 60,
    "gt": 62,
    "ndash": 8211,
    "mdash": 8212,
    "lsquo": 8216,
    "rsquo": 8217,
    "ldquo": 8220,
    "rdquo": 8221,
    "bull": 8226,
    "dagger": 8224,
    "Dagger": 8225,
    "prime": 8242,
    "Prime": 8243,
    "lsaquo": 8249,
    "rsaquo": 8250,
    "euro": 8364,
    "trade": 8482,
    "tilde": 732,
    "circ": 710,
    "spades": 9824,
    "clubs": 9827,
    "hearts": 9829,
    "diams": 9830,
    "loz": 9674,
    "larr": 8592,
    "rarr": 8594,
    "uarr": 8593,
    "darr": 8595,
    "harr": 8596,
    "not": 172,

    "iexcl": 161,
    "cent": 162,
    "pound": 163,
    "curren": 164,
    "yen": 165,
    "brvbar": 166,
    "sect": 167,
    "uml": 168,
    "ordf": 170,
    "laquo": 171,
    "shy": 173,
    "macr": 175,
    "deg": 176,
    "plusmn": 177,
    "acute": 180,
    "micro": 181,
    "para": 182,
    "middot": 183,
    "cedil": 184,
    "sup1": 185,
    "ordm": 186,
    "raquo": 187,
    "frac14": 188,
    "frac12": 189,
    "frac34": 190,
    "iquest": 191,
    "Agrave": 192,
    "Aacute": 193,
    "Acirc": 194,
    "Atilde": 195,
    "Auml": 196,
    "Aring": 197,
    "AElig": 198,
    "Ccedil": 199,
    "Egrave": 200,
    "Eacute": 201,
    "Ecirc": 202,
    "Euml": 203,
    "Igrave": 204,
    "Iacute": 205,
    "Icirc": 206,
    "Iuml": 207,
    "ETH": 208,
    "Ntilde": 209,
    "Ograve": 210,
    "Oacute": 211,
    "Ocirc": 212,
    "Otilde": 213,
    "Ouml": 214,
    "times": 215,
    "Oslash": 216,
    "Ugrave": 217,
    "Uacute": 218,
    "Ucirc": 219,
    "Uml": 220,
    "Yacute": 221,
    "THORN": 222,
    "szlig": 223,
    "agrave": 224,
    "aacute": 225,
    "acirc": 226,
    "atilde": 227,
    "auml": 228,
    "aring": 229,
    "aelig": 230,
    "ccedil": 231,
    "egrave": 232,
    "eacute": 233,
    "ecirc": 234,
    "euml": 235,
    "igrave": 236,
    "iacute": 237,
    "icirc": 238,
    "iuml": 239,
    "eth": 240,
    "ntilde": 241,
    "ograve": 242,
    "oacute": 243,
    "ocirc": 244,
    "otilde": 245,
    "ouml": 246,
    "divide": 247,
    "oslash": 248,
    "ugrave": 249,
    "uacute": 250,
    "ucirc": 251,
    "uuml": 252,
    "yacute": 253,
    "thorn": 254,
    "yuml": 255,
  };
  data = data.replace(/&(\w+);/g, function (str, name, offset, s) { return String.fromCharCode(nameNumericMap[name]) });

  this._searchService.AddDocumentText(this._uri, this._tags, this._favorite, this._title, this._description, data);
}

flockLuceneIndexer.prototype.TryToDelete = function (aURL) { 
    DEBUG ('TryToDelete ("'+aURL+'")');
    // get references to the services we use
    var flockHistorySearchService = 
        Components.classes['@flock.com/historysearch/service;1']
            .getService (Components.interfaces.flockIHistorySearchService);
    var flockFavoritesService = 
        Components.classes['@mozilla.org/rdf/datasource;1?name=flock-favorites']
            .getService (Components.interfaces.flockIFavoritesService);

    // we want to delete this url from the index if its not favoriteed 
    // or still in history
    history = this.historyRdf.GetSource (
        this.rdfService.GetResource ('http://home.netscape.com/NC-rdf#URL'),
        this.rdfService.GetResource (aURL), true);
    favorite = flockFavoritesService.isFavorite (aURL);
    if (history || favorite) {
        // its still in history or favoriteed
        DEBUG ('     cannot delete');
        return false;
    } else {
        flockHistorySearchService.DeleteDocument (aURL);
        DEBUG ('     can delete');
        return true;
    }
}
flockLuceneIndexer.prototype.Search = function (aQuery, aMaxResults, aListener) {
    DEBUG ('Search ("'+aQuery+'", '+aMaxResults+')');

    // get references to the services we use
    var flockHistorySearchService = 
        Components.classes['@flock.com/historysearch/service;1']
            .getService (Components.interfaces.flockIHistorySearchService);
    
    // create an in-memory RDF datasource to hold the results
    var rdf = Components.classes[
        "@mozilla.org/rdf/datasource;1?name=in-memory-datasource"]
            .createInstance(Components.interfaces.nsIRDFDataSource);

    // start the search

    flockHistorySearchService.Search (aQuery, aMaxResults, aListener, rdf);

    // and return the data source
    return rdf;
}

// the nsIClassInfo implementation
flockLuceneIndexer.prototype.flags = nsIClassInfo.SINGLETON;
flockLuceneIndexer.prototype.classDescription = "Flock Lucene Indexer";
flockLuceneIndexer.prototype.getInterfaces = function (count) {
    var interfaceList = [flockILuceneIndexer, nsIClassInfo, Components.interfaces.nsICacheListener, Components.interfaces.flockICacheDecompressListener];
    count.value = interfaceList.length;
    return interfaceList;
}
flockLuceneIndexer.prototype.getHelperForLanguage = function (count) { return null; }

// the nsISupports implementation
flockLuceneIndexer.prototype.QueryInterface =
function (iid) {
    if (!iid.equals(flockILuceneIndexer) && 
        !iid.equals(nsIRDFObserver) &&
        !iid.equals(nsIClassInfo) &&
        !iid.equals(nsISupports) &&
        !iid.equals(Components.interfaces.nsICacheListener) &&
        !iid.equals(Components.interfaces.flockICacheDecompressListener))
        throw Components.results.NS_ERROR_NO_INTERFACE;
    return this;
}



                           
function loadLibrary(aFilename) { // {{{
    var file = Components.classes["@mozilla.org/file/directory_service;1"]
        .getService(Components.interfaces.nsIProperties)
        .get("ComsD", Components.interfaces.nsIFile);
    file.append(aFilename);

    var ios = Components.classes["@mozilla.org/network/io-service;1"]
        .getService(Components.interfaces.nsIIOService);
    var fileHandler = ios.getProtocolHandler("file")
        .QueryInterface(Components.interfaces.nsIFileProtocolHandler);
    var spec = fileHandler.getURLSpecFromFile(file);

    var scriptLoader = Components.classes["@mozilla.org/moz/jssubscript-loader;1"]
        .getService(Components.interfaces.mozIJSSubScriptLoader);

    scriptLoader.loadSubScript(spec);
} // }}}

// load the XPCOM library
loadLibrary("flockXPCOMTemplate.js.lib");

var module = new xpcomModule (LUCENE_INDEXER_CID, LUCENE_INDEXER_CONTRACTID,
        'Flock Lucene Indexer Service',
        flockLuceneIndexer);
function NSGetModule(aCompMgr, aFileSpec) {
    return module;
};

