// vim: cindent foldmethod=marker

//
// BEGIN FLOCK GPL
// 
// Copyright Flock Inc. 2005-2006
// http://flock.com
// 
// This file may be used under the terms of of the
// GNU General Public License Version 2 or later (the "GPL"),
// http://www.gnu.org/licenses/gpl.html
// 
// Software distributed under the License is distributed on an "AS IS" basis,
// WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
// for the specific language governing rights and limitations under the
// License.
// 
// END FLOCK GPL
//

//function DEBUG(X) {}
function DEBUG(X) { debug ('flockLuceneIndexer.js: '+X+'\n'); }

// now - the XPCOM service
const flockIFavoriteProperties              = Components.interfaces.flockIFavoriteProperties;
const FIELD_WSID                            = flockIFavoriteProperties.FIELD_WSID;
const FIELD_TAGS                            = flockIFavoriteProperties.FIELD_TAGS;
const FIELD_PERMISSIONS                     = flockIFavoriteProperties.FIELD_PERMISSIONS;
const FIELD_NAME                            = flockIFavoriteProperties.FIELD_NAME;
const FIELD_SHORTCUTURL                     = flockIFavoriteProperties.FIELD_SHORTCUTURL;
const FIELD_DESCRIPTION                     = flockIFavoriteProperties.FIELD_DESCRIPTION;
const FIELD_NSYNC                           = flockIFavoriteProperties.FIELD_NSYNC;
const FIELD_LASTVISITDATE                   = flockIFavoriteProperties.FIELD_LASTVISITDATE;
const FIELD_LASTMODIFIEDDATE                = flockIFavoriteProperties.FIELD_LASTMODIFIEDDATE;
const FIELD_ADDEDDATE                       = flockIFavoriteProperties.FIELD_ADDEDDATE;

const LUCENE_INDEXER_CONTRACTID             = '@flock.com/lucene/indexer;1';
const LUCENE_INDEXER_CID                    = '{15a27de0-59ef-4774-8c76-e61211e55345}';
const flockILuceneIndexer                   = Components.interfaces.flockILuceneIndexer;
const nsIRDFObserver                        = Components.interfaces.nsIRDFObserver;
const nsIRDFContainer                       = Components.interfaces.nsIRDFContainer;
const nsIRDFNode                            = Components.interfaces.nsIRDFNode;
const nsIRDFResource                        = Components.interfaces.nsIRDFResource;
const nsIRDFLiteral                         = Components.interfaces.nsIRDFLiteral;
const nsIRDFInt                             = Components.interfaces.nsIRDFInt;
const nsIRDFDate                            = Components.interfaces.nsIRDFDate;
const nsIRDFBlob                            = Components.interfaces.nsIRDFBlob;/*}}}*/
const nsIClassInfo                          = Components.interfaces.nsIClassInfo;
const nsITimer                              = Components.interfaces.nsITimer;
const nsICache                              = Components.interfaces.nsICache;

const NAME_PROPERTY                         = 'http://home.netscape.com/NC-rdf#Name';
const CHILD_PROPERTY                        = 'http://home.netscape.com/NC-rdf#child';

const CID = '@flock.com/html2text;1';
const IID = Components.interfaces.flockIHTML2TEXT;

// a toString method for RDF nodes/*{{{*/
function rdf_repr (node) {
        try { 
            var value = node.QueryInterface (nsIRDFResource).ValueUTF8;
            value = value.replace ('http://www.w3.org/1999/02/22-rdf-syntax-ns#', 'W3:');
            value = value.replace ('http://home.netscape.com/NC-rdf#', 'NS:');
            value = value.replace ('http://developer.mozilla.org/rdf/vocabulary/forward-proxy#', 'FP:');
            value = value.replace ('http://flock.com/rdf#', 'FL:');
            return value;
        }
        catch (e) { }

        try { 
            var date = node.QueryInterface (nsIRDFDate);
            return 'date:'+date.Value;
        }
        catch (e) { }

        try { 
            var blob = node.QueryInterface (nsIRDFBlob);
            return 'blob: length'+blob.Length;
        }
        catch (e) { }

        try { 
            var int = node.QueryInterface (nsIRDFInt);
            return 'int:'+int.Value;
        }
        catch (e) { }

        try { 
            var literal = node.QueryInterface (nsIRDFLiteral);
            return '"'+literal.Value+'"';
        }
        catch (e) { }
        
        return 'unknown:'+node;
}/*}}}*/
function flockLuceneIndexer () {
    this.historyRdf =
        Components.classes['@mozilla.org/rdf/datasource;1?name=history']
            .getService (Components.interfaces.nsIRDFDataSource);
    this.rdfService = Components.classes['@mozilla.org/rdf/rdf-service;1']
            .getService (Components.interfaces.nsIRDFService);

    this.historyRoot = this.rdfService.GetResource ('NC:HistoryRoot');
    this.childResource = this.rdfService.GetResource (CHILD_PROPERTY);

    this.historyRdf.AddObserver (this);
}
flockLuceneIndexer.prototype.delayAddDocument = function (aURL) {
    // add it in five seconds
    var callback = new Object ();
    callback.indexer = this;
    callback.url = aURL;
    callback.notify = function (aTimer) {
        DEBUG ('timer callback!');
        this.indexer.AddDocument (this.url);
    }
    var timer = Components.classes["@mozilla.org/timer;1"]
            .createInstance(nsITimer);
    timer.initWithCallback (callback, 5000, nsITimer.TYPE_ONE_SHOT);
    DEBUG ('set up timer!');
}
// the nsIRDFObserver interface
flockLuceneIndexer.prototype.onAssert = function (ds, src, prop, tar) {/*{{{*/
    DEBUG ('HISTORY: onAssert ('+rdf_repr (src)+','+rdf_repr (prop)+','+rdf_repr (tar)+')');
    if (prop.ValueUTF8 == NAME_PROPERTY 
            && ds.HasAssertion (this.historyRoot, this.childResource, src, 
                true)) {
        this.delayAddDocument (src.ValueUTF8);
    }
}/*}}}*/
flockLuceneIndexer.prototype.onUnassert = function (ds, src, prop, tar) {/*{{{*/
    DEBUG ('HISTORY: onUnassert ('+rdf_repr (src)+','+rdf_repr (prop)+','+rdf_repr (tar)+')');
    if (src.ValueUTF8 == 'NC:HistoryRoot' &&
            prop.ValueUTF8 == 'http://home.netscape.com/NC-rdf#child') {
        // were removing a history entry!
        this.TryToDelete (tar.ValueUTF8);
    }

}/*}}}*/
flockLuceneIndexer.prototype.onChange = function (ds, src, prop, tar0, tar1) {/*{{{*/
    DEBUG ('HISTORY: onChange ('+rdf_repr (src)+','+rdf_repr (prop)+','+rdf_repr (tar0)+','+rdf_repr (tar1)+')');
    if (prop.ValueUTF8 == NAME_PROPERTY 
            && ds.HasAssertion (this.historyRoot, this.childResource, src, 
                true)) {
        this.delayAddDocument (src.ValueUTF8);
    }
}/*}}}*/
flockLuceneIndexer.prototype.onMove = function (ds, src0, src1, prop, tar) {/*{{{*/
    DEBUG ('HISTORY: onMove ('+rdf_repr (src0)+','+rdf_repr (src1)+','+rdf_repr (prop)+','+rdf_repr (tar)+')');
}/*}}}*/
flockLuceneIndexer.prototype.onBeginUpdateBatch = function (ds) {/*{{{*/
    DEBUG ('HISTORY: onBeginUpdateBatch()');
}/*}}}*/
flockLuceneIndexer.prototype.onEndUpdateBatch = function (ds) {/*{{{*/
    DEBUG ('HISTORY: onEndUpdateBatch()');
}/*}}}*/


// the flockILuceneIndexer implementation
flockLuceneIndexer.prototype.AddDocument = function (aURL) {
    DEBUG ('AddDocument ("'+aURL+'")');
    // XXX
    //return;
    
    // FIXME: we need to throttle the frequency with which we re-index pages.

    var h2t = Components.classes[CID].getService(IID);

    DEBUG ('flockLuceneIndexer.AddDocument ('+aURL+')');

    // get references to the services we use
    var flockHistorySearchService = 
        Components.classes['@flock.com/historysearch/service;1']
            .getService (Components.interfaces.flockIHistorySearchService);
    var flockFavoritesService = 
        Components.classes['@mozilla.org/rdf/datasource;1?name=flock-favorites']
            .getService (Components.interfaces.flockIFavoritesService);
    var cacheService = 
        Components.classes["@mozilla.org/network/cache-service;1"]
            .getService (Components.interfaces.nsICacheService);


    // get metadata from the favorites service
    var properties = flockFavoritesService.getFavoriteProperties (aURL);

    // title could be url if nothing better comes along
    // FIXME: instead we should handle empty titles in our templates
    var title = '';
    var tags = '';
    var description = '';
    var favorite = false;

    if (!flockFavoritesService.isFavorite(aURL)) {
        // this is not favoriteed
        // lets try to get the title from history
        DEBUG ('flockLuceneIndexer.AddDocument: no favorite properties - lets get what we can from history');
        var resource = this.historyRdf.GetSource (
            this.rdfService.GetResource ('http://home.netscape.com/NC-rdf#URL'),
            this.rdfService.GetResource (aURL), true);
        if (resource) {
            var title_node = this.historyRdf.GetTarget (resource,
                    this.rdfService.GetResource ('http://home.netscape.com/NC-rdf#Name'), true);
            if (title_node && title_node.QueryInterface (nsIRDFLiteral)) {
                title = title_node.Value;
            }
        }
    } else {
        // lets get title & tags from favorites
        if (properties.fields&Components.interfaces.flockIFavoriteProperties.FIELD_TAGS && properties.tags != null) {
            tags = properties.tags;
            DEBUG ('flockLuceneIndexer.AddDocument: favorite tags: '+tags);
        }

        if (properties.fields & Components.interfaces.flockIFavoriteProperties.FIELD_NAME) {
            // try to get the title from favorites
            title = properties.name;
            DEBUG ('flockLuceneIndexer.AddDocument: favorite title: '+title);
        }

        if (properties.fields & Components.interfaces.flockIFavoriteProperties.FIELD_DESCRIPTION) {
            // try to get the description from favorites
            description = properties.description;
            DEBUG ('flockLuceneIndexer.AddDocument: favorite description: '+description);
        }

        favorite = true;
    }


    // get the page contents from the cache service
    var cacheSession = cacheService.createSession ('HTTP', 0, true);
    cacheSession.doomEntriesIfExpired = false;
    var listener = new Object ();
    listener.onCacheEntryAvailable = function (descriptor, accessGranted, status) {
        try {
            var charset = descriptor.getMetaDataElement ('charset');
        } catch (e) {
            var charset = 'ISO-8859-1';
        }
        // FIXME: better header parsing
        // FIXME: handle other transfer encodings (compress?)
        var gzip = (descriptor.getMetaDataElement ('response-head').indexOf ('Content-Encoding: gzip') >=0);
        var istream = descriptor.openInputStream (0);
        if (gzip) {
            var cstream = Components.classes['@mozilla.org/streamconv;1?from=gzip&to=uncompressed'].createInstance (Components.interfaces.nsIStreamConverter);
            //istream = cstream.convert (istream, 'gzip', 'uncompressed', null);
            var listener = new Object ();
            listener.data = ''
            listener.onDataAvailable = function (request, context, inputstream, offset, count) {
                var sstream = Components.classes['@mozilla.org/scriptableinputstream;1'].createInstance (Components.interfaces.nsIScriptableInputStream);
                sstream.init (inputstream);
                listener.data += sstream.read (count);
            }
            listener.onStartRequest = function (aRequest, aContext) { }
            listener.onStopRequest = function (aRequest, aContext, aStatusCode) {
                var data;
                try {
                    var converter = Components.classes["@mozilla.org/intl/scriptableunicodeconverter"].createInstance(Components.interfaces.nsIScriptableUnicodeConverter);
                    converter.charset = charset;
                    data = converter.ConvertToUnicode(this.data);
                } catch (e) {
                    data = this.data;
                }
                flockHistorySearchService.AddDocumentText (aURL, tags, favorite, title, 
                        description, h2t.convert(data));
                istream.close ();
                descriptor.close ();
            }
            cstream.asyncConvertData ('gzip', 'uncompressed', listener, null);
            
            var pump = Components.classes['@mozilla.org/network/input-stream-pump;1'].createInstance (Components.interfaces.nsIInputStreamPump);
            pump.init (istream, 0, descriptor.dataSize, 0, 0, true);
            pump.asyncRead (cstream, null);
        } else {
            var sstream = Components.classes['@mozilla.org/scriptableinputstream;1'].createInstance (Components.interfaces.nsIScriptableInputStream);
            sstream.init (istream);
            var data = sstream.read (sstream.available ());
            try {
                var converter = Components.classes["@mozilla.org/intl/scriptableunicodeconverter"].createInstance(Components.interfaces.nsIScriptableUnicodeConverter);
                converter.charset = charset;
                data = converter.ConvertToUnicode(data);
            } catch (e) {
                /* do nothing, data is still set */
            }
            flockHistorySearchService.AddDocumentText (aURL, tags, favorite, title, 
                    description, h2t.convert(data));
            sstream.close ();
            descriptor.close ();
        }
    }
    try {
        cacheSession.asyncOpenCacheEntry (aURL, nsICache.ACCESS_READ, listener);
    } catch (e) {
        DEBUG ('failed to get page text: '+e);
        flockHistorySearchService.AddDocumentText (aURL, tags, favorite, title, description, '');
    }
}
flockLuceneIndexer.prototype.TryToDelete = function (aURL) { 
    DEBUG ('TryToDelete ("'+aURL+'")');
    // get references to the services we use
    var flockHistorySearchService = 
        Components.classes['@flock.com/historysearch/service;1']
            .getService (Components.interfaces.flockIHistorySearchService);
    var flockFavoritesService = 
        Components.classes['@mozilla.org/rdf/datasource;1?name=flock-favorites']
            .getService (Components.interfaces.flockIFavoritesService);

    // we want to delete this url from the index if its not favoriteed 
    // or still in history
    history = this.historyRdf.GetSource (
        this.rdfService.GetResource ('http://home.netscape.com/NC-rdf#URL'),
        this.rdfService.GetResource (aURL), true);
    favorite = flockFavoritesService.isFavorite (aURL);
    if (history || favorite) {
        // its still in history or favoriteed
        DEBUG ('     cannot delete');
        return false;
    } else {
        flockHistorySearchService.DeleteDocument (aURL);
        DEBUG ('     can delete');
        return true;
    }
}
flockLuceneIndexer.prototype.Search = function (aQuery, aMaxResults, aListener) {
    DEBUG ('Search ("'+aQuery+'", '+aMaxResults+')');

    // get references to the services we use
    var flockHistorySearchService = 
        Components.classes['@flock.com/historysearch/service;1']
            .getService (Components.interfaces.flockIHistorySearchService);
    
    // create an in-memory RDF datasource to hold the results
    var rdf = Components.classes[
        "@mozilla.org/rdf/datasource;1?name=in-memory-datasource"]
            .createInstance(Components.interfaces.nsIRDFDataSource);

    // start the search

    flockHistorySearchService.Search (aQuery, aMaxResults, aListener, rdf);

    // and return the data source
    return rdf;
}

// the nsIClassInfo implementation
flockLuceneIndexer.prototype.flags = nsIClassInfo.SINGLETON;
flockLuceneIndexer.prototype.classDescription = "Flock Lucene Indexer";
flockLuceneIndexer.prototype.getInterfaces = function (count) {
    var interfaceList = [flockILuceneIndexer, nsIClassInfo];
    count.value = interfaceList.length;
    return interfaceList;
}
flockLuceneIndexer.prototype.getHelperForLanguage = function (count) { return null; }

// the nsISupports implementation
flockLuceneIndexer.prototype.QueryInterface =
function (iid) {
    if (!iid.equals(flockILuceneIndexer) && 
        !iid.equals(nsIRDFObserver) &&
        !iid.equals(nsIClassInfo) &&
        !iid.equals(nsISupports))
        throw Components.results.NS_ERROR_NO_INTERFACE;
    return this;
}



                           
function loadLibrary(aFilename) { // {{{
    var file = Components.classes["@mozilla.org/file/directory_service;1"]
        .getService(Components.interfaces.nsIProperties)
        .get("ComsD", Components.interfaces.nsIFile);
    file.append(aFilename);

    var ios = Components.classes["@mozilla.org/network/io-service;1"]
        .getService(Components.interfaces.nsIIOService);
    var fileHandler = ios.getProtocolHandler("file")
        .QueryInterface(Components.interfaces.nsIFileProtocolHandler);
    var spec = fileHandler.getURLSpecFromFile(file);

    var scriptLoader = Components.classes["@mozilla.org/moz/jssubscript-loader;1"]
        .getService(Components.interfaces.mozIJSSubScriptLoader);

    scriptLoader.loadSubScript(spec);
} // }}}

// load the XPCOM library
loadLibrary("flockXPCOMTemplate.js.lib");

var module = new xpcomModule (LUCENE_INDEXER_CID, LUCENE_INDEXER_CONTRACTID,
        'Flock Lucene Indexer Service',
        flockLuceneIndexer);
function NSGetModule(aCompMgr, aFileSpec) {
    return module;
};

