// vim: ts=2 sw=2 expandtab cindent
//
// BEGIN FLOCK GPL
//
// Copyright Flock Inc. 2005-2009
// http://flock.com
//
// This file may be used under the terms of the
// GNU General Public License Version 2 or later (the "GPL"),
// http://www.gnu.org/licenses/gpl.html
//
// Software distributed under the License is distributed on an "AS IS" basis,
// WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
// for the specific language governing rights and limitations under the
// License.
//
// END FLOCK GPL
//

const Cc = Components.classes;
const Ci = Components.interfaces;
const Cr = Components.results;
const Cu = Components.utils;

Cu.import("resource:///modules/FlockStringBundleHelpers.jsm");

const CLASS_ID = Components.ID("{61F83B70-6B52-11DB-BD13-0800200C9A66}");
const CLASS_NAME = "Flock Web Detective";
const CONTRACT_ID = "@flock.com/web-detective;1";
const INTERFACES = [
  Components.interfaces.nsISupports,
  Components.interfaces.nsIClassInfo,
  Components.interfaces.nsIObserver,
  Components.interfaces.nsITimerCallback,
  Ci.flockIWebDetective
];

// from nspr's prio.h
const PR_RDONLY      = 0x01;
const PR_WRONLY      = 0x02;
const PR_RDWR        = 0x04;
const PR_CREATE_FILE = 0x08;
const PR_APPEND      = 0x10;
const PR_TRUNCATE    = 0x20;
const PR_SYNC        = 0x40;
const PR_EXCL        = 0x80;

const DEFAULT_UPDATE_SERVER = "https://extensions.flock.com/webdetective/kea/";
const DEFAULT_UPDATE_INTERVAL = 1; // This is in days

const DEFAULT_LANG_ID = "default";

const PREF_GENERAL_USERAGENT_LOCALE = "general.useragent.locale";


function getLocale() {
  var prefs = Cc["@mozilla.org/preferences-service;1"]
              .getService(Ci.nsIPrefService)
              .QueryInterface(Ci.nsIPrefBranch);

  try {
    var defaults = prefs.getDefaultBranch(null);
    return defaults.getCharPref(PREF_GENERAL_USERAGENT_LOCALE);
  } catch (ex) {
    return prefs.getCharPref(PREF_GENERAL_USERAGENT_LOCALE);
  }
}


// ===================================================
// ========== BEGIN flockWebDetective class ==========
// ===================================================

function flockWebDetective()
{
  this._logger = Cc["@flock.com/logger;1"]
                 .createInstance(Ci.flockILogger);
  this._logger.init("WebDetective");

  this.mCurrentLocale = getLocale();
  this.mCurrentBaseLocale = this.mCurrentLocale.replace(/-.*$/, "");

  // Associative array where keys are service names and values are XML Docs
  this.mRules = [];
  this.mRuleCache = [];

  // Associative array of version strings by service
  this.mVersions = [];

  // Associative array of strings by service
  this.mStrings = {};

  // Associative array of localized strings by service and locale
  this.mLocalizedStrings = {};

  // Associative array of session cookies by service
  this.mSessionCookies = [];

  // Associative array of detect files that we've loaded for a given service
  this.mDetectFiles = [];

  this.cookieMgr = Components.classes["@mozilla.org/cookiemanager;1"]
                             .getService(Components.interfaces.nsICookieManager);
  this.mEnabled = true;

  this._profiler = Cc["@flock.com/profiler;1"].getService(Ci.flockIProfiler);

  // Sandbox for safe evals
  this._sandbox = new Cu.Sandbox("about:blank");

  this.startUpdateService();
}

// BEGIN nsISupports interface
flockWebDetective.prototype.QueryInterface =
function flockWebDetective_QueryInterface(aIID)
{
  var interfaces = INTERFACES;
  for (var i in interfaces) {
    if (aIID.equals(interfaces[i])) {
      return this;
    }
  }
  throw Components.results.NS_ERROR_NO_INTERFACE;
};
// END nsISupports interface


// BEGIN nsIClassInfo interface
flockWebDetective.prototype.contractID = CONTRACT_ID;
flockWebDetective.prototype.classID = CLASS_ID;
flockWebDetective.prototype.classDescription = CLASS_NAME;
flockWebDetective.prototype.implementationLanguage = Components.interfaces.nsIProgrammingLanguage.JAVASCRIPT;
flockWebDetective.prototype.flags = Components.interfaces.nsIClassInfo.SINGLETON;

flockWebDetective.prototype.getInterfaces =
function flockWebDetective_getInterfaces(aCount)
{
  var interfaces = INTERFACES;
  aCount.value = interfaces.length;
  return interfaces;
};

flockWebDetective.prototype.getHelperForLanguage =
function flockWebDetective_getHelperForLanguage(aLanguage)
{
  return null;
};
// END nsIClassInfo interface


// BEGIN nsIObserver interface
flockWebDetective.prototype.observe =
function flockWebDetective_observe(aSubject, aTopic, aData)
{
};
// END nsIObserver interface


// BEGIN nsITimerCallback interface
flockWebDetective.prototype.notify =
function flockWebDetective_notify(timer)
{
  try {
    var prefs = Cc['@mozilla.org/preferences-service;1']
      .getService(Ci.nsIPrefBranch);
    var doUpdate = prefs.getBoolPref('flock.service.webdetective.update');
    if (!doUpdate) {
      return;
    }
  }
  catch (e) { }

  this.checkForUpdates();
};
// END nsITimerCallback interface


// BEGIN flockIWebDetective interface
flockWebDetective.prototype.detect =
function flockWebDetective_detect(aServiceName, aType, aDocument, aResults)
{
  this._logger.info(".detect('" + aServiceName + "', '" + aType + "')");
  return this.innerDetect(aServiceName, aType, aDocument, null, aResults);
};

flockWebDetective.prototype.asyncDetect =
function flockWebDetective_asyncDetect(aServiceName, aType, aDocContainer,
                                       aResults, aFlockListener, aTimerPeriod,
                                       aMaxTries)
{
  this._logger.info(".asyncDetect('" + aServiceName + "', '" + aType + "')");
  if (aDocContainer.contentDocument === undefined)
  {
    this._logger.error(".asyncDetect requires an object" +
                       " which contains a document.");
    aFlockListener.onError(null, "asyncDetect");
    return;
  }
  var inst = this;
  var timerCB = {
    tries: 0,
    notify: function wd_asyncDetect_notify(aTimer) {
      this.tries++;
      inst._logger.debug(".asyncDetect('" + aServiceName + "', '"
                                          + aType + "'): notify() try "
                                          + this.tries + " out of "
                                          + aMaxTries);
      if (this.tries > aMaxTries) {
        inst._logger.debug("async timer exceeded aMaxTries");
        aTimer.cancel();
        aFlockListener.onError(null, "asyncDetect");
        return;
      }
      if (inst.innerDetect(aServiceName, aType, aDocContainer.contentDocument,
                           null, aResults))
      {
        aTimer.cancel();
        aFlockListener.onSuccess(aResults, "asyncDetect");
      }
    }
  };
  var asyncTimer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
  asyncTimer.initWithCallback(timerCB, aTimerPeriod,
                              Ci.nsITimer.TYPE_REPEATING_SLACK);
};

flockWebDetective.prototype.detectForm =
function flockWebDetective_detectForm(aServiceName, aType, aForm, aResults)
{
  this._logger.info(".detectForm('" + aServiceName + "', '" + aType + "')");
  aForm.QueryInterface(Components.interfaces.nsIDOMHTMLFormElement);
  return this.innerDetect(aServiceName, aType, aForm.ownerDocument, aForm, aResults);
};

flockWebDetective.prototype.detectCookies =
function flockWebDetective_detectCookies(aServiceName, aType, aResults)
{
  this._logger.info(".detectCookies('" + aServiceName + "', '" + aType + "')");
  return this.innerDetect(aServiceName, aType, null, null, aResults);
};

flockWebDetective.prototype.detectNoDOM =
function flockWebDetective_detectNoDOM(aServiceName, aType, aURL, aDocumentText, aResults)
{
  this._logger.info(".detectNoDOM('" + aServiceName + "', '" + aType + "')");
  var doc = {
    noDOM: true,
    documentElement: {
      innerHTML: aDocumentText
    },
    URL: aURL
  };
  return this.innerDetect(aServiceName, aType, doc, null, aResults);
};

flockWebDetective.prototype.getSessionCookies =
function flockWebDetective_getSessionCookies(aServiceName)
{
  if (!this.mSessionCookies[aServiceName]) {
    this.loadSessionCookies(aServiceName);
  }
  if (!this.mSessionCookies[aServiceName]) {
    return null;
  }
  var wd = this;
  return {
    arr: wd.mSessionCookies[aServiceName],
    idx: 0,
    QueryInterface: function gsc_QueryInterface(aIID) {
      if ( !aIID.equals(Components.interfaces.nsISupports) &&
           !aIID.equals(Components.interfaces.nsISimpleEnumerator) )
      {
        throw Components.results.NS_ERROR_NO_INTERFACE;
      }
      return this;
    },
    hasMoreElements: function gsc_hasMoreElements() {
      return (this.idx < this.arr.length);
    },
    getNext: function gsc_getNext() {
      var c = this.arr[this.idx++];
      var cookie = {
        QueryInterface: function gscc_QueryInterface(aIID) {
          if ( !aIID.equals(Components.interfaces.nsISupports) &&
               !aIID.equals(Components.interfaces.nsICookie) )
          {
            throw Components.results.NS_ERROR_NO_INTERFACE;
          }
          return this;
        },
        host: c.host,
        name: c.name,
        path: c.path
      };
      return cookie.QueryInterface(Components.interfaces.nsICookie);
    }
  };
};

flockWebDetective.prototype.getString =
function flockWebDetective_getString(aServiceName, aURLName, aDefault)
{
  this._logger.info(".getString('" + aServiceName + "', '" + aURLName + "')");
  if (!this.mEnabled) {
    return aDefault;
  }
  if (!this.mRules[aServiceName]) {
    throw "No rules loaded for service: "+aServiceName;
  }
  if (!this.mStrings[aServiceName]) {
    this.loadStrings(aServiceName);
  }
  if (this.mStrings[aServiceName][aURLName]) {
    return this.mStrings[aServiceName][aURLName];
  }
  return aDefault;
};

flockWebDetective.prototype.getLocalizedString =
function flockWebDetective_getString(aServiceName, aURLName, aDefault)
{
  this._logger.info(".getLocalizedString('" + aServiceName + "', '"
                    + aURLName + "')");

  if (!this.mEnabled) {
    return aDefault;
  }

  if (!this.mRules[aServiceName]) {
    throw "No rules loaded for service: " + aServiceName;
  }

  if (!this.mLocalizedStrings[aServiceName]) {
    this.loadStrings(aServiceName);
  }

  var localizedStrings = this.mLocalizedStrings[aServiceName][aURLName];
  if (localizedStrings) {
    if (localizedStrings[this.mCurrentLocale]) {
      return localizedStrings[this.mCurrentLocale];
    } else if (localizedStrings[this.mCurrentBaseLocale]) {
      return localizedStrings[this.mCurrentBaseLocale];
    } else if (localizedStrings[DEFAULT_LANG_ID]) {
      return localizedStrings[DEFAULT_LANG_ID];
    }
  }

  return aDefault;
};

flockWebDetective.prototype.loadDetectFile =
function flockWebDetective_loadDetectFile(aDetectFile)
{
  this._logger.info(".loadDetectFile()");
  aDetectFile.QueryInterface(Ci.nsILocalFile);

  if ((!aDetectFile.exists()) || (!aDetectFile.isReadable())) {
    throw Cr.NS_ERROR_UNEXPECTED;
  }

  var fis = Cc["@mozilla.org/network/file-input-stream;1"]
            .createInstance(Ci.nsIFileInputStream);
  fis.init(aDetectFile, PR_RDONLY, 0, 0);

  var serviceName = this._loadInputStream(fis);

  // This will get used for updates
  if (serviceName) {
    this.mDetectFiles[serviceName] = aDetectFile;
  }
};

flockWebDetective.prototype.loadFromChrome =
function flockWebDetective_loadFromChrome(aChromeURL)
{
  this._logger.info(".loadFromChrome('" + aChromeURL + "')");
  var ios = Cc["@mozilla.org/network/io-service;1"]
            .createInstance(Ci.nsIIOService);
  var channel = ios.newChannel(aChromeURL, null, null);
  this._loadInputStream(channel.open());
};

flockWebDetective.prototype.listServices =
function flockWebDetective_listServices()
{
  var svcEnum = {
    _arr: [],
    QueryInterface: function lfc_QueryInterface(aIID) {
      if (aIID.equals(Ci.nsISupports)) {
        return this;
      }
      if (aIID.equals(Ci.nsISimpleEnumerator)) {
        return this;
      }
      throw Cr.NS_ERROR_NO_INTERFACE;
    },
    hasMoreElements: function lfc_hasMoreElements() {
      return (this._arr.length > 0);
    },
    getNext: function lfc_getNext() {
      return {
        QueryInterface: function lfcs_QueryInterface(aIID) {
          if (aIID.equals(Ci.nsISupports)) {
            return this;
          }
          if (aIID.equals(Ci.nsISupportsPrimitive)) {
            return this;
          }
          if (aIID.equals(Ci.nsISupportsString)) {
            return this;
          }
          throw Cr.NS_ERROR_NO_INTERFACE;
        },
        type: Ci.nsISupportsPrimitive.TYPE_STRING,
        data: this._arr.shift(),
        toString: function lfcs_toString() { return this.data; }
      };
    }
  };
  for (var svcName in this.mDetectFiles) {
    svcEnum._arr.push(svcName);
  }
  return svcEnum;
};

flockWebDetective.prototype.getVersionForService =
function flockWebDetective_getVersionForService(aServiceName)
{
  return (this.mVersions[aServiceName]) ? this.mVersions[aServiceName] : null;
};

flockWebDetective.prototype.testDomain =
function flockWebDetective_testDomain(aURL, aDomain)
{
  var uri = Cc["@mozilla.org/network/standard-url;1"]
            .createInstance(Ci.nsIURI);
  var host;
  try {
    uri.spec = aURL;
    host = uri.host;
  } catch (ex) {
    // There wasn't a host
    return false;
  }
  if (host == aDomain) {
    return true;
  }
  var idx = host.indexOf("." + aDomain);
  if (idx < 1) {
    return false;
  }
  // We now know that the host contains .<domain> as a substring, and we just
  // need to make sure that it properly ENDS with .<domain>
  return ((idx + aDomain.length + 1) == host.length);
};
// END flockIWebDetective interface


// BEGIN helper functions
flockWebDetective.prototype._loadInputStream =
function flockWebDetective__loadInputStream(aInputStream)
{
  aInputStream.QueryInterface(Ci.nsIInputStream);

  var prefs = Cc["@mozilla.org/preferences-service;1"]
              .getService(Ci.nsIPrefService).getBranch(null);
  if (prefs.getPrefType("flock.service.webdetective.enabled")) {
    this.mEnabled = prefs.getBoolPref("flock.service.webdetective.enabled");
  }
  if (!this.mEnabled) {
    return null;
  }

  // Load and parse the XML
  this._logger.debug("available bytes: " + aInputStream.available());
  var domParser = Cc["@mozilla.org/xmlextras/domparser;1"]
                  .createInstance(Ci.nsIDOMParser);
  var xmlDoc = domParser.parseFromStream(aInputStream, "UTF-8",
                                         aInputStream.available(),
                                         "text/xml");
  aInputStream.close();

  var serviceEl = xmlDoc.documentElement;
  if (serviceEl.tagName != "service") {
    this._logger.error("PARSE ERROR: no 'service' element found");
    throw Cr.NS_ERROR_UNEXPECTED;
  }
  var serviceName = serviceEl.getAttribute("name");
  if (!serviceName || (serviceName == "")) {
    this._logger.error("PARSE ERROR: 'service' element has no name");
    throw Cr.NS_ERROR_UNEXPECTED;
  }
  this.mRules[serviceName] = xmlDoc;
  if (serviceEl.hasAttribute("version")) {
    this.mVersions[serviceName] = serviceEl.getAttribute("version");
  }
  this._logger.info("loaded detection rules for service: " + serviceName);
  // Force the strings and cookies to be reloaded, since they may have changed
  this.mStrings[serviceName] = null;
  this.mLocalizedStrings[serviceName] = null;
  this.mSessionCookies[serviceName] = null;

  return serviceName;
};

flockWebDetective.prototype.innerDetect =
function flockWebDetective_innerDetect(aServiceName, aType, aDocument, aForm, aResults)
{
  if (!this.mEnabled) {
    return false;
  }
  var ruleMatch = false;
  var ruleDoc = this.mRules[aServiceName];
  if (ruleDoc) {
    // Cache the rules for faster retrieval next time
    if (!this.mRuleCache[aServiceName]) {
      this.mRuleCache[aServiceName] = [];
    }
    if (!this.mRuleCache[aServiceName][aType]) {
      this._logger.debug("Parsing and cacheing " + aServiceName
                         + " for rule(s) of type '" + aType + "'");
      this.mRuleCache[aServiceName][aType] =
        this.getRulesOfType(aServiceName, aType);
    }
    var rules = this.mRuleCache[aServiceName][aType];
    this._logger.debug("found " + rules.length +  " rule(s) of type '"
                       + aType + "'");
    var evtID = null;
    var r = -1;
    for each (var rule in rules) {
      var cookieFound = null;
      r++;
      if (evtID) {
        this._profiler.profileEventEnd(evtID, "");
      }
      evtID = this._profiler.profileEventStart(aServiceName + "-" + aType
                                               + "-webdict-rule" + r);
      var allCondsMatch = true;
      if (rule.conditionsEl) {

        if (!rule.conditions) {
          // This will be used for cacheing the conditions functions
          rule.conditions = [];
        }
        var conditions = rule.conditions;

        // First check the URL conditions
        if (!conditions["url"]) {
          // Cache the URL conditions for faster retrieval next time
          conditions["url"] = this.getURLConditionsForRule(rule);
        }
        var urlConds = conditions["url"];
        this._logger.debug("rule[" + r + "](" + aType + ") has "
                           + urlConds.length + " URL condition(s)");
        if (urlConds.length > 0) {
          if (aDocument && aDocument instanceof Components.interfaces.nsIDOMHTMLDocument) {
            var uri = makeURI(aDocument.URL);
            for (var c = 0; (c < urlConds.length) && allCondsMatch; c++) {
              if (!urlConds[c](uri)) {
                this._logger.debug("URL condition [" + c + "] failed");
                allCondsMatch = false;
              }
            }
            if (!allCondsMatch) {
              continue;
            }
          } else {
            // There is no URL, so URL conditions automatically fail
            allCondsMatch = false;
          }
        }
        if (!allCondsMatch) {
          continue;
        }

        // Check Form conditions
        if (!conditions["form"]) {
          // Cache the Form conditions for faster retrieval next time
          conditions["form"] = this.getFormConditionsForRule(rule, aForm);
        }
        var formConds = conditions["form"];
        this._logger.debug("rule[" + r + "](" + aType + ") has "
                           + formConds.length + " Form condition(s)");
        if (formConds.length > 0) {
          if (aForm) {
            for (var c = 0; (c < formConds.length) && allCondsMatch; c++) {
              if (!formConds[c](aForm)) {
                this._logger.debug("Form condition [" + c + "] failed");
                allCondsMatch = false;
              }
            }
          } else {
            // There's no Form, so Form conditions automatically fail
            allCondsMatch = false;
          }
        }
        if (!allCondsMatch) {
          continue;
        }

        // Next check the Document conditions
        if (!conditions["doc"]) {
          // Cache the Document conditions for faster retrieval next time
          conditions["doc"] = this.getDocConditionsForRule(rule);
        }
        var docConds = conditions["doc"];
        this._logger.debug("rule[" + r + "](" + aType + ") has "
                           + docConds.length + " Document condition(s)");
        if (docConds.length > 0) {
          if (aDocument) {
            for (var c = 0; (c < docConds.length) && allCondsMatch; c++) {
              if (!docConds[c](aDocument)) {
                this._logger.debug("Document condition [" + c + "] failed");
                allCondsMatch = false;
              }
            }
          } else {
            // There's no Document, so Document conditions automatically fail
            allCondsMatch = false;
          }
        }
        if (!allCondsMatch) {
          continue;
        }

        // Finally check the Cookie conditions
        if (!conditions["cookie"]) {
          // Cache the Cookie conditions for faster retrieval next time
          conditions["cookie"] = this.getCookieConditionsForRule(rule)
                                     .toString();
        }
        var cookieConds = conditions["cookie"];
        this._logger.debug("rule[" + r + "](" + aType + ") has "
                           + cookieConds + " Cookie condition(s)");
        if (cookieConds > 0) {
          cookieFound = this.getRelevantCookieForRule(rule);
          if (!cookieFound) {
            allCondsMatch = false;
          }
        }
      }

      if (allCondsMatch) {
        this._logger.debug("All conditions matched for rule[" + r + "]("
                           + aType + ")!");
        ruleMatch = true;
        if (aResults) {
          // Now get results
          this.getResultsForRule(aDocument, aForm, cookieFound, rule, aResults);
        }
        break;
      }
    }
    if (evtID) {
      this._profiler.profileEventEnd(evtID, "");
    }
  }
  return ruleMatch;
};

flockWebDetective.prototype.getRulesOfType =
function flockWebDetective_getRulesOfType(aServiceName, aType)
{
  var ruleDoc = this.mRules[aServiceName];
  var rules = [];
  var detectElements = ruleDoc.getElementsByTagName("detect");
  for (var i = 0; i < detectElements.length; i++) {
    var detect = detectElements.item(i).QueryInterface(Ci.nsIDOMElement);
    var ruleObj = {};
    ruleObj.domElem = detect;
    if (aType == detect.getAttribute("type")) {
      this._logger.debug("found <detect type='" + aType + "'>");
      ruleObj.type = aType;
      for (var j = 0; j < detect.childNodes.length; j++) {
        var child = detect.childNodes.item(j);
        try {
          child.QueryInterface(Components.interfaces.nsIDOMElement);
          if (child.tagName == "conditions") {
            ruleObj.conditionsEl = child;
          }
          if (child.tagName == "results") {
            ruleObj.resultsEl = child;
          }
        } catch (ex) {
          // Do nothing
        }
      }
      if (!ruleObj.conditionsEl) {
        ruleObj.conditionsEl = detect;
      }
      if (!ruleObj.resultsEl) {
        ruleObj.resultsEl = detect;
      }
      rules.push(ruleObj);
    }
  }
  return rules;
};

flockWebDetective.prototype.getURLConditionsForRule =
function flockWebDetective_getURLConditionsForRule(aRule)
{
  var conditions = [];

  // Look for the first "url" element (ignore any others)
  var urlEls = aRule.conditionsEl.getElementsByTagName("url");
  if (urlEls.length) {
    var urlEl = urlEls.item(0);
    conditions = this.addStandardConds(conditions, urlEl, function (aURI) { return aURI.spec; });
    if (urlEl.hasAttribute("domain")) {
      conditions[conditions.length] = function (aURI) {
        var domain = urlEl.getAttribute("domain");
        var host;
        try {
          host = aURI.host;
        } catch (ex) {
          // No host: e.g "about:neterror?e=dnsNotFound&u=http%3A//www...."
          return false;
        }
        if (host == domain) {
          return true;
        }
        var idx = host.indexOf("."+domain);
        if (idx < 1) {
          return false;
        }
        return ((idx + domain.length + 1) == host.length);
      };
    }
    var tagFuncs = [];
    tagFuncs["host"] = function (aURI) {
      try {
        return aURI.host;
      } catch (ex) {
        return null;
      }
    };
    tagFuncs["path"] = function (aURI) { return aURI.path; };
    tagFuncs["querystring"] = function (aURI) { return aURI.path.substring(aURI.path.indexOf("?")); };
    for (var tag in tagFuncs) {
      var elements = urlEl.getElementsByTagName(tag);
      for (var e = 0; e < elements.length; e++) {
        var elem = elements.item(e);
        conditions = this.addStandardConds(conditions, elem, tagFuncs[tag]);
      }
    }
    var regexpEls = urlEl.getElementsByTagName("regexp");
    for (var i = 0; i < regexpEls.length; i++) {
      var regexpEl = regexpEls.item(i);
      var rExprObj = Cu.evalInSandbox(this.getRegexpFromNode(regexpEl),
                                      this._sandbox);
      var inst = this;
      conditions[conditions.length] = function (aURI) {
        return inst._doRegexpMatchWithObj(aURI.spec,
                                          rExprObj,
                                          aRule.type,
                                          null,
                                          null);
      };
    }
  }
  return conditions;
};

flockWebDetective.prototype.getFormConditionsForRule =
function flockWebDetective_getFormConditionsForRule(aRule, aForm)
{
  var conditions = [];

  // Look for the first "form" element (ignore any others)
  var formEls = aRule.conditionsEl.getElementsByTagName("form");
  if (formEls.length) {
    var formEl = formEls.item(0);
    // Iterate through all the children of "form"
    for (var i = 0; i < formEl.childNodes.length; i++) {
      var formChild = formEl.childNodes.item(i);
      try {
        formChild.QueryInterface(Components.interfaces.nsIDOMElement);
      } catch (ex) {
        continue;
      }
      this._logger.debug("Found <conditions><form><" + formChild.tagName + ">");
      switch (formChild.tagName) {
        case "xpath":
          conditions[conditions.length] = this.createXPathFormCondition(formChild);
          break;
        case "field":
          conditions[conditions.length] = this.createFieldCondition(formChild);
          break;
      }
    }
  }
  return conditions;
};

flockWebDetective.prototype.createXPathFormCondition =
function flockWebDetective_createXPathFormCondition(aXPathNode)
{
  var inst = this;
  return function (aForm) {
    if (!aXPathNode.xpathExpr) {
      aXPathNode.xpathExpr = inst.getXPathExpression(aXPathNode);
    }
    if (!aXPathNode.xpathFunc) {
      aXPathNode.xpathFunc = inst._createXPathCondition(aXPathNode,
                                                        aXPathNode.xpathExpr);
    }
    if (!aForm.xpathPrefix) {
      aForm.xpathPrefix = inst.getXPathPrefix(aForm);
      inst._logger.debug("Got XPath prefix for this form: " + aForm.xpathPrefix);
    }
    return aXPathNode.xpathFunc(aForm.ownerDocument, aForm.xpathPrefix);
  };
};

const FIELD_ATTRIBUTES = [ "name", "type", "class" ];

flockWebDetective.prototype.createFieldCondition =
function flockWebDetective_createFieldCondition(aFieldNode)
{
  var inst = this;
  return function (aForm) {
    return (inst.getMatchingFormField(aFieldNode, aForm) != null);
  };
};

flockWebDetective.prototype.getMatchingFormField =
function flockWebDetective_getMatchingFormField(aFieldNode, aForm)
{
  var formFields = aForm.elements;
  for (var i = 0; i < formFields.length; i++) {
    var field = formFields.item(i)
      .QueryInterface(Components.interfaces.nsIDOMElement);
    if (aFieldNode.hasAttribute("tagname")) {
      if (field.tagName.toLowerCase() != aFieldNode.getAttribute("tagname").toLowerCase()) {
        // This form field does not match the pattern, so skip it
        this._logger.debug("tagname does NOT match [" + field.tagName + " != "
                           + aFieldNode.getAttribute("tagname") + "]");
        continue;
      }
    }
    if (aFieldNode.hasAttribute("fieldid")) {
      if ( !field.hasAttribute("id") ||
           (aFieldNode.getAttribute("fieldid") != field.getAttribute("id")) )
      {
        // This form field does not match the pattern, so skip it
        this._logger.debug("fieldid does NOT match");
        continue;
      }
      // The fields have matching ids!
    }
    var matchesAllAttributes = true;
    for (var j = 0; j < FIELD_ATTRIBUTES.length; j++) {
      var attr = FIELD_ATTRIBUTES[j];
      if (aFieldNode.hasAttribute(attr)) {
        if ( !field.hasAttribute(attr) ||
             (aFieldNode.getAttribute(attr) != field.getAttribute(attr)) )
        {
          // This form field does not match the pattern, so skip it
          this._logger.debug("'" + attr + "' does NOT match");
          matchesAllAttributes = false;
          break;
        }
      }
    }
    if (!matchesAllAttributes) {
      continue;
    }
    // This form field matches the pattern
    return field;
  }
  // Didn't find any fields matching the pattern
  return null;
};

flockWebDetective.prototype.getDocConditionsForRule =
function flockWebDetective_getDocConditionsForRule(aRule)
{
  var conditions = [];

  // Document conditions can occur either under a "document" element, or else
  // at the top level if compact syntax is being used.
  var docCondNodes = this.getDocSubNodes(aRule.conditionsEl);

  // Iterate through all the "document" sub nodes
  for (var i = 0; i < docCondNodes.length; i++) {
    var docChild = docCondNodes[i];
    try {
      docChild.QueryInterface(Components.interfaces.nsIDOMElement);
    } catch (ex) {
      continue;
    }
    this._logger.debug("Found <conditions><document><" + docChild.tagName + ">");
    switch (docChild.tagName) {
      case "xpath":
        var xpathExpr = this.getXPathExpression(docChild);
        if (xpathExpr) {
          conditions.push(this._createXPathCondition(docChild, xpathExpr));
        }
        break; // END "xpath" case

      case "regexp":
        var rExpr = this.getRegexpFromNode(docChild);
        if (this.isValidMatchRegexp(rExpr)) {
          var isMultiLine = (docChild.getAttribute("multiline") == "true");
          if (isMultiLine) {
            conditions[conditions.length] =
              this._createMultilineRegexpCondition(rExpr);
          } else {
            conditions[conditions.length] =
              this._createRegexpCondition(rExpr, aRule.type);
          }
        } else {
          this._logger.debug("Condition will fail due to INVALID regexp: "
                             + rExpr);
          conditions[conditions.length] = function (aDocument) {
            return false;
          };
        }
        break; // END "regexp" case
    }
  }
  return conditions;
};

/**
 * @param aXPathNode (in) nsIDOMElement <xpath>
 * @param aXPathExpr (in) XPath expression as a string
 * @return JS function that takes aDocument and aXPathPrefix params
 */
flockWebDetective.prototype._createXPathCondition =
function flockWebDetective__createXPathCondition(aXPathNode, aXPathExpr)
{
  var inst = this;
  var func = function (aDocument, aXPathPrefix) {
    if (aDocument.noDOM) {
      inst._logger.debug("Document has no DOM, can't use XPath!!");
      return false;
    }
    aDocument.QueryInterface(Components.interfaces.nsIDOMXPathEvaluator);
    var allXPath = (aXPathPrefix) ? aXPathPrefix+aXPathExpr : aXPathExpr;
    inst._logger.debug("Evaluating XPath statement: "+allXPath);
    var result;
    if (aDocument instanceof Components.interfaces.nsIDOMHTMLDocument) {
      result = aDocument.evaluate(allXPath,
                                  aDocument.body ? aDocument.body
                                                 : aDocument.documentElement,
                                  null,
                                  0,
                                  null);
    } else {
      inst._logger.debug(aDocument.documentElement.namespaceURI);
      var _xs = Components.classes["@mozilla.org/xmlextras/xmlserializer;1"]
                          .createInstance(Components.interfaces.nsIDOMSerializer);
      var xml = _xs.serializeToString(aDocument.documentElement);
      var namespace = "xmlns=\""+aDocument.documentElement.namespaceURI+"\"";
      var temp = xml.indexOf(namespace);
      aDocument.doc = null;
      //need to strip out the namespace otherwise aDocument.evaluate will fail
      if (temp > 0) {
        var new_xml = xml.substring(0, temp)
                    + xml.substring((temp + namespace.length), xml.length);
        var parser = Components.classes["@mozilla.org/xmlextras/domparser;1"]
                               .createInstance(Components.interfaces.nsIDOMParser);
        aDocument.doc = parser.parseFromString(new_xml, "text/xml");
      }
      result = aDocument.evaluate( allXPath,
                                   (aDocument.doc ? aDocument.doc : aDocument),
                                   null,
                                   Components.interfaces.nsIDOMXPathResult.ANY_TYPE,
                                   null );
    }
    try {
      result = result.QueryInterface(Components.interfaces.nsIDOMXPathResult);
    } catch (ex) {
      return false;
    }
    inst._logger.debug(" - got an nsIDOMXPathResult of type ["
                       + result.resultType + "]");
    switch (result.resultType) {
      case Components.interfaces.nsIDOMXPathResult.STRING_TYPE:
        inst._logger.debug("  result.stringValue = " + result.stringValue);
        break;
      case Components.interfaces.nsIDOMXPathResult.UNORDERED_NODE_ITERATOR_TYPE:
        var node = result.iterateNext();
        if (node instanceof Components.interfaces.nsIDOMNode) {
          // Found a matching node!
          inst._logger.debug(" - XPath matching node! " + node.nodeValue);
        } else {
          inst._logger.debug(" - NO match on XPath " + result);
          return false;
        }
        break;
      default:
        inst._logger.debug("WARNING!!! Unhandled XPath result type: "
                           + result.resultType);
        return false;
    }
    // At this point, the XPath has matched, but we still need to check if
    // there's a regexp subnode and make sure it matches too.
    var extract = null;
    if (aXPathNode.hasAttribute("extract")) {
      extract = aXPathNode.getAttribute("extract");
    }
    var snippets = inst.getXMLSnippetsFromXPath(aDocument, allXPath, extract);
    inst._logger.debug(" XPath got us " + snippets.length
                       + " snippets to check");
    return inst._testXPathRegexp(aXPathNode, snippets, "", null, null);
  };
  return func;
};

/**
 * Generates a function for testing a regular expression against each line of
 * a document in succession.
 */
flockWebDetective.prototype._createRegexpCondition =
function flockWebDetective__createRegexpCondition(aRegExp, aRuleType)
{
  var inst = this;
  var func = function createRegexpCond_inner(aDocument) {
    var html = aDocument.documentElement.innerHTML;
    var lines = html.split(/[\r\n]+/);
    var aRegExpObj = Cu.evalInSandbox(aRegExp, inst._sandbox);
    for (var i = 0; i < lines.length; i++) {
      if (inst._doRegexpMatchWithObj(lines[i], aRegExpObj, aRuleType, null, null)) {
        return true;
      }
    }
    return false;
  };
  return func;
};

/**
 * Generates a function for testing a "multiline" regular expression against
 * the entire text of a document.
 */
flockWebDetective.prototype._createMultilineRegexpCondition =
function flockWebDetective__createMultilineRegexpCondition(aRegExp, aRuleType)
{
  var inst = this;
  var func = function createMultilineRegexpCond_inner(aDocument) {
    var html = aDocument.documentElement.innerHTML;
    var oneString = html.replace(/[\r\n]/g, "");
    return inst._doRegexpMatch(oneString, aRegExp, aRuleType, null, null);
  };
  return func;
};

flockWebDetective.prototype.getCookieConditionsForRule =
function flockWebDetective_getCookieConditionsForRule(aRule)
{
  var cookieEls = aRule.conditionsEl.getElementsByTagName("cookie");
  for (var e = 0; e < cookieEls.length; e++) {
    var cookieEl = cookieEls.item(e);
    cookieEl.QueryInterface(Ci.nsIDOMElement);
    for (var i = 0; i < cookieEl.childNodes.length; i++) {
      var child = cookieEl.childNodes.item(i);
      if (child.nodeType == Ci.nsIDOMNode.CDATA_SECTION_NODE) {
        this._logger.debug("  - regexp: " + child.nodeValue);
        cookieEl.rExprObj = Cu.evalInSandbox(child.nodeValue, this._sandbox);
      }
    }
  }
  return cookieEls.length;
};

flockWebDetective.prototype.getRelevantCookieForRule =
function flockWebDetective_getRelevantCookieForRule(aRule)
{
  var cookieSv = Cc["@mozilla.org/cookieService;1"]
                  .getService(Ci.flockICookieService);
  var cookieEls = aRule.conditionsEl.getElementsByTagName("cookie");
  var ckStr, ckHost = "";
  var result = null;
  for (var e = cookieEls.length - 1; e >= 0 ; e--) {
    var cookieEl = cookieEls.item(e);
    var host = cookieEl.getAttribute("host");
    var cookieName = cookieEl.getAttribute("name");
    var lengthDiff = ckHost.length - host.length;
    // no need to re-get the cookie if it's a subdomain of the domain
    if (lengthDiff < 0 || ckHost.indexOf(host, lengthDiff) == -1) {
      ckHost = host;
      ckStr = cookieSv.getCookieStringAlways(makeURI("https://" + ckHost), null);
    }
    var index = ckStr ? ckStr.indexOf(cookieName + "=") : -1;
    var value = null;
    var noMatchFound = true;
    var cNoMatch = (cookieEl.getAttribute("nomatch") == "true");
    this._logger.debug("Testing cookie [" + host + "]["
                       + cookieName + "] against rule [" + cNoMatch + "]");
    if (index != -1) {
      index += cookieName.length + 1;
      var end = ckStr.indexOf(";", index);  
      value = unescape(ckStr.substring(index, end == -1 ? ckStr.length : end));
      this._logger.debug("Found relevant cookie: " + value);
      noMatchFound = cookieEl.rExprObj &&
                     !this._doRegexpMatchWithObj(value,
                                                 cookieEl.rExprObj,
                                                 aRule.type,
                                                 "",
                                                 null);
    }
    if (noMatchFound != cNoMatch) {
      this._logger.debug("Cookie condition [" + e + "] failed");
      return null;
    }
    if (cNoMatch || value) {
      result = {value: value, cookieEl: cookieEl};
      if (value) {
        break;
      }
    }
    ckHost = host;
    ckNames = cookieName;
  }
  return result;
};

flockWebDetective.prototype.addStandardConds =
function flockWebDetective_addStandardConds(aConditionsArray, aDOMElement, paramFunc)
{
  aDOMElement.QueryInterface(Components.interfaces.nsIDOMElement);
  var inst = this;
  if (aDOMElement.hasAttribute("equals")) {
    aConditionsArray[aConditionsArray.length] = function (param) {
      var input = paramFunc(param);
      inst._logger.debug("condition: " + input + " == "
                         + aDOMElement.getAttribute("equals"));
      return (input == aDOMElement.getAttribute("equals"));
    };
  }
  if (aDOMElement.hasAttribute("contains")) {
    aConditionsArray[aConditionsArray.length] = function (param) {
      var input = paramFunc(param);
      inst._logger.debug("condition: " + input + " ~= "
                         + aDOMElement.getAttribute("contains"));
      return (input.indexOf(aDOMElement.getAttribute("contains")) != -1);
    };
  }
  if (aDOMElement.hasAttribute("notcontains")) {
    aConditionsArray[aConditionsArray.length] = function wd_notcontains(aParam) {
      var input = paramFunc(aParam);
      inst._logger.debug("condition: " + input + " !~ "
                         + aDOMElement.getAttribute("notcontains"));
      return (input.indexOf(aDOMElement.getAttribute("notcontains")) == -1);
    };
  }
  return aConditionsArray;
};

flockWebDetective.prototype.getResultsForRule =
function flockWebDetective_getResultsForRule(aDocument,
                                             aForm,
                                             aCookieFound,
                                             aRule,
                                             aResults)
{
  this._logger.info(".getResultsForRule()");
  aResults.QueryInterface(Components.interfaces.nsIWritablePropertyBag2);
  if (!aRule.resultsEl) {
    return;
  }

  var child;
  var workingData;

  if (aDocument) {

    // Get URL results
    if (!aRule.urlResults) {
      aRule.urlResults = aRule.resultsEl.getElementsByTagName("url");
    }
    workingData = {};
    for (var u = 0; u < aRule.urlResults.length; u++) {
      var urlEl = aRule.urlResults.item(u);
      this._logger.debug("found <results><url>");
      for (var i = 0; i < urlEl.childNodes.length; i++) {
        child = urlEl.childNodes.item(i);
        try {
          child.QueryInterface(Components.interfaces.nsIDOMElement);
        } catch (ex) {
          continue;
        }
        this._logger.debug("found <results><url><" + child.tagName + ">");
        switch (child.tagName) {
          case "regexp":
            this._getRegexpResults(null,
                                   aDocument.URL,
                                   child,
                                   aRule.type,
                                   aResults,
                                   workingData);
            break;
        }
      }
    }

    // Get Document results
    workingData = {};
    if (!aRule.docResults) {
      aRule.docResults = this.getDocSubNodes(aRule.resultsEl);
    }
    for (var d = 0; d < aRule.docResults.length; d++) {
      child = aRule.docResults[d];
      try {
        child.QueryInterface(Components.interfaces.nsIDOMElement);
      } catch (ex) {
        continue;
      }
      this._logger.debug("found <results><document><" + child.tagName + ">");
      switch (child.tagName) {
        case "regexp":
          this._getRegexpResults(aDocument,
                                 aDocument.documentElement.innerHTML,
                                 child,
                                 aRule.type,
                                 aResults,
                                 workingData);
          break;
        case "xpath":
          this._getXPathResults(aDocument,
                                child,
                                aRule.type,
                                aResults,
                                workingData,
                                aDocument);
          break;
      }
    }
  }

  if (aForm) {
    // Get Form results
    workingData = {};
    var formEls = aRule.resultsEl.getElementsByTagName("form");
    for (var f = 0; f < formEls.length; f++) {
      var formEl = formEls.item(f);
      this._logger.debug("found <results><form>");
      for (var i = 0; i < formEl.childNodes.length; i++) {
        child = formEl.childNodes.item(i);
        try {
          child.QueryInterface(Components.interfaces.nsIDOMElement);
        } catch (ex) {
          continue;
        }
        this._logger.debug("found <results><form><" + child.tagName + ">");
        switch (child.tagName) {
          case "xpath":
            // Note that we don't cache form values as they are dynamic
            this._getXPathResults(aForm,
                                  child,
                                  aRule.type,
                                  aResults,
                                  workingData,
                                  null);
            break;
          case "field":
            this.getFieldResults(aForm, child, aResults);
            break;
        }
      }
    }
  }
  if (!aCookieFound || !aCookieFound.value) {
    return;
  }
  // Get Cookie results...
  workingData = {};
  this._getRegexpResults(null,
                         aCookieFound.value,
                         aCookieFound.cookieEl,
                         aRule.type,
                         aResults,
                         workingData);
};

flockWebDetective.prototype.getDocSubNodes =
function flockWebDetective_getDocSubNodes(aNode)
{
  var child;
  var resultNodes = [];
  var documentEls = aNode.getElementsByTagName("document");
  for (var i = 0; i < documentEls.length; i++) {
    var docEl = documentEls.item(i);
    for (var j = 0; j < docEl.childNodes.length; j++) {
      child = docEl.childNodes.item(j);
      try {
        child.QueryInterface(Components.interfaces.nsIDOMElement);
      } catch (ex) {
        continue;
      }
      resultNodes.push(child);
    }
  }
  // Any children of the result node that are NOT in [ conditions, url,
  // document, form, cookie ] are considered document conditions.
  for (var i = 0; i < aNode.childNodes.length; i++) {
    child = aNode.childNodes.item(i);
    try {
      child.QueryInterface(Components.interfaces.nsIDOMElement);
    } catch (ex) {
      continue;
    }
    switch (child.tagName.toLowerCase()) {
      case "conditions":
      case "url":
      case "document":
      case "form":
      case "cookie":
        continue;
      default:
        resultNodes.push(child);
    }
  }
  return resultNodes;
};

flockWebDetective.prototype._getRegexpResults =
function flockWebDetective__getRegexpResults(aDecorEl,
                                             aString,
                                             aRegexpNode,
                                             aRuleType,
                                             aResults,
                                             aData)
{
  // Cache the regexp
  if (!aRegexpNode.rExpr) {
    aRegexpNode.rExpr = this.getRegexpFromNode(aRegexpNode);
    aRegexpNode.rExprObj = Cu.evalInSandbox(aRegexpNode.rExpr, this._sandbox);
  }
  // Cache whether it is valid
  if (!aRegexpNode.isValid) {
    aRegexpNode.isValid = this.isValidMatchRegexp(aRegexpNode.rExpr);
  }
  if (!aRegexpNode.isValid) {
    this._logger.debug("Can't get results due to INVALID regexp: "
                       + aRegexpNode.rExpr);
    return;
  }
  // Cache the list of variables we are looking for
  if (!aRegexpNode.reVars) {
    aRegexpNode.reVars = [];
    aRegexpNode.postProcess = [];
    for (var re = 1; ; re++) {
      if (aRegexpNode.hasAttribute("re"+re)) {
        // Look for variable declarations as attributes
        aRegexpNode.reVars[re] = aRegexpNode.getAttribute("re"+re);
        continue;
      } else {
        // Look for variable declarations as elements
        var reTags = aRegexpNode.getElementsByTagName("re"+re);
        if (reTags.length) {
          var reTag = reTags.item(0);
          if (reTag.hasAttribute("name")) {
            aRegexpNode.reVars[re] = reTag.getAttribute("name");
            if (reTag.hasAttribute("processing")) {
              aRegexpNode.postProcess.push(reTag);
            }
            continue;
          }
        }
      }
      // There are no more variables to be gotten from this regexp
      break;
    }
  }
  // See if we have already decorated with the values for any of these vars
  var allDecorated = false;
  if (aDecorEl &&
      aDecorEl._flock_decorations &&
      aDecorEl._flock_decorations[aRuleType])
  {
    allDecorated = true;
    for (var i = 0; i < aRegexpNode.reVars.length; i++) {
      var val = aDecorEl._flock_decorations[aRuleType][aRegexpNode.reVars[i]];
      if (val) {
        this._logger.debug("Found a decoration for [" + aRuleType + "]["
                           + aRegexpNode.reVars[i] + "] = " + val);
        aResults.setPropertyAsAString(aRegexpNode.reVars[i], val);
      } else {
        allDecorated = false;
      }
    }
  }
  if (allDecorated) {
    this._logger.debug("Found all the vars as decorations! No need to run regexp...");
    return;
  }
  // Ensure that a value exists -- at least an empty string -- for each var
  for (var i = 0; i < aRegexpNode.reVars.length; i++) {
    try {
      aResults.getPropertyAsAString(aRegexpNode.reVars[i]);
    } catch (ex) {
      aResults.setPropertyAsAString(aRegexpNode.reVars[i], "");
    }
  }
  this._logger.debug("Getting results using regexp: " + aRegexpNode.rExpr);
  // Make sure we haven't passed in the node as aString
  if (aRegexpNode.rExpr && !(aString instanceof Ci.nsIDOMNode)) {
    var isMultiLine = (aRegexpNode.getAttribute("multiline") == "true");
    var isMultiValent = (aRegexpNode.getAttribute("multivalent") == "true");
    if (isMultiLine) {
      if (!aData.oneString) {
        aData.oneString = aString.replace(/[\r\n]/g, "");
      }
      this._doRegexpMatch(aData.oneString,
                          aRegexpNode.rExpr,
                          aRegexpNode.reVars,
                          aRuleType,
                          aResults,
                          aDecorEl);
    } else {
      // isMultiLine == false
      if (!aData.lines) {
        aData.lines = aString.split(/[\r\n]/);
      }
      this._logger.debug("There are " + aData.lines.length
                         + " lines to test with regexp");
      for (var line = 0; line < aData.lines.length; line++) {
        if (this._doRegexpMatchWithObj(aData.lines[line],
                                       aRegexpNode.rExprObj,
                                       aRegexpNode.reVars,
                                       aRuleType,
                                       aResults,
                                       aDecorEl,
                                       isMultiValent))
        {
          for (var i = 0; i < aRegexpNode.postProcess.length; i++) {
            this._postProcess(aRegexpNode.postProcess[i],
                              aRuleType,
                              aResults,
                              aDecorEl);
          }
          break;
        }
      }
    }
  }
};

flockWebDetective.prototype._setResult =
function flockWebDetective__setResult(aRuleType,
                                      aResults,
                                      aName,
                                      aValue,
                                      aDecorEl)
{
  // Check if we're passing back the node itself
  if (aValue instanceof Ci.nsIDOMNode) {
    aResults.setPropertyAsInterface(aName, aValue);
  } else {
    var convertedVal = this.htmlEntitiesDecode(aValue);
    aResults.setPropertyAsAString(aName, convertedVal);
    if (aDecorEl) {
      if (!aDecorEl._flock_decorations) {
        aDecorEl._flock_decorations = [];
      }
      if (!aDecorEl._flock_decorations[aRuleType]) {
        aDecorEl._flock_decorations[aRuleType] = [];
      }
      aDecorEl._flock_decorations[aRuleType][aName] = convertedVal;
    }
  }
};

flockWebDetective.prototype._postProcess =
function flockWebDetective__postProcess(aRENode, aRuleType, aResults, aDecorEl)
{
  var name = aRENode.getAttribute("name");
  this._logger.info("._postProcess('" + name + "')");
  var processes = aRENode.getAttribute("processing").split(",");
  for (var i = 0; i < processes.length; i++) {
    var oldVal;
    try {
      oldVal = aResults.getPropertyAsAString(name);
    } catch (ex) {
      // Was not a string so move on to next result
      continue;
    }
    var newVal = oldVal;
    this._logger.debug(" - process: " + processes[i]);
    switch (processes[i].toLowerCase()) {
      case "unescape":
        newVal = unescape(oldVal);
        break;
      case "toupper":
      case "touppercase":
        newVal = oldVal.toUpperCase();
        break;
      case "tolower":
      case "tolowercase":
        newVal = oldVal.toLowerCase();
        break;
      case "subst":
        // There must be a CDATA block with the substitution regexp
        var substRegexp = null;
        for (var j = 0; (j < aRENode.childNodes.length) && !substRegexp; j++) {
          var child = aRENode.childNodes.item(j);
          if (child.nodeType ==
              Components.interfaces.nsIDOMNode.CDATA_SECTION_NODE)
          {
            this._logger.debug("  - substitution regexp: " + child.nodeValue);
            substRegexp = this.parseSubstRegexp(child.nodeValue);
          }
        }
        if (substRegexp) {
          this._logger.debug("   pattern: " + substRegexp.pattern);
          this._logger.debug("   replacement: " + substRegexp.replace);
          // Convert to JS-compatible syntax
          var replacement = substRegexp.replace;
          for (var r = 1; r < 10; r++) {
            var replacementRE = "/\/"+r+"/g";
            replacement = replacement.replace(eval(replacementRE), "$"+r);
          }
          // Do the substitution
          newVal = oldVal.replace(eval(substRegexp.pattern), replacement);
        }
        break;
      default:
        this._logger.warning("WARNING: unhandled processing directive: "
                             + processes[i]);
    }
    this._logger.debug("   - new value: " + newVal);
    this._setResult(aRuleType, aResults, name, newVal, aDecorEl);
  }
};

flockWebDetective.prototype.parseSubstRegexp =
function flockWebDetective_parseSubstRegexp(aRegexpString)
{
  if (!aRegexpString.match(/^s\/((\\\/|[^\/])+)\/((\\\/|[^\/])*)\/[gim]{0,3}$/)) {
    this._logger.debug("INVALID substitution regexp: " + aRegexpString);
    return null;
  }
  return {
    pattern: "/"+RegExp.$1+"/",
    replace: RegExp.$3
  };
};

const TEST_FOR_ATTRIBS = [
  "class",
  "name",
  "value",
  "action",
  "method"
];

flockWebDetective.prototype.getXPathPrefix =
function flockWebDetective_getXPathPrefix(aHTMLElement)
{
  if (!aHTMLElement) {
    return "";
  }
  this._logger.info(".getXPathPrefix('" + aHTMLElement + "')");
  var el = aHTMLElement.QueryInterface(Components.interfaces.nsIDOMElement);
  var xpathExpr = "//"+el.tagName.toLowerCase();
  this._logger.debug(".getXPathPrefix" + xpathExpr);
  if (el.hasAttribute("id")) {
    xpathExpr += "[@id=\""+el.getAttribute("id")+"\"]";
  } else {
    for (var i = 0; i < TEST_FOR_ATTRIBS.length; i++) {
      var attrib = TEST_FOR_ATTRIBS[i];
      if (el.hasAttribute(attrib)) {
        xpathExpr += "[@"+attrib+"=\""+el.getAttribute(attrib)+"\"]";
      }
    }
    // This may not be enough to uniquely identify the node, so do the parent
    // as well...
    if (el.parentNode && (el.parentNode.tagName.toLowerCase() != "body")) {
      xpathExpr = this.getXPathPrefix(el.parentNode) + xpathExpr.substring(1);
    }
  }
  return xpathExpr;
};

flockWebDetective.prototype.getXPathExpression =
function flockWebDetective_getXPathExpression(aXPathNode)
{
  var xpathExpr = null;
  if (aXPathNode.hasAttribute("match")) {
    xpathExpr = aXPathNode.getAttribute("match");
  } else {
    for (var j = 0; (j < aXPathNode.childNodes.length) && !xpathExpr; j++) {
      if ( aXPathNode.childNodes.item(j).nodeType ==
           Components.interfaces.nsIDOMNode.CDATA_SECTION_NODE )
      {
        xpathExpr = aXPathNode.childNodes.item(j).nodeValue;
      }
    }
  }
  return xpathExpr;
};

flockWebDetective.prototype._getXPathResults =
function flockWebDetective__getXPathResults(aDocumentOrElement,
                                            aXPathNode,
                                            aRuleType,
                                            aResults,
                                            aData,
                                            aDecorEl)
{
  // First check to see if we have already cached this result as a decoration
  var name = null;
  if (aXPathNode.hasAttribute("name")) {
    name = aXPathNode.getAttribute("name");
    if (aDecorEl && 
        aDecorEl._flock_decorations &&
        aDecorEl._flock_decorations[aRuleType] &&
        aDecorEl._flock_decorations[aRuleType][name])
    {
      this._logger.debug("Using decorated value for '" + name + "'");
      aResults.setPropertyAsAString(name,
                                    aDecorEl._flock_decorations[aRuleType][name]);
      return;
    }
  }
  // No cached value, so press on
  var doc;
  var needPrefix = false;
  try {
    aDocumentOrElement.QueryInterface(Components.interfaces.nsIDOMDocument);
    doc = aDocumentOrElement;
  } catch (ex) {
    doc = aDocumentOrElement.ownerDocument;
    needPrefix = true;
  }
  if (!aDocumentOrElement.xpathPrefix && needPrefix) {
    aDocumentOrElement.xpathPrefix = this.getXPathPrefix(aDocumentOrElement);
  }
  if (!aDocumentOrElement.xpathPrefix) {
    aDocumentOrElement.xpathPrefix = "";
  }
  if (!aXPathNode.xpathExpr) {
    aXPathNode.xpathExpr = this.getXPathExpression(aXPathNode);
  }
  this._logger.debug("aDocumentOrElement.xpathPrefix = "
                     + aDocumentOrElement.xpathPrefix);
  var xpathExpr = aXPathNode.xpathExpr;
  if (aDocumentOrElement.xpathPrefix) {
    xpathExpr = aDocumentOrElement.xpathPrefix + aXPathNode.xpathExpr;
  }
  if (xpathExpr) {
    var extract = null;
    if (aXPathNode.hasAttribute("extract")) {
      extract = aXPathNode.getAttribute("extract");
    }
    var multivalent = false;
    if (aXPathNode.hasAttribute("multivalent")) {
      multivalent = aXPathNode.getAttribute("multivalent");
    }
    var snippets = this.getXMLSnippetsFromXPath(doc, xpathExpr, extract);
    this._logger.debug(" XPath got us " + snippets.length
                       + " snippets to check " + multivalent);
    if (snippets.length && name) {
      if (!multivalent) {
        this._setResult(aRuleType, aResults, name, snippets[0], aDecorEl);
      } else {
        this._setResult(aRuleType, aResults, name, snippets, aDecorEl);
      }
    }
    this._testXPathRegexp(aXPathNode, snippets, aRuleType, aResults, aData);
  }
};

/**
 * Check whether aXPathNode contains a <regexp> element, and test the regexp
 * against an array of text snippets.
 *
 * @param aXPathNode (in) nsIDOMElement <xpath> element
 * @param aSnippetsArray (in) JS array of text snippets to test the regexp on
 * @param aResults (in) nsIWritablePropertyBag2 which, if non-null, will be
 *        populated with any results
 * @param aData (in) working data object
 * @return TRUE if either there was no regexp, or else there was a regexp and it
 *         MATCHED; FALSE if there was a regexp and it did NOT match.
 */
flockWebDetective.prototype._testXPathRegexp =
function flockWebDetective__testXPathRegexp(aXPathNode,
                                            aSnippetsArray,
                                            aRuleType,
                                            aResults,
                                            aData)
{
  // Let's see if there's a regexp subnode
  for (var i = 0; i < aXPathNode.childNodes.length; i++) {
    var child = aXPathNode.childNodes.item(i);
    try {
      child.QueryInterface(Ci.nsIDOMElement);
    } catch (ex) {
      continue;
    }
    if (child.tagName == "regexp") {
      this._logger.debug(" This XPath statement has a Regexp too!");
      for (var j = 0; j < aSnippetsArray.length; j++) {
        if (aResults) {
          // We need to extract results from the regexp
          this._getRegexpResults(null,
                                 aSnippetsArray[j],
                                 child,
                                 aRuleType,
                                 aResults,
                                 aData);
        }
        var rExpr = this.getRegexpFromNode(child);
        if (this._doRegexpMatch(aSnippetsArray[j], rExpr, aRuleType, null, null)) {
          return true;
        }
      }
      // There *is* a <regexp> subnode, but it did not match any of the
      // snippets, so this test fails.
      return false;
    }
  }
  // There was NO <regexp> element, so the test succeeds by default
  return true;
};

const VALID_EXTRACT_VALUES = {
  "value": true,
  "nodeValue": true
};

flockWebDetective.prototype.getXMLSnippetsFromXPath =
function flockWebDetective_getXMLSnippetsFromXPath(aDocument, aXPathExpr, aExtract)
{
  this._logger.debug("Looking for snippets that match this XPath: "
                     + aXPathExpr + " " + aDocument);
  var snippets = [];
  if (aDocument.noDOM) {
    this._logger.debug("Document has no DOM, can't use XPath!!");
    return;
  }
  aDocument.QueryInterface(Components.interfaces.nsIDOMXPathEvaluator);
  var result;
  if (aDocument instanceof Components.interfaces.nsIDOMHTMLDocument) {
    result = aDocument.evaluate(aXPathExpr, aDocument.body, null, 0, null);
  } else {
    result = aDocument.evaluate( aXPathExpr,
                                 (aDocument.doc ? aDocument.doc : aDocument),
                                 null,
                                 Components.interfaces.nsIDOMXPathResult.ANY_TYPE,
                                 null );
  }
  try {
    result = result.QueryInterface(Components.interfaces.nsIDOMXPathResult);
  } catch (ex) {
    return;
  }
  switch (result.resultType) {
    case Components.interfaces.nsIDOMXPathResult.STRING_TYPE:
      this._logger.debug("  result.stringValue = " + result.stringValue);
      snippets.push(result.stringValue);
      break;
    case Components.interfaces.nsIDOMXPathResult.UNORDERED_NODE_ITERATOR_TYPE:
      var node;
      while (node = result.iterateNext()) {
        try {
          node.QueryInterface(Components.interfaces.nsIDOMNode);
        } catch (ex) {
          continue;
        }
        var value;
        if (node.hasChildNodes() &&
             (node.childNodes.length > 1 ||
              (node.childNodes.length == 1 &&
              !(node.firstChild instanceof Components.interfaces.nsIDOMText))))
        {
          var _xs = Components.classes["@mozilla.org/xmlextras/xmlserializer;1"]
                              .createInstance(Components.interfaces.nsIDOMSerializer);
          value = _xs.serializeToString(node);
        } else if ( node.childNodes.length == 1 &&
                    node.firstChild instanceof Components.interfaces.nsIDOMText )
        {
          value = node.firstChild.textContent;
        } else {
          value = node.nodeValue;
        }
        if (aExtract) {
          if (VALID_EXTRACT_VALUES[aExtract]) {
            value = node[aExtract];
          } else if ( aExtract.indexOf("attribute:") == 0 &&
                      node instanceof Components.interfaces.nsIDOMElement )
          {
            var attrName = aExtract.substring(10);
            value = node.getAttribute(attrName);
          } else if (aExtract == "this" && node instanceof Ci.nsIDOMNode) {
            // We're passing back the node here, not a string, so we need to
            // be careful later on when we're handling the snippets
            value = node;
          }
        }
        this._logger.debug(" - XPath matching node: " + value);
        snippets.push(value);
      }
      break;
    default:
      this._logger.warning("WARNING!!! Unhandled XPath result type: "
                           + result.resultType);
  }
  return snippets;
};

flockWebDetective.prototype.getFieldResults =
function flockWebDetective_getFieldResults(aForm, aFieldNode, aResults)
{
  var varname = null;
  if (aFieldNode.hasAttribute("extractas")) {
    varname = aFieldNode.getAttribute("extractas");
  } else if (aFieldNode.hasAttribute("name")) {
    varname = aFieldNode.getAttribute("name");
  } else if (aFieldNode.hasAttribute("fieldid")) {
    varname = aFieldNode.getAttribute("fieldid");
  }
  if (varname) {
    var field = this.getMatchingFormField(aFieldNode, aForm);
    if (field instanceof Components.interfaces.nsIDOMHTMLInputElement) {
      field.QueryInterface(Components.interfaces.nsIDOMHTMLInputElement);
      this._logger.debug(".getFieldResults() - extracting form field value ["
                         + varname + "] = " + field.value);
      aResults.setPropertyAsAString(varname, field.value);
    }
  }
};

flockWebDetective.prototype.getRegexpFromNode =
function flockWebDetective_getRegexpFromNode(aRegexpNode)
{
  var rExpr = null;
  if (aRegexpNode.hasAttribute("expression")) {
    rExpr = aRegexpNode.getAttribute("expression");
  } else {
    for (var j = 0; j < aRegexpNode.childNodes.length; j++) {
      if ( aRegexpNode.childNodes.item(j).nodeType ==
           Components.interfaces.nsIDOMNode.CDATA_SECTION_NODE )
      {
        rExpr = aRegexpNode.childNodes.item(j).nodeValue;
      }
    }
  }
  return rExpr;
};

/**
 * This does not do 'true' regexp validation, but rather just enough to ensure
 * that executing this string will not allow arbitrary code execution.
 */
flockWebDetective.prototype.isValidMatchRegexp =
function flockWebDetective_isValidMatchRegexp(aRegexpString)
{
  this._logger.info(".isValidMatchRegexp('" + aRegexpString + "')");
  return aRegexpString.match(/^\/(\\\/|[^\/])+\/[gim]{0,3}$/);
};

flockWebDetective.prototype._doRegexpMatchWithObj = 
function flockWebDetective__doRegexpMatchWithObj(aString,
                                                 aRegexpObj,
                                                 aResultFields,
                                                 aRuleType,
                                                 aResults,
                                                 aDecorEl,
                                                 aIsMultiValent)
{
  var regResults = aRegexpObj.exec(aString);
  if (regResults) {
    this._logger.debug("._doRegexpMatchWithObj() MATCH!");
    if (aResultFields && aResults) {
      for (var i = 1; i < aResultFields.length; i++) {
        var fieldName = aResultFields[i];
        if (aIsMultiValent) {
          for (var j = 1; ; j++) {
            try {
              aResults.getPropertyAsAString(fieldName+j);
            } catch (ex) {
              fieldName += j;
              break;
            }
          }
        }
        this._logger.debug("._doRegexpMatchWith() found [ " + fieldName + " ] = "
                           + regResults[i]);
        this._setResult(aRuleType, aResults, fieldName, regResults[i], aDecorEl);
      }
    }
    return true;
  }
  return false;
};

flockWebDetective.prototype._doRegexpMatch =
function flockWebDetective__doRegexpMatch(aString,
                                         aRegexp,
                                         aResultFields,
                                         aRuleType,
                                         aResults,
                                         aDecorEl,
                                         aIsMultiValent)
{
  return this._doRegexpMatchWithObj(aString,
                                    Cu.evalInSandbox(aRegexp, this._sandbox),
                                    aResultFields,
                                    aRuleType,
                                    aResults,
                                    aDecorEl,
                                    aIsMultiValent);
};

flockWebDetective.prototype.loadStrings =
function flockWebDetective_loadStrings(aServiceName)
{
  this.mStrings[aServiceName] = {};
  var localizedStrings = this.mLocalizedStrings[aServiceName] = {};
  var stringsEls = this.mRules[aServiceName].getElementsByTagName("strings");
  for (var i = 0; i < stringsEls.length; i++) {
    var stringsEl = stringsEls.item(i);
    stringsEl.QueryInterface(Components.interfaces.nsIDOMElement);
    var stringEls = stringsEl.getElementsByTagName("string");
    for (var j = 0; j < stringEls.length; j++) {
      var stringEl = stringEls.item(j);
      stringEl.QueryInterface(Components.interfaces.nsIDOMElement);
      if (stringEl.hasAttribute("name")) {
        var stringName = stringEl.getAttribute("name");
        var foundValue = null;
        if (stringEl.hasAttribute("value")) {
          foundValue = stringEl.getAttribute("value");
        }
        var foundLocale = null;
        if (stringEl.hasAttribute("locale")) {
          foundLocale = stringEl.getAttribute("locale");
        }
        var longestTextValue = "";
        for (var k = 0; (k < stringEl.childNodes.length) && !foundValue; k++) {
          var child = stringEl.childNodes.item(k);
          switch (child.nodeType) {
            case Components.interfaces.nsIDOMNode.TEXT_NODE:
              if (child.nodeValue.length > longestTextValue.length) {
                longestTextValue = child.nodeValue;
              }
              break;
            case Components.interfaces.nsIDOMNode.CDATA_SECTION_NODE:
              foundValue = stringEl.childNodes.item(k).nodeValue;
              break;
          }
        }
        if (!foundValue) {
          foundValue = longestTextValue;
        }
        if (foundLocale) {
          if (!localizedStrings[stringName]) {
            localizedStrings[stringName] = {};
          }
          localizedStrings[stringName][foundLocale] = foundValue;
          this._logger.debug(".loadStrings('" + aServiceName
                             + "'): found localized string '"
                             + stringName + "' ("
                             + foundLocale + "): "
                             + foundValue);
        } else {
          this.mStrings[aServiceName][stringName] = foundValue;
          this._logger.debug(".loadStrings('" + aServiceName
                             + "'): found string '"
                             + stringName + "': "
                             + foundValue);
        }
      }
    }
  }
};

flockWebDetective.prototype.loadSessionCookies =
function flockWebDetective_loadSessionCookies(aServiceName)
{
  this.mSessionCookies[aServiceName] = [];
  var cookiesEls = this.mRules[aServiceName].getElementsByTagName("sessioncookies");
  for (var i = 0; i < cookiesEls.length; i++) {
    var cookiesEl = cookiesEls.item(i);
    cookiesEl.QueryInterface(Components.interfaces.nsIDOMElement);
    var cookieEls = cookiesEl.getElementsByTagName("cookie");
    for (var j = 0; j < cookieEls.length; j++) {
      var cookieEl = cookieEls.item(j);
      cookieEl.QueryInterface(Components.interfaces.nsIDOMElement);
      var c = {
        host: cookieEl.getAttribute("host"),
        name: cookieEl.getAttribute("name"),
        path: cookieEl.getAttribute("path")
      };
      this._logger.debug(".loadSessionCookies('" + aServiceName + "'): host["
                         + c.host + "] name[" + c.name + "] path[" + c.path
                         + "]");
      this.mSessionCookies[aServiceName].push(c);
    }
  }
};

/* Basic implementation of decoding html entities. Ideally we need to map out
 * all special characters into their numerical equivalents and depending on
 * character set convert to appropriate symbol. Entity list HTML 4.01:
 * http://www.w3.org/TR/REC-html40/sgml/entities.html
 */
flockWebDetective.prototype.htmlEntitiesDecode =
function flockWebDetective_htmlEntitiesDecode(aInputStr) {
  var convertedStr = aInputStr;
  if (!convertedStr) {
    return "";
  }
  var regex;

  // First convert basic entities into numerical equivalents.
  var char_entities = { "quot" : "34",
                        "amp" : "38",
                        "lt" : "60",
                        "gt" : "62" };
  for (var i in char_entities) {
    regex = new RegExp("&" + i + ";", "g");
    convertedStr = convertedStr.replace(regex, "&#" + char_entities[i] + ";");
  }

  // Next convert numerical entities into real characters.
  var num_entities = { "34" : "\"",
                       "39" : "'",
                       "60" : "<",
                       "62" : ">"};
  for (var i in num_entities) {
    regex = new RegExp("&#0*" + i + ";", "g");
    convertedStr = convertedStr.replace(regex, num_entities[i]);
  }

  // Finally replace all numerical ampersand's.
  return convertedStr.replace(/&#0*38;/g, "&");
};
// END helper functions

// BEGIN update service functions

// This is patterned after nsSearchService.js:engineMetadataService
function makeURI(url) {
  var ios = Cc['@mozilla.org/network/io-service;1']
    .getService(Ci.nsIIOService);
  try {
    return ios.newURI(url, null, null);
  } catch (ex) { }

  return null;
}

function createStatement(dbconn, sql) {
  var stmt = dbconn.createStatement(sql);
  var wrapper = Cc["@mozilla.org/storage/statement-wrapper;1"]
    .createInstance(Ci.mozIStorageStatementWrapper);

  wrapper.initialize(stmt);
  return wrapper;
}

function UpdateMetadataStore() {
  this.init();
}

UpdateMetadataStore.prototype = {
  init: function UMS_init() {
    var dbfile = Cc['@mozilla.org/file/directory_service;1']
      .getService(Ci.nsIProperties).get('ProfD', Ci.nsIFile);
    dbfile.append('webdetective.sqlite');

    var storageService = Cc['@mozilla.org/storage/service;1']
      .getService(Ci.mozIStorageService);
    this.mDBConn = storageService.openDatabase(dbfile);

    var schema = 'id INTEGER PRIMARY KEY, servicename STRING, ' +
                 'name STRING, value STRING';

    try {
      this.mDBConn.createTable('webdetect_data', schema);
    }
    catch (e) { }

    this.mGetData = createStatement(this.mDBConn,
      'SELECT value FROM webdetect_data WHERE servicename = :servicename ' +
      'AND name = :name');
    this.mDeleteData = createStatement(this.mDBConn,
      'DELETE FROM webdetect_data WHERE servicename = :servicename ' +
      'AND name = :name');
    this.mInsertData = createStatement(this.mDBConn,
      'INSERT INTO webdetect_data (servicename, name, value) ' +
      'VALUES (:servicename, :name, :value)');
  },
  getAttr: function UMS_getAttr(serviceName, name) {
    name = name.toLowerCase();

    var stmt = this.mGetData;
    stmt.reset();
    var pp = stmt.params;
    pp.servicename = serviceName;
    pp.name = name;

    var value = null;
    if (stmt.step()) {
      value = stmt.row.value;
    }
    stmt.reset();
    return value;
  },
  setAttr: function UMS_setAttr(serviceName, name, value) {
    name = name.toLowerCase();

    this.mDBConn.beginTransaction();

    this.deleteServiceData(serviceName, name);

    var pp = this.mInsertData.params;
    pp.servicename = serviceName;
    pp.name = name;
    pp.value = value;
    this.mInsertData.step();
    this.mInsertData.reset();

    this.mDBConn.commitTransaction();
  },
  deleteServiceData: function UMS_deleteServiceData(serviceName, name) {
    name = name.toLowerCase();

    var pp = this.mDeleteData.params;
    pp.servicename = serviceName;
    pp.name = name;
    this.mDeleteData.step();
    this.mDeleteData.reset();
  }
};

flockWebDetective.prototype.startUpdateService =
function flockWebDetective_startUpdateService()
{
  this.updateMetadataStore = new UpdateMetadataStore();

  var tm = Cc['@mozilla.org/updates/timer-manager;1']
    .getService(Ci.nsIUpdateTimerManager);

  var prefs = Cc['@mozilla.org/preferences-service;1']
    .getService(Ci.nsIPrefBranch);
  var interval = prefs.getIntPref('flock.service.webdetective.updateinterval');

  var seconds = interval * 3600;
  tm.registerTimer('web-detective-update-timer', this, seconds);
};

/**
 * Stevo : This may be a bit clunky right now, we have 2 variables (updatesOk and fileCount) that
 *  are used to maintain if all updates are successful, and when to report back the results to aListener
 *  if aListener is valid. So for each service we send off a request and have a listener that onSuccess
 *  decrements the fileCount and when fileCount is <= 0 calls the onSuccess or onError depending on the
 *  state of updatesOk, onError will decrement fileCount and set updatesOk to false and when fileCount is <= 0
 *  it will report back onError to the aListener.
 */
flockWebDetective.prototype.checkForUpdates =
function flockWebDetective_checkForUpdates(forceUpdates, aFlockListener)
{
  // Updates will only be sought for services that have been instantiated and
  // have loaded their rules files into Web Detective.  So when we check for
  // updates, we need to make sure to explicitly instantiate all web service
  // components.
  var wsm = Cc["@flock.com/webservice-manager;1"]
            .getService(Ci.flockIWebServiceManager);
  wsm.instantiateAll();

  var now = Date.now();
  var updatesOk = true;  // If any updates fail then this will be false.
  var fileCount = 0;

  var serviceUpdateListener = {
    onSuccess : function cfu_onSuccess(aSubject, aTopic) {
      fileCount = fileCount - 1;
      if (fileCount <= 0) {
        if (aFlockListener) {
          if (updatesOk) {
            aFlockListener.onSuccess(null, null);
          } else {
            aFlockListener.onError(null, null);
          }
        }
      }
    },
    onError : function cfu_onError(aFlockError, aTopic) {
      fileCount = fileCount - 1;
      updatesOk = false;
      if (fileCount <= 0) {
        if (aFlockListener) {
          aFlockListener.onError(null, null);
        }
      }
    }
  };

  for (var serviceName in this.mDetectFiles) {
    if (forceUpdates) {
      this.updateMetadataStore.deleteServiceData(serviceName,
                                                 "updatelastmodified");
    } else {
      var expire = this.updateMetadataStore.getAttr(serviceName,
                                                    "updateexpire");
      if (expire && expire > now) {
        continue;
      }
    }

    fileCount = fileCount + 1;
    if (aFlockListener) {
      this.sendUpdateRequest(serviceName, serviceUpdateListener);
    } else {
      this.sendUpdateRequest(serviceName);
    }
  }
};

flockWebDetective.prototype.getUpdateInfo =
function flockWebDetective_getUpdateInfo(aServiceName)
{
  var url;
  var info;
  var updateEls = this.mRules[aServiceName].getElementsByTagName('update');
  for (var i = 0; i < updateEls.length; i++) {
    var updateEl = updateEls.item(i);
    updateEl.QueryInterface(Ci.nsIDOMElement);
    url = updateEl.getAttribute('url');
    var testURI = makeURI(url);
    if (testURI) {
      var interval = updateEl.getAttribute('interval');
      info = {
        url: testURI.spec,
        interval: interval ? interval : DEFAULT_UPDATE_INTERVAL
      };
      return info;
    }
  }

  url = DEFAULT_UPDATE_SERVER + this.mDetectFiles[aServiceName].leafName;
  info = {
    url: makeURI(url).spec,
    interval: DEFAULT_UPDATE_INTERVAL
  };
  return info;
};

flockWebDetective.prototype.sendUpdateRequest =
function flockWebDetective_sendUpdateRequest(aServiceName, aListener)
{
  var updateInfo = this.getUpdateInfo(aServiceName);

  var hr = Cc['@mozilla.org/xmlextras/xmlhttprequest;1']
    .createInstance(Ci.nsIXMLHttpRequest);
  hr.mozBackgroundRequest = true;
  hr.open('GET', updateInfo.url);

  var lastModified = this.updateMetadataStore.getAttr(aServiceName,
                                                      'updatelastmodified');
  hr.setRequestHeader('If-Modified-Since', lastModified);

  var self = this;
  var updateInterval = updateInfo.interval;

  hr.onload = function sur_onload(event) {
    var req = event.target;
    if (req.responseXML) {
      self.updateDetectFile(aServiceName, req.responseText, updateInterval);
      if (aListener) {
        aListener.onSuccess(null,aServiceName);
      }
    } else {
      if (aListener) {
        aListener.onError(null, aServiceName);
      }
    }
  };

  hr.send(null);
};

flockWebDetective.prototype.updateDetectFile =
function flockWebDetective_updateDetectFile(serviceName, contents, updateInterval)
{
  if (!contents) {
    this.updateMetadataStore.setAttr(serviceName, 'updateexpire',
                                     Date.now() + updateInterval * 86400000);
    return;
  }

  try {
    var detectFile = this.mDetectFiles[serviceName];

    var ostream = Cc['@mozilla.org/network/safe-file-output-stream;1']
      .createInstance(Ci.nsIFileOutputStream);
    ostream.init(detectFile, PR_WRONLY | PR_CREATE_FILE | PR_TRUNCATE, 0644, 0);

    var converter = Cc['@mozilla.org/intl/scriptableunicodeconverter']
      .createInstance(Ci.nsIScriptableUnicodeConverter);
    converter.charset = 'UTF-8';

    var convdata = converter.ConvertFromUnicode(contents) + converter.Finish();

    ostream.write(convdata, convdata.length);

    if (ostream instanceof Ci.nsISafeOutputStream) {
      ostream.finish();
    } else {
      ostream.close();
    }

    this.loadDetectFile(detectFile);

    var updateInfo = this.getUpdateInfo(serviceName);
    this.updateMetadataStore.setAttr(serviceName, 'updateexpire',
                                     Date.now() + updateInfo.interval * 86400000);

    this.updateMetadataStore.setAttr(serviceName, 'updatelastmodified',
                                     (new Date()).toUTCString());
  }
  catch (e) { }
};
// END update service functions

// ========== END flockWebDetective class ==========



// =========================================
// ========== BEGIN XPCOM Support ==========
// =========================================

var Module = {
  _firstTime: true,
  registerSelf: function(aCompMgr, aFileSpec, aLocation, aType)
  {
    if (this._firstTime) {
      this._firstTime = false;
      throw Components.results.NS_ERROR_FACTORY_REGISTER_AGAIN;
    }
    aCompMgr = aCompMgr.QueryInterface(Components.interfaces.nsIComponentRegistrar);
    aCompMgr.registerFactoryLocation(CLASS_ID, CLASS_NAME, CONTRACT_ID, aFileSpec, aLocation, aType);

    var categoryManager = Components.classes["@mozilla.org/categorymanager;1"]
                                    .getService(Components.interfaces.nsICategoryManager);
    categoryManager.addCategoryEntry("flock-startup", CLASS_NAME, "service," + CONTRACT_ID, true, true);
  },

  unregisterSelf: function(aCompMgr, aLocation, aType)
  {
    aCompMgr = aCompMgr.QueryInterface(Components.interfaces.nsIComponentRegistrar);
    aCompMgr.unregisterFactoryLocation(CLASS_ID, aLocation);
  },

  getClassObject: function(aCompMgr, aCID, aIID)
  {
    if (!aIID.equals(Components.interfaces.nsIFactory)) {
      throw Components.results.NS_ERROR_NOT_IMPLEMENTED;
    }
    if (aCID.equals(CLASS_ID)) {
      return Factory;
    }
    throw Components.results.NS_ERROR_NO_INTERFACE;
  },

  canUnload: function(aCompMgr) { return true; }
};

var Factory = {
  createInstance: function(aOuter, aIID)
  {
    if (aOuter != null) {
      throw Components.results.NS_ERROR_NO_AGGREGATION;
    }
    return (new flockWebDetective()).QueryInterface(aIID);
  }
};

function NSGetModule(aCompMgr, aFileSpec)
{
  return Module;
}

// ========== END XPCOM Support ==========
