// vim: ts=2 sw=2 expandtab cindent
//
// BEGIN FLOCK GPL
//
// Copyright Flock Inc. 2005-2008
// http://flock.com
//
// This file may be used under the terms of the
// GNU General Public License Version 2 or later (the "GPL"),
// http://www.gnu.org/licenses/gpl.html
//
// Software distributed under the License is distributed on an "AS IS" basis,
// WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
// for the specific language governing rights and limitations under the
// License.
//
// END FLOCK GPL
//

const Cc = Components.classes;
const Ci = Components.interfaces;
const Cr = Components.results;


const ENABLE_DEBUG = false; // switch to turn off slow debug code for production
function DEBUG(x) { if (ENABLE_DEBUG) debug("flockWebDetective: "+x+"\n"); }

const CLASS_ID = Components.ID("{61F83B70-6B52-11DB-BD13-0800200C9A66}");
const CLASS_NAME = "Flock Web Detective";
const CONTRACT_ID = "@flock.com/web-detective;1";
const INTERFACES = [
  Components.interfaces.nsISupports,
  Components.interfaces.nsIClassInfo,
  Components.interfaces.nsIObserver,
  Components.interfaces.nsITimerCallback,
  Ci.flockIWebDetective,
  Ci.flockIMigratable
];

// from nspr's prio.h
const PR_RDONLY      = 0x01;
const PR_WRONLY      = 0x02;
const PR_RDWR        = 0x04;
const PR_CREATE_FILE = 0x08;
const PR_APPEND      = 0x10;
const PR_TRUNCATE    = 0x20;
const PR_SYNC        = 0x40;
const PR_EXCL        = 0x80;

const DEFAULT_UPDATE_SERVER = "https://extensions.flock.com/webdetective/falcon/";
const DEFAULT_UPDATE_INTERVAL = 1; // This is in days


// ===================================================
// ========== BEGIN flockWebDetective class ==========
// ===================================================

function flockWebDetective()
{
  // Associative array where keys are service names and values are XML Docs
  this.mRules = [];

  // Associative array of version strings by service
  this.mVersions = [];

  // Associative array of strings by service
  this.mStrings = [];

  // Associative array of session cookies by service
  this.mSessionCookies = [];

  // Associative array of detect files that we've loaded for a given service
  this.mDetectFiles = [];

  this.cookieMgr = Components.classes["@mozilla.org/cookiemanager;1"]
                             .getService(Components.interfaces.nsICookieManager);
  this.mEnabled = true;

  this.startUpdateService();
}

// BEGIN nsISupports interface
flockWebDetective.prototype.QueryInterface =
function flockWebDetective_QueryInterface(aIID)
{
  var interfaces = INTERFACES;
  for (var i in interfaces) {
    if (aIID.equals(interfaces[i])) {
      return this;
    }
  }
  throw Components.results.NS_ERROR_NO_INTERFACE;
}
// END nsISupports interface


// BEGIN nsIClassInfo interface
flockWebDetective.prototype.contractID = CONTRACT_ID;
flockWebDetective.prototype.classID = CLASS_ID;
flockWebDetective.prototype.classDescription = CLASS_NAME;
flockWebDetective.prototype.implementationLanguage = Components.interfaces.nsIProgrammingLanguage.JAVASCRIPT;
flockWebDetective.prototype.flags = Components.interfaces.nsIClassInfo.SINGLETON;

flockWebDetective.prototype.getInterfaces =
function flockWebDetective_getInterfaces(aCount)
{
  var interfaces = INTERFACES;
  aCount.value = interfaces.length;
  return interfaces;
}

flockWebDetective.prototype.getHelperForLanguage =
function flockWebDetective_getHelperForLanguage(aLanguage)
{
  return null;
}
// END nsIClassInfo interface


// BEGIN nsIObserver interface
flockWebDetective.prototype.observe =
function flockWebDetective_observe(aSubject, aTopic, aData)
{
}
// END nsIObserver interface


// BEGIN nsITimerCallback interface
flockWebDetective.prototype.notify =
function flockWebDetective_notify(timer)
{
  try {
    var prefs = Cc['@mozilla.org/preferences-service;1']
      .getService(Ci.nsIPrefBranch)
    var doUpdate = prefs.getBoolPref('flock.service.webdetective.update');
    if (!doUpdate)
      return;
  }
  catch (e) { }

  this.checkForUpdates();
}
// END nsITimerCallback interface


// BEGIN flockIWebDetective interface
flockWebDetective.prototype.detect =
function flockWebDetective_detect(aServiceName, aType, aDocument, aResults)
{
  DEBUG("{flockIWebDetective}.detect('"+aServiceName+"', '"+aType+"')");
  return this.innerDetect(aServiceName, aType, aDocument, null, aResults);
}

flockWebDetective.prototype.detectForm =
function flockWebDetective_detectForm(aServiceName, aType, aForm, aResults)
{
  DEBUG("{flockIWebDetective}.detectForm('"+aServiceName+"', '"+aType+"')");
  aForm.QueryInterface(Components.interfaces.nsIDOMHTMLFormElement);
  return this.innerDetect(aServiceName, aType, aForm.ownerDocument, aForm, aResults);
}

flockWebDetective.prototype.detectCookies =
function flockWebDetective_detectCookies(aServiceName, aType, aResults)
{
  DEBUG("{flockIWebDetective}.detectCookies('"+aServiceName+"', '"+aType+"')");
  return this.innerDetect(aServiceName, aType, null, null, aResults);
}

flockWebDetective.prototype.detectNoDOM =
function flockWebDetective_detectNoDOM(aServiceName, aType, aURL, aDocumentText, aResults)
{
  DEBUG("{flockIWebDetective}.detectNoDOM('"+aServiceName+"', '"+aType+"')");
  //DEBUG(aDocumentText);
  var doc = {
    noDOM: true,
    documentElement: {
      innerHTML: aDocumentText
    },
    URL: aURL
  };
  return this.innerDetect(aServiceName, aType, doc, null, aResults);
}

flockWebDetective.prototype.getSessionCookies =
function flockWebDetective_getSessionCookies(aServiceName)
{
  if (!this.mSessionCookies[aServiceName]) {
    this.loadSessionCookies(aServiceName);
  }
  if (!this.mSessionCookies[aServiceName]) return null;
  var wd = this;
  return {
    arr: wd.mSessionCookies[aServiceName],
    idx: 0,
    QueryInterface: function (aIID) {
      if ( !aIID.equals(Components.interfaces.nsISupports) &&
           !aIID.equals(Components.interfaces.nsISimpleEnumerator) )
      {
        throw Components.results.NS_ERROR_NO_INTERFACE;
      }
      return this;
    },
    hasMoreElements: function () {
      return (this.idx < this.arr.length);
    },
    getNext: function () {
      var c = this.arr[this.idx++];
      var cookie = {
        QueryInterface: function (aIID) {
          if ( !aIID.equals(Components.interfaces.nsISupports) &&
               !aIID.equals(Components.interfaces.nsICookie) )
          {
            throw Components.results.NS_ERROR_NO_INTERFACE;
          }
          return this;
        },
        host: c.host,
        name: c.name,
        path: c.path
      };
      return cookie.QueryInterface(Components.interfaces.nsICookie);
    }
  };
}

flockWebDetective.prototype.getString =
function flockWebDetective_getString(aServiceName, aURLName, aDefault)
{
  DEBUG("{flockIWebDetective}.getString('"+aServiceName+"', '"+aURLName+"')");
  if (!this.mEnabled) return aDefault;
  if (!this.mRules[aServiceName]) {
    throw "No rules loaded for service: "+aServiceName;
  }
  if (!this.mStrings[aServiceName]) {
    this.loadStrings(aServiceName);
  }
  if (this.mStrings[aServiceName][aURLName]) {
    return this.mStrings[aServiceName][aURLName];
  }
  return aDefault;
}

flockWebDetective.prototype.loadDetectFile =
function flockWebDetective_loadDetectFile(aDetectFile)
{
  DEBUG("{flockIWebDetective}.loadDetectFile()");

  var prefs = Components.classes["@mozilla.org/preferences-service;1"]
                .getService(Components.interfaces.nsIPrefService)
                .getBranch(null);
  if (prefs.getPrefType("flock.service.webdetective.enabled")) {
    this.mEnabled = prefs.getBoolPref("flock.service.webdetective.enabled");
  }
  if (!this.mEnabled) return;

  aDetectFile.QueryInterface(Components.interfaces.nsILocalFile);

  if ((!aDetectFile.exists()) || (!aDetectFile.isReadable())) {
    throw Components.results.NS_ERROR_UNEXPECTED;
  }

  // Load and parse the XML
  var fis = Components.classes["@mozilla.org/network/file-input-stream;1"]
                      .createInstance(Components.interfaces.nsIFileInputStream);
  fis.init(aDetectFile, PR_RDONLY, 0, 0);
  DEBUG("available bytes: "+fis.available());
  var domParser = Components.classes["@mozilla.org/xmlextras/domparser;1"]
                            .createInstance(Components.interfaces.nsIDOMParser);
  var xmlDoc = domParser.parseFromStream(fis, "UTF-8", fis.available(), "text/xml");
  fis.close();

  var serviceEl = xmlDoc.documentElement;
  if (serviceEl.tagName != "service") {
    DEBUG("PARSE ERROR: no 'service' element found");
    throw Components.results.NS_ERROR_UNEXPECTED;
  }
  var serviceName = serviceEl.getAttribute("name");
  if (!serviceName || (serviceName == "")) {
    DEBUG("PARSE ERROR: 'service' element has no name");
    throw Components.results.NS_ERROR_UNEXPECTED;
  }
  this.mRules[serviceName] = xmlDoc;
  this.mDetectFiles[serviceName] = aDetectFile;
  if (serviceEl.hasAttribute("version")) {
    this.mVersions[serviceName] = serviceEl.getAttribute("version");
  }
  DEBUG("loaded detection rules for service: "+serviceName);
  // Force the strings and cookies to be reloaded, since they may have changed
  this.mStrings[serviceName] = null;
  this.mSessionCookies[serviceName] = null;
}

flockWebDetective.prototype.listServices =
function flockWebDetective_listServices()
{
  var svcEnum = {
    _arr: [],
    QueryInterface: function (aIID) {
      if (aIID.equals(Ci.nsISupports)) return this;
      if (aIID.equals(Ci.nsISimpleEnumerator)) return this;
      throw Cr.NS_ERROR_NO_INTERFACE;
    },
    hasMoreElements: function () {
      return (this._arr.length > 0);
    },
    getNext: function () {
      return {
        QueryInterface: function (aIID) {
          if (aIID.equals(Ci.nsISupports)) return this;
          if (aIID.equals(Ci.nsISupportsPrimitive)) return this;
          if (aIID.equals(Ci.nsISupportsString)) return this;
          throw Cr.NS_ERROR_NO_INTERFACE;
        },
        type: Ci.nsISupportsPrimitive.TYPE_STRING,
        data: this._arr.shift(),
        toString: function () { return this.data; }
      };
    }
  };
  for (var svcName in this.mDetectFiles) {
    svcEnum._arr.push(svcName);
  }
  return svcEnum;
}

flockWebDetective.prototype.getVersionForService =
function flockWebDetective_getVersionForService(aServiceName)
{
  return (this.mVersions[aServiceName]) ? this.mVersions[aServiceName] : null;
}

flockWebDetective.prototype.testDomain =
function flockWebDetective_testDomain(aURL, aDomain)
{
  var uri = Cc["@mozilla.org/network/standard-url;1"]
            .createInstance(Ci.nsIURI);
  var host;
  try {
    uri.spec = aURL;
    host = uri.host;
  } catch (ex) {
    // There wasn't a host
    return false;
  }
  if (host == aDomain) {
    return true;
  }
  var idx = host.indexOf("." + aDomain);
  if (idx < 1) {
    return false;
  }
  // We now know that the host contains .<domain> as a substring, and we just
  // need to make sure that it properly ENDS with .<domain>
  return ((idx + aDomain.length + 1) == host.length);
}
// END flockIWebDetective interface


// BEGIN flockIMigratable
flockWebDetective.prototype.__defineGetter__("migrationName",
  function flockWebDetective_getter_migrationName() {
    return "WebDetective";
  }
);

flockWebDetective.prototype.needsMigration =
function flockWebDetective_needsMigration(aOldVersion)
{
  if (aOldVersion.substr(0, 3) === "0.9") {
    // Migration from Cormorant
    return true;
  }
  return false;
};

flockWebDetective.prototype.startMigration =
function flockWebDetective_startMigration(aOldVersion, aListener)
{
  var ctxt = {
    oldVersion: aOldVersion,
    listener: aListener
  };

  if (aOldVersion.substr(0, 3) === "0.9") {
    // Migration from Cormorant: need to get handles to the <app>/res/detect as
    // well as the <profile>/detect directories so that we can copy files from
    // the former to the latter.
    var dirSvc = Cc["@mozilla.org/file/directory_service;1"]
                 .getService(Ci.nsIProperties);

    // Get the file spec for <app>/res/detect
    ctxt.appDetectDir = dirSvc.get("ARes", Ci.nsIFile);
    ctxt.appDetectDir.append("detect");

    // Get the file handle for <profile>/detect
    ctxt.profDetectDir = dirSvc.get("ProfD", Ci.nsIFile);
    ctxt.profDetectDir.append("detect");

    if (ctxt.profDetectDir.exists()) {
      aListener.onUpdate(0, "Migrating Web Detective files");
    }
  }

  return { wrappedJSObject: ctxt };
};

flockWebDetective.prototype.doMigrationWork =
function flockWebDetective_doMigrationWork(aCtxtWrapper)
{
  var ctxt = aCtxtWrapper.wrappedJSObject;

  if (ctxt.oldVersion.substr(0, 3) === "0.9") {
    // Migration from Cormorant: copy any .xml files from the <app>/res/detect
    // directory into the user's <profile>/detect directory, clobbering any
    // previous versions of those files that might exist there.
    var appDetectDir = ctxt.appDetectDir;
    var profDetectDir = ctxt.profDetectDir;

    if (!profDetectDir || !profDetectDir.exists()) {
      return false;
    }

    var dirEntries = appDetectDir.directoryEntries;
    while (dirEntries.hasMoreElements()) {
      var file = dirEntries.getNext().QueryInterface(Ci.nsIFile);

      if (file.isFile() &&
          file.path.substr(file.path.length - 4).toLowerCase() === ".xml")
      {
        // 'file' now represents a new XML file we want to copy into the
        // user's profile.  But we may need to delete the old version of the
        // file first.
        var oldFile = profDetectDir.clone();
        oldFile.append(file.leafName);

        if (oldFile.exists()) {
          DEBUG("removing old " + oldFile.leafName);
          oldFile.remove(false); // Non-recursive
        }

        DEBUG("copying " + file.leafName + " to " + profDetectDir.path);
        file.copyTo(profDetectDir, null);
      }
    }
  }

  return false;
};

flockWebDetective.prototype.finishMigration =
function flockWebDetective_finishMigration(aCtxtWrapper)
{
};
// END flockIMigratable


// BEGIN helper functions
flockWebDetective.prototype.innerDetect =
function flockWebDetective_innerDetect(aServiceName, aType, aDocument, aForm, aResults)
{
  if (!this.mEnabled) return false;
  var ruleMatch = false;
  var ruleDoc = this.mRules[aServiceName];
  if (ruleDoc) {
    // Cache the rules for faster retrieval next time
    if (!ruleDoc.rules) {
      ruleDoc.rules = [];
    }
    if (!ruleDoc.rules[aType]) {
      ruleDoc.rules[aType] = this.getRulesOfType(ruleDoc, aType);
    }
    var rules = ruleDoc.rules[aType];
    DEBUG("found "+rules.length+ " rule(s) of type '"+aType+"'");
    for (var r = 0; (r < rules.length) && !ruleMatch; r++) {
      var allCondsMatch = true;
      if (rules[r].conditionsEl) {

        if (!rules[r].conditions) {
          // This will be used for cacheing the conditions functions
          rules[r].conditions = [];
        }

        // First check the URL conditions
        if (!rules[r].conditions["url"]) {
          // Cache the URL conditions for faster retrieval next time
          rules[r].conditions["url"] = this.getURLConditionsForRule(rules[r]);
        }
        var urlConds = rules[r].conditions["url"];
        DEBUG("rule["+r+"]("+aType+") has "+urlConds.length+" URL condition(s)");
        if (urlConds.length > 0) {
          if (aDocument && aDocument instanceof Components.interfaces.nsIDOMHTMLDocument) {
            var ios = Components.classes["@mozilla.org/network/io-service;1"]
                                .getService(Components.interfaces.nsIIOService);
            var uri = ios.newURI(aDocument.URL, null, null);
            for (var c = 0; (c < urlConds.length) && allCondsMatch; c++) {
              if (!urlConds[c](uri)) {
                DEBUG("URL condition ["+c+"] failed");
                allCondsMatch = false;
              }
            }
            if (!allCondsMatch) continue;
          } else {
            // There is no URL, so URL conditions automatically fail
            allCondsMatch = false;
          }
        }
        if (!allCondsMatch) continue;

        // Check Form conditions
        if (!rules[r].conditions["form"]) {
          // Cache the Form conditions for faster retrieval next time
          rules[r].conditions["form"] = this.getFormConditionsForRule(rules[r], aForm);
        }
        var formConds = rules[r].conditions["form"];
        DEBUG("rule["+r+"]("+aType+") has "+formConds.length+" Form condition(s)");
        if (formConds.length > 0) {
          if (aForm) {
            for (var c = 0; (c < formConds.length) && allCondsMatch; c++) {
              if (!formConds[c](aForm)) {
                DEBUG("Form condition ["+c+"] failed");
                allCondsMatch = false;
              }
            }
          } else {
            // There's no Form, so Form conditions automatically fail
            allCondsMatch = false;
          }
        }
        if (!allCondsMatch) continue;

        // Next check the Document conditions
        if (!rules[r].conditions["doc"]) {
          // Cache the Document conditions for faster retrieval next time
          rules[r].conditions["doc"] = this.getDocConditionsForRule(rules[r]);
        }
        var docConds = rules[r].conditions["doc"];
        DEBUG("rule["+r+"]("+aType+") has "+docConds.length+" Document condition(s)");
        if (docConds.length > 0) {
          if (aDocument) {
            for (var c = 0; (c < docConds.length) && allCondsMatch; c++) {
              if (!docConds[c](aDocument)) {
                DEBUG("Document condition ["+c+"] failed");
                allCondsMatch = false;
              }
            }
          } else {
            // There's no Document, so Document conditions automatically fail
            allCondsMatch = false;
          }
        }
        if (!allCondsMatch) continue;

        // Finally check the Cookie conditions
        if (!rules[r].conditions["cookie"]) {
          // Cache the Cookie conditions for faster retrieval next time
          rules[r].conditions["cookie"] = this.getCookieConditionsForRule(rules[r]);
        }
        var cookieConds = rules[r].conditions["cookie"];
        DEBUG("rule["+r+"]("+aType+") has "+cookieConds.length+" Cookie condition(s)");
        if (cookieConds.length > 0) {
          var relevantCookies = this.getRelevantCookiesForRule(rules[r]);
          for (var c = 0; (c < cookieConds.length) && allCondsMatch; c++) {
            if (!cookieConds[c](relevantCookies)) {
              DEBUG("Cookie condition ["+c+"] failed");
              allCondsMatch = false;
            }
          }
        }
      }

      if (allCondsMatch) {
        DEBUG("All conditions matched for rule["+r+"]("+aType+")!");
        ruleMatch = true;
        if (!aResults) break;
        // Now get results
        this.getResultsForRule(aDocument, aForm, rules[r], aResults);
      }
    }
  }
  return ruleMatch;
}

flockWebDetective.prototype.getRulesOfType =
function flockWebDetective_getRulesOfType(aRulesDoc, aType)
{
  var rules = [];
  var detectElements = aRulesDoc.getElementsByTagName("detect");
  for (var i = 0; i < detectElements.length; i++) {
    var detect = detectElements.item(i);
    detect = detect.QueryInterface(Components.interfaces.nsIDOMElement);
    if (aType == detect.getAttribute("type")) {
      DEBUG("found <detect type='"+aType+"'>");
      for (var j = 0; j < detect.childNodes.length; j++) {
        var child = detect.childNodes.item(j);
        try {
          child.QueryInterface(Components.interfaces.nsIDOMElement);
          if (child.tagName == "conditions") {
            detect.conditionsEl = child;
          }
          if (child.tagName == "results") {
            detect.resultsEl = child;
          }
        } catch (ex) {
          // Do nothing
        }
      }
      if (!detect.conditionsEl) {
        detect.conditionsEl = detect;
      }
      if (!detect.resultsEl) {
        detect.resultsEl = detect;
      }
      rules[rules.length] = detect;
    }
  }
  return rules;
}

flockWebDetective.prototype.getURLConditionsForRule =
function flockWebDetective_getURLConditionsForRule(aRule)
{
  var conditions = [];

  // Look for the first "url" element (ignore any others)
  var urlEls = aRule.conditionsEl.getElementsByTagName("url");
  if (urlEls.length) {
    var urlEl = urlEls.item(0);
    conditions = this.addStandardConds(conditions, urlEl, function (aURI) { return aURI.spec; });
    if (urlEl.hasAttribute("domain")) {
      conditions[conditions.length] = function (aURI) {
        var domain = urlEl.getAttribute("domain");
        if (aURI.host == domain) return true;
        var idx = aURI.host.indexOf("."+domain);
        if (idx < 1) return false;
        return ((idx + domain.length + 1) == aURI.host.length);
      };
    }
    var tags = ["host", "path", "querystring"];
    var tagFuncs = [];
    tagFuncs["host"] = function (aURI) { return aURI.host; };
    tagFuncs["path"] = function (aURI) { return aURI.path; };
    tagFuncs["querystring"] = function (aURI) { return aURI.path.substring(aURI.path.indexOf("?")); };
    for (var tag in tagFuncs) {
      var elements = urlEl.getElementsByTagName(tag);
      for (var e = 0; e < elements.length; e++) {
        var elem = elements.item(e);
        conditions = this.addStandardConds(conditions, elem, tagFuncs[tag]);
      }
    }
    var regexpEls = urlEl.getElementsByTagName("regexp");
    for (var i = 0; i < regexpEls.length; i++) {
      var regexpEl = regexpEls.item(i);
      var rExpr = this.getRegexpFromNode(regexpEl);
      var inst = this;
      conditions[conditions.length] = function (aURI) {
        return inst.doRegexpMatch(aURI.spec, rExpr, null, null);
      };
    }
  }
  return conditions;
}

flockWebDetective.prototype.getFormConditionsForRule =
function flockWebDetective_getFormConditionsForRule(aRule, aForm)
{
  var conditions = [];

  // Look for the first "form" element (ignore any others)
  var formEls = aRule.conditionsEl.getElementsByTagName("form");
  if (formEls.length) {
    var formEl = formEls.item(0);
    // Iterate through all the children of "form"
    for (var i = 0; i < formEl.childNodes.length; i++) {
      var formChild = formEl.childNodes.item(i);
      try {
        formChild.QueryInterface(Components.interfaces.nsIDOMElement);
      } catch (ex) {
        continue;
      }
      DEBUG("Found <conditions><form><"+formChild.tagName+">");
      switch (formChild.tagName) {
        case "xpath":
        {
          conditions[conditions.length] = this.createXPathFormCondition(formChild);
        }; break;
        case "field":
        {
          conditions[conditions.length] = this.createFieldCondition(formChild);
        }; break;
      }
    }
  }
  return conditions;
}

flockWebDetective.prototype.createXPathFormCondition =
function flockWebDetective_createXPathFormCondition(aXPathNode)
{
  var inst = this;
  return function (aForm) {
    if (!aXPathNode.xpathExpr) {
      aXPathNode.xpathExpr = inst.getXPathExpression(aXPathNode);
    }
    if (!aXPathNode.xpathFunc) {
      aXPathNode.xpathFunc = inst.createXPathCondition(aXPathNode.xpathExpr);
    }
    if (!aForm.xpathPrefix) {
      aForm.xpathPrefix = inst.getXPathPrefix(aForm);
      DEBUG("Got XPath prefix for this form: "+aForm.xpathPrefix);
    }
    return aXPathNode.xpathFunc(aForm.ownerDocument, aForm.xpathPrefix);
  };
}

const FIELD_ATTRIBUTES = [ "name", "type", "class" ];

flockWebDetective.prototype.createFieldCondition =
function flockWebDetective_createFieldCondition(aFieldNode)
{
  var inst = this;
  return function (aForm) {
    return (inst.getMatchingFormField(aFieldNode, aForm) != null);
  };
}

flockWebDetective.prototype.getMatchingFormField =
function flockWebDetective_getMatchingFormField(aFieldNode, aForm)
{
  var formFields = aForm.elements;
  for (var i = 0; i < formFields.length; i++) {
    var field = formFields.item(i)
      .QueryInterface(Components.interfaces.nsIDOMElement);
    if (aFieldNode.hasAttribute("tagname")) {
      if (field.tagName.toLowerCase() != aFieldNode.getAttribute("tagname").toLowerCase()) {
        // This form field does not match the pattern, so skip it
        DEBUG("tagname does NOT match ["+field.tagName+" != "+aFieldNode.getAttribute("tagname")+"]");
        continue;
      }
    }
    if (aFieldNode.hasAttribute("fieldid")) {
      if ( !field.hasAttribute("id") ||
           (aFieldNode.getAttribute("fieldid") != field.getAttribute("id")) )
      {
        // This form field does not match the pattern, so skip it
        DEBUG("fieldid does NOT match");
        continue;
      }
      // The fields have matching ids!
    }
    var matchesAllAttributes = true;
    for (var j = 0; j < FIELD_ATTRIBUTES.length; j++) {
      var attr = FIELD_ATTRIBUTES[j];
      if (aFieldNode.hasAttribute(attr)) {
        if ( !field.hasAttribute(attr) ||
             (aFieldNode.getAttribute(attr) != field.getAttribute(attr)) )
        {
          // This form field does not match the pattern, so skip it
          DEBUG("'"+attr+"' does NOT match");
          matchesAllAttributes = false;
          break;
        }
      }
    }
    if (!matchesAllAttributes) continue;
    // This form field matches the pattern
    return field;
  }
  // Didn't find any fields matching the pattern
  return null;
}

flockWebDetective.prototype.getDocConditionsForRule =
function flockWebDetective_getDocConditionsForRule(aRule)
{
  var conditions = [];

  // Document conditions can occur either under a "document" element, or else
  // at the top level if compact syntax is being used.
  var docCondNodes = this.getDocSubNodes(aRule.conditionsEl);

  // Iterate through all the "document" sub nodes
  for (var i = 0; i < docCondNodes.length; i++) {
    var docChild = docCondNodes[i];
    try {
      docChild.QueryInterface(Components.interfaces.nsIDOMElement);
    } catch (ex) {
      continue;
    }
    DEBUG("Found <conditions><document><"+docChild.tagName+">");
    switch (docChild.tagName) {
      case "xpath":
      {
        var xpathExpr = this.getXPathExpression(docChild);
        if (xpathExpr) {
          conditions[conditions.length] = this.createXPathCondition(xpathExpr);
        }
      }; break; // END "xpath" case

      case "regexp":
      {
        var rExpr = this.getRegexpFromNode(docChild);
        if (this.isValidMatchRegexp(rExpr)) {
          var isMultiLine = (docChild.getAttribute("multiline") == "true");
          if (isMultiLine) {
            conditions[conditions.length] =
              this.createMultilineRegexpCondition(rExpr);
          } else {
            conditions[conditions.length] = this.createRegexpCondition(rExpr);
          }
        } else {
          DEBUG("Condition will fail due to INVALID regexp: "+rExpr);
          conditions[conditions.length] = function (aDocument) {
            return false;
          };
        }
      }; break; // END "regexp" case
    }
  }
  return conditions;
}

flockWebDetective.prototype.createXPathCondition =
function flockWebDetective_createXPathCondition(aXPathExpr)
{
  var func = function (aDocument, aXPathPrefix) {
    if (aDocument.noDOM) {
      DEBUG("Document has no DOM, can't use XPath!!");
      return false;
    }
    aDocument.QueryInterface(Components.interfaces.nsIDOMXPathEvaluator);
    var allXPath = (aXPathPrefix) ? aXPathPrefix+aXPathExpr : aXPathExpr;
    DEBUG("Evaluating XPath statement: "+allXPath);
    var result;
    if (aDocument instanceof Components.interfaces.nsIDOMHTMLDocument) {
      result = aDocument.evaluate(allXPath, aDocument.body, null, 0, null);
    } else {
      DEBUG(aDocument.documentElement.namespaceURI);
      var _xs = Components.classes["@mozilla.org/xmlextras/xmlserializer;1"]
                          .createInstance(Components.interfaces.nsIDOMSerializer);;
      var xml = _xs.serializeToString(aDocument.documentElement);
      var namespace = "xmlns=\""+aDocument.documentElement.namespaceURI+"\"";
      var temp = xml.indexOf(namespace);
      aDocument.doc = null;
      //need to strip out the namespace otherwise aDocument.evaluate will fail
      if (temp > 0) {
        var new_xml = xml.substring(0, temp)
                    + xml.substring((temp + namespace.length), xml.length);
        var parser = Components.classes["@mozilla.org/xmlextras/domparser;1"]
                               .createInstance(Components.interfaces.nsIDOMParser);
        aDocument.doc = parser.parseFromString(new_xml, "text/xml");
      }
      result = aDocument.evaluate( allXPath,
                                   (aDocument.doc ? aDocument.doc : aDocument),
                                   null,
                                   Components.interfaces.nsIDOMXPathResult.ANY_TYPE,
                                   null );
    }
    try {
      result = result.QueryInterface(Components.interfaces.nsIDOMXPathResult);
    } catch (ex) {
      return false;
    }
    DEBUG(" - got an nsIDOMXPathResult of type ["+result.resultType+"]");
    switch (result.resultType) {
      case Components.interfaces.nsIDOMXPathResult.STRING_TYPE:
      {
        DEBUG("  result.stringValue = "+result.stringValue);
        return true;
      }; break;
      case Components.interfaces.nsIDOMXPathResult.UNORDERED_NODE_ITERATOR_TYPE:
      {
        var node = result.iterateNext();
        try {
          node.QueryInterface(Components.interfaces.nsIDOMNode);
          DEBUG(" - XPath matching node!" + node.nodeValue);
          // Found a matching node!
          return true;
        } catch (ex) {
          DEBUG(ex);
          DEBUG(" - NO match on XPath " +result);
        }
        return false;
      }; break;
      default:
        DEBUG("WARNING!!! Unhandled XPath result type: "+result.resultType);
    }
    return false;
  };
  return func;
}

/**
 * Generates a function for testing a regular expression against each line of
 * a document in succession.
 */
flockWebDetective.prototype.createRegexpCondition =
function flockWebDetective_createRegexpCondition(aRegExp)
{
  var inst = this;
  var func = function createRegexpCond_inner(aDocument) {
    var html = aDocument.documentElement.innerHTML;
    var lines = html.split(/[\r\n]+/);
    for (var i = 0; i < lines.length; i++) {
      if (inst.doRegexpMatch(lines[i], aRegExp, null, null)) {
        return true;
      }
    }
    return false;
  };
  return func;
}

/**
 * Generates a function for testing a "multiline" regular expression against
 * the entire text of a document.
 */
flockWebDetective.prototype.createMultilineRegexpCondition =
function flockWebDetective_createMultilineRegexpCondition(aRegExp)
{
  var inst = this;
  var func = function createMultilineRegexpCond_inner(aDocument) {
    var html = aDocument.documentElement.innerHTML;
    var oneString = html.replace(/[\r\n]/g, "");
    return inst.doRegexpMatch(oneString, aRegExp, null, null);
  };
  return func;
}

flockWebDetective.prototype.getCookieConditionsForRule =
function flockWebDetective_getCookieConditionsForRule(aRule)
{
  var conditions = [];
  var cookieEls = aRule.conditionsEl.getElementsByTagName("cookie");
  for (var e = 0; e < cookieEls.length; e++) {
    var cookieEl = cookieEls.item(e);
    var cHost = null;
    if (cookieEl.hasAttribute("host")) {
      cHost = cookieEl.getAttribute("host");
    }
    var cName = null;
    if (cookieEl.hasAttribute("name")) {
      cName = cookieEl.getAttribute("name");
    }
    var cNoMatch = (cookieEl.getAttribute("nomatch") == "true");
    this.addCookieCondition(conditions, cHost, cName, cNoMatch);
  }
  return conditions;
}

flockWebDetective.prototype.addCookieCondition =
function flockWebDetective_addCookieCondition(aConditions, aHost, aName, aNoMatch)
{
  aConditions[aConditions.length] = function (aRelevantCookies) {
    var foundMatch = false;
    for (var c = 0; (c < aRelevantCookies.length) && !foundMatch; c++) {
      DEBUG("Testing cookie ["+aRelevantCookies[c].host+"]["+aRelevantCookies[c].name+"]");
      DEBUG(" against rule ["+aNoMatch+"]["+aHost+"]["+aName+"]");
      if ((aRelevantCookies[c].host == aHost) &&
          (aRelevantCookies[c].name == aName))
      {
        foundMatch = true;
      }
    }
    if (foundMatch && !aNoMatch) return true;
    if (!foundMatch && aNoMatch) return true;
    return false;
  };
}

flockWebDetective.prototype.getRelevantCookiesForRule =
function flockWebDetective_getRelevantCookiesForRule(aRule)
{
  var relevant = [];
  var cookieEls = aRule.conditionsEl.getElementsByTagName("cookie");
  for (var e = 0; e < cookieEls.length; e++) {
    var host = cookieEls.item(e).getAttribute("host");
    var name = cookieEls.item(e).getAttribute("name");
    //DEBUG("Cookies with host="+host+" and name="+name+" are relevant");
    if (!relevant[host]) {
      relevant[host] = [];
    }
    relevant[host][name] = true;
  }
  var relevantCookies = [];
  var cookEnum = this.cookieMgr.enumerator;
  var cookieCount = 0;
  while (cookEnum.hasMoreElements()) {
    var cookie = cookEnum.getNext()
                         .QueryInterface(Components.interfaces.nsICookie);
    cookieCount++;
    //DEBUG(" - filtering cookie ["+cookie.host+"]["+cookie.name+"]");
    if (relevant[cookie.host] && relevant[cookie.host][cookie.name]) {
      relevantCookies[relevantCookies.length] = cookie;
    }
  }
  DEBUG("Found "+relevantCookies.length+" relevant cookies out of "+cookieCount+" total");
  return relevantCookies;
}

flockWebDetective.prototype.addStandardConds =
function flockWebDetective_addStandardConds(aConditionsArray, aDOMElement, paramFunc)
{
  aDOMElement.QueryInterface(Components.interfaces.nsIDOMElement);
  if (aDOMElement.hasAttribute("equals")) {
    aConditionsArray[aConditionsArray.length] = function (param) {
      var input = paramFunc(param);
      DEBUG("condition: "+input+" == "+aDOMElement.getAttribute("equals"));
      return (input == aDOMElement.getAttribute("equals"));
    };
  }
  if (aDOMElement.hasAttribute("contains")) {
    aConditionsArray[aConditionsArray.length] = function (param) {
      var input = paramFunc(param);
      DEBUG("condition: "+input+" ~= "+aDOMElement.getAttribute("contains"));
      return (input.indexOf(aDOMElement.getAttribute("contains")) != -1);
    };
  }
  return aConditionsArray;
}

flockWebDetective.prototype.getResultsForRule =
function flockWebDetective_getResultsForRule(aDocument, aForm, aRule, aResults)
{
  DEBUG(".getResultsForRule()");
  aResults.QueryInterface(Components.interfaces.nsIWritablePropertyBag2);
  if (!aRule.resultsEl) return;

  if (aDocument) {

    // Get URL results
    if (!aRule.urlResults) {
      aRule.urlResults = aRule.resultsEl.getElementsByTagName("url");
    }
    var workingData = {};
    for (var u = 0; u < aRule.urlResults.length; u++) {
      var urlEl = aRule.urlResults.item(u);
      DEBUG("found <results><url>");
      for (var i = 0; i < urlEl.childNodes.length; i++) {
        var child = urlEl.childNodes.item(i);
        try {
          child.QueryInterface(Components.interfaces.nsIDOMElement);
        } catch (ex) {
          continue;
        }
        DEBUG("found <results><url><"+child.tagName+">");
        switch (child.tagName) {
          case "regexp":
          {
            this.getRegexpResults(null, aDocument.URL, child, aResults, workingData);
          }; break;
        }
      }
    }

    // Get Document results
    workingData = {};
    if (!aRule.docResults) {
      aRule.docResults = this.getDocSubNodes(aRule.resultsEl);
    }
    for (var d = 0; d < aRule.docResults.length; d++) {
      var child = aRule.docResults[d];
      try {
        child.QueryInterface(Components.interfaces.nsIDOMElement);
      } catch (ex) {
        continue;
      }
      DEBUG("found <results><document><"+child.tagName+">");
      switch (child.tagName) {
        case "regexp":
        {
          this.getRegexpResults(aDocument, aDocument.documentElement.innerHTML, child, aResults, workingData);
        }; break;
        case "xpath":
        {
          this.getXPathResults(aDocument, child, aResults, workingData);
        }; break;
      }
    }
  }

  if (aForm) {
    // Get Form results
    var workingData = {};
    var formEls = aRule.resultsEl.getElementsByTagName("form");
    for (var f = 0; f < formEls.length; f++) {
      var formEl = formEls.item(f);
      DEBUG("found <results><form>");
      for (var i = 0; i < formEl.childNodes.length; i++) {
        var child = formEl.childNodes.item(i);
        try {
          child.QueryInterface(Components.interfaces.nsIDOMElement);
        } catch (ex) {
          continue;
        }
        DEBUG("found <results><form><"+child.tagName+">");
        switch (child.tagName) {
          case "xpath":
          {
            this.getXPathResults(aForm, child, aResults, workingData);
          }; break;
          case "field":
          {
            this.getFieldResults(aForm, child, aResults);
          }; break;
        }
      }
    }
  }

  // TODO: Get Cookie results...
}

flockWebDetective.prototype.getDocSubNodes =
function flockWebDetective_getDocSubNodes(aNode)
{
  var resultNodes = [];
  var documentEls = aNode.getElementsByTagName("document");
  for (var i = 0; i < documentEls.length; i++) {
    var docEl = documentEls.item(i);
    for (var j = 0; j < docEl.childNodes.length; j++) {
      var child = docEl.childNodes.item(j);
      try {
        child.QueryInterface(Components.interfaces.nsIDOMElement);
      } catch (ex) {
        continue;
      }
      resultNodes.push(child);
    }
  }
  // Any children of the result node that are NOT in [ conditions, url,
  // document, form, cookie ] are considered document conditions.
  for (var i = 0; i < aNode.childNodes.length; i++) {
    var child = aNode.childNodes.item(i);
    try {
      child.QueryInterface(Components.interfaces.nsIDOMElement);
    } catch (ex) {
      continue;
    }
    switch (child.tagName.toLowerCase()) {
      case "conditions":
      case "url":
      case "document":
      case "form":
      case "cookie":
        continue;
      default:
        resultNodes.push(child);
    }
  }
  return resultNodes;
}

flockWebDetective.prototype.getRegexpResults =
function flockWebDetective_getRegexpResults(aDecorEl, aString, aRegexpNode, aResults, aData)
{
  // Cache the regexp
  if (!aRegexpNode.rExpr) {
    aRegexpNode.rExpr = this.getRegexpFromNode(aRegexpNode);
  }
  // Cache whether it is valid
  if (!aRegexpNode.isValid) {
    aRegexpNode.isValid = this.isValidMatchRegexp(aRegexpNode.rExpr);
  }
  if (!aRegexpNode.isValid) {
    DEBUG("Can't get results due to INVALID regexp: "+aRegexpNode.rExpr);
    return;
  }
  // Cache the list of variables we are looking for
  if (!aRegexpNode.reVars) {
    aRegexpNode.reVars = [];
    aRegexpNode.postProcess = [];
    for (var re = 1; ; re++) {
      if (aRegexpNode.hasAttribute("re"+re)) {
        // Look for variable declarations as attributes
        aRegexpNode.reVars[re] = aRegexpNode.getAttribute("re"+re);
        continue;
      } else {
        // Look for variable declarations as elements
        var reTags = aRegexpNode.getElementsByTagName("re"+re);
        if (reTags.length) {
          var reTag = reTags.item(0);
          if (reTag.hasAttribute("name")) {
            aRegexpNode.reVars[re] = reTag.getAttribute("name");
            if (reTag.hasAttribute("processing")) {
              aRegexpNode.postProcess.push(reTag);
            }
            continue;
          }
        }
      }
      // There are no more variables to be gotten from this regexp
      break;
    }
  }
  // See if we have already decorated with the values for any of these vars
  var allDecorated = false;
  if (aDecorEl && aDecorEl._flock_decorations) {
    allDecorated = true;
    for (var i = 0; i < aRegexpNode.reVars.length; i++) {
      var val = aDecorEl._flock_decorations[aRegexpNode.reVars[i]];
      if (val) {
        DEBUG("Found a decoration for ["+aRegexpNode.reVars[i]+"] = "+val);
        aResults.setPropertyAsAString(aRegexpNode.reVars[i], val);
      } else {
        allDecorated = false;
      }
    }
  }
  if (allDecorated) {
    DEBUG("Found all the vars as decorations! No need to run regexp...");
    return;
  }
  // Ensure that a value exists -- at least an empty string -- for each var
  for (var i = 0; i < aRegexpNode.reVars.length; i++) {
    try {
      aResults.getPropertyAsAString(aRegexpNode.reVars[i]);
    } catch (ex) {
      aResults.setPropertyAsAString(aRegexpNode.reVars[i], "");
    }
  }
  DEBUG("Getting results using regexp: "+aRegexpNode.rExpr);
  if (aRegexpNode.rExpr) {
    var isMultiLine = (aRegexpNode.getAttribute("multiline") == "true");
    var isMultiValent = (aRegexpNode.getAttribute("multivalent") == "true");
    if (isMultiLine) {
      if (!aData.oneString) {
        aData.oneString = aString.replace(/[\r\n]/g, "");
      }
      this.doRegexpMatch( aData.oneString,
                          aRegexpNode.rExpr,
                          aRegexpNode.reVars,
                          aResults,
                          aDecorEl );
    } else {
      // isMultiLine == false
      if (!aData.lines) {
        aData.lines = aString.split(/[\r\n]/);
      }
      DEBUG("There are "+aData.lines.length+" lines to test with regexp");
      for (var line = 0; line < aData.lines.length; line++) {
        //DEBUG("line "+line+" has "+aData.lines[line].length+" characters");
        if (this.doRegexpMatch( aData.lines[line],
                                aRegexpNode.rExpr,
                                aRegexpNode.reVars,
                                aResults,
                                aDecorEl,
                                isMultiValent ))
        {
          for (var i = 0; i < aRegexpNode.postProcess.length; i++) {
            this.postProcess(aRegexpNode.postProcess[i], aResults, aDecorEl);
          }
          break;
        }
      }
    }
  }
}

flockWebDetective.prototype.setResult =
function flockWebDetective_setResult(aResults, aName, aValue, aDecorEl)
{
  aResults.setPropertyAsAString(aName, aValue);
  if (aDecorEl) {
    if (!aDecorEl._flock_decorations) {
      aDecorEl._flock_decorations = [];
    }
    aDecorEl._flock_decorations[aName] = aValue;
  }
}

flockWebDetective.prototype.postProcess =
function flockWebDetective_postProcess(aRENode, aResults, aDecorEl)
{
  var name = aRENode.getAttribute("name");
  DEBUG(".postProcess('"+name+"')");
  var processes = aRENode.getAttribute("processing").split(",");
  for (var i = 0; i < processes.length; i++) {
    var oldVal = aResults.getPropertyAsAString(name);
    var newVal = oldVal;
    DEBUG(" - process: "+processes[i]);
    switch (processes[i].toLowerCase()) {
      case "unescape":
        newVal = unescape(oldVal);
        break;
      case "toupper":
      case "touppercase":
        newVal = oldVal.toUpperCase();
        break;
      case "tolower":
      case "tolowercase":
        newVal = oldVal.toLowerCase();
        break;
      case "subst":
        // There must be a CDATA block with the substitution regexp
        var substRegexp = null;
        for (var j = 0; (j < aRENode.childNodes.length) && !substRegexp; j++) {
          var child = aRENode.childNodes.item(j);
          if (child.nodeType ==
              Components.interfaces.nsIDOMNode.CDATA_SECTION_NODE)
          {
            DEBUG("  - substitution regexp: "+child.nodeValue);
            substRegexp = this.parseSubstRegexp(child.nodeValue);
          }
        }
        if (substRegexp) {
          DEBUG("   pattern: "+substRegexp.pattern);
          DEBUG("   replacement: "+substRegexp.replace);
          // Convert to JS-compatible syntax
          var replacement = substRegexp.replace;
          for (var r = 1; r < 10; r++) {
            var replacementRE = "/\/"+r+"/g";
            replacement = replacement.replace(eval(replacementRE), "$"+r);
          }
          // Do the substitution
          newVal = oldVal.replace(eval(substRegexp.pattern), replacement);
        }
        break;
      default:
        DEBUG("WARNING: unhandled processing directive: "+processes[i]);
    }
    DEBUG("   - new value: "+newVal);
    this.setResult(aResults, name, newVal, aDecorEl);
  }
}

flockWebDetective.prototype.parseSubstRegexp =
function flockWebDetective_parseSubstRegexp(aRegexpString)
{
  if (!aRegexpString.match(/^s\/((\\\/|[^\/])+)\/((\\\/|[^\/])*)\/[gim]{0,3}$/)) {
    DEBUG("INVALID substitution regexp: "+aRegexpString);
    return null;
  }
  return {
    pattern: "/"+RegExp.$1+"/",
    replace: RegExp.$3
  };
}

const TEST_FOR_ATTRIBS = [
  "class",
  "name",
  "value",
  "action",
  "method"
];

flockWebDetective.prototype.getXPathPrefix =
function flockWebDetective_getXPathPrefix(aHTMLElement)
{
  if (!aHTMLElement) return "";
  DEBUG(".getXPathPrefix('"+aHTMLElement+"')");
  var el = aHTMLElement.QueryInterface(Components.interfaces.nsIDOMElement);
  var doc = el.ownerDocument;
  var xpathExpr = "//"+el.tagName.toLowerCase();
  DEBUG(".getXPathPrefix" + xpathExpr);
  if (el.hasAttribute("id")) {
    xpathExpr += "[@id=\""+el.getAttribute("id")+"\"]";
  } else {
    for (var i = 0; i < TEST_FOR_ATTRIBS.length; i++) {
      var attrib = TEST_FOR_ATTRIBS[i];
      if (el.hasAttribute(attrib)) {
        xpathExpr += "[@"+attrib+"=\""+el.getAttribute(attrib)+"\"]";
      }
    }
    // This may not be enough to uniquely identify the node, so do the parent
    // as well...
    if (el.parentNode && (el.parentNode.tagName.toLowerCase() != "body")) {
      xpathExpr = this.getXPathPrefix(el.parentNode) + xpathExpr.substring(1);
    }
  }
  return xpathExpr;
}

flockWebDetective.prototype.getXPathExpression =
function flockWebDetective_getXPathExpression(aXPathNode)
{
  var xpathExpr = null;
  if (aXPathNode.hasAttribute("match")) {
    xpathExpr = aXPathNode.getAttribute("match");
  } else {
    for (var j = 0; (j < aXPathNode.childNodes.length) && !xpathExpr; j++) {
      if ( aXPathNode.childNodes.item(j).nodeType ==
           Components.interfaces.nsIDOMNode.CDATA_SECTION_NODE )
      {
        xpathExpr = aXPathNode.childNodes.item(j).nodeValue;
      }
    }
  }
  return xpathExpr;
}

flockWebDetective.prototype.getXPathResults =
function flockWebDetective_getXPathResults(aDocumentOrElement, aXPathNode, aResults, aData)
{
  // First check to see if we have already cached this result as a decoration
  // on aDocumentOrElement
  var name = null;
  if (aXPathNode.hasAttribute("name")) {
    name = aXPathNode.getAttribute("name");
    if ( aDocumentOrElement._flock_decorations &&
         aDocumentOrElement._flock_decorations[name] )
    {
      DEBUG("Using decorated value for '"+name+"'");
      aResults.setPropertyAsAString(name, aDocumentOrElement._flock_decorations[name]);
      return;
    }
  }
  // No cached value, so press on
  var doc;
  var needPrefix = false;
  try {
    aDocumentOrElement.QueryInterface(Components.interfaces.nsIDOMDocument);
    doc = aDocumentOrElement;
  } catch (ex) {
    doc = aDocumentOrElement.ownerDocument;
    needPrefix = true;
  }
  if (!aDocumentOrElement.xpathPrefix && needPrefix) {
    aDocumentOrElement.xpathPrefix = this.getXPathPrefix(aDocumentOrElement);
  }
  if (!aDocumentOrElement.xpathPrefix) {
    aDocumentOrElement.xpathPrefix = "";
  }
  if (!aXPathNode.xpathExpr) {
    aXPathNode.xpathExpr = this.getXPathExpression(aXPathNode);
  }
  DEBUG("aDocumentOrElement.xpathPrefix " + aDocumentOrElement.xpathPrefix);
  var xpathExpr = aXPathNode.xpathExpr;
  if (aDocumentOrElement.xpathPrefix) {
    xpathExpr = aDocumentOrElement.xpathPrefix + aXPathNode.xpathExpr;
  }
  if (xpathExpr) {
    var extract = null;
    if (aXPathNode.hasAttribute("extract")) {
      extract = aXPathNode.getAttribute("extract");
    }
    var multivalent = false;
    if (aXPathNode.hasAttribute("multivalent")) {
      multivalent = aXPathNode.getAttribute("multivalent");
    }
    var snippets = this.getXMLSnippetsFromXPath(doc, xpathExpr, extract);
    DEBUG(" XPath got us "+snippets.length+" snippets to check " + multivalent);
    if (snippets.length && name) {
      if (!multivalent) {
        this.setResult(aResults, name, snippets[0], aDocumentOrElement);
      } else {
        this.setResult(aResults, name, snippets, aDocumentOrElement);
      }
    }
    // Now let's see if there's a regexp subnode
    for (var i = 0; i < aXPathNode.childNodes.length; i++) {
      var child = aXPathNode.childNodes.item(i);
      try {
        child.QueryInterface(Components.interfaces.nsIDOMElement);
      } catch (ex) {
        continue;
      }
      if (child.tagName == "regexp") {
        DEBUG(" This XPath statement has a Regexp too!");
        for (var j = 0; j < snippets.length; j++) {
          this.getRegexpResults(aDocumentOrElement, snippets[j], child, aResults, aData);
        }
      }
    }
  }
}

const VALID_EXTRACT_VALUES = {
  "value": true,
  "nodeValue": true,
};

flockWebDetective.prototype.getXMLSnippetsFromXPath =
function flockWebDetective_getXMLSnippetsFromXPath(aDocument, aXPathExpr, aExtract)
{
  DEBUG("Looking for snippets that match this XPath: "+aXPathExpr+" "+aDocument);
  var snippets = [];
  if (aDocument.noDOM) {
    DEBUG("Document has no DOM, can't use XPath!!");
    return;
  }
  aDocument.QueryInterface(Components.interfaces.nsIDOMXPathEvaluator);
  var result;
  if (aDocument instanceof Components.interfaces.nsIDOMHTMLDocument) {
    result = aDocument.evaluate(aXPathExpr, aDocument.body, null, 0, null);
  } else {
    result = aDocument.evaluate( aXPathExpr,
                                 (aDocument.doc ? aDocument.doc : aDocument),
                                 null,
                                 Components.interfaces.nsIDOMXPathResult.ANY_TYPE,
                                 null );
  }
  try {
    result = result.QueryInterface(Components.interfaces.nsIDOMXPathResult);
  } catch (ex) {
    return;
  }
  switch (result.resultType) {
    case Components.interfaces.nsIDOMXPathResult.STRING_TYPE:
    {
      DEBUG("  result.stringValue = "+result.stringValue);
      snippets.push(result.stringValue);
    }; break;
    case Components.interfaces.nsIDOMXPathResult.UNORDERED_NODE_ITERATOR_TYPE:
    {
      var node;
      while (node = result.iterateNext()) {
        try {
          node.QueryInterface(Components.interfaces.nsIDOMNode);
        } catch (ex) {
          continue;
        }
        var value;
        if ( node.hasChildNodes() &&
             (node.childNodes.length > 1
              || (node.childNodes.length == 1
                && !node.firstChild instanceof Components.interfaces.nsIDOMText)) )
        {
          var _xs = Components.classes["@mozilla.org/xmlextras/xmlserializer;1"]
                              .createInstance(Components.interfaces.nsIDOMSerializer);
          value = _xs.serializeToString(node);
        } else if ( node.childNodes.length == 1 &&
                    node.firstChild instanceof Components.interfaces.nsIDOMText )
        {
          value = node.firstChild.textContent;
        } else {
          value = node.nodeValue;
        }
        if (aExtract) {
          if (VALID_EXTRACT_VALUES[aExtract]) {
            value = node[aExtract];
          } else if ( aExtract.indexOf("attribute:") == 0 &&
                      node instanceof Components.interfaces.nsIDOMElement )
          {
            var attrName = aExtract.substring(10);
            value = node.getAttribute(attrName);
          }
        }
        DEBUG(" - XPath matching node: "+ value);
        snippets.push(value);
      }
    }; break;
    default:
      DEBUG("WARNING!!! Unhandled XPath result type: "+result.resultType);
  }
  return snippets;
}

flockWebDetective.prototype.getFieldResults =
function flockWebDetective_getFieldResults(aForm, aFieldNode, aResults)
{
  var varname = null;
  if (aFieldNode.hasAttribute("extractas")) {
    varname = aFieldNode.getAttribute("extractas");
  } else if (aFieldNode.hasAttribute("name")) {
    varname = aFieldNode.getAttribute("name");
  } else if (aFieldNode.hasAttribute("fieldid")) {
    varname = aFieldNode.getAttribute("fieldid");
  }
  if (varname) {
    var field = this.getMatchingFormField(aFieldNode, aForm);
    if (field instanceof Components.interfaces.nsIDOMHTMLInputElement) {
      field.QueryInterface(Components.interfaces.nsIDOMHTMLInputElement);
      DEBUG(".getFieldResults() - extracting form field value ["+varname+"] = "+field.value);
      aResults.setPropertyAsAString(varname, field.value);
    }
  }
}

flockWebDetective.prototype.getRegexpFromNode =
function flockWebDetective_getRegexpFromNode(aRegexpNode)
{
  var rExpr = null;
  if (aRegexpNode.hasAttribute("expression")) {
    rExpr = aRegexpNode.getAttribute("expression");
  } else {
    for (var j = 0; j < aRegexpNode.childNodes.length; j++) {
      if ( aRegexpNode.childNodes.item(j).nodeType ==
           Components.interfaces.nsIDOMNode.CDATA_SECTION_NODE )
      {
        rExpr = aRegexpNode.childNodes.item(j).nodeValue;
      }
    }
  }
  return rExpr;
}

/**
 * This does not do 'true' regexp validation, but rather just enough to ensure
 * that executing this string will not allow arbitrary code execution.
 */
flockWebDetective.prototype.isValidMatchRegexp =
function flockWebDetective_isValidMatchRegexp(aRegexpString)
{
  DEBUG("Testing for validity: "+aRegexpString);
  return aRegexpString.match(/^\/(\\\/|[^\/])+\/[gim]{0,3}$/);
}

flockWebDetective.prototype.doRegexpMatch =
function flockWebDetective_doRegexpMatch( aString, aRegexp, aResultFields,
                                          aResults, aDecorEl, aIsMultiValent )
{
  //DEBUG(aString);
  if (aString.match(eval(aRegexp))) {
    DEBUG(".doRegexpMatch() MATCH!");
    if (aResultFields && aResults) {
      for (var i = 1; i < aResultFields.length; i++) {
        var fieldName = aResultFields[i];
        if (aIsMultiValent) {
          for (var j = 1; ; j++) {
            try {
              aResults.getPropertyAsAString(fieldName+j);
            } catch (ex) {
              fieldName += j;
              break;
            }
          }
        }
        DEBUG(".doRegexpMatch() found [ "+fieldName+" ] = "+eval("RegExp.$"+i));
        this.setResult(aResults, fieldName, eval("RegExp.$"+i), aDecorEl);
      }
    }
    return true;
  }
  //DEBUG(".doRegexpMatch() NO-match.");
  return false;
}

flockWebDetective.prototype.loadStrings =
function flockWebDetective_loadStrings(aServiceName)
{
  this.mStrings[aServiceName] = [];
  var stringsEls = this.mRules[aServiceName].getElementsByTagName("strings");
  for (var i = 0; i < stringsEls.length; i++) {
    var stringsEl = stringsEls.item(i);
    stringsEl.QueryInterface(Components.interfaces.nsIDOMElement);
    var stringEls = stringsEl.getElementsByTagName("string");
    for (var j = 0; j < stringEls.length; j++) {
      var stringEl = stringEls.item(j);
      stringEl.QueryInterface(Components.interfaces.nsIDOMElement);
      if (stringEl.hasAttribute("name")) {
        var stringName = stringEl.getAttribute("name");
        var foundValue = null;
        if (stringEl.hasAttribute("value")) {
          foundValue = stringEl.getAttribute("value");
        }
        var longestTextValue = "";
        for (var k = 0; (k < stringEl.childNodes.length) && !foundValue; k++) {
          var child = stringEl.childNodes.item(k);
          switch (child.nodeType) {
            case Components.interfaces.nsIDOMNode.TEXT_NODE:
              if (child.nodeValue.length > longestTextValue.length) {
                longestTextValue = child.nodeValue;
              }
              break;
            case Components.interfaces.nsIDOMNode.CDATA_SECTION_NODE:
              foundValue = stringEl.childNodes.item(k).nodeValue;
              break;
          }
        }
        if (!foundValue) {
          foundValue = longestTextValue;
        }
        this.mStrings[aServiceName][stringName] = foundValue;
        DEBUG( ".loadStrings('"+aServiceName+"'): found string '"+stringName
               +"': "+this.mStrings[aServiceName][stringName] );
      }
    }
  }
}

flockWebDetective.prototype.loadSessionCookies =
function flockWebDetective_loadSessionCookies(aServiceName)
{
  this.mSessionCookies[aServiceName] = [];
  var cookiesEls = this.mRules[aServiceName].getElementsByTagName("sessioncookies");
  for (var i = 0; i < cookiesEls.length; i++) {
    var cookiesEl = cookiesEls.item(i);
    cookiesEl.QueryInterface(Components.interfaces.nsIDOMElement);
    var cookieEls = cookiesEl.getElementsByTagName("cookie");
    for (var j = 0; j < cookieEls.length; j++) {
      var cookieEl = cookieEls.item(j);
      cookieEl.QueryInterface(Components.interfaces.nsIDOMElement);
      var c = {
        host: cookieEl.getAttribute("host"),
        name: cookieEl.getAttribute("name"),
        path: cookieEl.getAttribute("path")
      };
      DEBUG( ".loadSessionCookies('"+aServiceName+"'): host["+c.host+"] name["
             +c.name+"] path["+c.path+"]" );
      this.mSessionCookies[aServiceName].push(c);
    }
  }
}
// END helper functions

// BEGIN update service functions

// This is patterned after nsSearchService.js:engineMetadataService
function makeURI(url) {
  var ios = Cc['@mozilla.org/network/io-service;1']
    .getService(Ci.nsIIOService);
  try {
    return ios.newURI(url, null, null);
  } catch (ex) { }

  return null;
}

function createStatement(dbconn, sql) {
  var stmt = dbconn.createStatement(sql);
  var wrapper = Cc["@mozilla.org/storage/statement-wrapper;1"]
    .createInstance(Ci.mozIStorageStatementWrapper);

  wrapper.initialize(stmt);
  return wrapper;
}

function UpdateMetadataStore() {
  this.init();
}

UpdateMetadataStore.prototype = {
  init: function UMS_init() {
    var dbfile = Cc['@mozilla.org/file/directory_service;1']
      .getService(Ci.nsIProperties).get('ProfD', Ci.nsIFile);
    dbfile.append('webdetective.sqlite');

    var storageService = Cc['@mozilla.org/storage/service;1']
      .getService(Ci.mozIStorageService);
    this.mDBConn = storageService.openDatabase(dbfile);

    var schema = 'id INTEGER PRIMARY KEY, servicename STRING, ' +
                 'name STRING, value STRING';

    try {
      this.mDBConn.createTable('webdetect_data', schema);
    }
    catch (e) { }

    this.mGetData = createStatement(this.mDBConn,
      'SELECT value FROM webdetect_data WHERE servicename = :servicename ' +
      'AND name = :name');
    this.mDeleteData = createStatement(this.mDBConn,
      'DELETE FROM webdetect_data WHERE servicename = :servicename ' +
      'AND name = :name');
    this.mInsertData = createStatement(this.mDBConn,
      'INSERT INTO webdetect_data (servicename, name, value) ' +
      'VALUES (:servicename, :name, :value)');
  },
  getAttr: function UMS_getAttr(serviceName, name) {
    name = name.toLowerCase();

    var stmt = this.mGetData;
    stmt.reset();
    var pp = stmt.params;
    pp.servicename = serviceName;
    pp.name = name;

    var value = null;
    if (stmt.step())
      value = stmt.row.value;
    stmt.reset();
    return value;
  },
  setAttr: function UMS_setAttr(serviceName, name, value) {
    name = name.toLowerCase();

    this.mDBConn.beginTransaction();

    this.deleteServiceData(serviceName, name);

    pp = this.mInsertData.params;
    pp.servicename = serviceName;
    pp.name = name;
    pp.value = value;
    this.mInsertData.step();
    this.mInsertData.reset();

    this.mDBConn.commitTransaction();
  },
  deleteServiceData: function UMS_deleteServiceData(serviceName, name) {
    name = name.toLowerCase();

    var pp = this.mDeleteData.params;
    pp.servicename = serviceName;
    pp.name = name;
    this.mDeleteData.step();
    this.mDeleteData.reset();
  },
}

flockWebDetective.prototype.startUpdateService =
function flockWebDetective_startUpdateService()
{
  this.updateMetadataStore = new UpdateMetadataStore();

  var tm = Cc['@mozilla.org/updates/timer-manager;1']
    .getService(Ci.nsIUpdateTimerManager);

  var prefs = Cc['@mozilla.org/preferences-service;1']
    .getService(Ci.nsIPrefBranch)
  var interval = prefs.getIntPref('flock.service.webdetective.updateinterval');

  var seconds = interval * 3600;
  tm.registerTimer('web-detective-update-timer', this, seconds);
}

/**
 * Stevo : This may be a bit clunky right now, we have 2 variables (updatesOk and fileCount) that
 *  are used to maintain if all updates are successful, and when to report back the results to aListener
 *  if aListener is valid. So for each service we send off a request and have a listener that onSuccess
 *  decrements the fileCount and when fileCount is <= 0 calls the onSuccess or onError depending on the
 *  state of updatesOk, onError will decrement fileCount and set updatesOk to false and when fileCount is <= 0
 *  it will report back onError to the aListener.
 */
flockWebDetective.prototype.checkForUpdates =
function flockWebDetective_checkForUpdates(/*boolean */forceUpdates, /*flockIListener */aListener)
{
  // Updates will only be sought for services that have been instantiated and
  // have loaded their rules files into Web Detective.  So when we check for
  // updates, we need to make sure to explicitly instantiate all web service
  // components.
  var wsm = Cc["@flock.com/webservice-manager;1"]
            .getService(Ci.flockIWebServiceManager);
  wsm.instantiateAll();

  var now = Date.now();
  var updatesOk = true;  // If any updates fail then this will be false.
  var fileCount = 0;

  var serviceUpdateListener = {
    onSuccess : function(aSubject, aTopic) {
      fileCount = fileCount - 1;
      if (fileCount <= 0) {
        if (aListener) {
          if (updatesOk) {
            aListener.onSuccess(null, null);
          } else {
            aListener.onError(null, null, null);
          }
        }
      }
    },
    onError : function(aSubject, aTopic, aError) {
      fileCount = fileCount - 1;
      updatesOk = false;
      if (fileCount <= 0) {
        if (aListener) {
          aListener.onError(null, null, null);
        }
      }
    }
  };

  for (var serviceName in this.mDetectFiles) {
    if (forceUpdates) {
      this.updateMetadataStore.deleteServiceData(serviceName,
                                                 "updatelastmodified");
    } else {
      var expire = this.updateMetadataStore.getAttr(serviceName,
                                                    "updateexpire");
      if (expire && expire > now)
        continue;
    }

    fileCount = fileCount + 1;
    if (aListener) {
      aListener.onStart(null, serviceName);
      this.sendUpdateRequest(serviceName, serviceUpdateListener);
    } else {
      this.sendUpdateRequest(serviceName);
    }
  }
}

flockWebDetective.prototype.getUpdateInfo =
function flockWebDetective_getUpdateInfo(aServiceName)
{
  var updateEls = this.mRules[aServiceName].getElementsByTagName('update');
  for (var i = 0; i < updateEls.length; i++) {
    var updateEl = updateEls.item(i);
    updateEl.QueryInterface(Ci.nsIDOMElement);
    var url = updateEl.getAttribute('url');
    var testURI = makeURI(url);
    if (testURI) {
      var interval = updateEl.getAttribute('interval');
      var info = {
        url: testURI.spec,
        interval: interval ? interval : DEFAULT_UPDATE_INTERVAL
      };
      return info;
    }
  }

  var url = DEFAULT_UPDATE_SERVER + this.mDetectFiles[aServiceName].leafName;
  var info = {
    url: makeURI(url).spec,
    interval: DEFAULT_UPDATE_INTERVAL
  };
  return info;
}

flockWebDetective.prototype.sendUpdateRequest =
function flockWebDetective_sendUpdateRequest(aServiceName, aListener)
{
  var updateInfo = this.getUpdateInfo(aServiceName);

  var hr = Cc['@mozilla.org/xmlextras/xmlhttprequest;1']
    .createInstance(Ci.nsIXMLHttpRequest);
  hr.backgroundRequest = true;
  hr.open('GET', updateInfo.url);

  var lastModified = this.updateMetadataStore.getAttr(aServiceName,
                                                      'updatelastmodified');
  hr.setRequestHeader('If-Modified-Since', lastModified);

  var self = this;
  var updateInterval = updateInfo.interval;

  hr.onload = function(event) {
    var req = event.target;
    if (req.responseXML) {
      self.updateDetectFile(aServiceName, req.responseText, updateInterval);
      if (aListener) {
        aListener.onSuccess(null,aServiceName);
      }
    } else {
      if (aListener) {
        aListener.onError(null,aServiceName,null);
      }
    }
  }

  hr.send(null);
}

flockWebDetective.prototype.updateDetectFile =
function flockWebDetective_updateDetectFile(serviceName, contents, updateInterval)
{
  if (!contents) {
    this.updateMetadataStore.setAttr(serviceName, 'updateexpire',
                                     Date.now() + updateInterval * 86400000);
    return;
  }

  try {
    var detectFile = this.mDetectFiles[serviceName];

    var ostream = Cc['@mozilla.org/network/safe-file-output-stream;1']
      .createInstance(Ci.nsIFileOutputStream);
    ostream.init(detectFile, PR_WRONLY | PR_CREATE_FILE | PR_TRUNCATE, 0644, 0);

    var converter = Cc['@mozilla.org/intl/scriptableunicodeconverter']
      .createInstance(Ci.nsIScriptableUnicodeConverter);
    converter.charset = 'UTF-8';

    var convdata = converter.ConvertFromUnicode(contents) + converter.Finish();

    ostream.write(convdata, convdata.length);

    if (ostream instanceof Ci.nsISafeOutputStream) {
      ostream.finish();
    } else {
      ostream.close();
    }

    this.loadDetectFile(detectFile);

    var updateInfo = this.getUpdateInfo(serviceName);
    this.updateMetadataStore.setAttr(serviceName, 'updateexpire',
                                     Date.now() + updateInfo.interval * 86400000);

    this.updateMetadataStore.setAttr(serviceName, 'updatelastmodified',
                                     (new Date()).toUTCString());
  }
  catch (e) { }
}
// END update service functions

// ========== END flockWebDetective class ==========



// =========================================
// ========== BEGIN XPCOM Support ==========
// =========================================

var Module = {
  _firstTime: true,
  registerSelf: function(aCompMgr, aFileSpec, aLocation, aType)
  {
    if (this._firstTime) {
      this._firstTime = false;
      throw Components.results.NS_ERROR_FACTORY_REGISTER_AGAIN;
    }
    aCompMgr = aCompMgr.QueryInterface(Components.interfaces.nsIComponentRegistrar);
    aCompMgr.registerFactoryLocation(CLASS_ID, CLASS_NAME, CONTRACT_ID, aFileSpec, aLocation, aType);

    var categoryManager = Components.classes["@mozilla.org/categorymanager;1"]
                                    .getService(Components.interfaces.nsICategoryManager);
    categoryManager.addCategoryEntry("flock-startup", CLASS_NAME, "service," + CONTRACT_ID, true, true);
    categoryManager.addCategoryEntry("flockMigratable", CLASS_NAME, CONTRACT_ID, true, true);
  },

  unregisterSelf: function(aCompMgr, aLocation, aType)
  {
    aCompMgr = aCompMgr.QueryInterface(Components.interfaces.nsIComponentRegistrar);
    aCompMgr.unregisterFactoryLocation(CLASS_ID, aLocation);
  },

  getClassObject: function(aCompMgr, aCID, aIID)
  {
    if (!aIID.equals(Components.interfaces.nsIFactory)) {
      throw Components.results.NS_ERROR_NOT_IMPLEMENTED;
    }
    if (aCID.equals(CLASS_ID)) {
      return Factory;
    }
    throw Components.results.NS_ERROR_NO_INTERFACE;
  },

  canUnload: function(aCompMgr) { return true; }
};

var Factory = {
  createInstance: function(aOuter, aIID)
  {
    if (aOuter != null) {
      throw Components.results.NS_ERROR_NO_AGGREGATION;
    }
    return (new flockWebDetective()).QueryInterface(aIID);
  }
};

function NSGetModule(aCompMgr, aFileSpec)
{
  return Module;
}

// ========== END XPCOM Support ==========
