#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""simple parser / string tokenizer
rather than returning a list of token types etc, we simple return a list of tokens...
each tokenizing function takes a string as input and returns a list of tokens
"""

# Copyright 2002, 2003 St James Software
# 
# This file is part of translate.
#
# translate is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
# 
# translate is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with translate; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

def stringeval(text):
  """takes away repeated quotes (escapes) and returns the string represented by the text"""
  stringchar = text[0]
  if text[-1] != stringchar or stringchar not in ("'",'"'):
    # scratch your head
    raise ValueError, "error parsing escaped string: %r" % text
  return text[1:-1].replace(stringchar+stringchar,stringchar)

def stringquote(text):
  """escapes quotes as neccessary and returns a string representing the text"""
  if "'" in text:
    if '"' in text:
      return '"' + text.replace('"', '""') + '"'
    else:
      return '"' + text + '"'
  else:
    return "'" + text + "'"

class ParserError(ValueError):
  """Intelligent parser error"""
  def __init__(self, parser, message, tokennum):
    """takes a message and the number of the token that caused the error"""
    tokenpos = parser.findtokenpos(tokennum)
    line, charpos = parser.getlinepos(tokenpos)
    ValueError.__init__(self, "%s at line %d, char %d (token %r)" % \
        (message, line, charpos, parser.tokens[tokennum]))
    self.parser = parser
    self.tokennum = tokennum

class SimpleParser:
  """this is a simple parser"""
  def __init__(self, defaulttokenlist=None, whitespacechars=" \t\r\n", includewhitespacetokens=0):
    if defaulttokenlist is None:
      self.defaulttokenlist = ['<=', '>=', '==', '!=', '+=', '-=', '*=', '/=', '<>']
      self.defaulttokenlist.extend('(),[]:=+-')
    else:
      self.defaulttokenlist = defaulttokenlist
    self.whitespacechars = whitespacechars
    self.includewhitespacetokens = includewhitespacetokens
    self.standardtokenizers = [self.stringtokenize, self.removewhitespace, self.separatetokens]
    self.quotechars = ('"', "'")
    self.endquotechars = {'"':'"',"'":"'"}
    self.stringescaping = 1

  def stringtokenize(self, text):
    """makes strings in text into tokens..."""
    tokens = []
    laststart = 0
    instring = 0
    endstringchar, escapechar = '', '\\'
    gotclose, gotescape = 0, 0
    for pos in range(len(text)):
      char = text[pos]
      if instring:
        if self.stringescaping and (gotescape or char == escapechar) and not gotclose:
          gotescape = not gotescape
        elif char == endstringchar:
          gotclose = not gotclose
        elif gotclose:
          tokens.append(text[laststart:pos])
          instring, laststart, endstringchar = 0, pos, ''
      if not instring:
        if char in self.quotechars:
          if pos > laststart: tokens.append(text[laststart:pos])
          instring, laststart, endstringchar, gotclose = 1, pos, self.endquotechars[char], 0
    if laststart < len(text): tokens.append(text[laststart:])
    return tokens

  def keeptogether(self, text):
    """checks whether a token should be kept together"""
    return self.isstringtoken(text)

  def isstringtoken(self, text):
    """checks whether a token is a string token"""
    return text[:1] in self.quotechars

  def separatetokens(self, text, tokenlist = None):
    """this separates out tokens in tokenlist from whitespace etc"""
    if self.keeptogether(text): return [text]
    if tokenlist is None:
      tokenlist = self.defaulttokenlist
    # loop through and put tokens into a list
    tokens = []
    pos = 0
    laststart = 0
    lentext = len(text)
    while pos < lentext:
      foundtoken = 0
      for token in tokenlist:
        lentoken = len(token)
        if text[pos:pos+lentoken] == token:
          if laststart < pos: tokens.append(text[laststart:pos])
          tokens.append(token)
          pos += lentoken
          foundtoken, laststart = 1, pos
          break
      if not foundtoken: pos += 1
    if laststart < lentext: tokens.append(text[laststart:])
    return tokens

  def removewhitespace(self, text):
    """this removes whitespace but lets it separate things out into separate tokens"""
    if self.keeptogether(text): return [text]
    # loop through and put tokens into a list
    tokens = []
    pos = 0
    inwhitespace = 0
    laststart = 0
    for pos in range(len(text)):
      char = text[pos]
      if inwhitespace:
        if char not in self.whitespacechars:
          if laststart < pos and self.includewhitespacetokens: tokens.append(text[laststart:pos])
          inwhitespace, laststart = 0, pos
      else:
        if char in self.whitespacechars:
          if laststart < pos: tokens.append(text[laststart:pos])
          inwhitespace, laststart = 1, pos
    if laststart < len(text) and (not inwhitespace or self.includewhitespacetokens):
      tokens.append(text[laststart:])
    return tokens

  def applytokenizer(self, inputlist, tokenizer):
    """apply a tokenizer to a set of text, flattening the result"""
    tokenizedlists = [tokenizer(text) for text in inputlist]
    joined = []
    map(joined.extend, tokenizedlists)
    return joined

  def applytokenizers(self, inputlist, tokenizers):
    """apply a set of tokenizers to a set of text, flattening each time"""
    for tokenizer in tokenizers:
      inputlist = self.applytokenizer(inputlist, tokenizer)
    return inputlist

  def tokenize(self, source, tokenizers=None):
    """tokenize the text string with the standard tokenizers"""
    self.source = source
    if tokenizers is None:
      tokenizers = self.standardtokenizers
    self.tokens = self.applytokenizers([self.source], tokenizers)
    return self.tokens

  def findtokenpos(self, tokennum):
    """finds the position of the given token in the text"""
    currenttokenpos = 0
    for currenttokennum in range(tokennum+1):
      currenttokenpos = self.source.find(self.tokens[currenttokennum], currenttokenpos)
    return currenttokenpos

  def getlinepos(self, tokenpos):
    """finds the line and character position of the given character"""
    sourcecut = self.source[:tokenpos]
    line = sourcecut.count("\n")+1
    charpos = tokenpos - sourcecut.rfind("\n")
    return line, charpos

  def raiseerror(self, message, tokennum):
    """raises a ParserError"""
    raise ParserError(self, message, tokennum)


