/** -- $Id: HyphenationTree.java,v 1.1 2000/12/12 19:18:49 fotis Exp $ --

 ============================================================================
                   The Apache Software License, Version 1.1
 ============================================================================

    Copyright (C) 1999 The Apache Software Foundation. All rights reserved.

 Redistribution and use in source and binary forms, with or without modifica-
 tion, are permitted provided that the following conditions are met:

 1. Redistributions of  source code must  retain the above copyright  notice,
    this list of conditions and the following disclaimer.

 2. Redistributions in binary form must reproduce the above copyright notice,
    this list of conditions and the following disclaimer in the documentation
    and/or other materials provided with the distribution.

 3. The end-user documentation included with the redistribution, if any, must
    include  the following  acknowledgment:  "This product includes  software
    developed  by the  Apache Software Foundation  (http://www.apache.org/)."
    Alternately, this  acknowledgment may  appear in the software itself,  if
    and wherever such third-party acknowledgments normally appear.

 4. The names "Fop" and  "Apache Software Foundation"  must not be used to
    endorse  or promote  products derived  from this  software without  prior
    written permission. For written permission, please contact
    apache@apache.org.

 5. Products  derived from this software may not  be called "Apache", nor may
    "Apache" appear  in their name,  without prior written permission  of the
    Apache Software Foundation.

 THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES,
 INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND
 FITNESS  FOR A PARTICULAR  PURPOSE ARE  DISCLAIMED.  IN NO  EVENT SHALL  THE
 APACHE SOFTWARE  FOUNDATION  OR ITS CONTRIBUTORS  BE LIABLE FOR  ANY DIRECT,
 INDIRECT, INCIDENTAL, SPECIAL,  EXEMPLARY, OR CONSEQUENTIAL  DAMAGES (INCLU-
 DING, BUT NOT LIMITED TO, PROCUREMENT  OF SUBSTITUTE GOODS OR SERVICES; LOSS
 OF USE, DATA, OR  PROFITS; OR BUSINESS  INTERRUPTION)  HOWEVER CAUSED AND ON
 ANY  THEORY OF LIABILITY,  WHETHER  IN CONTRACT,  STRICT LIABILITY,  OR TORT
 (INCLUDING  NEGLIGENCE OR  OTHERWISE) ARISING IN  ANY WAY OUT OF THE  USE OF
 THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 This software  consists of voluntary contributions made  by many individuals
 on  behalf of the Apache Software  Foundation and was  originally created by
 James Tauber <jtauber@jtauber.com>. For more  information on the Apache
 Software Foundation, please see <http://www.apache.org/>.

 */

package org.apache.fop.layout.hyphenation;

import java.io.*;
import java.util.Vector;
import java.util.Hashtable;

/**
 * This tree structure stores the hyphenation patterns in an efficient
 * way for fast lookup. It provides the provides the method to
 * hyphenate a word.
 *
 * @author Carlos Villegas <cav@uniscope.co.jp>
 */
public class HyphenationTree extends TernaryTree
   implements PatternConsumer, Serializable
{

      /** value space: stores the inteletter values */
      protected ByteVector vspace;
      /** This map stores hyphenation exceptions */
      protected Hashtable stoplist;
      /** This map stores the character classes */
      protected TernaryTree classmap;
      /** Temporary map to store interletter values on pattern loading. */
      private transient TernaryTree ivalues;

      public HyphenationTree()
      {
         stoplist = new Hashtable(23); // usually a small table
         classmap = new TernaryTree();
         vspace = new ByteVector();
         vspace.alloc(1); // this reserves index 0, which we don't use
      }
      
      /**
       * Packs the values by storing them in 4 bits, two values into a byte
       * Values range is from 0 to 9. We use zero as terminator,
       * so we'll add 1 to the value.
       * @param values a string of digits from '0' to '9' representing the
       *        interletter values.
       * @return the index into the vspace array where the packed values
       *        are stored.
       */
      protected int packValues(String values)
      {
         int i, n = values.length();
         int m = (n & 1) == 1 ? (n>>1)+2 : (n>>1)+1;
         int offset = vspace.alloc(m);
         byte[] va = vspace.getArray();
         for(i=0; i<n; i++) {
            int j = i>>1;
            byte v = (byte)((values.charAt(i) - '0' + 1) & 0x0f);
            if ( (i&1) == 1 )
               va[j+offset] = (byte)( va[j+offset] | v );
            else
               va[j+offset] = (byte)(v << 4);  // big endian
         }
         va[m-1+offset] = 0; // terminator
         return offset;
      }

      protected String unpackValues(int k)
      {
         StringBuffer buf = new StringBuffer();
         byte v = vspace.get(k++);
         while( v != 0 ) {
            char c = (char)((v >>> 4) -1 + '0');
            buf.append(c);
            c = (char)(v & 0x0f);
            if ( c == 0 ) break;
            c = (char)(c - 1 + '0');
            buf.append(c);
            v = vspace.get(k++);
         }
         return buf.toString();
      }

      /**
       * Read hyphenation patterns from an XML file.
       */
      public void loadPatterns(String filename)
         throws HyphenationException
      {
         PatternParser pp = new PatternParser(this);
         ivalues = new TernaryTree();

         pp.parse(filename);

         // patterns/values should be now in the tree
         // let's optimize a bit
         trimToSize();
         vspace.trimToSize();
         classmap.trimToSize();
         
         // get rid of the auxiliary map
         ivalues = null;
      }

      public String findPattern(String pat)
      {
         int k = super.find(pat);
         if ( k >= 0 )
            return unpackValues(k);
         return "";
      }

      /**
       * String compare, returns 0 if equal or
       * t is a substring of s
       */
      protected int hstrcmp(char[] s, int si, char[] t, int ti)
      {
         for ( ; s[si] == t[ti]; si++, ti++)
            if (s[si] == 0)
               return 0;
         if ( t[ti] == 0 )
            return 0;
         return s[si] - t[ti];
      }

      protected byte[] getValues(int k)
      {
         StringBuffer buf = new StringBuffer();
         byte v = vspace.get(k++);
         while( v != 0 ) {
            char c = (char)((v >>> 4) - 1);
            buf.append(c);
            c = (char)(v & 0x0f);
            if ( c == 0 ) break;
            c = (char)(c - 1);
            buf.append(c);
            v = vspace.get(k++);
         }
         byte[] res = new byte[buf.length()];
         for(int i=0;i<res.length;i++)
            res[i] = (byte)buf.charAt(i);
         return res;
      }
      
      /**
       * <p>Search for all possible partial matches of word starting
       * at index an update interletter values. In other words, it
       * does something like:</p>
       * <code>
       *  for(i=0; i<patterns.length; i++) {
       *        if ( word.substring(index).startsWidth(patterns[i]) )
       *          update_interletter_values(patterns[i]);
       *  }
       * </code>
       * <p>But it is done in an efficient way since the patterns are
       * stored in a ternary tree. In fact, this is the whole purpose
       * of having the tree: doing this search without having to test
       * every single pattern. The number of patterns for languages
       * such as English range from 4000 to 10000. Thus, doing thousands
       * of string comparisons for each word to hyphenate would be
       * really slow without the tree. The tradeoff is memory, but
       * using a ternary tree instead of a trie, almost halves the
       * the memory used by Lout or TeX. It's also faster than using
       * a hash table</p>
       * @param word null terminated word to match
       * @param index start index from word
       * @param il interletter values array to update
       */
      protected void searchPatterns(char[] word, int index, byte[] il)
      {
         byte[] values;
         int i=index;
         char p, q;
         char sp = word[i];
         p = root;

         while( p != 0 ) {
            if (sc[p] == 0xFFFF) {
               if ( hstrcmp(word, i, kv.getArray(), lo[p]) == 0 ) {
                  values = getValues(eq[p]);  // data pointer is in eq[]
                  int j=index;
                  for(int k=0; k<values.length; k++) {
                     if ( values[k] > il[j] )
                        il[j] = values[k];
                     j++;
                  }
               }
               return;
            }
            int d = sp - sc[p];
            if ( d == 0 ) {
               if ( sp == 0 )
                  break;
               sp = word[++i];
               p = eq[p];
               q = p;

               // look for a pattern ending at this position by searching for
               // the null char ( splitchar == 0 )
               while ( q != 0 ) {
                  if ( sc[q] == 0xFFFF ) // stop at compressed branch
                     break;
                  if ( sc[q] == 0 ) {
                     values = getValues(eq[q]);
                     int j=index;
                     for(int k=0; k<values.length; k++) {
                        if ( values[k] > il[j] )
                           il[j] = values[k];
                        j++;
                     }
                     break;
                  } else {
                     q = lo[q];
                     /** actually the code should be:
                         q = sc[q] < 0 ? hi[q] : lo[q];
                         but java chars are unsigned
                     */
                  }
               }
            } else
               p = d < 0 ? lo[p] : hi[p];
         }
      }

      /**
       * Hyphenate word and return a Hyphenation object.
       * @param word the word to be hyphenated
       * @param remainCharCount Minimum number of characters allowed
       *  before the hyphenation point.
       * @param pushCharCount Minimum number of characters allowed after
       *  the hyphenation point.
       * @return a {@link Hyphenation Hyphenation} object representing
       * the hyphenated word or null if word is not hyphenated.
       */
      public Hyphenation hyphenate(String word, int remainCharCount, int pushCharCount)
      {
         char[] w = word.toCharArray();
         return hyphenate(w, 0, w.length, remainCharCount, pushCharCount);
      }
      
      /**
       * Hyphenate word and return an array of hyphenation points.
       * @param w char array that contains the word
       * @param offset Offset to first character in word
       * @param len Length of word
       * @param remainCharCount Minimum number of characters allowed
       *  before the hyphenation point.
       * @param pushCharCount Minimum number of characters allowed after
       *  the hyphenation point.
       * @return a {@link Hyphenation Hyphenation} object representing
       * the hyphenated word or null if word is not hyphenated.
       */
      public Hyphenation hyphenate(char[] w, int offset, int len,
                              int remainCharCount, int pushCharCount)
      {
         int i;
         char[] word = new char[len+3];

         // normalize word
         char[] c = new char[2];
         for(i=1; i<=len; i++) {
            c[0] = w[offset+i-1];
            int nc = classmap.find(c,0);
            if ( nc < 0 ) { // found a non-letter character, abort
               return null;
            }
            word[i] = (char)nc;
         }
         int[] result = new int[len+1];
         int k=0;

         // check exception list first
         String sw = new String(word,1,len);
         if ( stoplist.containsKey(sw) ) {
            // assume only simple hyphens (Hyphen.pre="-", Hyphen.post = Hyphen.no = null)
            Vector hw = (Vector)stoplist.get(sw);
            int j = 0;
            for(i=0; i<hw.size(); i++) {
               Object o = hw.elementAt(i);
               if ( o instanceof String ) {
                  j += ((String)o).length();
                  if ( j >= remainCharCount && j < (len - pushCharCount) )
                     result[k++] = j;
               }
            }
         } else {
            
            // use algorithm to get hyphenation points
            word[0] = '.';  // word start marker            
            word[len+1] = '.'; // word end marker
            word[len+2] = 0;  // null terminated
            byte[] il = new byte[len+3]; // initialized to zero
            for(i=0; i<len+1; i++)
               searchPatterns(word, i, il);

            // hyphenation points are located where interletter value is odd
            for(i=0; i<len; i++) {
               if ( ((il[i+1] & 1) == 1) && i >= remainCharCount
                    && i < (len-pushCharCount) ) {
                  result[k++] = i;
               }
            }
         }

         if ( k > 0 ) {
            // trim result array
            int[] res = new int[k];
            System.arraycopy(result, 0, res, 0, k);
            return new Hyphenation(new String(w,offset,len), res);
         } else {
            return null;
         }
      }      

      /**
       * Add a character class to the tree. It is used by 
       * {@link PatternParser PatternParser} as callback to
       * add character classes. Character classes define the
       * valid word characters for hyphenation. If a word contains
       * a character not defined in any of the classes, it is not hyphenated.
       * It also defines a way to normalize the characters in order
       * to compare them with the stored patterns. Usually pattern
       * files use only lower case characters, in this case a class
       * for letter 'a', for example, should be defined as "aA", the first
       * character being the normalization char.
       */
      public void addClass(String chargroup)
      {
         if ( chargroup.length() > 0 ) {
            char equivChar = chargroup.charAt(0);
            char[] key = new char[2];
            key[1] = 0;
            for(int i=0; i<chargroup.length(); i++ ) {
               key[0] = chargroup.charAt(i);
               classmap.insert(key, 0, equivChar);
            }
         }     
      }

      /**
       * Add an exception to the tree. It is used by
       * {@link PatternParser PatternParser} class as callback to
       * store the hyphenation exceptions.
       * @param word normalized word
       * @param hyphenatedword a vector of alternating strings and
       * {@link Hyphen hyphen} objects.
       */
      public void addException(String word, Vector hyphenatedword)
      {
         stoplist.put(word, hyphenatedword);
      }

      /**
       * Add a pattern to the tree. Mainly, to be used by
       * {@link PatternParser PatternParser} class as callback to
       * add a pattern to the tree.
       * @param pattern the hyphenation pattern
       * @param ivalue interletter weight values indicating the
       * desirability and priority of hyphenating at a given point
       * within the pattern. It should contain only digit characters.
       * (i.e. '0' to '9').
       */
      public void addPattern(String pattern, String ivalue)
      {
         int k = ivalues.find(ivalue);
         if ( k <= 0 ) {
            k = packValues(ivalue);
            ivalues.insert(ivalue, (char)k);
         }
         insert(pattern, (char)k);
      }

      public void printStats()
      {
         System.out.println("Value space size = " + Integer.toString(vspace.length()));
         super.printStats();
         
      }
      
      public static void main(String[] argv)
         throws Exception
      {
         HyphenationTree ht = null;
         int minCharCount = 2;
         BufferedReader in
            = new BufferedReader(new InputStreamReader(System.in));
         for(;;) {
            System.out.print("l:\tload patterns from XML\nL:\tload patterns from serialized object\ns:\tset minimun character count\nw:\twrite hyphenation tree to object file\nh:\thyphenate\nf:\tfind pattern\nb:\tbenchmark\nq:\tquit\n\nCommand:");
            String token = in.readLine().trim();
            if ( token.equals("f") ) {
               System.out.print("Pattern: ");
               token = in.readLine().trim();
               System.out.println("Values: " + ht.findPattern(token));
            } else if ( token.equals("s")) {
               System.out.print("Minimun value: " );
               token = in.readLine().trim();
               minCharCount = Integer.parseInt(token);
            } else if ( token.equals("l") ) {
               ht = new HyphenationTree();
               System.out.print("XML file name: ");
               token = in.readLine().trim();
               ht.loadPatterns(token);
            } else if ( token.equals("L") ) {
               ObjectInputStream ois = null;
               System.out.print("Object file name: ");
               token = in.readLine().trim();
               try {
                  ois = new ObjectInputStream(new FileInputStream(token));
                  ht = (HyphenationTree)ois.readObject();
               }
               catch (Exception e) {
                  e.printStackTrace();
               }
               finally {
                  if ( ois != null ) {
                     try { ois.close(); }
                     catch (IOException e) { }
                  }
               }
            } else if ( token.equals("w") ) {
               System.out.print("Object file name: ");
               token = in.readLine().trim();
               ObjectOutputStream oos = null;
               try {
                  oos = new ObjectOutputStream(new FileOutputStream(token));
                  oos.writeObject(ht);
               }
               catch (Exception e) {
                  e.printStackTrace();
               }
               finally {
                  if ( oos != null ) {
                     try { oos.flush(); }
                     catch (IOException e) {}
                     try { oos.close(); }
                     catch (IOException e) {}
                  }
               }
            } else if ( token.equals("h") ) {
               System.out.print("Word: ");
               token = in.readLine().trim();
               System.out.print("Hyphenation points: ");
               System.out.println(ht.hyphenate(token,minCharCount,minCharCount));
            } else if ( token.equals("b") ) {
               if ( ht == null ) {
                  System.out.println("No patterns has been loaded.");
                  break;
               }
               System.out.print("Word list filename: ");
               token = in.readLine().trim();
               long starttime = 0;
               int counter = 0;;
               try {
                  BufferedReader reader = new BufferedReader ( new FileReader(token));
                  String line;

                  starttime = System.currentTimeMillis();
                  while ((line = reader.readLine())!= null) {
                     //System.out.print("\nline: ");
                     Hyphenation hyp = ht.hyphenate(line,minCharCount,minCharCount);
                     if (hyp != null) {
                        String hword = hyp.toString();
                        //System.out.println(line);
                        //System.out.println(hword);
                     } else {
                        //System.out.println("No hyphenation");
                     }
                     counter++;
                  }
               } catch (Exception ioe) {
                  System.out.println("Exception " + ioe);
                  ioe.printStackTrace();
               }
               long endtime = System.currentTimeMillis();
               long result = endtime - starttime;
               System.out.println(counter+ " words in " + result + " Millisekunden hyphenated");
               
            } else if ( token.equals("q") )
               break;
         }

      }
}
