    /*
     *  call-seq:
     *     Token.new(text, start, end, pos_inc = 1) -> new Token
     *
     *  Creates a new token setting the text, start and end offsets of the token
     *  and the position increment for the token.
     *
     *  The position increment is usually set to 1 but you can set it to other
     *  values as needed.  For example, if you have a stop word filter you will be
     *  skipping tokens. Let's say you have the stop words "the" and "and" and you
     *  parse the title "The Old Man and the Sea". The terms "Old", "Man" and
     *  "Sea" will have the position increments 2, 1 and 3 respectively.
     *
     *  Another reason you might want to vary the position increment is if you are
     *  adding synonyms to the index. For example let's say you have the synonym
     *  group "quick", "fast" and "speedy". When tokenizing the phrase "Next day
     *  speedy delivery", you'll add "speedy" first with a position increment of 1
     *  and then "fast" and "quick" with position increments of 0 since they are
     *  represented in the same position.
     *
     *  The offset set values +start+ and +end+ should be byte offsets, not
     *  character offsets. This makes it easy to use those offsets to quickly
     *  access the token in the input string and also to insert highlighting tags
     *  when necessary.
     *
     *  text::       the main text for the token.
     *  start::      the start offset of the token in bytes.
     *  end::        the end offset of the token in bytes.
     *  pos_inc::    the position increment of a token. See above.
     *  return::     a newly created and assigned Token object
     */
    static VALUE
    frt_token_init(int argc, VALUE *argv, VALUE self)
    {
        RToken *token;
        VALUE rtext, rstart, rend, rpos_inc, rtype;
        GET_TK(token, self);
        token->pos_inc = 1;
        switch (rb_scan_args(argc, argv, "32", &rtext, &rstart,
                             &rend, &rpos_inc, &rtype)) {
            case 5: /* type gets ignored at this stage */
            case 4: token->pos_inc = FIX2INT(rpos_inc);
        }
        token->text = rb_obj_as_string(rtext);
        token->start = FIX2INT(rstart);
        token->end = FIX2INT(rend);
        return self;
    }