# include/xapian/unicode.h

Go to the documentation of this file.

    00001
    00004 /* Copyright (C) 2006,2007 Olly Betts
    00005  *
    00006  * This program is free software; you can redistribute it and/or modify
    00007  * it under the terms of the GNU General Public License as published by
    00008  * the Free Software Foundation; either version 2 of the License, or
    00009  * (at your option) any later version.
    00010  *
    00011  * This program is distributed in the hope that it will be useful,
    00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
    00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    00014  * GNU General Public License for more details.
    00015  *
    00016  * You should have received a copy of the GNU General Public License
    00017  * along with this program; if not, write to the Free Software
    00018  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
    00019  */
    00020
    00021 // FIXME: Nail down API details (what to expose; what names to use).
    00022
    00023 #ifndef XAPIAN_INCLUDED_UNICODE_H
    00024 #define XAPIAN_INCLUDED_UNICODE_H
    00025
    00026 #include <xapian/visibility.h>
    00027
    00028 #include <string>
    00029
    00030 namespace Xapian {
    00031
    00035 class XAPIAN_VISIBILITY_DEFAULT Utf8Iterator {
    00036     const unsigned char *p;
    00037     const unsigned char *end;
    00038     mutable unsigned seqlen;
    00039
    00040     void calculate_sequence_length() const;
    00041
    00042     unsigned get_char() const;
    00043
    00044     Utf8Iterator(const unsigned char *p_, const unsigned char *end_, unsigned seqlen_)
    00045         : p(p_), end(end_), seqlen(seqlen_) { }
    00046
    00047   public:
    00049     const char * raw() const {
    00050         return reinterpret_cast<const char *>(p ? p : end);
    00051     }
    00052
    00054     size_t left() const { return p ? end - p : 0; }
    00055
    00067     void assign(const char *p_, size_t len) {
    00068         if (len) {
    00069             p = reinterpret_cast<const unsigned char*>(p_);
    00070             end = p + len;
    00071             seqlen = 0;
    00072         } else {
    00073             p = NULL;
    00074         }
    00075     }
    00076
    00087     void assign(const std::string &s) { assign(s.data(), s.size()); }
    00088
    00097     explicit Utf8Iterator(const char *p_);
    00098
    00109     Utf8Iterator(const char *p_, size_t len) { assign(p_, len); }
    00110
    00120     Utf8Iterator(const std::string &s) { assign(s.data(), s.size()); }
    00121
    00127     Utf8Iterator() : p(NULL), end(0), seqlen(0) { }
    00128
    00133     unsigned operator*() const;
    00134
    00139     Utf8Iterator operator++(int) {
    00140         // If we've not calculated seqlen yet, do so.
    00141         if (seqlen == 0) calculate_sequence_length();
    00142         const unsigned char *old_p = p;
    00143         unsigned old_seqlen = seqlen;
    00144         p += seqlen;
    00145         if (p == end) p = NULL;
    00146         seqlen = 0;
    00147         return Utf8Iterator(old_p, end, old_seqlen);
    00148     }
    00149
    00154     Utf8Iterator & operator++() {
    00155         if (seqlen == 0) calculate_sequence_length();
    00156         p += seqlen;
    00157         if (p == end) p = NULL;
    00158         seqlen = 0;
    00159         return *this;
    00160     }
    00161
    00166     bool operator==(const Utf8Iterator &other) const { return p == other.p; }
    00167
    00172     bool operator!=(const Utf8Iterator &other) const { return p != other.p; }
    00173
    00175
    00176     typedef std::input_iterator_tag iterator_category;
    00177     typedef unsigned value_type;
    00178     typedef size_t difference_type;
    00179     typedef const unsigned * pointer;
    00180     typedef const unsigned & reference;
    00182 };
    00183
    00184 namespace Unicode {
    00185
    00187 typedef enum {
    00188     UNASSIGNED,
    00189     UPPERCASE_LETTER,
    00190     LOWERCASE_LETTER,
    00191     TITLECASE_LETTER,
    00192     MODIFIER_LETTER,
    00193     OTHER_LETTER,
    00194     NON_SPACING_MARK,
    00195     ENCLOSING_MARK,
    00196     COMBINING_SPACING_MARK,
    00197     DECIMAL_DIGIT_NUMBER,
    00198     LETTER_NUMBER,
    00199     OTHER_NUMBER,
    00200     SPACE_SEPARATOR,
    00201     LINE_SEPARATOR,
    00202     PARAGRAPH_SEPARATOR,
    00203     CONTROL,
    00204     FORMAT,
    00205     PRIVATE_USE,
    00206     SURROGATE,
    00207     CONNECTOR_PUNCTUATION,
    00208     DASH_PUNCTUATION,
    00209     OPEN_PUNCTUATION,
    00210     CLOSE_PUNCTUATION,
    00211     INITIAL_QUOTE_PUNCTUATION,
    00212     FINAL_QUOTE_PUNCTUATION,
    00213     OTHER_PUNCTUATION,
    00214     MATH_SYMBOL,
    00215     CURRENCY_SYMBOL,
    00216     MODIFIER_SYMBOL,
    00217     OTHER_SYMBOL
    00218 } category;
    00219
    00220 namespace Internal {
    00221     /* Extract the information about a character from the Unicode character
    00222      * tables.
    00223      *
    00224      * ch must be a valid Unicode character value (i.e. < 0x110000)
    00225      */
    00226     XAPIAN_VISIBILITY_DEFAULT
    00227     int get_character_info(unsigned ch);
    00228
    00230     inline int get_case_type(int info) { return ((info & 0xe0) >> 5); }
    00231
    00233     inline category get_category(int info) { return static_cast<category>(info & 0x1f); }
    00234
    00236     inline int get_delta(int info) {
    00237         /* It's implementation defined if sign extension happens on right shift
    00238          * of a signed int, hence the conditional (hopefully the compiler will
    00239          * spot this and optimise it to a sign-extending shift on architectures
    00240          * with a suitable instruction).
    00241          */
    00242         return (info >= 0) ? (info >> 15) : (~(~info >> 15));
    00243     }
    00244 }
    00245
    00255 XAPIAN_VISIBILITY_DEFAULT
    00256 unsigned nonascii_to_utf8(unsigned ch, char * buf);
    00257
    00265 inline unsigned to_utf8(unsigned ch, char *buf) {
    00266     if (ch < 128) {
    00267         *buf = static_cast<unsigned char>(ch);
    00268         return 1;
    00269     }
    00270     return Xapian::Unicode::nonascii_to_utf8(ch, buf);
    00271 }
    00272
    00276 inline void append_utf8(std::string &s, unsigned ch) {
    00277     char buf[4];
    00278     s.append(buf, to_utf8(ch, buf));
    00279 }
    00280
    00282 inline category get_category(unsigned ch) {
    00283     // Categorise non-Unicode values as UNASSIGNED.
    00284     if (ch >= 0x110000) return Xapian::Unicode::UNASSIGNED;
    00285     return Internal::get_category(Internal::get_character_info(ch));
    00286 }
    00287
    00289 inline bool is_wordchar(unsigned ch) {
    00290     const unsigned int WORDCHAR_MASK =
    00291             (1 << Xapian::Unicode::UPPERCASE_LETTER) |
    00292             (1 << Xapian::Unicode::LOWERCASE_LETTER) |
    00293             (1 << Xapian::Unicode::TITLECASE_LETTER) |
    00294             (1 << Xapian::Unicode::MODIFIER_LETTER) |
    00295             (1 << Xapian::Unicode::OTHER_LETTER) |
    00296             (1 << Xapian::Unicode::DECIMAL_DIGIT_NUMBER) |
    00297             (1 << Xapian::Unicode::LETTER_NUMBER) |
    00298             (1 << Xapian::Unicode::OTHER_NUMBER) |
    00299             (1 << Xapian::Unicode::CONNECTOR_PUNCTUATION);
    00300     return ((WORDCHAR_MASK >> get_category(ch)) & 1);
    00301 }
    00302
    00304 inline bool is_whitespace(unsigned ch) {
    00305     const unsigned int WHITESPACE_MASK =
    00306             (1 << Xapian::Unicode::CONTROL) | // For TAB, CR, LF, FF.
    00307             (1 << Xapian::Unicode::SPACE_SEPARATOR) |
    00308             (1 << Xapian::Unicode::LINE_SEPARATOR) |
    00309             (1 << Xapian::Unicode::PARAGRAPH_SEPARATOR);
    00310     return ((WHITESPACE_MASK >> get_category(ch)) & 1);
    00311 }
    00312
    00314 inline bool is_currency(unsigned ch) {
    00315     return (get_category(ch) == Xapian::Unicode::CURRENCY_SYMBOL);
    00316 }
    00317
    00319 inline unsigned tolower(unsigned ch) {
    00320     int info;
    00321     // Leave non-Unicode values unchanged.
    00322     if (ch >= 0x110000 || !(Internal::get_case_type((info = Xapian::Unicode::Internal::get_character_info(ch))) & 2))
    00323         return ch;
    00324     return ch + Internal::get_delta(info);
    00325 }
    00326
    00328 inline std::string
    00329 tolower(const std::string &term)
    00330 {
    00331     std::string result;
    00332     result.reserve(term.size());
    00333     for (Utf8Iterator i(term); i != Utf8Iterator(); ++i) {
    00334         append_utf8(result, tolower(*i));
    00335     }
    00336     return result;
    00337 }
    00338
    00339 }
    00340
    00341 }
    00342
    00343 #endif // XAPIAN_INCLUDED_UNICODE_H