# include/xapian/unicode.h

Go to the documentation of this file.

    00001
    00004 /* Copyright (C) 2006,2007,2008 Olly Betts
    00005  *
    00006  * This program is free software; you can redistribute it and/or modify
    00007  * it under the terms of the GNU General Public License as published by
    00008  * the Free Software Foundation; either version 2 of the License, or
    00009  * (at your option) any later version.
    00010  *
    00011  * This program is distributed in the hope that it will be useful,
    00012  * but WITHOUT ANY WARRANTY; without even the implied warranty of
    00013  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    00014  * GNU General Public License for more details.
    00015  *
    00016  * You should have received a copy of the GNU General Public License
    00017  * along with this program; if not, write to the Free Software
    00018  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
    00019  */
    00020
    00021 #ifndef XAPIAN_INCLUDED_UNICODE_H
    00022 #define XAPIAN_INCLUDED_UNICODE_H
    00023
    00024 #include <xapian/visibility.h>
    00025
    00026 #include <string>
    00027
    00028 namespace Xapian {
    00029
    00033 class XAPIAN_VISIBILITY_DEFAULT Utf8Iterator {
    00034     const unsigned char *p;
    00035     const unsigned char *end;
    00036     mutable unsigned seqlen;
    00037
    00038     void calculate_sequence_length() const;
    00039
    00040     unsigned get_char() const;
    00041
    00042     Utf8Iterator(const unsigned char *p_, const unsigned char *end_, unsigned seqlen_)
    00043         : p(p_), end(end_), seqlen(seqlen_) { }
    00044
    00045   public:
    00047     const char * raw() const {
    00048         return reinterpret_cast<const char *>(p ? p : end);
    00049     }
    00050
    00052     size_t left() const { return p ? end - p : 0; }
    00053
    00065     void assign(const char *p_, size_t len) {
    00066         if (len) {
    00067             p = reinterpret_cast<const unsigned char*>(p_);
    00068             end = p + len;
    00069             seqlen = 0;
    00070         } else {
    00071             p = NULL;
    00072         }
    00073     }
    00074
    00085     void assign(const std::string &s) { assign(s.data(), s.size()); }
    00086
    00095     explicit Utf8Iterator(const char *p_);
    00096
    00107     Utf8Iterator(const char *p_, size_t len) { assign(p_, len); }
    00108
    00118     Utf8Iterator(const std::string &s) { assign(s.data(), s.size()); }
    00119
    00125     Utf8Iterator() : p(NULL), end(0), seqlen(0) { }
    00126
    00131     unsigned operator*() const;
    00132
    00137     Utf8Iterator operator++(int) {
    00138         // If we've not calculated seqlen yet, do so.
    00139         if (seqlen == 0) calculate_sequence_length();
    00140         const unsigned char *old_p = p;
    00141         unsigned old_seqlen = seqlen;
    00142         p += seqlen;
    00143         if (p == end) p = NULL;
    00144         seqlen = 0;
    00145         return Utf8Iterator(old_p, end, old_seqlen);
    00146     }
    00147
    00152     Utf8Iterator & operator++() {
    00153         if (seqlen == 0) calculate_sequence_length();
    00154         p += seqlen;
    00155         if (p == end) p = NULL;
    00156         seqlen = 0;
    00157         return *this;
    00158     }
    00159
    00164     bool operator==(const Utf8Iterator &other) const { return p == other.p; }
    00165
    00170     bool operator!=(const Utf8Iterator &other) const { return p != other.p; }
    00171
    00173
    00174     typedef std::input_iterator_tag iterator_category;
    00175     typedef unsigned value_type;
    00176     typedef size_t difference_type;
    00177     typedef const unsigned * pointer;
    00178     typedef const unsigned & reference;
    00180 };
    00181
    00182 namespace Unicode {
    00183
    00185 typedef enum {
    00186     UNASSIGNED,
    00187     UPPERCASE_LETTER,
    00188     LOWERCASE_LETTER,
    00189     TITLECASE_LETTER,
    00190     MODIFIER_LETTER,
    00191     OTHER_LETTER,
    00192     NON_SPACING_MARK,
    00193     ENCLOSING_MARK,
    00194     COMBINING_SPACING_MARK,
    00195     DECIMAL_DIGIT_NUMBER,
    00196     LETTER_NUMBER,
    00197     OTHER_NUMBER,
    00198     SPACE_SEPARATOR,
    00199     LINE_SEPARATOR,
    00200     PARAGRAPH_SEPARATOR,
    00201     CONTROL,
    00202     FORMAT,
    00203     PRIVATE_USE,
    00204     SURROGATE,
    00205     CONNECTOR_PUNCTUATION,
    00206     DASH_PUNCTUATION,
    00207     OPEN_PUNCTUATION,
    00208     CLOSE_PUNCTUATION,
    00209     INITIAL_QUOTE_PUNCTUATION,
    00210     FINAL_QUOTE_PUNCTUATION,
    00211     OTHER_PUNCTUATION,
    00212     MATH_SYMBOL,
    00213     CURRENCY_SYMBOL,
    00214     MODIFIER_SYMBOL,
    00215     OTHER_SYMBOL
    00216 } category;
    00217
    00218 namespace Internal {
    00224     XAPIAN_VISIBILITY_DEFAULT
    00225     int get_character_info(unsigned ch);
    00226
    00230     inline int get_case_type(int info) { return ((info & 0xe0) >> 5); }
    00231
    00233     inline category get_category(int info) { return static_cast<category>(info & 0x1f); }
    00234
    00238     inline int get_delta(int info) {
    00239         /* It's implementation defined if sign extension happens on right shift
    00240          * of a signed int, hence the conditional (hopefully the compiler will
    00241          * spot this and optimise it to a sign-extending shift on architectures
    00242          * with a suitable instruction).
    00243          */
    00244         return (info >= 0) ? (info >> 15) : (~(~info >> 15));
    00245     }
    00246 }
    00247
    00257 XAPIAN_VISIBILITY_DEFAULT
    00258 unsigned nonascii_to_utf8(unsigned ch, char * buf);
    00259
    00267 inline unsigned to_utf8(unsigned ch, char *buf) {
    00268     if (ch < 128) {
    00269         *buf = static_cast<unsigned char>(ch);
    00270         return 1;
    00271     }
    00272     return Xapian::Unicode::nonascii_to_utf8(ch, buf);
    00273 }
    00274
    00278 inline void append_utf8(std::string &s, unsigned ch) {
    00279     char buf[4];
    00280     s.append(buf, to_utf8(ch, buf));
    00281 }
    00282
    00284 inline category get_category(unsigned ch) {
    00285     // Categorise non-Unicode values as UNASSIGNED.
    00286     if (ch >= 0x110000) return Xapian::Unicode::UNASSIGNED;
    00287     return Internal::get_category(Internal::get_character_info(ch));
    00288 }
    00289
    00291 inline bool is_wordchar(unsigned ch) {
    00292     const unsigned int WORDCHAR_MASK =
    00293             (1 << Xapian::Unicode::UPPERCASE_LETTER) |
    00294             (1 << Xapian::Unicode::LOWERCASE_LETTER) |
    00295             (1 << Xapian::Unicode::TITLECASE_LETTER) |
    00296             (1 << Xapian::Unicode::MODIFIER_LETTER) |
    00297             (1 << Xapian::Unicode::OTHER_LETTER) |
    00298             (1 << Xapian::Unicode::DECIMAL_DIGIT_NUMBER) |
    00299             (1 << Xapian::Unicode::LETTER_NUMBER) |
    00300             (1 << Xapian::Unicode::OTHER_NUMBER) |
    00301             (1 << Xapian::Unicode::CONNECTOR_PUNCTUATION);
    00302     return ((WORDCHAR_MASK >> get_category(ch)) & 1);
    00303 }
    00304
    00306 inline bool is_whitespace(unsigned ch) {
    00307     const unsigned int WHITESPACE_MASK =
    00308             (1 << Xapian::Unicode::CONTROL) | // For TAB, CR, LF, FF.
    00309             (1 << Xapian::Unicode::SPACE_SEPARATOR) |
    00310             (1 << Xapian::Unicode::LINE_SEPARATOR) |
    00311             (1 << Xapian::Unicode::PARAGRAPH_SEPARATOR);
    00312     return ((WHITESPACE_MASK >> get_category(ch)) & 1);
    00313 }
    00314
    00316 inline bool is_currency(unsigned ch) {
    00317     return (get_category(ch) == Xapian::Unicode::CURRENCY_SYMBOL);
    00318 }
    00319
    00321 inline unsigned tolower(unsigned ch) {
    00322     int info;
    00323     // Leave non-Unicode values unchanged.
    00324     if (ch >= 0x110000 || !(Internal::get_case_type((info = Xapian::Unicode::Internal::get_character_info(ch))) & 2))
    00325         return ch;
    00326     return ch + Internal::get_delta(info);
    00327 }
    00328
    00330 inline unsigned toupper(unsigned ch) {
    00331     int info;
    00332     // Leave non-Unicode values unchanged.
    00333     if (ch >= 0x110000 || !(Internal::get_case_type((info = Xapian::Unicode::Internal::get_character_info(ch))) & 4))
    00334         return ch;
    00335     return ch - Internal::get_delta(info);
    00336 }
    00337
    00339 inline std::string
    00340 tolower(const std::string &term)
    00341 {
    00342     std::string result;
    00343     result.reserve(term.size());
    00344     for (Utf8Iterator i(term); i != Utf8Iterator(); ++i) {
    00345         append_utf8(result, tolower(*i));
    00346     }
    00347     return result;
    00348 }
    00349
    00351 inline std::string
    00352 toupper(const std::string &term)
    00353 {
    00354     std::string result;
    00355     result.reserve(term.size());
    00356     for (Utf8Iterator i(term); i != Utf8Iterator(); ++i) {
    00357         append_utf8(result, toupper(*i));
    00358     }
    00359     return result;
    00360 }
    00361
    00362 }
    00363
    00364 }
    00365
    00366 #endif // XAPIAN_INCLUDED_UNICODE_H