MySQL  8.0.21
Source Code Documentation
mb_wc.h
Go to the documentation of this file.
1 #ifndef MB_WC_INCLUDED
2 #define MB_WC_INCLUDED
3 
4 /* Copyright (c) 2016, Oracle and/or its affiliates. All rights reserved.
5 
6  This library is free software; you can redistribute it and/or
7  modify it under the terms of the GNU Library General Public
8  License, version 2.0, as published by the Free Software Foundation.
9 
10  This library is also distributed with certain software (including
11  but not limited to OpenSSL) that is licensed under separate terms,
12  as designated in a particular file or component or in included license
13  documentation. The authors of MySQL hereby grant you an additional
14  permission to link the library and your derivative works with the
15  separately licensed software that they have included with MySQL.
16 
17  This library is distributed in the hope that it will be useful,
18  but WITHOUT ANY WARRANTY; without even the implied warranty of
19  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
20  Library General Public License, version 2.0, for more details.
21 
22  You should have received a copy of the GNU Library General Public
23  License along with this library; if not, write to the Free
24  Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston,
25  MA 02110-1301 USA */
26 
27 /**
28  @file mb_wc.h
29 
30  Definitions of mb_wc (multibyte to wide character, ie., effectively
31  “parse a UTF-8 character”) functions for UTF-8 (both three- and four-byte).
32  These are available both as inline functions, as C-style thunks so that they
33  can fit into MY_CHARSET_HANDLER, and as functors.
34 
35  The functors exist so that you can specialize a class on them and get them
36  inlined instead of having to call them through the function pointer in
37  MY_CHARSET_HANDLER; mb_wc is in itself so cheap (the most common case is
38  just a single byte load and a predictable compare) that the call overhead
39  in a tight loop is significant, and these routines tend to take up a lot
40  of CPU time when sorting. Typically, at the outermost level, you'd simply
41  compare cs->cset->mb_wc with my_mb_wc_{utf8,utf8mb4}_thunk, and if so,
42  instantiate your function with the given class. If it doesn't match,
43  you can use Mb_wc_through_function_pointer, which calls through the
44  function pointer as usual. (It will cache the function pointer for you,
45  which is typically faster than looking it up all the time -- the compiler
46  cannot always figure out on its own that it doesn't change.)
47 
48  The Mb_wc_* classes should be sent by _value_, not by reference, since
49  they are never larger than two pointers (and usually simply zero).
50 */
51 
52 #include "m_ctype.h"
53 #include "my_compiler.h"
54 #include "my_config.h"
55 
56 template <bool RANGE_CHECK, bool SUPPORT_MB4>
57 static int my_mb_wc_utf8_prototype(my_wc_t *pwc, const uchar *s,
58  const uchar *e);
59 
60 static int my_mb_wc_utf8(my_wc_t *pwc, const uchar *s, const uchar *e);
61 static int my_mb_wc_utf8mb4(my_wc_t *pwc, const uchar *s, const uchar *e);
62 
63 /**
64  Functor that converts a UTF-8 multibyte sequence (up to three bytes)
65  to a wide character.
66 */
67 struct Mb_wc_utf8 {
69 
71  int operator()(my_wc_t *pwc, const uchar *s, const uchar *e) const {
72  return my_mb_wc_utf8(pwc, s, e);
73  }
74 };
75 
76 /**
77  Functor that converts a UTF-8 multibyte sequence (up to four bytes)
78  to a wide character.
79 */
80 struct Mb_wc_utf8mb4 {
82 
84  int operator()(my_wc_t *pwc, const uchar *s, const uchar *e) const {
85  return my_mb_wc_utf8mb4(pwc, s, e);
86  }
87 };
88 
89 /**
90  Functor that uses a function pointer to convert a multibyte sequence
91  to a wide character.
92 */
94  public:
96  : m_funcptr(cs->cset->mb_wc), m_cs(cs) {}
97 
98  int operator()(my_wc_t *pwc, const uchar *s, const uchar *e) const {
99  return m_funcptr(m_cs, pwc, s, e);
100  }
101 
102  private:
103  typedef int (*mbwc_func_t)(const CHARSET_INFO *, my_wc_t *, const uchar *,
104  const uchar *);
105 
106  const mbwc_func_t m_funcptr;
107  const CHARSET_INFO *const m_cs;
108 };
109 
110 template <bool RANGE_CHECK, bool SUPPORT_MB4>
112  const uchar *e) {
113  if (RANGE_CHECK && s >= e) return MY_CS_TOOSMALL;
114 
115  uchar c = s[0];
116  if (c < 0x80) {
117  *pwc = c;
118  return 1;
119  }
120 
121  if (c < 0xe0) {
122  if (c < 0xc2) // Resulting code point would be less than 0x80.
123  return MY_CS_ILSEQ;
124 
125  if (RANGE_CHECK && s + 2 > e) return MY_CS_TOOSMALL2;
126 
127  if ((s[1] & 0xc0) != 0x80) // Next byte must be a continuation byte.
128  return MY_CS_ILSEQ;
129 
130  *pwc = ((my_wc_t)(c & 0x1f) << 6) + (my_wc_t)(s[1] & 0x3f);
131  return 2;
132  }
133 
134  if (c < 0xf0) {
135  if (RANGE_CHECK && s + 3 > e) return MY_CS_TOOSMALL3;
136 
137  // Next two bytes must be continuation bytes.
138  uint16 two_bytes;
139  memcpy(&two_bytes, s + 1, sizeof(two_bytes));
140  if ((two_bytes & 0xc0c0) != 0x8080) // Endianness does not matter.
141  return MY_CS_ILSEQ;
142 
143  *pwc = ((my_wc_t)(c & 0x0f) << 12) + ((my_wc_t)(s[1] & 0x3f) << 6) +
144  (my_wc_t)(s[2] & 0x3f);
145  if (*pwc < 0x800) return MY_CS_ILSEQ;
146  /*
147  According to RFC 3629, UTF-8 should prohibit characters between
148  U+D800 and U+DFFF, which are reserved for surrogate pairs and do
149  not directly represent characters.
150  */
151  if (*pwc >= 0xd800 && *pwc <= 0xdfff) return MY_CS_ILSEQ;
152  return 3;
153  }
154 
155  if (SUPPORT_MB4) {
156  if (RANGE_CHECK && s + 4 > e) /* We need 4 characters */
157  return MY_CS_TOOSMALL4;
158 
159  /*
160  This byte must be of the form 11110xxx, and the next three bytes
161  must be continuation bytes.
162  */
163  uint32 four_bytes;
164  memcpy(&four_bytes, s, sizeof(four_bytes));
165 #ifdef WORDS_BIGENDIAN
166  if ((four_bytes & 0xf8c0c0c0) != 0xf0808080)
167 #else
168  if ((four_bytes & 0xc0c0c0f8) != 0x808080f0)
169 #endif
170  return MY_CS_ILSEQ;
171 
172  *pwc = ((my_wc_t)(c & 0x07) << 18) + ((my_wc_t)(s[1] & 0x3f) << 12) +
173  ((my_wc_t)(s[2] & 0x3f) << 6) + (my_wc_t)(s[3] & 0x3f);
174  if (*pwc < 0x10000 || *pwc > 0x10ffff) return MY_CS_ILSEQ;
175  return 4;
176  }
177 
178  return MY_CS_ILSEQ;
179 }
180 
181 /**
182  Parses a single UTF-8 character from a byte string.
183 
184  @param[out] pwc the parsed character, if any
185  @param s the string to read from
186  @param e the end of the string; will not read past this
187 
188  @return the number of bytes read from s, or a value <= 0 for failure
189  (see m_ctype.h)
190 */
191 static inline int my_mb_wc_utf8(my_wc_t *pwc, const uchar *s, const uchar *e) {
192  return my_mb_wc_utf8_prototype</*RANGE_CHECK=*/true, /*SUPPORT_MB4=*/false>(
193  pwc, s, e);
194 }
195 
196 /**
197  Parses a single UTF-8 character from a byte string. The difference
198  between this and my_mb_wc_utf8 is that this function also can handle
199  four-byte UTF-8 characters.
200 
201  @param[out] pwc the parsed character, if any
202  @param s the string to read from
203  @param e the end of the string; will not read past this
204 
205  @return the number of bytes read from s, or a value <= 0 for failure
206  (see m_ctype.h)
207 */
208 static ALWAYS_INLINE int my_mb_wc_utf8mb4(my_wc_t *pwc, const uchar *s,
209  const uchar *e) {
210  return my_mb_wc_utf8_prototype</*RANGE_CHECK=*/true, /*SUPPORT_MB4=*/true>(
211  pwc, s, e);
212 }
213 
214 // Non-inlined versions of the above. These are used as function pointers
215 // in MY_CHARSET_HANDLER structs, and you can compare againt them to see
216 // if using the Mb_wc_utf8* functors would be appropriate.
217 
218 extern "C" int my_mb_wc_utf8_thunk(const CHARSET_INFO *cs, my_wc_t *pwc,
219  const uchar *s, const uchar *e);
220 
221 extern "C" int my_mb_wc_utf8mb4_thunk(const CHARSET_INFO *cs, my_wc_t *pwc,
222  const uchar *s, const uchar *e);
223 
224 #endif // MB_WC_INCLUDED
unsigned char uchar
Definition: my_inttypes.h:51
Functor that converts a UTF-8 multibyte sequence (up to three bytes) to a wide character.
Definition: mb_wc.h:67
ALWAYS_INLINE int operator()(my_wc_t *pwc, const uchar *s, const uchar *e) const
Definition: mb_wc.h:84
static int my_mb_wc_utf8(my_wc_t *pwc, const uchar *s, const uchar *e)
Parses a single UTF-8 character from a byte string.
Definition: mb_wc.h:191
const CHARSET_INFO *const m_cs
Definition: mb_wc.h:107
#define MY_CS_TOOSMALL3
Definition: m_ctype.h:97
Mb_wc_utf8mb4()
Definition: mb_wc.h:81
Functor that converts a UTF-8 multibyte sequence (up to four bytes) to a wide character.
Definition: mb_wc.h:80
#define MY_CS_TOOSMALL
Definition: m_ctype.h:95
Mb_wc_through_function_pointer(const CHARSET_INFO *cs)
Definition: mb_wc.h:95
int operator()(my_wc_t *pwc, const uchar *s, const uchar *e) const
Definition: mb_wc.h:98
Header for compiler-dependent features.
uint16_t uint16
Definition: my_inttypes.h:64
ALWAYS_INLINE int operator()(my_wc_t *pwc, const uchar *s, const uchar *e) const
Definition: mb_wc.h:71
uint32_t uint32
Definition: my_inttypes.h:66
static int my_mb_wc_utf8_prototype(my_wc_t *pwc, const uchar *s, const uchar *e)
#define MY_CS_ILSEQ
Definition: m_ctype.h:93
Definition: m_ctype.h:359
#define MY_CS_TOOSMALL2
Definition: m_ctype.h:96
int my_mb_wc_utf8_thunk(const CHARSET_INFO *cs, my_wc_t *pwc, const uchar *s, const uchar *e)
A thunk to be able to use my_mb_wc_utf8 in MY_CHARSET_HANDLER structs.
Definition: ctype-utf8.cc:7206
Mb_wc_utf8()
Definition: mb_wc.h:68
Functor that uses a function pointer to convert a multibyte sequence to a wide character.
Definition: mb_wc.h:93
static int my_mb_wc_utf8mb4(my_wc_t *pwc, const uchar *s, const uchar *e)
Parses a single UTF-8 character from a byte string.
Definition: mb_wc.h:208
A better implementation of the UNIX ctype(3) library.
const mbwc_func_t m_funcptr
Definition: mb_wc.h:106
int my_mb_wc_utf8mb4_thunk(const CHARSET_INFO *cs, my_wc_t *pwc, const uchar *s, const uchar *e)
A thunk to be able to use my_mb_wc_utf8mb4 in MY_CHARSET_HANDLER structs.
Definition: ctype-utf8.cc:7222
#define ALWAYS_INLINE
Definition: my_compiler.h:109
ulong my_wc_t
Our own version of wchar_t, ie., a type that holds a single Unicode code point ("wide character")...
Definition: m_ctype.h:58
#define MY_CS_TOOSMALL4
Definition: m_ctype.h:99