MySQL  8.0.27
Source Code Documentation
json_binary.h
Go to the documentation of this file.
1 #ifndef JSON_BINARY_INCLUDED
2 #define JSON_BINARY_INCLUDED
3 
4 /* Copyright (c) 2015, 2021, Oracle and/or its affiliates.
5 
6  This program is free software; you can redistribute it and/or modify
7  it under the terms of the GNU General Public License, version 2.0,
8  as published by the Free Software Foundation.
9 
10  This program is also distributed with certain software (including
11  but not limited to OpenSSL) that is licensed under separate terms,
12  as designated in a particular file or component or in included license
13  documentation. The authors of MySQL hereby grant you an additional
14  permission to link the program and your derivative works with the
15  separately licensed software that they have included with MySQL.
16 
17  This program is distributed in the hope that it will be useful,
18  but WITHOUT ANY WARRANTY; without even the implied warranty of
19  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20  GNU General Public License, version 2.0, for more details.
21 
22  You should have received a copy of the GNU General Public License
23  along with this program; if not, write to the Free Software
24  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
25 
26 /**
27  @file
28 
29  This file specifies the interface for serializing JSON values into
30  binary representation, and for reading values back from the binary
31  representation.
32 
33  The binary format is as follows:
34 
35  Each JSON value (scalar, object or array) has a one byte type
36  identifier followed by the actual value.
37 
38  If the value is a JSON object, its binary representation will have a
39  header that contains:
40 
41  - the member count
42  - the size of the binary value in bytes
43  - a list of pointers to each key
44  - a list of pointers to each value
45 
46  The actual keys and values will come after the header, in the same
47  order as in the header.
48 
49  Similarly, if the value is a JSON array, the binary representation
50  will have a header with
51 
52  - the element count
53  - the size of the binary value in bytes
54  - a list of pointers to each value
55 
56  followed by the actual values, in the same order as in the header.
57 
58  @verbatim
59  doc ::= type value
60 
61  type ::=
62  0x00 | // small JSON object
63  0x01 | // large JSON object
64  0x02 | // small JSON array
65  0x03 | // large JSON array
66  0x04 | // literal (true/false/null)
67  0x05 | // int16
68  0x06 | // uint16
69  0x07 | // int32
70  0x08 | // uint32
71  0x09 | // int64
72  0x0a | // uint64
73  0x0b | // double
74  0x0c | // utf8mb4 string
75  0x0f // custom data (any MySQL data type)
76 
77  value ::=
78  object |
79  array |
80  literal |
81  number |
82  string |
83  custom-data
84 
85  object ::= element-count size key-entry* value-entry* key* value*
86 
87  array ::= element-count size value-entry* value*
88 
89  // number of members in object or number of elements in array
90  element-count ::=
91  uint16 | // if used in small JSON object/array
92  uint32 // if used in large JSON object/array
93 
94  // number of bytes in the binary representation of the object or array
95  size ::=
96  uint16 | // if used in small JSON object/array
97  uint32 // if used in large JSON object/array
98 
99  key-entry ::= key-offset key-length
100 
101  key-offset ::=
102  uint16 | // if used in small JSON object
103  uint32 // if used in large JSON object
104 
105  key-length ::= uint16 // key length must be less than 64KB
106 
107  value-entry ::= type offset-or-inlined-value
108 
109  // This field holds either the offset to where the value is stored,
110  // or the value itself if it is small enough to be inlined (that is,
111  // if it is a JSON literal or a small enough [u]int).
112  offset-or-inlined-value ::=
113  uint16 | // if used in small JSON object/array
114  uint32 // if used in large JSON object/array
115 
116  key ::= utf8mb4-data
117 
118  literal ::=
119  0x00 | // JSON null literal
120  0x01 | // JSON true literal
121  0x02 | // JSON false literal
122 
123  number ::= .... // little-endian format for [u]int(16|32|64), whereas
124  // double is stored in a platform-independent, eight-byte
125  // format using float8store()
126 
127  string ::= data-length utf8mb4-data
128 
129  custom-data ::= custom-type data-length binary-data
130 
131  custom-type ::= uint8 // type identifier that matches the
132  // internal enum_field_types enum
133 
134  data-length ::= uint8* // If the high bit of a byte is 1, the length
135  // field is continued in the next byte,
136  // otherwise it is the last byte of the length
137  // field. So we need 1 byte to represent
138  // lengths up to 127, 2 bytes to represent
139  // lengths up to 16383, and so on...
140  @endverbatim
141 */
142 
143 #include <stddef.h>
144 #include <cassert>
145 #include <cstdint>
146 #include <string>
147 
148 /*
149  This file is part of our public interface for the 'json_binary' library.
150  This means that #includes of MySQL headers here should be limited to
151  files that are INSTALLed.
152 */
153 #include "field_types.h" // enum_field_types
154 
155 #ifdef MYSQL_SERVER
156 class Field_json;
157 class Json_dom;
158 class Json_wrapper;
159 class THD;
160 #endif
161 
162 class String;
163 
164 #if defined(WIN32) && defined(EXPORT_JSON_FUNCTIONS)
165 #define EXPORT_JSON_FUNCTION __declspec(dllexport)
166 #else
167 #define EXPORT_JSON_FUNCTION
168 #endif
169 
170 namespace json_binary {
171 
172 /**
173  Serialize the JSON document represented by dom to binary format in
174  the destination string, replacing any content already in the
175  destination string.
176 
177  @param[in] thd THD handle
178  @param[in] dom the input DOM tree
179  @param[in,out] dest the destination string
180  @retval false on success
181  @retval true if an error occurred
182 */
183 #ifdef MYSQL_SERVER
184 bool serialize(const THD *thd, const Json_dom *dom, String *dest);
185 #endif
186 
187 /**
188  Class used for reading JSON values that are stored in the binary
189  format. Values are parsed lazily, so that only the parts of the
190  value that are interesting to the caller, are read. Array elements
191  can be looked up in constant time using the element() function.
192  Object members can be looked up in O(log n) time using the lookup()
193  function.
194 */
195 class Value {
196  public:
197  enum enum_type : uint8_t {
208  ERROR /* Not really a type. Used to signal that an
209  error was detected. */
210  };
211 
212  /**
213  Does this value, and all of its members, represent a valid JSON
214  value?
215  */
217  bool is_valid() const;
218 
220  enum_type type() const { return m_type; }
221 
222  /// Does this value use the large storage format?
224  bool large_format() const { return m_large; }
225 
226  /**
227  Get a pointer to the beginning of the STRING or OPAQUE data
228  represented by this instance.
229  */
231  const char *get_data() const {
232  assert(m_type == STRING || m_type == OPAQUE);
233  return m_data;
234  }
235 
236  /**
237  Get the length in bytes of the STRING or OPAQUE value represented by
238  this instance.
239  */
241  uint32_t get_data_length() const {
242  assert(m_type == STRING || m_type == OPAQUE);
243  return m_length;
244  }
245 
246  /** Get the value of an INT. */
248  int64_t get_int64() const {
249  assert(m_type == INT);
250  return m_int_value;
251  }
252 
253  /** Get the value of a UINT. */
255  uint64_t get_uint64() const {
256  assert(m_type == UINT);
257  return static_cast<uint64_t>(m_int_value);
258  }
259 
260  /** Get the value of a DOUBLE. */
262  double get_double() const {
263  assert(m_type == DOUBLE);
264  return m_double_value;
265  }
266 
267  /**
268  Get the number of elements in an array, or the number of members in
269  an object.
270  */
272  uint32_t element_count() const {
273  assert(m_type == ARRAY || m_type == OBJECT);
274  return m_element_count;
275  }
276 
277  /**
278  Get the MySQL field type of an opaque value. Identifies the type of
279  the value stored in the data portion of an opaque value.
280  */
283  assert(m_type == OPAQUE);
284  return m_field_type;
285  }
286 
288  Value element(size_t pos) const;
289 
291  Value key(size_t pos) const;
292 
294  Value lookup(const char *key, size_t length) const;
295 
297  Value lookup(const std::string &key) const {
298  return lookup(key.c_str(), key.length());
299  }
300 
302  size_t lookup_index(const char *key, size_t length) const;
303 
305  size_t lookup_index(const std::string &key) const {
306  return lookup_index(key.c_str(), key.length());
307  }
308 
310  bool is_backed_by(const String *str) const;
311 
312 #ifdef MYSQL_SERVER
313  bool raw_binary(const THD *thd, String *buf) const;
314  bool get_free_space(const THD *thd, size_t *space) const;
315  bool update_in_shadow(const Field_json *field, size_t pos,
316  Json_wrapper *new_value, size_t data_offset,
317  size_t data_length, const char *original,
318  char *destination, bool *changed) const;
319  bool remove_in_shadow(const Field_json *field, size_t pos,
320  const char *original, char *destination) const;
321 #endif
323  bool has_space(size_t pos, size_t needed, size_t *offset) const;
324 
325  /** Constructor for values that represent literals or errors. */
327  explicit Value(enum_type t) : m_data(nullptr), m_type(t) {
328  assert(t == LITERAL_NULL || t == LITERAL_TRUE || t == LITERAL_FALSE ||
329  t == ERROR);
330  }
331 
332  /** Constructor for values that represent ints or uints. */
334  explicit Value(enum_type t, int64_t val) : m_int_value(val), m_type(t) {
335  assert(t == INT || t == UINT);
336  }
337 
338  /** Constructor for values that represent doubles. */
340  explicit Value(double val) : m_double_value(val), m_type(DOUBLE) {}
341 
342  /** Constructor for values that represent strings. */
344  Value(const char *data, uint32_t len)
345  : m_data(data), m_length(len), m_type(STRING) {}
346 
347  /**
348  Constructor for values that represent arrays or objects.
349 
350  @param t type
351  @param data pointer to the start of the binary representation
352  @param bytes the number of bytes in the binary representation of the value
353  @param element_count the number of elements or members in the value
354  @param large true if the value should be stored in the large
355  storage format with 4 byte offsets instead of 2 byte offsets
356  */
358  Value(enum_type t, const char *data, uint32_t bytes, uint32_t element_count,
359  bool large)
360  : m_data(data),
362  m_length(bytes),
363  m_type(t),
364  m_large(large) {
365  assert(t == ARRAY || t == OBJECT);
366  }
367 
368  /** Constructor for values that represent opaque data. */
370  Value(enum_field_types ft, const char *data, uint32_t len)
371  : m_data(data), m_length(len), m_field_type(ft), m_type(OPAQUE) {}
372 
373  /** Empty constructor. Produces a value that represents an error condition. */
375  Value() : Value(ERROR) {}
376 
377  /** Is this value an array? */
379  bool is_array() const { return m_type == ARRAY; }
380 
381  /** Is this value an object? */
383  bool is_object() const { return m_type == OBJECT; }
384 
385  /**
386  Format the JSON value to an external JSON string in buffer in
387  the format of ISO/IEC 10646.
388 
389  @param[in,out] buffer the formatted string is appended, so make sure
390  the length is set correctly before calling
391 
392  @return false formatting went well, else true
393  */
395  bool to_std_string(std::string *buffer) const;
396 
397  /**
398  Format the JSON value to an external JSON string in buffer in the format of
399  ISO/IEC 10646. Add newlines and indentation for readability.
400 
401  @param[in,out] buffer the buffer that receives the formatted string
402  (the string is appended, so make sure the length
403  is set correctly before calling)
404 
405  @retval false on success
406  @retval true on error
407  */
409  bool to_pretty_std_string(std::string *buffer) const;
410 
411  /**
412  Compare two Values
413  @note This function is limited to scalars only, for objects/arrays it
414  asserts. The main purpose is to separate old/new scalar values for updates
415  on multi-valued indexes.
416  @returns
417  -1 this < val
418  0 this == val
419  1 this > val
420  */
421 #ifdef MYSQL_SERVER
422  int eq(const Value &val) const;
423 #endif
424 
425  private:
426  /*
427  Instances use only one of m_data, m_int_value and m_double_value,
428  so keep them in a union to save space in memory.
429  */
430  union {
431  /**
432  Pointer to the start of the binary representation of the value. Only
433  used by STRING, OPAQUE, OBJECT and ARRAY.
434 
435  The memory pointed to by this member is not owned by this Value
436  object. Callers that create Value objects must make sure that the
437  memory is not freed as long as the Value object is alive.
438  */
439  const char *m_data;
440  /** The value if the type is INT or UINT. */
441  int64_t m_int_value;
442  /** The value if the type is DOUBLE. */
444  };
445 
446  /**
447  Element count for arrays and objects. Unused for other types.
448  */
449  uint32_t m_element_count;
450 
451  /**
452  The full length (in bytes) of the binary representation of an array or
453  object, or the length of a string or opaque value. Unused for other types.
454  */
455  uint32_t m_length;
456 
457  /**
458  The MySQL field type of the value, in case the type of the value is
459  OPAQUE. Otherwise, it is unused.
460  */
462 
463  /** The JSON type of the value. */
465 
466  /**
467  True if an array or an object uses the large storage format with 4
468  byte offsets instead of 2 byte offsets.
469  */
470  bool m_large;
471 
472  size_t key_entry_offset(size_t pos) const;
473  size_t value_entry_offset(size_t pos) const;
474  bool first_value_offset(size_t *offset) const;
475  bool element_offsets(size_t pos, size_t *start, size_t *end,
476  bool *inlined) const;
477 };
478 
479 /**
480  Parse a JSON binary document.
481 
482  @param[in] data a pointer to the binary data
483  @param[in] len the size of the binary document in bytes
484  @return an object that allows access to the contents of the document
485 */
487 Value parse_binary(const char *data, size_t len);
488 
489 /**
490  How much space is needed for a JSON value when it is stored in the binary
491  format.
492 
493  @param[in] thd THD handle
494  @param[in] value the JSON value to add to a document
495  @param[in] large true if the large storage format is used
496  @param[out] needed gets set to the amount of bytes needed to store
497  the value
498  @retval false if successful
499  @retval true if an error occurred while calculating the needed space
500 */
501 #ifdef MYSQL_SERVER
502 bool space_needed(const THD *thd, const Json_wrapper *value, bool large,
503  size_t *needed);
504 #endif
505 
506 /**
507  Apply a function to every value in a JSON document. That is, apply
508  the function to the root node of the JSON document, to all its
509  children, grandchildren and so on.
510 
511  @param value the root of the JSON document
512  @param func the function to apply
513  @retval true if the processing was stopped
514  @retval false if the processing was completed
515 
516  @tparam Func a functor type that takes a #json_binary::Value
517  parameter and returns a `bool` which is `true` if the processing
518  should stop or `false` if the processing should continue with the
519  next node
520 */
521 template <typename Func>
522 bool for_each_node(const Value &value, const Func &func) {
523  if (func(value)) return true;
524 
525  if (value.is_array() || value.is_object())
526  for (size_t i = 0, size = value.element_count(); i < size; ++i)
527  if (for_each_node(value.element(i), func)) return true;
528 
529  return false;
530 }
531 } // namespace json_binary
532 
533 #endif /* JSON_BINARY_INCLUDED */
A field that stores a JSON value.
Definition: field.h:3941
JSON DOM abstract base class.
Definition: json_dom.h:171
Abstraction for accessing JSON values irrespective of whether they are (started out as) binary JSON v...
Definition: json_dom.h:1167
Using this class is fraught with peril, and you need to be very careful when doing so.
Definition: sql_string.h:165
For each client connection we create a separate thread with THD serving as a thread/connection descri...
Definition: sql_class.h:821
Class used for reading JSON values that are stored in the binary format.
Definition: json_binary.h:195
double m_double_value
The value if the type is DOUBLE.
Definition: json_binary.h:443
EXPORT_JSON_FUNCTION uint32_t element_count() const
Get the number of elements in an array, or the number of members in an object.
Definition: json_binary.h:272
EXPORT_JSON_FUNCTION Value(enum_field_types ft, const char *data, uint32_t len)
Constructor for values that represent opaque data.
Definition: json_binary.h:370
EXPORT_JSON_FUNCTION Value key(size_t pos) const
Get the key of the member stored at the specified position in a JSON object.
Definition: json_binary.cc:1129
EXPORT_JSON_FUNCTION bool to_pretty_std_string(std::string *buffer) const
Format the JSON value to an external JSON string in buffer in the format of ISO/IEC 10646.
Definition: json_binary.cc:2113
EXPORT_JSON_FUNCTION Value lookup(const std::string &key) const
Definition: json_binary.h:297
EXPORT_JSON_FUNCTION enum_field_types field_type() const
Get the MySQL field type of an opaque value.
Definition: json_binary.h:282
EXPORT_JSON_FUNCTION Value(enum_type t, const char *data, uint32_t bytes, uint32_t element_count, bool large)
Constructor for values that represent arrays or objects.
Definition: json_binary.h:358
EXPORT_JSON_FUNCTION int64_t get_int64() const
Get the value of an INT.
Definition: json_binary.h:248
EXPORT_JSON_FUNCTION bool to_std_string(std::string *buffer) const
Format the JSON value to an external JSON string in buffer in the format of ISO/IEC 10646.
Definition: json_binary.cc:2102
bool m_large
True if an array or an object uses the large storage format with 4 byte offsets instead of 2 byte off...
Definition: json_binary.h:470
EXPORT_JSON_FUNCTION Value(const char *data, uint32_t len)
Constructor for values that represent strings.
Definition: json_binary.h:344
int64_t m_int_value
The value if the type is INT or UINT.
Definition: json_binary.h:441
EXPORT_JSON_FUNCTION uint32_t get_data_length() const
Get the length in bytes of the STRING or OPAQUE value represented by this instance.
Definition: json_binary.h:241
size_t value_entry_offset(size_t pos) const
Get the offset of the value entry that describes the element at a given position in this array or obj...
Definition: json_binary.cc:1522
bool first_value_offset(size_t *offset) const
Find the lowest possible offset where a value can be located inside this array or object.
Definition: json_binary.cc:1376
EXPORT_JSON_FUNCTION bool is_object() const
Is this value an object?
Definition: json_binary.h:383
EXPORT_JSON_FUNCTION size_t lookup_index(const std::string &key) const
Definition: json_binary.h:305
EXPORT_JSON_FUNCTION Value(double val)
Constructor for values that represent doubles.
Definition: json_binary.h:340
uint32_t m_length
The full length (in bytes) of the binary representation of an array or object, or the length of a str...
Definition: json_binary.h:455
enum_type
Definition: json_binary.h:197
@ DOUBLE
Definition: json_binary.h:203
@ LITERAL_FALSE
Definition: json_binary.h:206
@ INT
Definition: json_binary.h:201
@ STRING
Definition: json_binary.h:200
@ OBJECT
Definition: json_binary.h:198
@ LITERAL_TRUE
Definition: json_binary.h:205
@ ERROR
Definition: json_binary.h:208
@ LITERAL_NULL
Definition: json_binary.h:204
@ UINT
Definition: json_binary.h:202
@ OPAQUE
Definition: json_binary.h:207
@ ARRAY
Definition: json_binary.h:199
EXPORT_JSON_FUNCTION Value()
Empty constructor.
Definition: json_binary.h:375
EXPORT_JSON_FUNCTION Value lookup(const char *key, size_t length) const
Get the value associated with the specified key in a JSON object.
Definition: json_binary.cc:1167
EXPORT_JSON_FUNCTION const char * get_data() const
Get a pointer to the beginning of the STRING or OPAQUE data represented by this instance.
Definition: json_binary.h:231
bool get_free_space(const THD *thd, size_t *space) const
Get the amount of unused space in the binary representation of this value.
Definition: json_binary.cc:1965
EXPORT_JSON_FUNCTION enum_type type() const
Definition: json_binary.h:220
EXPORT_JSON_FUNCTION bool has_space(size_t pos, size_t needed, size_t *offset) const
Does this array or object have enough space to replace the value at the given position with another v...
Definition: json_binary.cc:1408
int eq(const Value &val) const
Compare two Values.
Definition: json_binary.cc:2057
EXPORT_JSON_FUNCTION Value(enum_type t)
Constructor for values that represent literals or errors.
Definition: json_binary.h:327
enum_field_types m_field_type
The MySQL field type of the value, in case the type of the value is OPAQUE.
Definition: json_binary.h:461
EXPORT_JSON_FUNCTION size_t lookup_index(const char *key, size_t length) const
Get the index of the element with the specified key in a JSON object.
Definition: json_binary.cc:1181
EXPORT_JSON_FUNCTION bool is_array() const
Is this value an array?
Definition: json_binary.h:379
EXPORT_JSON_FUNCTION Value element(size_t pos) const
Get the element at the specified position of a JSON array or a JSON object.
Definition: json_binary.cc:1091
EXPORT_JSON_FUNCTION bool large_format() const
Does this value use the large storage format?
Definition: json_binary.h:224
EXPORT_JSON_FUNCTION bool is_backed_by(const String *str) const
Is this binary value pointing to data that is contained in the specified string.
Definition: json_binary.cc:1228
enum_type m_type
The JSON type of the value.
Definition: json_binary.h:464
bool update_in_shadow(const Field_json *field, size_t pos, Json_wrapper *new_value, size_t data_offset, size_t data_length, const char *original, char *destination, bool *changed) const
Update a value in an array or object.
Definition: json_binary.cc:1725
bool remove_in_shadow(const Field_json *field, size_t pos, const char *original, char *destination) const
Remove a value from an array or object.
Definition: json_binary.cc:1912
const char * m_data
Pointer to the start of the binary representation of the value.
Definition: json_binary.h:439
EXPORT_JSON_FUNCTION uint64_t get_uint64() const
Get the value of a UINT.
Definition: json_binary.h:255
size_t key_entry_offset(size_t pos) const
Get the offset of the key entry that describes the key of the member at a given position in this obje...
Definition: json_binary.cc:1509
EXPORT_JSON_FUNCTION bool is_valid() const
Does this value, and all of its members, represent a valid JSON value?
Definition: json_binary.cc:865
bool element_offsets(size_t pos, size_t *start, size_t *end, bool *inlined) const
Find the start offset and the end offset of the specified element.
Definition: json_binary.cc:1319
EXPORT_JSON_FUNCTION Value(enum_type t, int64_t val)
Constructor for values that represent ints or uints.
Definition: json_binary.h:334
bool raw_binary(const THD *thd, String *buf) const
Copy the binary representation of this value into a buffer, replacing the contents of the receiving b...
Definition: json_binary.cc:1254
uint32_t m_element_count
Element count for arrays and objects.
Definition: json_binary.h:449
EXPORT_JSON_FUNCTION double get_double() const
Get the value of a DOUBLE.
Definition: json_binary.h:262
Dialog Client Authentication nullptr
Definition: dialog.cc:352
char * pos
Definition: do_ctype.cc:76
This file contains the field type.
enum_field_types
Column types for MySQL.
Definition: field_types.h:57
static void start(mysql_harness::PluginFuncEnv *env)
Definition: http_auth_backend_plugin.cc:165
#define EXPORT_JSON_FUNCTION
Definition: json_binary.h:167
uint32 element_count
Definition: my_tree.h:51
std::string str(const mysqlrouter::ConfigGenerator::Options::Endpoint &ep)
Definition: config_generator.cc:1056
Definition: buf0block_hint.cc:29
bool length(const dd::Spatial_reference_system *srs, const Geometry *g1, double *length, bool *null) noexcept
Computes the length of linestrings and multilinestrings.
Definition: length.cc:75
Definition: json_binary.cc:98
bool space_needed(const THD *thd, const Json_wrapper *value, bool large, size_t *needed)
How much space is needed for a JSON value when it is stored in the binary format.
Definition: json_binary.cc:1536
bool serialize(const THD *thd, const Json_dom *dom, String *dest)
Serialize the JSON document represented by dom to binary format in the destination string,...
Definition: json_binary.cc:130
bool for_each_node(const Value &value, const Func &func)
Apply a function to every value in a JSON document.
Definition: json_binary.h:522
Value parse_binary(const char *data, size_t len)
Parse a JSON binary document.
Definition: json_binary.cc:1066
mutable_buffer buffer(void *p, size_t n) noexcept
Definition: buffer.h:391
Cursor end()
A past-the-end Cursor.
Definition: rules_table_service.cc:191
const string value("\"Value\"")