MySQL 8.1.0
Source Code Documentation
json_binary.h
Go to the documentation of this file.
1#ifndef JSON_BINARY_INCLUDED
2#define JSON_BINARY_INCLUDED
3
4/* Copyright (c) 2015, 2023, Oracle and/or its affiliates.
5
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License, version 2.0,
8 as published by the Free Software Foundation.
9
10 This program is also distributed with certain software (including
11 but not limited to OpenSSL) that is licensed under separate terms,
12 as designated in a particular file or component or in included license
13 documentation. The authors of MySQL hereby grant you an additional
14 permission to link the program and your derivative works with the
15 separately licensed software that they have included with MySQL.
16
17 This program is distributed in the hope that it will be useful,
18 but WITHOUT ANY WARRANTY; without even the implied warranty of
19 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 GNU General Public License, version 2.0, for more details.
21
22 You should have received a copy of the GNU General Public License
23 along with this program; if not, write to the Free Software
24 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
25
26/**
27 @file
28
29 This file specifies the interface for serializing JSON values into
30 binary representation, and for reading values back from the binary
31 representation.
32
33 The binary format is as follows:
34
35 Each JSON value (scalar, object or array) has a one byte type
36 identifier followed by the actual value.
37
38 If the value is a JSON object, its binary representation will have a
39 header that contains:
40
41 - the member count
42 - the size of the binary value in bytes
43 - a list of pointers to each key
44 - a list of pointers to each value
45
46 The actual keys and values will come after the header, in the same
47 order as in the header.
48
49 Similarly, if the value is a JSON array, the binary representation
50 will have a header with
51
52 - the element count
53 - the size of the binary value in bytes
54 - a list of pointers to each value
55
56 followed by the actual values, in the same order as in the header.
57
58 @verbatim
59 doc ::= type value
60
61 type ::=
62 0x00 | // small JSON object
63 0x01 | // large JSON object
64 0x02 | // small JSON array
65 0x03 | // large JSON array
66 0x04 | // literal (true/false/null)
67 0x05 | // int16
68 0x06 | // uint16
69 0x07 | // int32
70 0x08 | // uint32
71 0x09 | // int64
72 0x0a | // uint64
73 0x0b | // double
74 0x0c | // utf8mb4 string
75 0x0f // custom data (any MySQL data type)
76
77 value ::=
78 object |
79 array |
80 literal |
81 number |
82 string |
83 custom-data
84
85 object ::= element-count size key-entry* value-entry* key* value*
86
87 array ::= element-count size value-entry* value*
88
89 // number of members in object or number of elements in array
90 element-count ::=
91 uint16 | // if used in small JSON object/array
92 uint32 // if used in large JSON object/array
93
94 // number of bytes in the binary representation of the object or array
95 size ::=
96 uint16 | // if used in small JSON object/array
97 uint32 // if used in large JSON object/array
98
99 key-entry ::= key-offset key-length
100
101 key-offset ::=
102 uint16 | // if used in small JSON object
103 uint32 // if used in large JSON object
104
105 key-length ::= uint16 // key length must be less than 64KB
106
107 value-entry ::= type offset-or-inlined-value
108
109 // This field holds either the offset to where the value is stored,
110 // or the value itself if it is small enough to be inlined (that is,
111 // if it is a JSON literal or a small enough [u]int).
112 offset-or-inlined-value ::=
113 uint16 | // if used in small JSON object/array
114 uint32 // if used in large JSON object/array
115
116 key ::= utf8mb4-data
117
118 literal ::=
119 0x00 | // JSON null literal
120 0x01 | // JSON true literal
121 0x02 | // JSON false literal
122
123 number ::= .... // little-endian format for [u]int(16|32|64), whereas
124 // double is stored in a platform-independent, eight-byte
125 // format using float8store()
126
127 string ::= data-length utf8mb4-data
128
129 custom-data ::= custom-type data-length binary-data
130
131 custom-type ::= uint8 // type identifier that matches the
132 // internal enum_field_types enum
133
134 data-length ::= uint8* // If the high bit of a byte is 1, the length
135 // field is continued in the next byte,
136 // otherwise it is the last byte of the length
137 // field. So we need 1 byte to represent
138 // lengths up to 127, 2 bytes to represent
139 // lengths up to 16383, and so on...
140 @endverbatim
141*/
142
143#include <stddef.h>
144#include <cassert>
145#include <cstdint>
146#include <functional>
147#include <string>
148
149/*
150 This file is part of our public interface for the 'json_binary' library.
151 This means that #includes of MySQL headers here should be limited to
152 files that are INSTALLed.
153*/
154#include "field_types.h" // enum_field_types
156
157#ifdef MYSQL_SERVER
158class Field_json;
159class Json_dom;
160class Json_wrapper;
161class THD;
162#endif
163
164class String;
165
166#if defined(WIN32) && defined(EXPORT_JSON_FUNCTIONS)
167#define EXPORT_JSON_FUNCTION __declspec(dllexport)
168#else
169#define EXPORT_JSON_FUNCTION
170#endif
171
172namespace json_binary {
173
174/**
175 Serialize the JSON document represented by dom to binary format in
176 the destination string, replacing any content already in the
177 destination string.
178
179 @param[in] thd THD handle
180 @param[in] dom the input DOM tree
181 @param[in,out] dest the destination string
182 @retval false on success
183 @retval true if an error occurred
184*/
185#ifdef MYSQL_SERVER
186bool serialize(const THD *thd, const Json_dom *dom, String *dest);
187#endif
188
189/**
190 Class used for reading JSON values that are stored in the binary
191 format. Values are parsed lazily, so that only the parts of the
192 value that are interesting to the caller, are read. Array elements
193 can be looked up in constant time using the element() function.
194 Object members can be looked up in O(log n) time using the lookup()
195 function.
196*/
197class Value {
198 public:
199 enum enum_type : uint8_t {
210 ERROR /* Not really a type. Used to signal that an
211 error was detected. */
212 };
213
214 /**
215 Does this value, and all of its members, represent a valid JSON
216 value?
217 */
219 bool is_valid() const;
220
222 enum_type type() const { return m_type; }
223
224 /// Does this value use the large storage format?
226 bool large_format() const { return m_large; }
227
228 /**
229 Get a pointer to the beginning of the STRING or OPAQUE data
230 represented by this instance.
231 */
233 const char *get_data() const {
234 assert(m_type == STRING || m_type == OPAQUE);
235 return m_data;
236 }
237
238 /**
239 Get the length in bytes of the STRING or OPAQUE value represented by
240 this instance.
241 */
243 uint32_t get_data_length() const {
244 assert(m_type == STRING || m_type == OPAQUE);
245 return m_length;
246 }
247
248 /** Get the value of an INT. */
250 int64_t get_int64() const {
251 assert(m_type == INT);
252 return m_int_value;
253 }
254
255 /** Get the value of a UINT. */
257 uint64_t get_uint64() const {
258 assert(m_type == UINT);
259 return static_cast<uint64_t>(m_int_value);
260 }
261
262 /** Get the value of a DOUBLE. */
264 double get_double() const {
265 assert(m_type == DOUBLE);
266 return m_double_value;
267 }
268
269 /**
270 Get the number of elements in an array, or the number of members in
271 an object.
272 */
274 uint32_t element_count() const {
275 assert(m_type == ARRAY || m_type == OBJECT);
276 return m_element_count;
277 }
278
279 /**
280 Get the MySQL field type of an opaque value. Identifies the type of
281 the value stored in the data portion of an opaque value.
282 */
285 assert(m_type == OPAQUE);
286 return m_field_type;
287 }
288
290 Value element(size_t pos) const;
291
293 Value key(size_t pos) const;
294
296 Value lookup(const char *key, size_t length) const;
297
299 Value lookup(const std::string &key) const {
300 return lookup(key.c_str(), key.length());
301 }
302
304 size_t lookup_index(const char *key, size_t length) const;
305
307 size_t lookup_index(const std::string &key) const {
308 return lookup_index(key.c_str(), key.length());
309 }
310
312 bool is_backed_by(const String *str) const;
313
314#ifdef MYSQL_SERVER
315 bool raw_binary(const THD *thd, String *buf) const;
316 bool get_free_space(const THD *thd, size_t *space) const;
317 bool update_in_shadow(const Field_json *field, size_t pos,
318 Json_wrapper *new_value, size_t data_offset,
319 size_t data_length, const char *original,
320 char *destination, bool *changed) const;
321 bool remove_in_shadow(const Field_json *field, size_t pos,
322 const char *original, char *destination) const;
323#endif
325 bool has_space(size_t pos, size_t needed, size_t *offset) const;
326
327 /** Constructor for values that represent literals or errors. */
329 explicit Value(enum_type t) : m_data(nullptr), m_type(t) {
330 assert(t == LITERAL_NULL || t == LITERAL_TRUE || t == LITERAL_FALSE ||
331 t == ERROR);
332 }
333
334 /** Constructor for values that represent ints or uints. */
336 explicit Value(enum_type t, int64_t val) : m_int_value(val), m_type(t) {
337 assert(t == INT || t == UINT);
338 }
339
340 /** Constructor for values that represent doubles. */
342 explicit Value(double val) : m_double_value(val), m_type(DOUBLE) {}
343
344 /** Constructor for values that represent strings. */
346 Value(const char *data, uint32_t len)
347 : m_data(data), m_length(len), m_type(STRING) {}
348
349 /**
350 Constructor for values that represent arrays or objects.
351
352 @param t type
353 @param data pointer to the start of the binary representation
354 @param bytes the number of bytes in the binary representation of the value
355 @param element_count the number of elements or members in the value
356 @param large true if the value should be stored in the large
357 storage format with 4 byte offsets instead of 2 byte offsets
358 */
360 Value(enum_type t, const char *data, uint32_t bytes, uint32_t element_count,
361 bool large)
362 : m_data(data),
364 m_length(bytes),
365 m_type(t),
366 m_large(large) {
367 assert(t == ARRAY || t == OBJECT);
368 }
369
370 /** Constructor for values that represent opaque data. */
372 Value(enum_field_types ft, const char *data, uint32_t len)
373 : m_data(data), m_length(len), m_field_type(ft), m_type(OPAQUE) {}
374
375 /** Empty constructor. Produces a value that represents an error condition. */
378
379 /** Is this value an array? */
381 bool is_array() const { return m_type == ARRAY; }
382
383 /** Is this value an object? */
385 bool is_object() const { return m_type == OBJECT; }
386
387 /**
388 Format the JSON value to an external JSON string in buffer in
389 the format of ISO/IEC 10646.
390
391 @param[in,out] buffer the formatted string is appended, so make sure
392 the length is set correctly before calling
393 @param[in] depth_handler Pointer to a function that should handle error
394 occurred when depth is exceeded.
395
396 @return false formatting went well, else true
397 */
399 bool to_std_string(std::string *buffer,
400 const JsonDocumentDepthHandler &depth_handler) const;
401
402 /**
403 Format the JSON value to an external JSON string in buffer in the format of
404 ISO/IEC 10646. Add newlines and indentation for readability.
405
406 @param[in,out] buffer the buffer that receives the formatted string
407 (the string is appended, so make sure the length
408 is set correctly before calling)
409 @param[in] depth_handler Pointer to a function that should handle error
410 occurred when depth is exceeded.
411
412 @retval false on success
413 @retval true on error
414 */
417 std::string *buffer, const JsonDocumentDepthHandler &depth_handler) const;
418
419 /**
420 Compare two Values
421 @note This function is limited to scalars only, for objects/arrays it
422 asserts. The main purpose is to separate old/new scalar values for updates
423 on multi-valued indexes.
424 @returns
425 -1 this < val
426 0 this == val
427 1 this > val
428 */
429#ifdef MYSQL_SERVER
430 int eq(const Value &val) const;
431#endif
432
433 private:
434 /*
435 Instances use only one of m_data, m_int_value and m_double_value,
436 so keep them in a union to save space in memory.
437 */
438 union {
439 /**
440 Pointer to the start of the binary representation of the value. Only
441 used by STRING, OPAQUE, OBJECT and ARRAY.
442
443 The memory pointed to by this member is not owned by this Value
444 object. Callers that create Value objects must make sure that the
445 memory is not freed as long as the Value object is alive.
446 */
447 const char *m_data;
448 /** The value if the type is INT or UINT. */
449 int64_t m_int_value;
450 /** The value if the type is DOUBLE. */
452 };
453
454 /**
455 Element count for arrays and objects. Unused for other types.
456 */
458
459 /**
460 The full length (in bytes) of the binary representation of an array or
461 object, or the length of a string or opaque value. Unused for other types.
462 */
463 uint32_t m_length;
464
465 /**
466 The MySQL field type of the value, in case the type of the value is
467 OPAQUE. Otherwise, it is unused.
468 */
470
471 /** The JSON type of the value. */
473
474 /**
475 True if an array or an object uses the large storage format with 4
476 byte offsets instead of 2 byte offsets.
477 */
479
480 size_t key_entry_offset(size_t pos) const;
481 size_t value_entry_offset(size_t pos) const;
482 bool first_value_offset(size_t *offset) const;
483 bool element_offsets(size_t pos, size_t *start, size_t *end,
484 bool *inlined) const;
485};
486
487/**
488 Parse a JSON binary document.
489
490 @param[in] data a pointer to the binary data
491 @param[in] len the size of the binary document in bytes
492 @return an object that allows access to the contents of the document
493*/
495Value parse_binary(const char *data, size_t len);
496
497/**
498 How much space is needed for a JSON value when it is stored in the binary
499 format.
500
501 @param[in] thd THD handle
502 @param[in] value the JSON value to add to a document
503 @param[in] large true if the large storage format is used
504 @param[out] needed gets set to the amount of bytes needed to store
505 the value
506 @retval false if successful
507 @retval true if an error occurred while calculating the needed space
508*/
509#ifdef MYSQL_SERVER
510bool space_needed(const THD *thd, const Json_wrapper *value, bool large,
511 size_t *needed);
512#endif
513
514/**
515 Apply a function to every value in a JSON document. That is, apply
516 the function to the root node of the JSON document, to all its
517 children, grandchildren and so on.
518
519 @param value the root of the JSON document
520 @param func the function to apply
521 @retval true if the processing was stopped
522 @retval false if the processing was completed
523
524 @tparam Func a functor type that takes a #json_binary::Value
525 parameter and returns a `bool` which is `true` if the processing
526 should stop or `false` if the processing should continue with the
527 next node
528*/
529template <typename Func>
530bool for_each_node(const Value &value, const Func &func) {
531 if (func(value)) return true;
532
533 if (value.is_array() || value.is_object())
534 for (size_t i = 0, size = value.element_count(); i < size; ++i)
535 if (for_each_node(value.element(i), func)) return true;
536
537 return false;
538}
539} // namespace json_binary
540
541#endif /* JSON_BINARY_INCLUDED */
A field that stores a JSON value.
Definition: field.h:3931
JSON DOM abstract base class.
Definition: json_dom.h:174
Abstraction for accessing JSON values irrespective of whether they are (started out as) binary JSON v...
Definition: json_dom.h:1168
Using this class is fraught with peril, and you need to be very careful when doing so.
Definition: sql_string.h:166
For each client connection we create a separate thread with THD serving as a thread/connection descri...
Definition: sql_lexer_thd.h:33
Class used for reading JSON values that are stored in the binary format.
Definition: json_binary.h:197
double m_double_value
The value if the type is DOUBLE.
Definition: json_binary.h:451
EXPORT_JSON_FUNCTION Value key(size_t pos) const
Get the key of the member stored at the specified position in a JSON object.
Definition: json_binary.cc:1132
EXPORT_JSON_FUNCTION uint32_t element_count() const
Get the number of elements in an array, or the number of members in an object.
Definition: json_binary.h:274
EXPORT_JSON_FUNCTION const char * get_data() const
Get a pointer to the beginning of the STRING or OPAQUE data represented by this instance.
Definition: json_binary.h:233
EXPORT_JSON_FUNCTION Value(enum_field_types ft, const char *data, uint32_t len)
Constructor for values that represent opaque data.
Definition: json_binary.h:372
bool update_in_shadow(const Field_json *field, size_t pos, Json_wrapper *new_value, size_t data_offset, size_t data_length, const char *original, char *destination, bool *changed) const
Update a value in an array or object.
Definition: json_binary.cc:1729
EXPORT_JSON_FUNCTION Value lookup(const std::string &key) const
Definition: json_binary.h:299
bool first_value_offset(size_t *offset) const
Find the lowest possible offset where a value can be located inside this array or object.
Definition: json_binary.cc:1380
EXPORT_JSON_FUNCTION enum_field_types field_type() const
Get the MySQL field type of an opaque value.
Definition: json_binary.h:284
EXPORT_JSON_FUNCTION Value(enum_type t, const char *data, uint32_t bytes, uint32_t element_count, bool large)
Constructor for values that represent arrays or objects.
Definition: json_binary.h:360
EXPORT_JSON_FUNCTION size_t lookup_index(const char *key, size_t length) const
Get the index of the element with the specified key in a JSON object.
Definition: json_binary.cc:1184
EXPORT_JSON_FUNCTION int64_t get_int64() const
Get the value of an INT.
Definition: json_binary.h:250
bool m_large
True if an array or an object uses the large storage format with 4 byte offsets instead of 2 byte off...
Definition: json_binary.h:478
EXPORT_JSON_FUNCTION Value(const char *data, uint32_t len)
Constructor for values that represent strings.
Definition: json_binary.h:346
int64_t m_int_value
The value if the type is INT or UINT.
Definition: json_binary.h:449
bool remove_in_shadow(const Field_json *field, size_t pos, const char *original, char *destination) const
Remove a value from an array or object.
Definition: json_binary.cc:1916
EXPORT_JSON_FUNCTION Value element(size_t pos) const
Get the element at the specified position of a JSON array or a JSON object.
Definition: json_binary.cc:1093
bool raw_binary(const THD *thd, String *buf) const
Copy the binary representation of this value into a buffer, replacing the contents of the receiving b...
Definition: json_binary.cc:1258
EXPORT_JSON_FUNCTION uint32_t get_data_length() const
Get the length in bytes of the STRING or OPAQUE value represented by this instance.
Definition: json_binary.h:243
EXPORT_JSON_FUNCTION bool is_object() const
Is this value an object?
Definition: json_binary.h:385
EXPORT_JSON_FUNCTION size_t lookup_index(const std::string &key) const
Definition: json_binary.h:307
EXPORT_JSON_FUNCTION bool has_space(size_t pos, size_t needed, size_t *offset) const
Does this array or object have enough space to replace the value at the given position with another v...
Definition: json_binary.cc:1412
EXPORT_JSON_FUNCTION bool is_valid() const
Does this value, and all of its members, represent a valid JSON value?
Definition: json_binary.cc:866
EXPORT_JSON_FUNCTION Value(double val)
Constructor for values that represent doubles.
Definition: json_binary.h:342
uint32_t m_length
The full length (in bytes) of the binary representation of an array or object, or the length of a str...
Definition: json_binary.h:463
enum_type
Definition: json_binary.h:199
@ DOUBLE
Definition: json_binary.h:205
@ LITERAL_FALSE
Definition: json_binary.h:208
@ INT
Definition: json_binary.h:203
@ STRING
Definition: json_binary.h:202
@ OBJECT
Definition: json_binary.h:200
@ LITERAL_TRUE
Definition: json_binary.h:207
@ ERROR
Definition: json_binary.h:210
@ LITERAL_NULL
Definition: json_binary.h:206
@ UINT
Definition: json_binary.h:204
@ OPAQUE
Definition: json_binary.h:209
@ ARRAY
Definition: json_binary.h:201
size_t key_entry_offset(size_t pos) const
Get the offset of the key entry that describes the key of the member at a given position in this obje...
Definition: json_binary.cc:1513
EXPORT_JSON_FUNCTION Value()
Empty constructor.
Definition: json_binary.h:377
EXPORT_JSON_FUNCTION enum_type type() const
Definition: json_binary.h:222
EXPORT_JSON_FUNCTION bool to_std_string(std::string *buffer, const JsonDocumentDepthHandler &depth_handler) const
Format the JSON value to an external JSON string in buffer in the format of ISO/IEC 10646.
Definition: json_binary.cc:2106
EXPORT_JSON_FUNCTION Value(enum_type t)
Constructor for values that represent literals or errors.
Definition: json_binary.h:329
enum_field_types m_field_type
The MySQL field type of the value, in case the type of the value is OPAQUE.
Definition: json_binary.h:469
bool element_offsets(size_t pos, size_t *start, size_t *end, bool *inlined) const
Find the start offset and the end offset of the specified element.
Definition: json_binary.cc:1323
size_t value_entry_offset(size_t pos) const
Get the offset of the value entry that describes the element at a given position in this array or obj...
Definition: json_binary.cc:1526
EXPORT_JSON_FUNCTION bool to_pretty_std_string(std::string *buffer, const JsonDocumentDepthHandler &depth_handler) const
Format the JSON value to an external JSON string in buffer in the format of ISO/IEC 10646.
Definition: json_binary.cc:2118
EXPORT_JSON_FUNCTION bool is_array() const
Is this value an array?
Definition: json_binary.h:381
EXPORT_JSON_FUNCTION bool large_format() const
Does this value use the large storage format?
Definition: json_binary.h:226
bool get_free_space(const THD *thd, size_t *space) const
Get the amount of unused space in the binary representation of this value.
Definition: json_binary.cc:1969
enum_type m_type
The JSON type of the value.
Definition: json_binary.h:472
const char * m_data
Pointer to the start of the binary representation of the value.
Definition: json_binary.h:447
EXPORT_JSON_FUNCTION uint64_t get_uint64() const
Get the value of a UINT.
Definition: json_binary.h:257
int eq(const Value &val) const
Compare two Values.
Definition: json_binary.cc:2061
EXPORT_JSON_FUNCTION Value(enum_type t, int64_t val)
Constructor for values that represent ints or uints.
Definition: json_binary.h:336
EXPORT_JSON_FUNCTION bool is_backed_by(const String *str) const
Is this binary value pointing to data that is contained in the specified string.
Definition: json_binary.cc:1232
uint32_t m_element_count
Element count for arrays and objects.
Definition: json_binary.h:457
EXPORT_JSON_FUNCTION double get_double() const
Get the value of a DOUBLE.
Definition: json_binary.h:264
EXPORT_JSON_FUNCTION Value lookup(const char *key, size_t length) const
Get the value associated with the specified key in a JSON object.
Definition: json_binary.cc:1170
Fido Client Authentication nullptr
Definition: fido_client_plugin.cc:221
This file contains the field type.
enum_field_types
Column types for MySQL Note: Keep include/mysql/components/services/bits/stored_program_bits....
Definition: field_types.h:54
static void start(mysql_harness::PluginFuncEnv *env)
Definition: http_auth_backend_plugin.cc:176
#define EXPORT_JSON_FUNCTION
Definition: json_binary.h:169
std::function< void()> JsonDocumentDepthHandler
Definition: json_error_handler.h:31
uint32 element_count
Definition: my_tree.h:51
std::string str(const mysqlrouter::ConfigGenerator::Options::Endpoint &ep)
Definition: config_generator.cc:1063
Definition: buf0block_hint.cc:29
bool length(const dd::Spatial_reference_system *srs, const Geometry *g1, double *length, bool *null) noexcept
Computes the length of linestrings and multilinestrings.
Definition: length.cc:75
Definition: json_binary.cc:99
bool space_needed(const THD *thd, const Json_wrapper *value, bool large, size_t *needed)
How much space is needed for a JSON value when it is stored in the binary format.
Definition: json_binary.cc:1540
bool serialize(const THD *thd, const Json_dom *dom, String *dest)
Serialize the JSON document represented by dom to binary format in the destination string,...
Definition: json_binary.cc:131
bool for_each_node(const Value &value, const Func &func)
Apply a function to every value in a JSON document.
Definition: json_binary.h:530
Value parse_binary(const char *data, size_t len)
Parse a JSON binary document.
Definition: json_binary.cc:1068
mutable_buffer buffer(void *p, size_t n) noexcept
Definition: buffer.h:419