MySQL  8.0.17
Source Code Documentation
rpl_utility.h
Go to the documentation of this file.
1 /* Copyright (c) 2006, 2019, Oracle and/or its affiliates. All rights reserved.
2 
3  This program is free software; you can redistribute it and/or modify
4  it under the terms of the GNU General Public License, version 2.0,
5  as published by the Free Software Foundation.
6 
7  This program is also distributed with certain software (including
8  but not limited to OpenSSL) that is licensed under separate terms,
9  as designated in a particular file or component or in included license
10  documentation. The authors of MySQL hereby grant you an additional
11  permission to link the program and your derivative works with the
12  separately licensed software that they have included with MySQL.
13 
14  This program is distributed in the hope that it will be useful,
15  but WITHOUT ANY WARRANTY; without even the implied warranty of
16  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  GNU General Public License, version 2.0, for more details.
18 
19  You should have received a copy of the GNU General Public License
20  along with this program; if not, write to the Free Software
21  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
22 
23 #ifndef RPL_UTILITY_H
24 #define RPL_UTILITY_H
25 
26 #ifndef __cplusplus
27 #error "Don't include this C++ header file from a non-C++ file!"
28 #endif
29 
30 #include <sys/types.h>
31 #include <unordered_map>
32 
33 #include "field_types.h" // enum_field_types
34 #include "my_dbug.h"
35 #include "my_inttypes.h"
36 #include "my_macros.h"
37 #include "sql/psi_memory_key.h"
38 
39 struct MY_BITMAP;
40 
41 #ifdef MYSQL_SERVER
42 #include <memory>
43 
44 #include "map_helpers.h"
45 #include "prealloced_array.h" // Prealloced_array
46 #include "sql/table.h" // TABLE_LIST
47 
48 class Log_event;
49 class Relay_log_info;
50 class THD;
51 
52 /**
53  Hash table used when applying row events on the slave and there is
54  no index on the slave's table.
55  */
56 
57 struct HASH_ROW_POS {
58  /**
59  Points at the position where the row starts in the
60  event buffer (ie, area in memory before unpacking takes
61  place).
62  */
63  const uchar *bi_start;
64  const uchar *bi_ends;
65 };
66 
67 struct HASH_ROW_ENTRY;
68 
70  void operator()(HASH_ROW_ENTRY *entry) const;
71 };
72 
73 /**
74  Internal structure that acts as a preamble for HASH_ROW_POS
75  in memory structure.
76 
77  Allocation is done in Hash_slave_rows::make_entry as part of
78  the entry allocation.
79  */
81  HASH_ROW_PREAMBLE() = default;
82  /*
83  The actual key.
84  */
86 
87  /**
88  The search state used to iterate over multiple entries for a
89  given key.
90  */
92  uint, std::unique_ptr<HASH_ROW_ENTRY, hash_slave_rows_free_entry>>::
93  const_iterator search_state;
94 
95  /**
96  Wether this search_state is usable or not.
97  */
99 };
100 
104 };
105 
107  public:
108  /**
109  Allocates an empty entry to be added to the hash table.
110  It should be called before calling member function @c put.
111 
112  @returns NULL if a problem occurred, a valid pointer otherwise.
113  */
115 
116  /**
117  Allocates an entry to be added to the hash table. It should be
118  called before calling member function @c put.
119 
120  @param bi_start the position to where in the rows buffer the
121  before image begins.
122  @param bi_ends the position to where in the rows buffer the
123  before image ends.
124  @returns NULL if a problem occurred, a valid pointer otherwise.
125  */
126  HASH_ROW_ENTRY *make_entry(const uchar *bi_start, const uchar *bi_ends);
127 
128  /**
129  Puts data into the hash table. It calculates the key taking
130  the data on @c TABLE::record as the input for hash computation.
131 
132  @param table The table holding the buffer used to calculate the
133  key, ie, table->record[0].
134  @param cols The read_set bitmap signaling which columns are used.
135  @param entry The entry with the values to store.
136 
137  @returns true if something went wrong, false otherwise.
138  */
139  bool put(TABLE *table, MY_BITMAP *cols, HASH_ROW_ENTRY *entry);
140 
141  /**
142  Gets the entry, from the hash table, that matches the data in
143  table->record[0] and signaled using cols.
144 
145  @param table The table holding the buffer containing data used to
146  make the entry lookup.
147  @param cols Bitmap signaling which columns, from
148  table->record[0], should be used.
149 
150  @returns a pointer that will hold a reference to the entry
151  found. If the entry is not found then NULL shall be
152  returned.
153  */
154  HASH_ROW_ENTRY *get(TABLE *table, MY_BITMAP *cols);
155 
156  /**
157  Gets the entry that stands next to the one pointed to by
158  *entry. Before calling this member function, the entry that one
159  uses as parameter must have: 1. been obtained through get() or
160  next() invocations; and 2. must have not been used before in a
161  next() operation.
162 
163  @param[in,out] entry contains a pointer to an entry that we can
164  use to search for another adjacent entry
165  (ie, that shares the same key).
166 
167  @returns true if something went wrong, false otherwise. In the
168  case that this entry was already used in a next()
169  operation this member function returns true and does not
170  update the pointer.
171  */
172  bool next(HASH_ROW_ENTRY **entry);
173 
174  /**
175  Deletes the entry pointed by entry. It also frees memory used
176  holding entry contents. This is the way to release memeory
177  used for entry, freeing it explicitly with my_free will cause
178  undefined behavior.
179 
180  @param entry Pointer to the entry to be deleted.
181  @returns true if something went wrong, false otherwise.
182  */
183  bool del(HASH_ROW_ENTRY *entry);
184 
185  /**
186  Initializes the hash table.
187 
188  @returns true if something went wrong, false otherwise.
189  */
190  bool init(void);
191 
192  /**
193  De-initializes the hash table.
194 
195  @returns true if something went wrong, false otherwise.
196  */
197  bool deinit(void);
198 
199  /**
200  Checks if the hash table is empty or not.
201 
202  @returns true if the hash table has zero entries, false otherwise.
203  */
204  bool is_empty(void);
205 
206  /**
207  Returns the number of entries in the hash table.
208 
209  @returns the number of entries in the hash table.
210  */
211  int size();
212 
213  private:
214  /**
215  The hashtable itself.
216  */
218  uint, std::unique_ptr<HASH_ROW_ENTRY, hash_slave_rows_free_entry>>
220 
221  /**
222  Auxiliary and internal method used to create an hash key, based on
223  the data in table->record[0] buffer and signaled as used in cols.
224 
225  @param table The table that is being scanned
226  @param cols The read_set bitmap signaling which columns are used.
227 
228  @returns the hash key created.
229  */
230  uint make_hash_key(TABLE *table, MY_BITMAP *cols);
231 };
232 
233 #endif
234 
235 /**
236  A table definition from the master.
237 
238  The responsibilities of this class is:
239  - Extract and decode table definition data from the table map event
240  - Check if table definition in table map is compatible with table
241  definition on slave
242  - expose the type information so that it can be used when encoding
243  or decoding row event data.
244 */
245 class table_def {
246  public:
247  /**
248  No-op constructor. Instances of RPL_TABLE_LIST are created by first
249  allocating memory, then placement-new-ing an RPL_TABLE_LIST object
250  containing an uninitialized table_def object which is only conditionally
251  initialized. See Table_map_log_event::do_apply_event().
252  */
254 
255  /**
256  Constructor.
257 
258  @param types Array of types, each stored as a byte
259  @param size Number of elements in array 'types'
260  @param field_metadata Array of extra information about fields
261  @param metadata_size Size of the field_metadata array
262  @param null_bitmap The bitmap of fields that can be null
263  @param flags Table flags
264  */
265  table_def(unsigned char *types, ulong size, uchar *field_metadata,
266  int metadata_size, uchar *null_bitmap, uint16 flags);
267 
268  ~table_def();
269 
270  /**
271  Return the number of fields there is type data for.
272 
273  @return The number of fields that there is type data for.
274  */
275  ulong size() const { return m_size; }
276 
277  /*
278  Returns internal binlog type code for one field,
279  without translation to real types.
280  */
282  return static_cast<enum_field_types>(m_type[index]);
283  }
284 
285  /// Return the number of JSON columns in this table.
286  int json_column_count() const {
287  // Cache in member field to make successive calls faster.
288  if (m_json_column_count == -1) {
289  int c = 0;
290  for (uint i = 0; i < size(); i++)
291  if (type(i) == MYSQL_TYPE_JSON) c++;
293  }
294  return m_json_column_count;
295  }
296 
297  /*
298  Return a representation of the type data for one field.
299 
300  @param index Field index to return data for
301 
302  @return Will return a representation of the type data for field
303  <code>index</code>. Currently, only the type identifier is
304  returned.
305  */
308  /*
309  If the source type is MYSQL_TYPE_STRING, it can in reality be
310  either MYSQL_TYPE_STRING, MYSQL_TYPE_ENUM, or MYSQL_TYPE_SET, so
311  we might need to modify the type to get the real type.
312  */
313  enum_field_types source_type = binlog_type(index);
314  uint source_metadata = m_field_metadata[index];
315  switch (source_type) {
316  case MYSQL_TYPE_STRING: {
317  int real_type = source_metadata >> 8;
318  if (real_type == MYSQL_TYPE_ENUM || real_type == MYSQL_TYPE_SET)
319  source_type = static_cast<enum_field_types>(real_type);
320  break;
321  }
322 
323  /*
324  This type has not been used since before row-based replication,
325  so we can safely assume that it really is MYSQL_TYPE_NEWDATE.
326  */
327  case MYSQL_TYPE_DATE:
328  source_type = MYSQL_TYPE_NEWDATE;
329  break;
330 
331  default:
332  /* Do nothing */
333  break;
334  }
335 
336  return source_type;
337  }
338 
339  /*
340  This function allows callers to get the extra field data from the
341  table map for a given field. If there is no metadata for that field
342  or there is no extra metadata at all, the function returns 0.
343 
344  The function returns the value for the field metadata for column at
345  position indicated by index. As mentioned, if the field was a type
346  that stores field metadata, that value is returned else zero (0) is
347  returned. This method is used in the unpack() methods of the
348  corresponding fields to properly extract the data from the binary log
349  in the event that the master's field is smaller than the slave.
350  */
354  return m_field_metadata[index];
355  else
356  return 0;
357  }
358 
359  /**
360  Returns whether or not the field at `index` is a typed array.
361  */
362  bool is_array(uint index) const {
365  return m_is_array[index];
366  else
367  return false;
368  }
369 
370  /*
371  This function returns whether the field on the master can be null.
372  This value is derived from field->maybe_null().
373  */
374  bool maybe_null(uint index) const {
376  return ((m_null_bits[(index / 8)] & (1 << (index % 8))) ==
377  (1 << (index % 8)));
378  }
379 
380  /*
381  This function returns the field size in raw bytes based on the type
382  and the encoded field data from the master's raw data. This method can
383  be used for situations where the slave needs to skip a column (e.g.,
384  WL#3915) or needs to advance the pointer for the fields in the raw
385  data from the master to a specific column.
386  */
387  uint32 calc_field_size(uint col, const uchar *master_data) const;
388 
389 #ifdef MYSQL_SERVER
390  /**
391  Decide if the table definition is compatible with a table.
392 
393  Compare the definition with a table to see if it is compatible
394  with it.
395 
396  A table definition is compatible with a table if:
397  - The columns types of the table definition is a (not
398  necessarily proper) prefix of the column type of the table.
399 
400  - The other way around.
401 
402  - Each column on the master that also exists on the slave can be
403  converted according to the current settings of @c
404  SLAVE_TYPE_CONVERSIONS.
405 
406  @param thd Current thread
407  @param rli Pointer to relay log info
408  @param table Pointer to table to compare with.
409 
410  @param[out] conv_table_var Pointer to temporary table for holding
411  conversion table.
412 
413  @retval 1 if the table definition is not compatible with @c table
414  @retval 0 if the table definition is compatible with @c table
415  */
416  bool compatible_with(THD *thd, Relay_log_info *rli, TABLE *table,
417  TABLE **conv_table_var) const;
418 
419  /**
420  Create a virtual in-memory temporary table structure.
421 
422  The table structure has records and field array so that a row can
423  be unpacked into the record for further processing.
424 
425  In the virtual table, each field that requires conversion will
426  have a non-NULL value, while fields that do not require
427  conversion will have a NULL value.
428 
429  Some information that is missing in the events, such as the
430  character set for string types, are taken from the table that the
431  field is going to be pushed into, so the target table that the data
432  eventually need to be pushed into need to be supplied.
433 
434  @param thd Thread to allocate memory from.
435  @param rli Relay log info structure, for error reporting.
436  @param target_table Target table for fields.
437 
438  @return A pointer to a temporary table with memory allocated in the
439  thread's memroot, NULL if the table could not be created
440  */
442  TABLE *target_table) const;
443 #endif
444 
445  private:
446  ulong m_size; // Number of elements in the types array
447  unsigned char *m_type; // Array of type descriptors
451  uint16 m_flags; // Table flags
453  mutable int m_json_column_count; // Number of JSON columns
454  bool *m_is_array;
455 };
456 
457 #ifdef MYSQL_SERVER
458 /**
459  Extend the normal table list with a few new fields needed by the
460  slave thread, but nowhere else.
461  */
462 struct RPL_TABLE_LIST : public TABLE_LIST {
463  RPL_TABLE_LIST(const char *db_name_arg, size_t db_length_arg,
464  const char *table_name_arg, size_t table_name_length_arg,
465  const char *alias_arg, enum thr_lock_type lock_type_arg)
466  : TABLE_LIST(nullptr, db_name_arg, db_length_arg, table_name_arg,
467  table_name_length_arg, alias_arg, lock_type_arg) {}
468 
472 };
473 
475  private:
477 
478  public:
481  /* queue for exection at Query-log-event time prior the Query */
482  int add(Log_event *ev);
483  bool is_empty();
484  bool execute(Relay_log_info *rli);
485  void rewind();
486 };
487 
488 #endif
489 
490 /**
491  Decode field metadata from a char buffer (serialized form) into an int
492  (packed form).
493 
494  @note On little-endian platforms (e.g Intel) this function effectively
495  inverts order of bytes compared to what Field::save_field_metadata()
496  writes. E.g for MYSQL_TYPE_NEWDECIMAL save_field_metadata writes precision
497  into the first byte and decimals into the second, this function puts
498  precision into the second byte and decimals into the first. This layout
499  is expected by replication code that reads metadata in the uint form.
500  Due to this design feature show_sql_type() can't correctly print
501  immediate output of save_field_metadata(), this function have to be used
502  as translator.
503 
504  @param buffer Field metadata, in the character stream form produced by
505  save_field_metadata.
506  @param binlog_type The type of the field, in the form returned by
507  Field::binlog_type and stored in Table_map_log_event.
508  @retval pair where:
509  - the first component is the length of the metadata within 'buffer',
510  i.e., how much the buffer pointer should move forward in order to skip it.
511  - the second component is pair containing:
512  - the metadata, encoded as an 'uint', in the form required by e.g.
513  show_sql_type.
514  - bool indicating whether the field is array (true) or a scalar (false)
515 */
516 
517 std::pair<my_off_t, std::pair<uint, bool>> read_field_metadata(
518  const uchar *metadata_ptr, enum_field_types type);
519 
520 // NB. number of printed bit values is limited to sizeof(buf) - 1
521 #define DBUG_PRINT_BITSET(N, FRM, BS) \
522  do { \
523  char buf[256]; \
524  uint i; \
525  for (i = 0; i < MY_MIN(sizeof(buf) - 1, (BS)->n_bits); i++) \
526  buf[i] = bitmap_is_set((BS), i) ? '1' : '0'; \
527  buf[i] = '\0'; \
528  DBUG_PRINT((N), ((FRM), buf)); \
529  } while (0)
530 
531 #endif /* RPL_UTILITY_H */
bool next(HASH_ROW_ENTRY **entry)
Gets the entry that stands next to the one pointed to by entry.
Definition: rpl_utility.cc:995
bool del(HASH_ROW_ENTRY *entry)
Deletes the entry pointed by entry.
Definition: rpl_utility.cc:1036
Some common macros.
This file contains the field type.
unsigned char uchar
Definition: my_inttypes.h:51
table_def()
No-op constructor.
Definition: rpl_utility.h:253
bool compatible_with(THD *thd, Relay_log_info *rli, TABLE *table, TABLE **conv_table_var) const
Decide if the table definition is compatible with a table.
Definition: rpl_utility.cc:459
bool put(TABLE *table, MY_BITMAP *cols, HASH_ROW_ENTRY *entry)
Puts data into the hash table.
Definition: rpl_utility.cc:949
bool is_empty(void)
Checks if the hash table is empty or not.
Definition: rpl_utility.cc:883
Internal structure that acts as a preamble for HASH_ROW_POS in memory structure.
Definition: rpl_utility.h:80
bool is_array(uint index) const
Returns whether or not the field at index is a typed array.
Definition: rpl_utility.h:362
Internal to MySQL.
Definition: field_types.h:71
TABLE * create_conversion_table(THD *thd, Relay_log_info *rli, TABLE *target_table) const
Create a virtual in-memory temporary table structure.
Definition: rpl_utility.cc:594
uint32 calc_field_size(uint col, const uchar *master_data) const
Definition: rpl_utility.cc:91
void rewind()
Definition: rpl_utility.cc:1175
Some integer typedefs for easier portability.
void operator()(HASH_ROW_ENTRY *entry) const
Utility methods for handling row based operations.
Definition: rpl_utility.cc:870
bool is_empty()
Definition: rpl_utility.cc:1159
Definition: field_types.h:80
Definition: rpl_utility.h:106
PSI_memory_key key_memory_HASH_ROW_ENTRY
Definition: psi_memory_key.cc:50
RPL_TABLE_LIST(const char *db_name_arg, size_t db_length_arg, const char *table_name_arg, size_t table_name_length_arg, const char *alias_arg, enum thr_lock_type lock_type_arg)
Definition: rpl_utility.h:463
enum_field_types binlog_type(ulong index) const
Definition: rpl_utility.h:281
TABLE * m_conv_table
Definition: rpl_utility.h:471
Hash table used when applying row events on the slave and there is no index on the slave&#39;s table...
Definition: rpl_utility.h:57
Deferred_log_events()
Definition: rpl_utility.cc:1148
std::pair< my_off_t, std::pair< uint, bool > > read_field_metadata(const uchar *metadata_ptr, enum_field_types type)
Decode field metadata from a char buffer (serialized form) into an int (packed form).
Definition: rpl_utility.cc:740
Definition: table.h:1294
uint16 m_flags
Definition: rpl_utility.h:451
HASH_ROW_PREAMBLE()=default
uint * m_field_metadata
Definition: rpl_utility.h:449
int m_json_column_count
Definition: rpl_utility.h:453
Used for replication only.
Definition: field_types.h:78
ulong size() const
Return the number of fields there is type data for.
Definition: rpl_utility.h:275
Definition: field_types.h:81
#define DBUG_ASSERT(A)
Definition: my_dbug.h:183
bool maybe_null(uint index) const
Definition: rpl_utility.h:374
HASH_ROW_ENTRY * make_entry()
Allocates an empty entry to be added to the hash table.
Definition: rpl_utility.cc:899
bool execute(Relay_log_info *rli)
Definition: rpl_utility.cc:1161
char * index(const char *, int c)
Definition: mysql.cc:2847
Definition: rpl_rli.h:164
uint hash_value
Definition: rpl_utility.h:85
thr_lock_type
Definition: thr_lock.h:50
bool m_tabledef_valid
Definition: rpl_utility.h:469
Definition: my_bitmap.h:42
~table_def()
Definition: rpl_utility.cc:854
HASH_ROW_POS * positions
Definition: rpl_utility.h:103
This is the abstract base class for binary log events.
Definition: log_event.h:517
table_def m_tabledef
Definition: rpl_utility.h:470
Definition: rpl_utility.h:69
std::unordered_multimap, but with my_malloc, so that you can track the memory used using PSI memory k...
Definition: map_helpers.h:215
enum_field_types
Column types for MySQL.
Definition: field_types.h:52
const uchar * bi_ends
Definition: rpl_utility.h:64
uint16_t uint16
Definition: my_inttypes.h:60
unsigned int uint
Definition: uca-dump.cc:29
Definition: field_types.h:87
uint m_field_metadata_size
Definition: rpl_utility.h:448
uint32_t uint32
Definition: my_inttypes.h:62
ulong m_size
Definition: rpl_utility.h:446
int add(Log_event *ev)
Definition: rpl_utility.cc:1153
uint field_metadata(uint index) const
Definition: rpl_utility.h:351
enum_field_types type(ulong index) const
Definition: rpl_utility.h:306
uint make_hash_key(TABLE *table, MY_BITMAP *cols)
Auxiliary and internal method used to create an hash key, based on the data in table->record[0] buffe...
Definition: rpl_utility.cc:1044
bool * m_is_array
Definition: rpl_utility.h:454
int type
Definition: http_common.h:411
bool init(void)
Initializes the hash table.
Definition: rpl_utility.cc:889
bool is_search_state_inited
Wether this search_state is usable or not.
Definition: rpl_utility.h:98
const uchar * bi_start
Points at the position where the row starts in the event buffer (ie, area in memory before unpacking ...
Definition: rpl_utility.h:63
int size()
Returns the number of entries in the hash table.
Definition: rpl_utility.cc:897
uchar * m_null_bits
Definition: rpl_utility.h:450
HASH_ROW_PREAMBLE * preamble
Definition: rpl_utility.h:102
uchar * m_memory
Definition: rpl_utility.h:452
A table definition from the master.
Definition: rpl_utility.h:245
Definition: field_types.h:67
Prealloced_array< Log_event *, 32 > m_array
Definition: rpl_utility.h:476
Definition: table.h:2442
Extend the normal table list with a few new fields needed by the slave thread, but nowhere else...
Definition: rpl_utility.h:462
static int flags[50]
Definition: hp_test1.cc:39
unsigned char * m_type
Definition: rpl_utility.h:447
malloc_unordered_multimap< uint, std::unique_ptr< HASH_ROW_ENTRY, hash_slave_rows_free_entry > >::const_iterator search_state
The search state used to iterate over multiple entries for a given key.
Definition: rpl_utility.h:93
Definition: rpl_utility.h:101
~Deferred_log_events()
Definition: rpl_utility.cc:1151
Definition: rpl_utility.h:474
unsigned long ulong
Definition: my_inttypes.h:48
Definition: completion_hash.h:34
bool deinit(void)
De-initializes the hash table.
Definition: rpl_utility.cc:891
For each client connection we create a separate thread with THD serving as a thread/connection descri...
Definition: sql_class.h:777
int json_column_count() const
Return the number of JSON columns in this table.
Definition: rpl_utility.h:286
malloc_unordered_multimap< uint, std::unique_ptr< HASH_ROW_ENTRY, hash_slave_rows_free_entry > > m_hash
The hashtable itself.
Definition: rpl_utility.h:219