MySQL  8.0.16
Source Code Documentation
rpl_utility.h
Go to the documentation of this file.
1 /* Copyright (c) 2006, 2019, Oracle and/or its affiliates. All rights reserved.
2 
3  This program is free software; you can redistribute it and/or modify
4  it under the terms of the GNU General Public License, version 2.0,
5  as published by the Free Software Foundation.
6 
7  This program is also distributed with certain software (including
8  but not limited to OpenSSL) that is licensed under separate terms,
9  as designated in a particular file or component or in included license
10  documentation. The authors of MySQL hereby grant you an additional
11  permission to link the program and your derivative works with the
12  separately licensed software that they have included with MySQL.
13 
14  This program is distributed in the hope that it will be useful,
15  but WITHOUT ANY WARRANTY; without even the implied warranty of
16  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  GNU General Public License, version 2.0, for more details.
18 
19  You should have received a copy of the GNU General Public License
20  along with this program; if not, write to the Free Software
21  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
22 
23 #ifndef RPL_UTILITY_H
24 #define RPL_UTILITY_H
25 
26 #ifndef __cplusplus
27 #error "Don't include this C++ header file from a non-C++ file!"
28 #endif
29 
30 #include <sys/types.h>
31 #include <unordered_map>
32 
33 #include "field_types.h" // enum_field_types
34 #include "my_dbug.h"
35 #include "my_inttypes.h"
36 #include "my_macros.h"
37 #include "sql/psi_memory_key.h"
38 
39 struct MY_BITMAP;
40 
41 #ifdef MYSQL_SERVER
42 #include <memory>
43 
44 #include "map_helpers.h"
45 #include "prealloced_array.h" // Prealloced_array
46 #include "sql/table.h" // TABLE_LIST
47 
48 class Log_event;
49 class Relay_log_info;
50 class THD;
51 
52 /**
53  Hash table used when applying row events on the slave and there is
54  no index on the slave's table.
55  */
56 
57 struct HASH_ROW_POS {
58  /**
59  Points at the position where the row starts in the
60  event buffer (ie, area in memory before unpacking takes
61  place).
62  */
63  const uchar *bi_start;
64  const uchar *bi_ends;
65 };
66 
67 struct HASH_ROW_ENTRY;
68 
70  void operator()(HASH_ROW_ENTRY *entry) const;
71 };
72 
73 /**
74  Internal structure that acts as a preamble for HASH_ROW_POS
75  in memory structure.
76 
77  Allocation is done in Hash_slave_rows::make_entry as part of
78  the entry allocation.
79  */
81  HASH_ROW_PREAMBLE() = default;
82  /*
83  The actual key.
84  */
86 
87  /**
88  The search state used to iterate over multiple entries for a
89  given key.
90  */
92  uint, std::unique_ptr<HASH_ROW_ENTRY, hash_slave_rows_free_entry>>::
93  const_iterator search_state;
94 
95  /**
96  Wether this search_state is usable or not.
97  */
99 };
100 
104 };
105 
107  public:
108  /**
109  Allocates an empty entry to be added to the hash table.
110  It should be called before calling member function @c put.
111 
112  @returns NULL if a problem occured, a valid pointer otherwise.
113  */
115 
116  /**
117  Allocates an entry to be added to the hash table. It should be
118  called before calling member function @c put.
119 
120  @param bi_start the position to where in the rows buffer the
121  before image begins.
122  @param bi_ends the position to where in the rows buffer the
123  before image ends.
124  @returns NULL if a problem occured, a valid pointer otherwise.
125  */
126  HASH_ROW_ENTRY *make_entry(const uchar *bi_start, const uchar *bi_ends);
127 
128  /**
129  Puts data into the hash table. It calculates the key taking
130  the data on @c TABLE::record as the input for hash computation.
131 
132  @param table The table holding the buffer used to calculate the
133  key, ie, table->record[0].
134  @param cols The read_set bitmap signaling which columns are used.
135  @param entry The entry with the values to store.
136 
137  @returns true if something went wrong, false otherwise.
138  */
139  bool put(TABLE *table, MY_BITMAP *cols, HASH_ROW_ENTRY *entry);
140 
141  /**
142  Gets the entry, from the hash table, that matches the data in
143  table->record[0] and signaled using cols.
144 
145  @param table The table holding the buffer containing data used to
146  make the entry lookup.
147  @param cols Bitmap signaling which columns, from
148  table->record[0], should be used.
149 
150  @returns a pointer that will hold a reference to the entry
151  found. If the entry is not found then NULL shall be
152  returned.
153  */
154  HASH_ROW_ENTRY *get(TABLE *table, MY_BITMAP *cols);
155 
156  /**
157  Gets the entry that stands next to the one pointed to by
158  *entry. Before calling this member function, the entry that one
159  uses as parameter must have: 1. been obtained through get() or
160  next() invocations; and 2. must have not been used before in a
161  next() operation.
162 
163  @param[in,out] entry contains a pointer to an entry that we can
164  use to search for another adjacent entry
165  (ie, that shares the same key).
166 
167  @returns true if something went wrong, false otherwise. In the
168  case that this entry was already used in a next()
169  operation this member function returns true and does not
170  update the pointer.
171  */
172  bool next(HASH_ROW_ENTRY **entry);
173 
174  /**
175  Deletes the entry pointed by entry. It also frees memory used
176  holding entry contents. This is the way to release memeory
177  used for entry, freeing it explicitly with my_free will cause
178  undefined behavior.
179 
180  @param entry Pointer to the entry to be deleted.
181  @returns true if something went wrong, false otherwise.
182  */
183  bool del(HASH_ROW_ENTRY *entry);
184 
185  /**
186  Initializes the hash table.
187 
188  @returns true if something went wrong, false otherwise.
189  */
190  bool init(void);
191 
192  /**
193  De-initializes the hash table.
194 
195  @returns true if something went wrong, false otherwise.
196  */
197  bool deinit(void);
198 
199  /**
200  Checks if the hash table is empty or not.
201 
202  @returns true if the hash table has zero entries, false otherwise.
203  */
204  bool is_empty(void);
205 
206  /**
207  Returns the number of entries in the hash table.
208 
209  @returns the number of entries in the hash table.
210  */
211  int size();
212 
213  private:
214  /**
215  The hashtable itself.
216  */
218  uint, std::unique_ptr<HASH_ROW_ENTRY, hash_slave_rows_free_entry>>
220 
221  /**
222  Auxiliary and internal method used to create an hash key, based on
223  the data in table->record[0] buffer and signaled as used in cols.
224 
225  @param table The table that is being scanned
226  @param cols The read_set bitmap signaling which columns are used.
227 
228  @returns the hash key created.
229  */
230  uint make_hash_key(TABLE *table, MY_BITMAP *cols);
231 };
232 
233 #endif
234 
235 /**
236  A table definition from the master.
237 
238  The responsibilities of this class is:
239  - Extract and decode table definition data from the table map event
240  - Check if table definition in table map is compatible with table
241  definition on slave
242  - expose the type information so that it can be used when encoding
243  or decoding row event data.
244 */
245 class table_def {
246  public:
247  /**
248  No-op constructor. Instances of RPL_TABLE_LIST are created by first
249  allocating memory, then placement-new-ing an RPL_TABLE_LIST object
250  containing an uninitialized table_def object which is only conditionally
251  initialized. See Table_map_log_event::do_apply_event().
252  */
254 
255  /**
256  Constructor.
257 
258  @param types Array of types, each stored as a byte
259  @param size Number of elements in array 'types'
260  @param field_metadata Array of extra information about fields
261  @param metadata_size Size of the field_metadata array
262  @param null_bitmap The bitmap of fields that can be null
263  @param flags Table flags
264  */
265  table_def(unsigned char *types, ulong size, uchar *field_metadata,
266  int metadata_size, uchar *null_bitmap, uint16 flags);
267 
268  ~table_def();
269 
270  /**
271  Return the number of fields there is type data for.
272 
273  @return The number of fields that there is type data for.
274  */
275  ulong size() const { return m_size; }
276 
277  /*
278  Returns internal binlog type code for one field,
279  without translation to real types.
280  */
282  return static_cast<enum_field_types>(m_type[index]);
283  }
284 
285  /// Return the number of JSON columns in this table.
286  int json_column_count() const {
287  // Cache in member field to make successive calls faster.
288  if (m_json_column_count == -1) {
289  int c = 0;
290  for (uint i = 0; i < size(); i++)
291  if (type(i) == MYSQL_TYPE_JSON) c++;
293  }
294  return m_json_column_count;
295  }
296 
297  /*
298  Return a representation of the type data for one field.
299 
300  @param index Field index to return data for
301 
302  @return Will return a representation of the type data for field
303  <code>index</code>. Currently, only the type identifier is
304  returned.
305  */
308  /*
309  If the source type is MYSQL_TYPE_STRING, it can in reality be
310  either MYSQL_TYPE_STRING, MYSQL_TYPE_ENUM, or MYSQL_TYPE_SET, so
311  we might need to modify the type to get the real type.
312  */
313  enum_field_types source_type = binlog_type(index);
314  uint16 source_metadata = m_field_metadata[index];
315  switch (source_type) {
316  case MYSQL_TYPE_STRING: {
317  int real_type = source_metadata >> 8;
318  if (real_type == MYSQL_TYPE_ENUM || real_type == MYSQL_TYPE_SET)
319  source_type = static_cast<enum_field_types>(real_type);
320  break;
321  }
322 
323  /*
324  This type has not been used since before row-based replication,
325  so we can safely assume that it really is MYSQL_TYPE_NEWDATE.
326  */
327  case MYSQL_TYPE_DATE:
328  source_type = MYSQL_TYPE_NEWDATE;
329  break;
330 
331  default:
332  /* Do nothing */
333  break;
334  }
335 
336  return source_type;
337  }
338 
339  /*
340  This function allows callers to get the extra field data from the
341  table map for a given field. If there is no metadata for that field
342  or there is no extra metadata at all, the function returns 0.
343 
344  The function returns the value for the field metadata for column at
345  position indicated by index. As mentioned, if the field was a type
346  that stores field metadata, that value is returned else zero (0) is
347  returned. This method is used in the unpack() methods of the
348  corresponding fields to properly extract the data from the binary log
349  in the event that the master's field is smaller than the slave.
350  */
354  return m_field_metadata[index];
355  else
356  return 0;
357  }
358 
359  /*
360  This function returns whether the field on the master can be null.
361  This value is derived from field->maybe_null().
362  */
363  bool maybe_null(uint index) const {
365  return ((m_null_bits[(index / 8)] & (1 << (index % 8))) ==
366  (1 << (index % 8)));
367  }
368 
369  /*
370  This function returns the field size in raw bytes based on the type
371  and the encoded field data from the master's raw data. This method can
372  be used for situations where the slave needs to skip a column (e.g.,
373  WL#3915) or needs to advance the pointer for the fields in the raw
374  data from the master to a specific column.
375  */
376  uint32 calc_field_size(uint col, uchar *master_data) const;
377 
378 #ifdef MYSQL_SERVER
379  /**
380  Decide if the table definition is compatible with a table.
381 
382  Compare the definition with a table to see if it is compatible
383  with it.
384 
385  A table definition is compatible with a table if:
386  - The columns types of the table definition is a (not
387  necessarily proper) prefix of the column type of the table.
388 
389  - The other way around.
390 
391  - Each column on the master that also exists on the slave can be
392  converted according to the current settings of @c
393  SLAVE_TYPE_CONVERSIONS.
394 
395  @param thd Current thread
396  @param rli Pointer to relay log info
397  @param table Pointer to table to compare with.
398 
399  @param[out] conv_table_var Pointer to temporary table for holding
400  conversion table.
401 
402  @retval 1 if the table definition is not compatible with @c table
403  @retval 0 if the table definition is compatible with @c table
404  */
405  bool compatible_with(THD *thd, Relay_log_info *rli, TABLE *table,
406  TABLE **conv_table_var) const;
407 
408  /**
409  Create a virtual in-memory temporary table structure.
410 
411  The table structure has records and field array so that a row can
412  be unpacked into the record for further processing.
413 
414  In the virtual table, each field that requires conversion will
415  have a non-NULL value, while fields that do not require
416  conversion will have a NULL value.
417 
418  Some information that is missing in the events, such as the
419  character set for string types, are taken from the table that the
420  field is going to be pushed into, so the target table that the data
421  eventually need to be pushed into need to be supplied.
422 
423  @param thd Thread to allocate memory from.
424  @param rli Relay log info structure, for error reporting.
425  @param target_table Target table for fields.
426 
427  @return A pointer to a temporary table with memory allocated in the
428  thread's memroot, NULL if the table could not be created
429  */
431  TABLE *target_table) const;
432 #endif
433 
434  private:
435  ulong m_size; // Number of elements in the types array
436  unsigned char *m_type; // Array of type descriptors
440  uint16 m_flags; // Table flags
442  mutable int m_json_column_count; // Number of JSON columns
443 };
444 
445 #ifdef MYSQL_SERVER
446 /**
447  Extend the normal table list with a few new fields needed by the
448  slave thread, but nowhere else.
449  */
450 struct RPL_TABLE_LIST : public TABLE_LIST {
451  RPL_TABLE_LIST(const char *db_name_arg, size_t db_length_arg,
452  const char *table_name_arg, size_t table_name_length_arg,
453  const char *alias_arg, enum thr_lock_type lock_type_arg)
454  : TABLE_LIST(nullptr, db_name_arg, db_length_arg, table_name_arg,
455  table_name_length_arg, alias_arg, lock_type_arg) {}
456 
460 };
461 
463  private:
465 
466  public:
469  /* queue for exection at Query-log-event time prior the Query */
470  int add(Log_event *ev);
471  bool is_empty();
472  bool execute(Relay_log_info *rli);
473  void rewind();
474 };
475 
476 #endif
477 
478 // NB. number of printed bit values is limited to sizeof(buf) - 1
479 #define DBUG_PRINT_BITSET(N, FRM, BS) \
480  do { \
481  char buf[256]; \
482  uint i; \
483  for (i = 0; i < MY_MIN(sizeof(buf) - 1, (BS)->n_bits); i++) \
484  buf[i] = bitmap_is_set((BS), i) ? '1' : '0'; \
485  buf[i] = '\0'; \
486  DBUG_PRINT((N), ((FRM), buf)); \
487  } while (0)
488 
489 #endif /* RPL_UTILITY_H */
bool next(HASH_ROW_ENTRY **entry)
Gets the entry that stands next to the one pointed to by entry.
Definition: rpl_utility.cc:943
bool del(HASH_ROW_ENTRY *entry)
Deletes the entry pointed by entry.
Definition: rpl_utility.cc:984
Some common macros.
This file contains the field type.
unsigned char uchar
Definition: my_inttypes.h:49
table_def()
No-op constructor.
Definition: rpl_utility.h:253
uint16 field_metadata(uint index) const
Definition: rpl_utility.h:351
bool compatible_with(THD *thd, Relay_log_info *rli, TABLE *table, TABLE **conv_table_var) const
Decide if the table definition is compatible with a table.
Definition: rpl_utility.cc:450
bool put(TABLE *table, MY_BITMAP *cols, HASH_ROW_ENTRY *entry)
Puts data into the hash table.
Definition: rpl_utility.cc:897
bool is_empty(void)
Checks if the hash table is empty or not.
Definition: rpl_utility.cc:833
Internal structure that acts as a preamble for HASH_ROW_POS in memory structure.
Definition: rpl_utility.h:80
Internal to MySQL.
Definition: field_types.h:71
TABLE * create_conversion_table(THD *thd, Relay_log_info *rli, TABLE *target_table) const
Create a virtual in-memory temporary table structure.
Definition: rpl_utility.cc:589
void rewind()
Definition: rpl_utility.cc:1122
Some integer typedefs for easier portability.
void operator()(HASH_ROW_ENTRY *entry) const
Utility methods for handling row based operations.
Definition: rpl_utility.cc:820
bool is_empty()
Definition: rpl_utility.cc:1106
Definition: field_types.h:79
Definition: rpl_utility.h:106
PSI_memory_key key_memory_HASH_ROW_ENTRY
Definition: psi_memory_key.cc:50
RPL_TABLE_LIST(const char *db_name_arg, size_t db_length_arg, const char *table_name_arg, size_t table_name_length_arg, const char *alias_arg, enum thr_lock_type lock_type_arg)
Definition: rpl_utility.h:451
enum_field_types binlog_type(ulong index) const
Definition: rpl_utility.h:281
TABLE * m_conv_table
Definition: rpl_utility.h:459
Hash table used when applying row events on the slave and there is no index on the slave&#39;s table...
Definition: rpl_utility.h:57
Deferred_log_events()
Definition: rpl_utility.cc:1095
Definition: table.h:1294
uint16 m_flags
Definition: rpl_utility.h:440
HASH_ROW_PREAMBLE()=default
uint32 calc_field_size(uint col, uchar *master_data) const
Definition: rpl_utility.cc:91
int m_json_column_count
Definition: rpl_utility.h:442
Definition: field_types.h:77
ulong size() const
Return the number of fields there is type data for.
Definition: rpl_utility.h:275
Definition: field_types.h:80
#define DBUG_ASSERT(A)
Definition: my_dbug.h:128
bool maybe_null(uint index) const
Definition: rpl_utility.h:363
HASH_ROW_ENTRY * make_entry()
Allocates an empty entry to be added to the hash table.
Definition: rpl_utility.cc:849
bool execute(Relay_log_info *rli)
Definition: rpl_utility.cc:1108
char * index(const char *, int c)
Definition: mysql.cc:2840
Definition: rpl_rli.h:164
uint hash_value
Definition: rpl_utility.h:85
thr_lock_type
Definition: thr_lock.h:50
bool m_tabledef_valid
Definition: rpl_utility.h:457
Definition: my_bitmap.h:42
~table_def()
Definition: rpl_utility.cc:804
HASH_ROW_POS * positions
Definition: rpl_utility.h:103
This is the abstract base class for binary log events.
Definition: log_event.h:517
table_def m_tabledef
Definition: rpl_utility.h:458
Definition: rpl_utility.h:69
std::unordered_multimap, but with my_malloc, so that you can track the memory used using PSI memory k...
Definition: map_helpers.h:215
enum_field_types
Column types for MySQL.
Definition: field_types.h:52
const uchar * bi_ends
Definition: rpl_utility.h:64
unsigned int uint
Definition: uca-dump.cc:29
Definition: field_types.h:86
uint m_field_metadata_size
Definition: rpl_utility.h:437
ulong m_size
Definition: rpl_utility.h:435
int add(Log_event *ev)
Definition: rpl_utility.cc:1100
enum_field_types type(ulong index) const
Definition: rpl_utility.h:306
uint make_hash_key(TABLE *table, MY_BITMAP *cols)
Auxiliary and internal method used to create an hash key, based on the data in table->record[0] buffe...
Definition: rpl_utility.cc:992
bool init(void)
Initializes the hash table.
Definition: rpl_utility.cc:839
bool is_search_state_inited
Wether this search_state is usable or not.
Definition: rpl_utility.h:98
const uchar * bi_start
Points at the position where the row starts in the event buffer (ie, area in memory before unpacking ...
Definition: rpl_utility.h:63
int size()
Returns the number of entries in the hash table.
Definition: rpl_utility.cc:847
uchar * m_null_bits
Definition: rpl_utility.h:439
HASH_ROW_PREAMBLE * preamble
Definition: rpl_utility.h:102
uchar * m_memory
Definition: rpl_utility.h:441
A table definition from the master.
Definition: rpl_utility.h:245
Definition: field_types.h:67
Prealloced_array< Log_event *, 32 > m_array
Definition: rpl_utility.h:464
Definition: table.h:2439
Extend the normal table list with a few new fields needed by the slave thread, but nowhere else...
Definition: rpl_utility.h:450
static int flags[50]
Definition: hp_test1.cc:39
unsigned char * m_type
Definition: rpl_utility.h:436
malloc_unordered_multimap< uint, std::unique_ptr< HASH_ROW_ENTRY, hash_slave_rows_free_entry > >::const_iterator search_state
The search state used to iterate over multiple entries for a given key.
Definition: rpl_utility.h:93
Definition: rpl_utility.h:101
uint16 * m_field_metadata
Definition: rpl_utility.h:438
~Deferred_log_events()
Definition: rpl_utility.cc:1098
Definition: rpl_utility.h:462
unsigned short uint16
Definition: my_inttypes.h:53
unsigned long ulong
Definition: my_inttypes.h:46
Definition: completion_hash.h:34
bool deinit(void)
De-initializes the hash table.
Definition: rpl_utility.cc:841
For each client connection we create a separate thread with THD serving as a thread/connection descri...
Definition: sql_class.h:776
int json_column_count() const
Return the number of JSON columns in this table.
Definition: rpl_utility.h:286
malloc_unordered_multimap< uint, std::unique_ptr< HASH_ROW_ENTRY, hash_slave_rows_free_entry > > m_hash
The hashtable itself.
Definition: rpl_utility.h:219