MySQL 8.2.0
Source Code Documentation
clone0repl.h
Go to the documentation of this file.
1/*****************************************************************************
2
3Copyright (c) 2018, 2023, Oracle and/or its affiliates.
4
5This program is free software; you can redistribute it and/or modify it under
6the terms of the GNU General Public License, version 2.0, as published by the
7Free Software Foundation.
8
9This program is also distributed with certain software (including but not
10limited to OpenSSL) that is licensed under separate terms, as designated in a
11particular file or component or in included license documentation. The authors
12of MySQL hereby grant you an additional permission to link the program and
13your derivative works with the separately licensed software that they have
14included with MySQL.
15
16This program is distributed in the hope that it will be useful, but WITHOUT
17ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0,
19for more details.
20
21You should have received a copy of the GNU General Public License along with
22this program; if not, write to the Free Software Foundation, Inc.,
2351 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
24
25*****************************************************************************/
26
27/** @file include/clone0repl.h
28 GTID persistence interface
29
30 *******************************************************/
31
32#ifndef CLONE_REPL_INCLUDE
33#define CLONE_REPL_INCLUDE
34
35#include <vector>
36#include "clone0monitor.h"
37#include "os0thread-create.h"
38#include "sql/rpl_gtid.h"
39#include "srv0srv.h"
40#include "srv0start.h"
41#include "trx0sys.h"
42
44
45/** Serialized GTID information size */
46static const size_t GTID_INFO_SIZE = 64;
47
48/** GTID format version. */
49static const uint32_t GTID_VERSION = 1;
50
51/** Serialized GTID */
52using Gtid_info = std::array<unsigned char, GTID_INFO_SIZE>;
53
54/** List of GTIDs */
55using Gitd_info_list = std::vector<Gtid_info>;
56
57/** GTID descriptor with version information. */
58struct Gtid_desc {
59 /** If GTID descriptor is set. */
61 /** Serialized GTID information. */
63 /* GTID version. */
64 uint32_t m_version;
65};
66
67/** Persist GTID along with transaction commit */
69 public:
70 /** Constructor: start gtid thread */
73 /* No background is created yet. */
74 m_thread_active.store(false);
75 m_gtid_trx_no.store(0);
76 m_flush_number.store(0);
77 m_explicit_request.store(false);
78 m_active_number.store(m_flush_number.load() + 1);
79 /* We accept GTID even before the background service is started. This
80 is needed because we add GTIDs from undo log during recovery. */
81 m_active.store(true);
82 m_num_gtid_mem.store(0);
83 m_flush_in_progress.store(false);
84 m_close_thread.store(false);
85 }
86
87 /** Destructor: stop gtid thread */
89 ut_ad(!m_thread_active.load());
90 stop();
92 }
93
94 /** Start GTID persistence and background thread.
95 @return true, if successful. */
96 bool start();
97
98 /* Stop GTID persistence. */
99 void stop();
100
101 /* Wait for immediate flush.
102 @param[in] compress_gtid request GTID compression.
103 @param[in] early_timeout don't wait long if flush is blocked.
104 @param[in] cbk alert callback for long wait. */
105 void wait_flush(bool compress_gtid, bool early_timeout, Clone_Alert_Func cbk);
106
107 /**@return true, if GTID persistence is active. */
108 bool is_active() const { return (m_active.load()); }
109
110 /**@return true, if GTID thread is active. */
111 bool is_thread_active() const { return (m_thread_active.load()); }
112
113 /** Get oldest transaction number for which GTID is not persisted to table.
114 Transactions committed after this point should not be purged.
115 @return oldest transaction number. */
117 trx_id_t ret_no = m_gtid_trx_no.load();
118 /* Should never be zero. It can be set to max only before
119 GTID persister is active and no GTID is persisted. */
121 if (ret_no == TRX_ID_MAX) {
123 ut_ad(m_num_gtid_mem.load() == 0);
124 } else if (m_num_gtid_mem.load() == 0) {
125 /* For all transactions that are committed before this function is called
126 have their GTID flushed if flush is not in progress. "flush not in
127 progress" is sufficient but not necessary condition here. This is mainly
128 for cases when there is no GTID and purge doesn't need to wait. */
129 if (!m_flush_in_progress.load()) {
130 ret_no = TRX_ID_MAX;
131 }
132 }
133 return (ret_no);
134 }
135
136 /** Set oldest transaction number for which GTID is not persisted to table.
137 This is set during recovery from persisted value.
138 @param[in] max_trx_no transaction number */
140 ib::info(ER_IB_CLONE_GTID_PERSIST)
141 << "GTID recovery trx_no: " << max_trx_no;
142 /* Zero is special value. It is from old database without GTID
143 persistence. */
144 if (max_trx_no == 0) {
145 max_trx_no = TRX_ID_MAX;
146 }
147 m_gtid_trx_no.store(max_trx_no);
148 }
149
150 /** Get transaction GTID information.
151 @param[in,out] trx innodb transaction
152 @param[out] gtid_desc descriptor with serialized GTID */
153 void get_gtid_info(trx_t *trx, Gtid_desc &gtid_desc);
154
155 /** Set transaction flag to persist GTID and check if space need to be
156 allocated for GTID.
157 @param[in,out] trx current innodb transaction
158 @param[in] prepare if operation is Prepare
159 @param[in] rollback if operation is Rollback
160 @param[out] set_explicit if explicitly set to persist GTID
161 @return true, if undo space needs to be allocated. */
162 bool trx_check_set(trx_t *trx, bool prepare, bool rollback,
163 bool &set_explicit);
164
165 /** Check if current transaction has GTID.
166 @param[in] trx innodb transaction
167 @param[in,out] thd session THD
168 @param[out] passed_check true if transaction is good for GTID
169 @return true, if transaction has valid GTID. */
170 bool has_gtid(trx_t *trx, THD *&thd, bool &passed_check);
171
172 /** Check if GTID persistence is set
173 @param[in] trx current innnodb transaction
174 @return GTID storage type. */
176
177 /** Set or reset GTID persist flag in THD.
178 @param[in,out] trx current innnodb transaction
179 @param[in] set true, if need to set */
180 void set_persist_gtid(trx_t *trx, bool set);
181
182 /** Add GTID to in memory list.
183 @param[in] gtid_desc Descriptor with serialized GTID */
184 void add(const Gtid_desc &gtid_desc);
185
186 /** Write GTIDs periodically to disk table. */
187 void periodic_write();
188
189 /** Write GTIDs of non Innodb transactions to table. */
190 int write_other_gtids();
191
192 /** Disable copy construction */
194
195 /** Disable assignment */
197
198 private:
199 /** Check if GTID needs to persist at XA prepare.
200 @param[in] thd session THD
201 @param[in,out] trx current innnodb transaction
202 @param[in] found_gtid session is owning GTID
203 @param[in,out] alloc in:transaction checks are passed
204 out:GTID space need to be allocated
205 @return true, if GTID needs to be persisted */
206 bool check_gtid_prepare(THD *thd, trx_t *trx, bool found_gtid, bool &alloc);
207
208 /** Check if GTID needs to persist at commit.
209 @param[in] thd session THD
210 @param[in] found_gtid session is owning GTID
211 @param[out] set_explicit if explicitly set to persist GTID
212 @return true, if GTID needs to be persisted */
213 bool check_gtid_commit(THD *thd, bool found_gtid, bool &set_explicit);
214
215 /** Check if GTID needs to persist at rollback.
216 @param[in] thd session THD
217 @param[in,out] trx current innnodb transaction
218 @param[in] found_gtid session is owning GTID
219 @return true, if GTID needs to be persisted */
220 bool check_gtid_rollback(THD *thd, trx_t *trx, bool found_gtid);
221
222 /** Wait for gtid thread to start, finish or flush.
223 @param[in] start if waiting for start
224 @param[in] flush wait for immediate flush
225 @param[in] flush_number wait flush to reach this number
226 @param[in] compress wait also for compression
227 @param[in] early_timeout don't wait long if flush is blocked
228 @param[in] cbk alert callback for long wait
229 @return true if successful. */
230 bool wait_thread(bool start, bool flush, uint64_t flush_number, bool compress,
231 bool early_timeout, Clone_Alert_Func cbk);
232
233 /** @return current active GTID list */
236 return (get_list(m_active_number));
237 }
238
239 /** @return GTID list by number.
240 @param[in] list_number list number
241 @return GTID list reference. */
242 Gitd_info_list &get_list(uint64_t list_number) {
243 int list_index = (list_number & static_cast<uint64_t>(1));
244 return (m_gtids[list_index]);
245 }
246
247 /** Check if we need to skip write or compression based on debug variables.
248 @param[in] compression check for compression
249 @return true, if we should skip. */
250 bool debug_skip_write(bool compression);
251
252 /** Request immediate flush of all GTIDs accumulated.
253 @param[in] compress request compression of GTID table
254 @return flush list number to track and wait for flush to complete. */
257 /* We want to flush all GTIDs. */
258 uint64_t request_number = m_active_number.load();
259 /* If no GTIDs added to active, wait for previous index. */
260 if (m_num_gtid_mem.load() == 0) {
261 ut_a(request_number > 0);
262 --request_number;
263 }
264 m_flush_request_number = request_number;
266
267 if (compress) {
268 m_explicit_request.store(true);
269 }
270 return (request_number);
271 }
272
273 /** Check if flush has finished up to a list number.
274 @param[in] request_number flush request number
275 @return true, if it is already flushed. */
276 bool check_flushed(uint64_t request_number) const {
277 return (m_flush_number >= request_number);
278 }
279
280 /** @return true, iff background needs to flush immediately. */
281 bool flush_immediate() const {
283 }
284
285 /** Check if GTID compression is necessary based on threshold.
286 @return true, if GTID table needs to be compressed. */
287 bool check_compress();
288
289 /** Switch active GTID list. */
291 /* Switch active list under transaction system mutex. */
293 uint64_t flush_number = m_active_number;
296 m_num_gtid_mem.store(0);
297#ifdef UNIV_DEBUG
298 /* The new active list must have no elements. */
299 auto &active_list = get_active_list();
300 ut_ad(active_list.size() == 0);
301#endif
302 return (flush_number);
303 }
304
305 /** Persist GTID to gtid_executed table.
306 @param[in] flush_list_number list number to flush
307 @param[in,out] table_gtid_set GTIDs in table during recovery
308 @param[in,out] sid_map SID map for GTIDs
309 @return mysql error code. */
310 int write_to_table(uint64_t flush_list_number, Gtid_set &table_gtid_set,
311 Sid_map &sid_map);
312
313 /** Update transaction number up to which GTIDs are flushed to table.
314 @param[in] new_gtid_trx_no GTID transaction number */
315 void update_gtid_trx_no(trx_id_t new_gtid_trx_no);
316
317 /** Write all GTIDs to table and update GTID transaction number.
318 @param[in,out] thd current session thread */
319 void flush_gtids(THD *thd);
320
321 /** @return true iff number of GTIDs in active list exceeded threshold. */
323
324 private:
325 /** Time threshold to trigger persisting GTID. Insert GTID once per 1k
326 transactions or every 100 millisecond. */
327 static constexpr std::chrono::milliseconds s_time_threshold{100};
328
329 /** Threshold for the count for compressing GTID. */
330 const static uint32_t s_compression_threshold = 50;
331
332 /** Number of transaction/GTID threshold for writing to disk table. */
333 const static int s_gtid_threshold = 1024;
334
335 /** Maximum Number of transaction/GTID to hold. Transaction commits
336 must wait beyond this point. Not expected to happen as GTIDs are
337 compressed and written together. */
338 const static int s_max_gtid_threshold = 1024 * 1024;
339
340 /** Two lists of GTID. One of them is active where running transactions
341 add their GTIDs. Other list is used to persist them to table from time
342 to time. */
344
345 /** Number of the current GTID list. Increased when list is switched */
346 std::atomic<uint64_t> m_active_number;
347
348 /** Number up to which GTIDs are flushed. Increased when list is flushed.*/
349 std::atomic<uint64_t> m_flush_number;
350
351 /** If explicit request to flush is made. */
352 std::atomic<bool> m_explicit_request;
353
354 /** Number for which last flush request was made. */
356
357 /** Event for GTID background thread. */
359
360 /** Counter to keep track of the number of writes till it reaches
361 compression threshold. */
363
364 /** Counter to keep number of GTIDs flushed before compression. */
366
367 /* Oldest transaction number for which GTID is not persisted. */
368 std::atomic<uint64_t> m_gtid_trx_no;
369
370 /** Number of GTID accumulated in memory */
371 std::atomic<int> m_num_gtid_mem;
372
373 /** Flush of GTID is in progress. */
374 std::atomic<bool> m_flush_in_progress;
375
376 /** Set to true, when the background thread is asked to exit. */
377 std::atomic<bool> m_close_thread;
378
379 /** true, if background thread is active.*/
380 std::atomic<bool> m_thread_active;
381
382 /** true, if GTID persistence is active.*/
383 std::atomic<bool> m_active;
384};
385
386#endif /* CLONE_REPL_INCLUDE */
Persist GTID along with transaction commit.
Definition: clone0repl.h:68
std::atomic< bool > m_flush_in_progress
Flush of GTID is in progress.
Definition: clone0repl.h:374
trx_id_t get_oldest_trx_no()
Get oldest transaction number for which GTID is not persisted to table.
Definition: clone0repl.h:116
static const int s_max_gtid_threshold
Maximum Number of transaction/GTID to hold.
Definition: clone0repl.h:338
Clone_persist_gtid & operator=(Clone_persist_gtid const &)=delete
Disable assignment.
bool wait_thread(bool start, bool flush, uint64_t flush_number, bool compress, bool early_timeout, Clone_Alert_Func cbk)
Wait for gtid thread to start, finish or flush.
Definition: clone0repl.cc:628
bool start()
Start GTID persistence and background thread.
Definition: clone0repl.cc:696
Gitd_info_list & get_active_list()
Definition: clone0repl.h:234
void stop()
Definition: clone0repl.cc:715
bool has_gtid(trx_t *trx, THD *&thd, bool &passed_check)
Check if current transaction has GTID.
Definition: clone0repl.cc:278
trx_undo_t::Gtid_storage persists_gtid(const trx_t *trx)
Check if GTID persistence is set.
Definition: clone0repl.cc:85
static const int s_gtid_threshold
Number of transaction/GTID threshold for writing to disk table.
Definition: clone0repl.h:333
void set_persist_gtid(trx_t *trx, bool set)
Set or reset GTID persist flag in THD.
Definition: clone0repl.cc:106
std::atomic< bool > m_explicit_request
If explicit request to flush is made.
Definition: clone0repl.h:352
bool is_active() const
Definition: clone0repl.h:108
void update_gtid_trx_no(trx_id_t new_gtid_trx_no)
Update transaction number up to which GTIDs are flushed to table.
Definition: clone0repl.cc:461
static constexpr std::chrono::milliseconds s_time_threshold
Time threshold to trigger persisting GTID.
Definition: clone0repl.h:327
std::atomic< int > m_num_gtid_mem
Number of GTID accumulated in memory.
Definition: clone0repl.h:371
Gitd_info_list m_gtids[2]
Two lists of GTID.
Definition: clone0repl.h:343
bool check_gtid_rollback(THD *thd, trx_t *trx, bool found_gtid)
Check if GTID needs to persist at rollback.
Definition: clone0repl.cc:240
bool check_flushed(uint64_t request_number) const
Check if flush has finished up to a list number.
Definition: clone0repl.h:276
bool debug_skip_write(bool compression)
Check if we need to skip write or compression based on debug variables.
Definition: clone0repl.cc:397
void periodic_write()
Write GTIDs periodically to disk table.
Definition: clone0repl.cc:564
Clone_persist_gtid()
Constructor: start gtid thread.
Definition: clone0repl.h:71
std::atomic< uint64_t > m_active_number
Number of the current GTID list.
Definition: clone0repl.h:346
os_event_t m_event
Event for GTID background thread.
Definition: clone0repl.h:358
uint64_t m_flush_request_number
Number for which last flush request was made.
Definition: clone0repl.h:355
bool check_compress()
Check if GTID compression is necessary based on threshold.
Definition: clone0repl.cc:373
void set_oldest_trx_no_recovery(trx_id_t max_trx_no)
Set oldest transaction number for which GTID is not persisted to table.
Definition: clone0repl.h:139
uint32_t m_compression_counter
Counter to keep track of the number of writes till it reaches compression threshold.
Definition: clone0repl.h:362
static const uint32_t s_compression_threshold
Threshold for the count for compressing GTID.
Definition: clone0repl.h:330
int write_to_table(uint64_t flush_list_number, Gtid_set &table_gtid_set, Sid_map &sid_map)
Persist GTID to gtid_executed table.
Definition: clone0repl.cc:408
void flush_gtids(THD *thd)
Write all GTIDs to table and update GTID transaction number.
Definition: clone0repl.cc:478
bool check_max_gtid_threshold()
Definition: clone0repl.cc:556
bool trx_check_set(trx_t *trx, bool prepare, bool rollback, bool &set_explicit)
Set transaction flag to persist GTID and check if space need to be allocated for GTID.
Definition: clone0repl.cc:160
bool check_gtid_prepare(THD *thd, trx_t *trx, bool found_gtid, bool &alloc)
Check if GTID needs to persist at XA prepare.
Definition: clone0repl.cc:189
std::atomic< bool > m_thread_active
true, if background thread is active.
Definition: clone0repl.h:380
Gitd_info_list & get_list(uint64_t list_number)
Definition: clone0repl.h:242
std::atomic< uint64_t > m_gtid_trx_no
Definition: clone0repl.h:368
~Clone_persist_gtid()
Destructor: stop gtid thread.
Definition: clone0repl.h:88
Clone_persist_gtid(Clone_persist_gtid const &)=delete
Disable copy construction.
void add(const Gtid_desc &gtid_desc)
Add GTID to in memory list.
Definition: clone0repl.cc:43
std::atomic< bool > m_close_thread
Set to true, when the background thread is asked to exit.
Definition: clone0repl.h:377
uint64_t request_immediate_flush(bool compress)
Request immediate flush of all GTIDs accumulated.
Definition: clone0repl.h:255
uint64_t switch_active_list()
Switch active GTID list.
Definition: clone0repl.h:290
void get_gtid_info(trx_t *trx, Gtid_desc &gtid_desc)
Get transaction GTID information.
Definition: clone0repl.cc:334
int write_other_gtids()
Write GTIDs of non Innodb transactions to table.
Definition: clone0repl.cc:365
bool flush_immediate() const
Definition: clone0repl.h:281
std::atomic< bool > m_active
true, if GTID persistence is active.
Definition: clone0repl.h:383
std::atomic< uint64_t > m_flush_number
Number up to which GTIDs are flushed.
Definition: clone0repl.h:349
void wait_flush(bool compress_gtid, bool early_timeout, Clone_Alert_Func cbk)
Definition: clone0repl.cc:724
bool check_gtid_commit(THD *thd, bool found_gtid, bool &set_explicit)
Check if GTID needs to persist at commit.
Definition: clone0repl.cc:227
uint32_t m_compression_gtid_counter
Counter to keep number of GTIDs flushed before compression.
Definition: clone0repl.h:365
bool is_thread_active() const
Definition: clone0repl.h:111
Represents a set of GTIDs.
Definition: rpl_gtid.h:1466
Represents a bidirectional map between SID and SIDNO.
Definition: rpl_gtid.h:734
For each client connection we create a separate thread with THD serving as a thread/connection descri...
Definition: sql_lexer_thd.h:35
Performance Schema stage instrumentation to monitor clone progress.
std::function< int()> Clone_Alert_Func
Function to alert caller for long wait.
Definition: clone0monitor.h:42
static const size_t GTID_INFO_SIZE
Serialized GTID information size.
Definition: clone0repl.h:46
static const uint32_t GTID_VERSION
GTID format version.
Definition: clone0repl.h:49
std::vector< Gtid_info > Gitd_info_list
List of GTIDs.
Definition: clone0repl.h:55
std::array< unsigned char, GTID_INFO_SIZE > Gtid_info
Serialized GTID.
Definition: clone0repl.h:52
static int compress(PACK_MRG_INFO *file, char *join_name)
Definition: myisampack.cc:466
Log info(cout, "NOTE")
static mysql_service_status_t flush(reference_caching_cache cache) noexcept
Definition: component.cc:113
std::set< Key, Compare, ut::allocator< Key > > set
Specialization of set which uses ut_allocator.
Definition: ut0new.h:2881
os_event_t os_event_create()
Creates an event semaphore, i.e., a semaphore which may just have two states: signaled and nonsignale...
Definition: os0event.cc:527
void os_event_destroy(os_event_t &event)
Frees an event object.
Definition: os0event.cc:594
The interface to the threading wrapper.
static bool rollback(THD *thd)
Abort the current statement and transaction.
Definition: sql_cmd_srs.cc:139
The server main program.
@ SRV_FORCE_NO_UNDO_LOG_SCAN
do not look at undo logs when starting the database: InnoDB will treat even incomplete transactions a...
Definition: srv0srv.h:935
ulong srv_force_recovery
Normally 0.
Definition: srv0srv.cc:534
Starts the Innobase database server.
GTID descriptor with version information.
Definition: clone0repl.h:58
uint32_t m_version
Definition: clone0repl.h:64
bool m_is_set
If GTID descriptor is set.
Definition: clone0repl.h:60
Gtid_info m_info
Serialized GTID information.
Definition: clone0repl.h:62
InnoDB condition variable.
Definition: os0event.cc:62
Definition: trx0trx.h:683
Gtid_storage
Undo log may could be allocated to store transaction GTIDs.
Definition: trx0undo.h:340
Transaction system.
static void trx_sys_serialisation_mutex_exit()
Release the trx_sys->serialisation_mutex.
Definition: trx0sys.h:620
static void trx_sys_serialisation_mutex_enter()
Acquire the trx_sys->serialisation_mutex.
Definition: trx0sys.h:615
static bool trx_sys_serialisation_mutex_own()
Test if trx_sys->serialisation_mutex is owned.
Definition: trx0sys.h:609
constexpr trx_id_t TRX_ID_MAX
Maximum transaction identifier.
Definition: trx0types.h:144
ib_id_t trx_id_t
Transaction identifier (DB_TRX_ID, DATA_TRX_ID)
Definition: trx0types.h:137
#define ut_ad(EXPR)
Debug assertion.
Definition: ut0dbg.h:68
#define ut_a(EXPR)
Abort execution if EXPR does not evaluate to nonzero.
Definition: ut0dbg.h:56
static void prepare(pax_msg *p, pax_op op)
Definition: xcom_base.cc:1587