MySQL  8.0.20
Source Code Documentation
pipeline_stats.h
Go to the documentation of this file.
1 /* Copyright (c) 2016, 2020, Oracle and/or its affiliates. All rights reserved.
2 
3  This program is free software; you can redistribute it and/or modify
4  it under the terms of the GNU General Public License, version 2.0,
5  as published by the Free Software Foundation.
6 
7  This program is also distributed with certain software (including
8  but not limited to OpenSSL) that is licensed under separate terms,
9  as designated in a particular file or component or in included license
10  documentation. The authors of MySQL hereby grant you an additional
11  permission to link the program and your derivative works with the
12  separately licensed software that they have included with MySQL.
13 
14  This program is distributed in the hope that it will be useful,
15  but WITHOUT ANY WARRANTY; without even the implied warranty of
16  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17  GNU General Public License, version 2.0, for more details.
18 
19  You should have received a copy of the GNU General Public License
20  along with this program; if not, write to the Free Software
21  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
22 
23 #ifndef PIPELINE_STATS_INCLUDED
24 #define PIPELINE_STATS_INCLUDED
25 
26 #include <map>
27 #include <string>
28 #include <vector>
29 
31 #include "my_inttypes.h"
34 
35 /**
36  Flow control modes:
37  FCM_DISABLED flow control disabled
38  FCM_QUOTA introduces a delay only on transactions the exceed a quota
39 */
41 
42 /**
43  @class Pipeline_stats_member_message
44 
45  Describes all statistics sent by members.
46 */
48  public:
50  // This type should not be used anywhere.
52 
53  // Length of the payload item: 4 bytes
55 
56  // Length of the payload item: 4 bytes
58 
59  // Length of the payload item: 8 bytes
61 
62  // Length of the payload item: 8 bytes
64 
65  // Length of the payload item: 8 bytes
67 
68  // Length of the payload item: 8 bytes
70 
71  // Length of the payload item: 8 bytes
73 
74  // Length of the payload item: variable
76 
77  // Length of the payload item: variable
79 
80  // Length of the payload item: 8 bytes
82 
83  // Length of the payload item: 1 byte
85 
86  // Length of the payload item: 1 byte
88 
89  // No valid type codes can appear after this one.
90  PIT_MAX = 13
91  };
92 
93  /**
94  Message constructor
95 
96  @param[in] transactions_waiting_certification
97  Number of transactions pending certification
98  @param[in] transactions_waiting_apply
99  Number of remote transactions waiting apply
100  @param[in] transactions_certified
101  Number of transactions already certified
102  @param[in] transactions_applied
103  Number of remote transactions applied
104  @param[in] transactions_local
105  Number of local transactions
106  @param[in] transactions_negative_certified
107  Number of transactions that were negatively certified
108  @param[in] transactions_rows_validating
109  Number of transactions with which certification will be done
110  against
111  @param[in] transaction_gtids
112  Flag to indicate whether or not the transaction ids have been
113  updated
114  @param[in] transactions_committed_all_members
115  Set of transactions committed on all members
116  @param[in] transactions_last_conflict_free
117  Latest transaction certified without conflicts
118  @param[in] transactions_local_rollback
119  Number of local transactions that were negatively certified
120  @param[in] mode
121  Flow-control mode
122  */
124  int32 transactions_waiting_certification,
125  int32 transactions_waiting_apply, int64 transactions_certified,
126  int64 transactions_applied, int64 transactions_local,
127  int64 transactions_negative_certified, int64 transactions_rows_validating,
128  bool transaction_gtids,
129  const std::string &transactions_committed_all_members,
130  const std::string &transactions_last_conflict_free,
131  int64 transactions_local_rollback, Flow_control_mode mode);
132 
133  /**
134  Message constructor for raw data
135 
136  @param[in] buf raw data
137  @param[in] len raw length
138  */
139  Pipeline_stats_member_message(const unsigned char *buf, size_t len);
140 
141  /**
142  Message destructor
143  */
145 
146  /**
147  Get transactions waiting certification counter value.
148 
149  @return the counter value
150  */
152 
153  /**
154  Get transactions waiting apply counter value.
155 
156  @return the counter value
157  */
159 
160  /**
161  Get transactions certified.
162 
163  @return the counter value
164  */
166 
167  /**
168  Get transactions applied.
169 
170  @return the counter value
171  */
173 
174  /**
175  Get local transactions that member tried to commmit.
176 
177  @return the counter value
178  */
180 
181  /**
182  Get negatively certfied transaction by member.
183 
184  @return the counter value
185  */
187 
188  /**
189  Get size of conflict detection database.
190 
191  @return the counter value
192  */
194 
195  /**
196  Returns a flag indicating whether or not the GTIDs on this stats message
197  are updated/present.
198 
199  @return the flag indicating the presence of valid GTIDs on this message.
200  */
201  bool get_transation_gtids_present() const;
202 
203  /**
204  Get set of stable group transactions.
205 
206  @return the transaction identifier.
207  */
208  const std::string &get_transaction_committed_all_members();
209 
210  /**
211  Get last positive certified transaction.
212 
213  @return the transaction identifier.
214  */
215  const std::string &get_transaction_last_conflict_free();
216 
217  /**
218  Get local transactions rolled back by the member.
219 
220  @return the transaction identifiers.
221  */
223 
224  /**
225  Get flow-control mode of member.
226 
227  @return the mode value
228  */
230 
231  protected:
232  /**
233  Encodes the message contents for transmission.
234 
235  @param[out] buffer the message buffer to be written
236  */
237  void encode_payload(std::vector<unsigned char> *buffer) const;
238 
239  /**
240  Message decoding method
241 
242  @param[in] buffer the received data
243  @param[in] end the end of the buffer
244  */
245  void decode_payload(const unsigned char *buffer, const unsigned char *end);
246 
247  private:
260 };
261 
262 /**
263  @class Pipeline_stats_member_collector
264 
265  The pipeline collector for the local member stats.
266 */
268  public:
269  /**
270  Default constructor.
271  */
273 
274  /**
275  Destructor.
276  */
278 
279  /**
280  Increment transactions waiting apply counter value.
281  */
282  void increment_transactions_waiting_apply();
283 
284  /**
285  Decrement transactions waiting apply counter value.
286  */
287  void decrement_transactions_waiting_apply();
288 
289  /**
290  Increment transactions certified counter value.
291  */
292  void increment_transactions_certified();
293 
294  /**
295  Increment transactions applied counter value.
296  */
297  void increment_transactions_applied();
298 
299  /**
300  Increment local transactions counter value.
301  */
302  void increment_transactions_local();
303 
304  /**
305  Increment local rollback transactions counter value.
306  */
307  void increment_transactions_local_rollback();
308 
309  /**
310  Send member statistics to group.
311  */
312  void send_stats_member_message(Flow_control_mode mode);
313 
314  /**
315  Increment local recovery transactions counter value.
316  */
317  void increment_transactions_applied_during_recovery();
318 
319  /**
320  @returns transactions waiting to be applied during recovery.
321  */
322  uint64 get_transactions_waiting_apply_during_recovery();
323 
324  /**
325  Increment delivered transactions during recovery counter value.
326  */
327  void increment_transactions_delivered_during_recovery();
328 
329  /**
330  Increment certified transactions during recovery counter value.
331  */
332  void increment_transactions_certified_during_recovery();
333 
334  /**
335  Increment negatively certified transactions during recovery counter value.
336  */
337  void increment_transactions_certified_negatively_during_recovery();
338 
339  /**
340  @returns transactions waiting to be certified during recovery.
341  */
342  uint64 get_transactions_waiting_certification_during_recovery();
343 
344  /**
345  Compute the transactions applied during last flow-control tick
346  while the member is in recovery.
347  */
348  void compute_transactions_deltas_during_recovery();
349 
350  /**
351  @returns transactions applied during last flow-control tick
352  while the member is in recovery.
353  */
354  uint64 get_delta_transactions_applied_during_recovery();
355 
356  /**
357  @returns transactions waiting to be applied.
358  */
360 
361  /**
362  @returns transactions certified.
363  */
365 
366  /**
367  @returns transactions applied of local member.
368  */
370 
371  /**
372  @returns local transactions proposed by member.
373  */
375 
376  /**
377  @returns local transactions rollback due to Negative certification
378  */
380 
381  /**
382  Send Transaction Identifiers or not.
383  Once Transactions identifiers are sent, variable will be reset to FALSE
384  So need to set each time Transactions identifiers needs to be transmitted
385  */
386  void set_send_transaction_identifiers();
387 
388  private:
389  std::atomic<int32> m_transactions_waiting_apply;
390  std::atomic<int64> m_transactions_certified;
391  std::atomic<int64> m_transactions_applied;
392  std::atomic<int64> m_transactions_local;
393  std::atomic<int64> m_transactions_local_rollback;
394  /* Includes both positively and negatively certified. */
401 
404 };
405 
406 /**
407  @class Pipeline_member_stats
408 
409  Computed statistics per member.
410 */
412  public:
413  /**
414  Default constructor.
415  */
417 
418  /**
419  Constructor.
420  */
422 
423  /**
424  Constructor.
425  */
427  ulonglong applier_queue, ulonglong negative_certified,
428  ulonglong certificatin_size);
429 
430  /**
431  Updates member statistics with a new message from the network
432  */
433  void update_member_stats(Pipeline_stats_member_message &msg, uint64 stamp);
434 
435  /**
436  Returns true if the node is behind on some user-defined criteria
437  */
438  bool is_flow_control_needed();
439 
440  /**
441  Get transactions waiting certification counter value.
442 
443  @return the counter value
444  */
446 
447  /**
448  Get transactions waiting apply counter value.
449 
450  @return the counter value
451  */
453 
454  /**
455  Get transactions certified counter value.
456 
457  @return the counter value
458  */
460 
461  /**
462  Get transactions applied counter value.
463 
464  @return the counter value
465  */
467 
468  /**
469  Get local member transactions proposed counter value.
470 
471  @return the counter value
472  */
474 
475  /**
476  Get transactions negatively certified.
477 
478  @return the counter value
479  */
481 
482  /**
483  Get certification database counter value.
484 
485  @return the counter value
486  */
488 
489  /**
490  Get the stable group transactions.
491  */
492  void get_transaction_committed_all_members(std::string &value);
493 
494  /**
495  Set the stable group transactions.
496  */
497  void set_transaction_committed_all_members(char *str, size_t len);
498 
499  /**
500  Get the last positive certified transaction.
501  */
502  void get_transaction_last_conflict_free(std::string &value);
503 
504  /**
505  Set the last positive certified transaction.
506  */
507  void set_transaction_last_conflict_free(std::string &value);
508 
509  /**
510  Get local member transactions negatively certified.
511 
512  @return the counter value
513  */
515 
516  /**
517  Get transactions certified since last stats message.
518 
519  @return the counter value
520  */
521  int64 get_delta_transactions_certified();
522 
523  /**
524  Get transactions applied since last stats message.
525 
526  @return the counter value
527  */
528  int64 get_delta_transactions_applied();
529 
530  /**
531  Get local transactions that member tried to commmit
532  since last stats message.
533 
534  @return the counter value
535  */
536  int64 get_delta_transactions_local();
537 
538  /**
539  Get flow_control_mode of a member.
540 
541  @return the mode value
542  */
544 
545  /**
546  Get the last stats update stamp.
547 
548  @return the counter value
549  */
550  uint64 get_stamp();
551 
552 #ifndef DBUG_OFF
553  void debug(const char *member, int64 quota_size, int64 quota_used);
554 #endif
555 
556  private:
572 };
573 
574 /**
575  Data type that holds all members stats.
576  The key value is the GCS member_id.
577 */
578 typedef std::map<std::string, Pipeline_member_stats> Flow_control_module_info;
579 
580 /**
581  @class Flow_control_module
582 
583  The pipeline stats aggregator of all group members stats and
584  flow control module.
585 */
587  public:
588  static const int64 MAXTPS;
589 
590  /**
591  Default constructor.
592  */
594 
595  /**
596  Destructor.
597  */
598  virtual ~Flow_control_module();
599 
600  /**
601  Handles a Pipeline_stats_message, updating the
602  Flow_control_module_info and the delay, if needed.
603 
604  @param[in] data the packet data
605  @param[in] len the packet length
606  @param[in] member_id the GCS member_id which sent the message
607 
608  @return the operation status
609  @retval 0 OK
610  @retval !=0 Error on queue
611  */
612  int handle_stats_data(const uchar *data, size_t len,
613  const std::string &member_id);
614 
615  /**
616  Evaluate the information received in the last flow control period
617  and adjust the system parameters accordingly
618  */
619  void flow_control_step(Pipeline_stats_member_collector *);
620 
621  /**
622  Returns copy of individual member stats information.
623  @note Its caller responsibility to clean up allocated memory.
624 
625  @param[in] member_id GCS Type Member Id, i.e. format HOST:PORT
626  @return the reference to class Pipeline_member_stats of memberID
627  storing network(GCS Broadcasted) received information
628  */
629  Pipeline_member_stats *get_pipeline_stats(const std::string &member_id);
630 
631  /**
632  Compute and wait the amount of time in microseconds that must
633  be elapsed before a new message is sent.
634  If there is no need to wait, the method returns immediately.
635 
636  @return the wait time
637  @retval 0 No wait was done
638  @retval >0 The wait time
639  */
640  int32 do_wait();
641 
642  private:
645 
647  /*
648  A rw lock to protect the Flow_control_module_info map.
649  */
651 
652  /*
653  Number of members that did have waiting transactions on
654  certification and/or apply.
655  */
656  std::atomic<int32> m_holds_in_period;
657 
658  /*
659  FCM_QUOTA
660  */
661  std::atomic<int64> m_quota_used;
662  std::atomic<int64> m_quota_size;
663 
664  /*
665  Counter incremented on every flow control step.
666  */
668 
669  /*
670  Remaining seconds to skip flow-control steps
671  */
673 };
674 
675 #endif /* PIPELINE_STATS_INCLUDED */
enum_payload_item_type
Definition: pipeline_stats.h:49
unsigned long long int ulonglong
Definition: my_inttypes.h:55
int64 m_transactions_rows_validating
Definition: pipeline_stats.h:566
unsigned char uchar
Definition: my_inttypes.h:51
int64 m_transactions_certified
Definition: pipeline_stats.h:559
bool m_transaction_gtids_present
Definition: pipeline_stats.h:255
int64 m_transactions_negative_certified
Definition: pipeline_stats.h:253
uint64 m_stamp
Definition: pipeline_stats.h:667
std::string m_transactions_committed_all_members
Definition: pipeline_stats.h:567
The pipeline stats aggregator of all group members stats and flow control module. ...
Definition: pipeline_stats.h:586
uint64 m_stamp
Definition: pipeline_stats.h:571
std::atomic< int32 > m_holds_in_period
Definition: pipeline_stats.h:656
int64 get_transactions_certified()
Get transactions certified.
Definition: pipeline_stats.cc:113
char buffer[STRING_BUFFER]
Definition: test_sql_9_sessions.cc:57
Definition: pipeline_stats.h:90
An instrumented cond structure.
Definition: mysql_cond_bits.h:49
Some integer typedefs for easier portability.
static const int64 MAXTPS
Definition: pipeline_stats.h:588
std::atomic< int64 > m_transactions_local
Definition: pipeline_stats.h:392
std::atomic< int64 > m_transactions_local_rollback
Definition: pipeline_stats.h:393
Checkable_rwlock * m_flow_control_module_info_lock
Definition: pipeline_stats.h:650
Flow_control_mode
Flow control modes: FCM_DISABLED flow control disabled FCM_QUOTA introduces a delay only on transacti...
Definition: pipeline_stats.h:40
int32 m_transactions_waiting_certification
Definition: pipeline_stats.h:248
int64 get_transactions_rows_validating()
Get size of conflict detection database.
Definition: pipeline_stats.cc:138
int32 get_transactions_waiting_apply()
Get transactions waiting apply counter value.
Definition: pipeline_stats.cc:118
This is the base GCS plugin message.
Definition: gcs_plugin_messages.h:63
int64 m_transactions_certified
Definition: pipeline_stats.h:250
int64 get_transactions_negative_certified()
Get negatively certfied transaction by member.
Definition: pipeline_stats.cc:133
mysql_mutex_t m_flow_control_lock
Definition: pipeline_stats.h:643
Describes all statistics sent by members.
Definition: pipeline_stats.h:47
std::atomic< uint64 > m_transactions_applied_during_recovery
Definition: pipeline_stats.h:397
int64 m_transactions_negative_certified
Definition: pipeline_stats.h:565
int64 m_delta_transactions_applied
Definition: pipeline_stats.h:562
Pipeline_stats_member_message(int32 transactions_waiting_certification, int32 transactions_waiting_apply, int64 transactions_certified, int64 transactions_applied, int64 transactions_local, int64 transactions_negative_certified, int64 transactions_rows_validating, bool transaction_gtids, const std::string &transactions_committed_all_members, const std::string &transactions_last_conflict_free, int64 transactions_local_rollback, Flow_control_mode mode)
Message constructor.
Definition: pipeline_stats.cc:66
std::atomic< int64 > m_transactions_applied
Definition: pipeline_stats.h:391
std::atomic< int32 > m_transactions_waiting_apply
Definition: pipeline_stats.h:389
bool send_transaction_identifiers
Definition: pipeline_stats.h:402
void decode_payload(const unsigned char *buffer, const unsigned char *end)
Message decoding method.
Definition: pipeline_stats.cc:226
int64 m_transactions_rows_validating
Definition: pipeline_stats.h:254
int64 m_transactions_local_rollback
Definition: pipeline_stats.h:258
int32 m_transactions_waiting_certification
Definition: pipeline_stats.h:557
Flow_control_mode m_flow_control_mode
Definition: pipeline_stats.h:570
int64 m_transactions_local
Definition: pipeline_stats.h:563
int64 m_transactions_applied
Definition: pipeline_stats.h:251
const std::string & get_transaction_committed_all_members()
Get set of stable group transactions.
Definition: pipeline_stats.cc:153
Flow_control_mode m_flow_control_mode
Definition: pipeline_stats.h:259
mysql_cond_t m_flow_control_cond
Definition: pipeline_stats.h:644
Definition: pipeline_stats.h:51
mysql_mutex_t m_transactions_waiting_apply_lock
Definition: pipeline_stats.h:403
int64 m_transactions_local
Definition: pipeline_stats.h:252
Cursor end()
A past-the-end Cursor.
Definition: rules_table_service.cc:188
void encode_payload(std::vector< unsigned char > *buffer) const
Encodes the message contents for transmission.
Definition: pipeline_stats.cc:169
int64_t int64
Definition: my_inttypes.h:67
std::atomic< uint64 > m_delta_transactions_applied_during_recovery
Definition: pipeline_stats.h:399
The pipeline collector for the local member stats.
Definition: pipeline_stats.h:267
Flow_control_mode get_flow_control_mode()
Get flow-control mode of member.
Definition: pipeline_stats.cc:164
bool get_transation_gtids_present() const
Returns a flag indicating whether or not the GTIDs on this stats message are updated/present.
Definition: pipeline_stats.cc:143
virtual ~Pipeline_stats_member_message()
Message destructor.
Definition: pipeline_stats.cc:106
std::map< std::string, Pipeline_member_stats > Flow_control_module_info
Data type that holds all members stats.
Definition: pipeline_stats.h:578
std::string m_transaction_last_conflict_free
Definition: pipeline_stats.h:257
char msg[1024]
Definition: test_sql_9_sessions.cc:281
std::atomic< int64 > m_quota_size
Definition: pipeline_stats.h:662
int64 m_delta_transactions_certified
Definition: pipeline_stats.h:560
Definition: pipeline_stats.h:40
std::atomic< int64 > m_transactions_certified
Definition: pipeline_stats.h:390
int64 get_transactions_applied()
Get transactions applied.
Definition: pipeline_stats.cc:123
int32 get_transactions_waiting_certification()
Get transactions waiting certification counter value.
Definition: pipeline_stats.cc:108
std::atomic< int64 > m_quota_used
Definition: pipeline_stats.h:661
const std::string & get_transaction_last_conflict_free()
Get last positive certified transaction.
Definition: pipeline_stats.cc:159
Definition: pipeline_stats.h:40
An instrumented mutex structure.
Definition: mysql_mutex_bits.h:49
int32 m_transactions_waiting_apply
Definition: pipeline_stats.h:249
std::string m_transaction_last_conflict_free
Definition: pipeline_stats.h:568
int64 m_transactions_local_rollback
Definition: pipeline_stats.h:569
int32_t int32
Definition: my_inttypes.h:65
const string value("\alue\)
int seconds_to_skip
Definition: pipeline_stats.h:672
int64 get_transactions_local()
Get local transactions that member tried to commmit.
Definition: pipeline_stats.cc:128
uint64 m_previous_transactions_applied_during_recovery
Definition: pipeline_stats.h:398
This has the functionality of mysql_rwlock_t, with two differences:
Definition: rpl_gtid.h:431
std::atomic< uint64 > m_transactions_delivered_during_recovery
Definition: pipeline_stats.h:400
int32 m_transactions_waiting_apply
Definition: pipeline_stats.h:558
std::atomic< uint64 > m_transactions_certified_during_recovery
Definition: pipeline_stats.h:395
int64 m_delta_transactions_local
Definition: pipeline_stats.h:564
std::string m_transactions_committed_all_members
Definition: pipeline_stats.h:256
int64 m_transactions_applied
Definition: pipeline_stats.h:561
uint64_t uint64
Definition: my_inttypes.h:68
int64 get_transactions_local_rollback()
Get local transactions rolled back by the member.
Definition: pipeline_stats.cc:147
Computed statistics per member.
Definition: pipeline_stats.h:411
std::atomic< uint64 > m_transactions_certified_negatively_during_recovery
Definition: pipeline_stats.h:396
Flow_control_module_info m_info
Definition: pipeline_stats.h:646