MySQL  8.0.21
Source Code Documentation
arch0arch.h
Go to the documentation of this file.
1 /*****************************************************************************
2 
3 Copyright (c) 2017, 2020, Oracle and/or its affiliates. All Rights Reserved.
4 
5 This program is free software; you can redistribute it and/or modify it under
6 the terms of the GNU General Public License, version 2.0, as published by the
7 Free Software Foundation.
8 
9 This program is also distributed with certain software (including but not
10 limited to OpenSSL) that is licensed under separate terms, as designated in a
11 particular file or component or in included license documentation. The authors
12 of MySQL hereby grant you an additional permission to link the program and
13 your derivative works with the separately licensed software that they have
14 included with MySQL.
15 
16 This program is distributed in the hope that it will be useful, but WITHOUT
17 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18 FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0,
19 for more details.
20 
21 You should have received a copy of the GNU General Public License along with
22 this program; if not, write to the Free Software Foundation, Inc.,
23 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
24 
25 *****************************************************************************/
26 
27 /** @file include/arch0arch.h
28  Common interface for redo log and dirty page archiver system
29 
30  *******************************************************/
31 
32 #ifndef ARCH_ARCH_INCLUDE
33 #define ARCH_ARCH_INCLUDE
34 
36 #include "log0log.h"
37 #include "ut0mutex.h"
38 
39 #include <list>
40 
41 /** @name Archive file name prefix and constant length parameters. */
42 /* @{ */
43 /** Archive directory prefix */
44 const char ARCH_DIR[] = OS_FILE_PREFIX "ib_archive";
45 
46 /** Archive Log group directory prefix */
47 const char ARCH_LOG_DIR[] = "log_group_";
48 
49 /** Archive Page group directory prefix */
50 const char ARCH_PAGE_DIR[] = "page_group_";
51 
52 /** Archive log file prefix */
53 const char ARCH_LOG_FILE[] = "ib_log_";
54 
55 /** Archive page file prefix */
56 const char ARCH_PAGE_FILE[] = "ib_page_";
57 
58 /** //@} */
59 
60 /** File name for the durable file which indicates whether a group was made
61 durable or not. Required to differentiate durable group from group left over by
62 crash during clone operation. */
63 constexpr char ARCH_PAGE_GROUP_DURABLE_FILE_NAME[] = "durable";
64 
65 /** Byte length for printing LSN.
66 Each archive group name is appended with start LSN */
68 
69 /** Max string length for archive log file name */
71  sizeof(ARCH_DIR) + 1 + sizeof(ARCH_LOG_DIR) + MAX_LSN_DECIMAL_DIGIT + 1 +
73 
74 /** Max string length for archive page file name */
76  sizeof(ARCH_DIR) + 1 + sizeof(ARCH_PAGE_DIR) + MAX_LSN_DECIMAL_DIGIT + 1 +
78 
79 /** Max string length for archive group directory name */
81  sizeof(ARCH_DIR) + 1 + sizeof(ARCH_PAGE_DIR) + MAX_LSN_DECIMAL_DIGIT + 1;
82 
83 /** Log archiver background thread */
84 void log_archiver_thread();
85 
86 /** Archiver thread event to signal that data is available */
88 
89 /** Memory block size */
90 constexpr uint ARCH_PAGE_BLK_SIZE = UNIV_PAGE_SIZE_DEF;
91 
92 /** Archiver client state.
93 Archiver clients request archiving for specific interval using
94 the start and stop interfaces. During this time the client is
95 attached to global Archiver system. A client copies archived
96 data for the interval after calling stop. System keeps the data
97 till the time client object is destroyed.
98 
99 @startuml
100 
101  state ARCH_CLIENT_STATE_INIT
102  state ARCH_CLIENT_STATE_STARTED
103  state ARCH_CLIENT_STATE_STOPPED
104 
105  [*] -down-> ARCH_CLIENT_STATE_INIT
106  ARCH_CLIENT_STATE_INIT -down-> ARCH_CLIENT_STATE_STARTED : Attach and start \
107  archiving
108  ARCH_CLIENT_STATE_STARTED -right-> ARCH_CLIENT_STATE_STOPPED : Stop \
109  archiving
110  ARCH_CLIENT_STATE_STOPPED -down-> [*] : Detach client
111 
112 @enduml */
114  /** Client is initialized */
116 
117  /** Archiving started by client */
119 
120  /** Archiving stopped by client */
122 };
123 
124 /** Remove files related to page and log archiving.
125 @param[in] file_path path to the file
126 @param[in] file_name name of the file */
127 void arch_remove_file(const char *file_path, const char *file_name);
128 
129 /** Remove group directory and the files related to page and log archiving.
130 @param[in] dir_path path to the directory
131 @param[in] dir_name directory name */
132 void arch_remove_dir(const char *dir_path, const char *dir_name);
133 
134 /** Archiver system state.
135 Archiver state changes are triggered by client request to start or
136 stop archiving and system wide events like shutdown fatal error etc.
137 Following diagram shows the state transfer.
138 
139 @startuml
140 
141  state ARCH_STATE_INIT
142  state ARCH_STATE_ACTIVE
143  state ARCH_STATE_PREPARE_IDLE
144  state ARCH_STATE_IDLE
145  state ARCH_STATE_ABORT
146 
147  [*] -down-> ARCH_STATE_INIT
148  ARCH_STATE_INIT -down-> ARCH_STATE_ACTIVE : Start archiving
149  ARCH_STATE_ACTIVE -right-> ARCH_STATE_PREPARE_IDLE : Stop archiving
150  ARCH_STATE_PREPARE_IDLE -right-> ARCH_STATE_IDLE : All data archived
151  ARCH_STATE_IDLE -down-> ARCH_STATE_ABORT : Shutdown or Fatal Error
152  ARCH_STATE_PREPARE_IDLE --> ARCH_STATE_ACTIVE : Resume archiving
153  ARCH_STATE_IDLE --> ARCH_STATE_ACTIVE : Start archiving
154  ARCH_STATE_ABORT -down-> [*]
155 
156 @enduml */
158  /** Archiver is initialized */
160 
161  /** Archiver is active and archiving data */
163 
164  /** Archiver is processing last data chunks before idle state */
166 
167  /** Archiver is idle */
169 
170  /** Server is in read only mode, and hence the archiver */
172 
173  /** Archiver is aborted */
175 };
176 
177 /** Archived data block state.
178 A data block is a block in memory that holds dirty page IDs before persisting
179 into disk. Shown below is the state transfer diagram for a data block.
180 
181 @startuml
182 
183  state ARCH_BLOCK_INIT
184  state ARCH_BLOCK_ACTIVE
185  state ARCH_BLOCK_READY_TO_FLUSH
186  state ARCH_BLOCK_FLUSHED
187 
188  [*] -down-> ARCH_BLOCK_INIT
189  ARCH_BLOCK_INIT -> ARCH_BLOCK_ACTIVE : Writing page ID
190  ARCH_BLOCK_ACTIVE -> ARCH_BLOCK_READY_TO_FLUSH : Block is full
191  ARCH_BLOCK_READY_TO_FLUSH -> ARCH_BLOCK_FLUSHED : Block is flushed
192  ARCH_BLOCK_FLUSHED --> ARCH_BLOCK_ACTIVE : Writing page ID
193  ARCH_BLOCK_FLUSHED -down-> [*]
194 
195 @enduml */
197  /** Data block is initialized */
199 
200  /** Data block is active and having data */
202 
203  /** Data block is full but not flushed to disk */
205 
206  /** Data block is flushed and can be reused */
208 };
209 
210 /** Archiver block type */
212  /* Block which holds reset information */
214 
215  /* Block which holds archived page IDs */
217 };
218 
219 /** Archiver block flush type */
221  /** Flush when block is full */
223 
224  /** Flush partial block.
225  Needed for persistent page tracking. */
227 };
228 
229 /** Page Archive doublewrite buffer block offsets */
231  /** Archive doublewrite buffer page offset for RESET page. */
233 
234  /* Archive doublewrite buffer page offset for FULL FLUSH page. */
236 
237  /* Archive doublewrite buffer page offset for PARTIAL FLUSH page. */
239 };
240 
241 /** Initialize Page and Log archiver system
242 @return error code */
244 
245 /** Free Page and Log archiver system */
246 void arch_free();
247 
248 /** Start log archiver background thread.
249 @return error code */
251 
252 /** Start page archiver background thread.
253 @return error code */
255 
256 /** Archiver thread event to signal that data is available */
258 
259 /** Page archiver background thread */
260 void page_archiver_thread();
261 
262 /** Wakes up archiver threads.
263 @return true iff any thread was still alive */
264 bool arch_wake_threads();
265 
266 /** Forward declarations */
267 class Arch_Group;
268 class Arch_Log_Sys;
269 class Arch_Dblwr_Ctx;
270 struct Arch_Recv_Group_Info;
271 
272 /** Position in page ID archiving system */
274  /** Initialize a position */
275  void init();
276 
277  /** Position in the beginning of next block */
278  void set_next();
279 
280  /** Unique block number */
281  uint64_t m_block_num;
282 
283  /** Offset within a block */
285 
287  if (m_block_num < pos.m_block_num ||
288  (m_block_num == pos.m_block_num && m_offset <= pos.m_offset)) {
289  return (true);
290  }
291  return (false);
292  }
293 };
294 
295 /** Structure which represents a point in a file. */
296 struct Arch_Point {
297  /** LSN of the point */
299 
300  /** Position of the point */
302 };
303 
304 /* Structure which represents a file in a group and its reset points. */
306  /* Initialize the structure. */
307  void init();
308 
309  /* Index of the file in the group */
310  uint m_file_index{0};
311 
312  /* LSN of the first reset point in the vector of reset points this
313  structure maintains. Treated as the file LSN. */
314  lsn_t m_lsn{LSN_MAX};
315 
316  /* Vector of reset points which belong to this file */
317  std::vector<Arch_Point> m_start_point;
318 };
319 
320 /* Structure representing list of archived files. */
321 using Arch_Reset = std::deque<Arch_Reset_File>;
322 
323 /** In memory data block in Page ID archiving system */
324 class Arch_Block {
325  public:
326  /** Constructor: Initialize elements
327  @param[in] blk_buf buffer for data block
328  @param[in] size buffer size
329  @param[in] type block type */
331  : m_data(blk_buf), m_size(size), m_type(type) {}
332 
333  /** Do a deep copy of the members of the block passed as the parameter.
334  @note This member needs to be updated whenever a new data member is added to
335  this class. */
336  void copy_data(const Arch_Block *block);
337 
338  /** Set the block ready to begin writing page ID
339  @param[in] pos position to initiate block number */
340  void begin_write(Arch_Page_Pos pos);
341 
342  /** End writing to a block.
343  Change state to #ARCH_BLOCK_READY_TO_FLUSH */
344  void end_write();
345 
346  /** Check if block is initialised or not.
347  @return true if it has been initialised, else false */
348  bool is_init() const { return (m_state == ARCH_BLOCK_INIT); }
349 
350  bool is_active() const { return (m_state == ARCH_BLOCK_ACTIVE); }
351  /** Check if the block can be flushed or not.
352  @return true, if the block cannot be flushed */
353  bool is_flushable() const { return (m_state != ARCH_BLOCK_READY_TO_FLUSH); }
354 
355  /** Set current block flushed.
356  Must hold page archiver sys operation mutex. */
357  void set_flushed() { m_state = ARCH_BLOCK_FLUSHED; }
358 
359  /** Add page ID to current block
360  @param[in] page page from buffer pool
361  @param[in] pos Archiver current position
362  @return true, if successful
363  false, if no more space in current block */
364  bool add_page(buf_page_t *page, Arch_Page_Pos *pos);
365 
366  /* Add reset information to the current reset block.
367  @param[in] reset_lsn reset lsn info
368  @param[in] reset_pos reset pos info which needs to be added
369  to the current reset block */
370  void add_reset(lsn_t reset_lsn, Arch_Page_Pos reset_pos);
371 
372  /** Copy page Ids from this block at read position to a buffer.
373  @param[in] read_pos current read position
374  @param[in] read_len length of data to copy
375  @param[out] read_buff buffer to copy page IDs.
376  Caller must allocate the buffer.
377  @return true, if successful
378  false, if block is already overwritten */
379  bool get_data(Arch_Page_Pos *read_pos, uint read_len, byte *read_buff);
380 
381  /** Copy page Ids from a buffer to this block.
382  @param[in] read_len length of data to copy
383  @param[in] read_buff buffer to copy page IDs from
384  @param[in] read_offset offset from where to write
385  @return true if successful */
386  bool set_data(uint read_len, byte *read_buff, uint read_offset);
387 
388  /** Flush this block to the file group
389  @param[in] file_group current archive group
390  @param[in] type flush type
391  @return error code. */
393 
394  /* Update the block header with the given LSN
395  @param[in] stop_lsn stop LSN to update in the block header
396  @param[in] reset_lsn reset LSN to update in the blk header */
397  void update_block_header(lsn_t stop_lsn, lsn_t reset_lsn);
398 
399  void read(Arch_Group *group, uint64_t offset);
400 
401  /** Set the data length of the block.
402  @param[in] data_len data length */
403  void set_data_len(uint data_len) { m_data_len = data_len; }
404 
405  /** @return data length of the block. */
406  uint get_data_len() const { return (m_data_len); }
407 
408  /** @return block number of the block. */
409  uint64_t get_number() const { return (m_number); }
410 
411  /** @return stop lsn */
412  lsn_t get_stop_lsn() const { return (m_stop_lsn); }
413 
414  /** Get oldest LSN among the pages that are added to this block
415  @return oldest LSN in block pages */
416  lsn_t get_oldest_lsn() const { return (m_oldest_lsn); }
417 
418  /** Get current state of the block
419  @return block state */
420  Arch_Blk_State get_state() const { return (m_state); }
421 
422  /** Check if the block contains only zeroes.
423  @param[in] block block data
424  @return true if block is filled with zeroes. */
425  static bool is_zeroes(const byte *block);
426 
427  /** Check if the block data is valid.
428  @param[in] block block to be validated
429  @return true if it's a valid block, else false */
430  static bool validate(byte *block);
431 
432  /** Get file index of the file the block belongs to.
433  @return file index */
434  static uint get_file_index(uint64_t block_num);
435 
436  /** Get block type from the block header.
437  @param[in] block block from where to get the type
438  @return block type */
439  static uint get_type(byte *block);
440 
441  /** Get block data length from the block header.
442  @param[in] block block from where to get the data length
443  @return block data length */
444  static uint get_data_len(byte *block);
445 
446  /** Get the stop lsn stored in the block header.
447  @param[in] block block from where to fetch the stop lsn
448  @return stop lsn */
449  static lsn_t get_stop_lsn(byte *block);
450 
451  /** Get the block number from the block header.
452  @param[in] block block from where to fetch the block number
453  @return block number */
454  static uint64_t get_block_number(byte *block);
455 
456  /** Get the reset lsn stored in the block header.
457  @param[in] block block from where to fetch the reset lsn
458  @return reset lsn */
459  static lsn_t get_reset_lsn(byte *block);
460 
461  /** Get the checksum stored in the block header.
462  @param[in] block block from where to fetch the checksum
463  @return checksum */
464  static uint32_t get_checksum(byte *block);
465 
466  /** Fetch the offset for a block in the archive file.
467  @param[in] block_num block number
468  @param[in] type type of block
469  @return file offset of the block */
470  static uint64_t get_file_offset(uint64_t block_num, Arch_Blk_Type type);
471 
472  private:
473  /* @note member function copy_data needs to be updated whenever a new data
474  member is added to this class. */
475 
476  /** Block data buffer */
478 
479  /** Block data length in bytes */
480  uint m_data_len{};
481 
482  /** Total block size in bytes */
484 
485  /** State of the block. */
487 
488  /** Unique block number */
489  uint64_t m_number{};
490 
491  /** Type of block. */
493 
494  /** Checkpoint lsn at the time the last page ID was added to the
495  block. */
496  lsn_t m_stop_lsn{LSN_MAX};
497 
498  /** Oldest LSN of all the page IDs added to the block since the last
499  * checkpoint */
500  lsn_t m_oldest_lsn{LSN_MAX};
501 
502  /** Start LSN or the last reset LSN of the group */
503  lsn_t m_reset_lsn{LSN_MAX};
504 };
505 
506 /** Archiver file context.
507 Represents a set of fixed size files within a group */
509  public:
510  /** Constructor: Initialize members */
511  Arch_File_Ctx() { m_file.m_file = OS_FILE_CLOSED; }
512 
513  /** Destructor: Close open file and free resources */
515  close();
516 
517  if (m_name_buf != nullptr) {
518  ut_free(m_name_buf);
519  }
520  }
521 
522  /** Initializes archiver file context.
523  @param[in] path path to the file
524  @param[in] base_dir directory name prefix
525  @param[in] base_file file name prefix
526  @param[in] num_files initial number of files
527  @param[in] file_size file size in bytes
528  @return error code. */
529  dberr_t init(const char *path, const char *base_dir, const char *base_file,
530  uint num_files, uint64_t file_size);
531 
532  /** Open a file at specific index
533  @param[in] read_only open in read only mode
534  @param[in] start_lsn start lsn for the group
535  @param[in] file_index index of the file within the group which needs
536  to be opened
537  @param[in] file_offset start offset
538  @return error code. */
539  dberr_t open(bool read_only, lsn_t start_lsn, uint file_index,
540  uint64_t file_offset);
541 
542  /** Add a new file and open
543  @param[in] start_lsn start lsn for the group
544  @param[in] file_offset start offset
545  @return error code. */
546  dberr_t open_new(lsn_t start_lsn, uint64_t file_offset);
547 
548  /** Open next file for read
549  @param[in] start_lsn start lsn for the group
550  @param[in] file_offset start offset
551  @return error code. */
552  dberr_t open_next(lsn_t start_lsn, uint64_t file_offset);
553 
554  /** Read data from the current file that is open.
555  Caller must ensure that the size is within the limits of current file
556  context.
557  @param[in,out] to_buffer read data into this buffer
558  @param[in] offset file offset from where to read
559  @param[in] size size of data to read in bytes
560  @return error code */
561  dberr_t read(byte *to_buffer, const uint64_t offset, const uint size);
562 
563  /** Write data to this file context from the given file offset.
564  Data source is another file context or buffer. If buffer is NULL, data is
565  copied from input file context. Caller must ensure that the size is within
566  the limits of current file for both source and destination file context.
567  @param[in] from_file file context to copy data from
568  @param[in] from_buffer buffer to copy data or NULL
569  @param[in] offset file offset from where to write
570  @param[in] size size of data to copy in bytes
571  @return error code */
572  dberr_t write(Arch_File_Ctx *from_file, byte *from_buffer, uint offset,
573  uint size);
574 
575  /** Write data to this file context from the current offset.
576  Data source is another file context or buffer. If buffer is NULL, data is
577  copied from input file context. Caller must ensure that the size is within
578  the limits of current file for both source and destination file context.
579  @param[in] from_file file context to copy data from
580  @param[in] from_buffer buffer to copy data or NULL
581  @param[in] size size of data to copy in bytes
582  @return error code */
583  dberr_t write(Arch_File_Ctx *from_file, byte *from_buffer, uint size);
584 
585  /** Flush file. */
586  void flush() {
587  if (m_file.m_file != OS_FILE_CLOSED) {
588  os_file_flush(m_file);
589  }
590  }
591 
592  /** Close file, if open */
593  void close() {
594  if (m_file.m_file != OS_FILE_CLOSED) {
595  os_file_close(m_file);
596  m_file.m_file = OS_FILE_CLOSED;
597  }
598  }
599 
600  /** Check if file is closed
601  @return true, if file is closed */
602  bool is_closed() const { return (m_file.m_file == OS_FILE_CLOSED); }
603 
604  /** Check how much is left in current file
605  @return length left in bytes */
606  uint64_t bytes_left() const {
607  ut_ad(m_size >= m_offset);
608  return (m_size - m_offset);
609  }
610 
611  /** Construct file name at specific index
612  @param[in] idx file index
613  @param[in] dir_lsn lsn of the group
614  @param[out] buffer file name including path.
615  The buffer is allocated by caller.
616  @param[in] length buffer length */
617  void build_name(uint idx, lsn_t dir_lsn, char *buffer, uint length);
618 
619  /** Construct group directory name
620  @param[in] dir_lsn lsn of the group
621  @param[out] buffer directory name.
622  The buffer is allocated by caller.
623  @param[in] length buffer length */
624  void build_dir_name(lsn_t dir_lsn, char *buffer, uint length);
625 
626  /** Get the logical size of a file.
627  @return logical file size. */
628  uint64_t get_size() const { return (m_size); }
629 
630  /* Fetch offset of the file open in this context.
631  @return file offset */
632  uint64_t get_offset() const { return (m_offset); }
633 
634  /** Get number of files
635  @return current file count */
636  uint get_count() const { return (m_count); }
637 
638  /** Get the physical size of a file that is open in this context.
639  @return physical file size */
640  uint64_t get_phy_size() const {
641  ut_ad(m_name_buf != nullptr);
642  os_file_size_t file_size = os_file_get_size(m_name_buf);
643  return (file_size.m_total_size);
644  }
645 
646  /** Update stop lsn of a file in the group.
647  @param[in] file_index file_index the current write_pos belongs to
648  @param[in] stop_lsn stop point */
649  void update_stop_point(uint file_index, lsn_t stop_lsn);
650 
651 #ifdef UNIV_DEBUG
652  /** Print recovery related data.
653  @param[in] file_start_index file index from where to begin */
654  void recovery_reset_print(uint file_start_index);
655 
656  /** Check if the information maintained in the memory is the same
657  as the information maintained in the files.
658  @return true if both sets of information are the same
659  @param[in] group group whose file is being validated
660  @param[in] file_index index of the file which is being validated
661  @param[in] start_lsn
662  @param[in,out] reset_count count of files which has been validated
663  @return true if both the sets of information are the same. */
664  bool validate(Arch_Group *group, uint file_index, lsn_t start_lsn,
665  uint &reset_count);
666 #endif
667 
668  /** Update the reset information in the in-memory structure that we maintain
669  for faster access.
670  @param[in] lsn lsn at the time of reset
671  @param[in] pos pos at the time of reset
672  @retval true if the reset point was saved
673  @retval false if the reset point wasn't saved because it was already saved */
674  void save_reset_point_in_mem(lsn_t lsn, Arch_Page_Pos pos);
675 
676  /** Find the appropriate reset LSN that is less than or equal to the
677  given lsn and fetch the reset point.
678  @param[in] check_lsn LSN to be searched against
679  @param[out] reset_point reset position of the fetched reset point
680  @return true if the search was successful. */
681  bool find_reset_point(lsn_t check_lsn, Arch_Point &reset_point);
682 
683  /** Find the first stop LSN that is greater than the given LSN and fetch
684  the stop point.
685  @param[in] group the group whose stop_point we're interested in
686  @param[in] check_lsn LSN to be searched against
687  @param[out] stop_point stop point
688  @param[in] last_pos position of the last block in the group;
689  m_write_pos if group is active and m_stop_pos if not
690  @return true if the search was successful. */
691  bool find_stop_point(Arch_Group *group, lsn_t check_lsn,
692  Arch_Point &stop_point, Arch_Page_Pos last_pos);
693 
694  /** Delete a single file belonging to the specified file index.
695  @param[in] file_index file index of the file which needs to be deleted
696  @param[in] begin_lsn group's start lsn
697  @return true if successful, else false. */
698  bool delete_file(uint file_index, lsn_t begin_lsn);
699 
700  /** Delete all files for this archive group
701  @param[in] begin_lsn group's start lsn */
702  void delete_files(lsn_t begin_lsn);
703 
704  /** Purge archived files until the specified purge LSN.
705  @param[in] begin_lsn start LSN of the group
706  @param[in] end_lsn end LSN of the group
707  @param[in] purge_lsn purge LSN until which files needs to be purged
708  @return LSN until which purging was successful
709  @retval LSN_MAX if there was no purging done. */
710  lsn_t purge(lsn_t begin_lsn, lsn_t end_lsn, lsn_t purge_lsn);
711 
712  /** Fetch the last reset file and last stop point info during recovery
713  @param[out] reset_file last reset file to be updated
714  @param[out] stop_lsn last stop lsn to be updated */
715  void recovery_fetch_info(Arch_Reset_File &reset_file, lsn_t &stop_lsn) {
716  if (m_reset.size() != 0) {
717  reset_file = m_reset.back();
718  }
719 
720  stop_lsn = get_last_stop_point();
721  }
722 
723  /** Fetch the status of the page tracking system.
724  @param[out] status vector of a pair of (ID, bool) where ID is the
725  start/stop point and bool is true if the ID is a start point else false */
726  void get_status(std::vector<std::pair<lsn_t, bool>> &status) {
727  for (auto reset_file : m_reset) {
728  for (auto reset_point : reset_file.m_start_point) {
729  status.push_back(std::make_pair(reset_point.lsn, true));
730  }
731  }
732  }
733 
734  /** @return the stop_point which was stored last */
736  if (m_stop_points.size() == 0) {
737  return (LSN_MAX);
738  }
739 
740  return (m_stop_points.back());
741  }
742 
743  /** Fetch the reset points pertaining to a file.
744  @param[in] file_index file index of the file from which reset points
745  needs to be fetched
746  @param[in,out] reset_pos Update the reset_pos while fetching the
747  reset points
748  @return error code. */
749  dberr_t fetch_reset_points(uint file_index, Arch_Page_Pos &reset_pos);
750 
751  /** Fetch the stop lsn pertaining to a file.
752  @param[in] last_file true if the file for which the stop point is
753  being fetched for is the last file
754  @param[in,out] write_pos Update the write_pos while fetching the
755  stop points
756  @return error code. */
757  dberr_t fetch_stop_points(bool last_file, Arch_Page_Pos &write_pos);
758 
759  private:
760 #ifdef UNIV_DEBUG
761  /** Check if the reset information maintained in the memory is the same
762  as the information maintained in the given file.
763  @param[in] file file descriptor
764  @param[in] file_index index of the file
765  @param[in,out] reset_count number of files processed containing
766  reset data
767  @return true if both sets of information are the same */
768  bool validate_reset_block_in_file(pfs_os_file_t file, uint file_index,
769  uint &reset_count);
770 
771  /** Check if the stop LSN maintained in the memory is the same as the
772  information maintained in the files.
773  @param[in] group group whose file is being validated
774  @param[in] file file descriptor
775  @param[in] file_index index of the file for which the validation is
776  happening
777  @return true if both the sets of information are the same. */
778  bool validate_stop_point_in_file(Arch_Group *group, pfs_os_file_t file,
779  uint file_index);
780 #endif
781 
782  /** Fetch reset lsn of a particular reset point pertaining to a file.
783  @param[in] block_num block number where the reset occurred.
784  @return reset lsn */
785  lsn_t fetch_reset_lsn(uint64_t block_num);
786 
787  private:
788  /** File name buffer.
789  Used if caller doesn't allocate buffer. */
790  char *m_name_buf{nullptr};
791 
792  /** File name buffer length */
793  uint m_name_len{};
794 
795  /** Fixed length part of the file.
796  Path ended with directory separator. */
797  uint m_base_len{};
798 
799  /** Fixed part of the path to file */
800  const char *m_path_name{nullptr};
801 
802  /** Directory name prefix */
803  const char *m_dir_name{nullptr};
804 
805  /** File name prefix */
806  const char *m_file_name{nullptr};
807 
808  /** Current file descriptor */
810 
811  /** File index within the archive group */
812  uint m_index{};
813 
814  /** Current number of files in the archive group */
815  uint m_count{};
816 
817  /** Current file offset */
818  uint64_t m_offset{};
819 
820  /** File size limit in bytes */
821  uint64_t m_size{};
822 
823  /** Queue of file structure holding reset information pertaining to
824  their respective files in a group.
825  Protected by Arch_Page_Sys::m_mutex and Arch_Page_Sys::m_oper_mutex.
826  @note used only by the page archiver */
828 
829  /** Vector of stop points corresponding to a file.
830  Stop point refers to the stop lsn (checkpoint lsn) until which the pages are
831  guaranteed to be tracked in a file. Each block in a file maintains this
832  information.
833  Protected by Arch_Page_Sys::m_oper_mutex.
834  @note used only by the page archiver */
835  std::vector<lsn_t> m_stop_points;
836 };
837 
838 /** Contiguous archived data for redo log or page tracking.
839 If there is a gap, that is if archiving is stopped and started, a new
840 group is created. */
841 class Arch_Group {
842  public:
843  /** Constructor: Initialize members
844  @param[in] start_lsn start LSN for the group
845  @param[in] header_len length of header for archived files
846  @param[in] mutex archive system mutex from caller */
847  Arch_Group(lsn_t start_lsn, uint header_len, ib_mutex_t *mutex)
848  : m_begin_lsn(start_lsn),
849  m_header_len(header_len)
850 #ifdef UNIV_DEBUG
851  ,
852  m_arch_mutex(mutex)
853 #endif /* UNIV_DEBUG */
854  {
855  m_active_file.m_file = OS_FILE_CLOSED;
856  m_durable_file.m_file = OS_FILE_CLOSED;
857  m_stop_pos.init();
858  }
859 
860  /** Destructor: Delete all files for non-durable archiving. */
861  ~Arch_Group();
862 
863  /** Initialize the doublewrite buffer file context for the archive group.
864  @param[in] path path to the file
865  @param[in] base_file file name prefix
866  @param[in] num_files initial number of files
867  @param[in] file_size file size in bytes
868  @return error code. */
869  static dberr_t init_dblwr_file_ctx(const char *path, const char *base_file,
870  uint num_files, uint64_t file_size);
871 
872  /** Initialize the file context for the archive group.
873  File context keeps the archived data in files on disk. There
874  is one file context for a archive group.
875  @param[in] path path to the file
876  @param[in] base_dir directory name prefix
877  @param[in] base_file file name prefix
878  @param[in] num_files initial number of files
879  @param[in] file_size file size in bytes
880  @return error code. */
881  dberr_t init_file_ctx(const char *path, const char *base_dir,
882  const char *base_file, uint num_files,
883  uint64_t file_size) {
884  return (m_file_ctx.init(path, base_dir, base_file, num_files, file_size));
885  }
886 
887  /* Close the file contexts when they're not required anymore. */
889  m_file_ctx.close();
890 
891  if (m_durable_file.m_file != OS_FILE_CLOSED) {
892  os_file_close(m_durable_file);
893  m_durable_file.m_file = OS_FILE_CLOSED;
894  }
895  }
896 
897  /** Mark archive group inactive.
898  A group is marked inactive by archiver background before entering
899  into idle state ARCH_STATE_IDLE.
900  @param[in] end_lsn lsn where redo archiving is stopped */
901  void disable(lsn_t end_lsn) {
902  m_is_active = false;
903 
904  if (end_lsn != LSN_MAX) {
905  m_end_lsn = end_lsn;
906  }
907  }
908 
909  /** Attach a client to the archive group.
910  @param[in] is_durable true, if durable tracking is requested */
911  void attach(bool is_durable) {
912  ut_ad(mutex_own(m_arch_mutex));
913  ++m_num_active;
914 
915  if (is_durable) {
916  ++m_dur_ref_count;
917  } else {
918  ++m_ref_count;
919  }
920  }
921 
922  /** Detach a client when archiving is stopped by the client.
923  The client still has reference to the group so that the group
924  is not destroyed when it retrieves the archived data. The
925  reference is removed later by #Arch_Group::release.
926  @param[in] stop_lsn archive stop lsn for client
927  @param[in] stop_pos archive stop position for client. Used only by
928  the page_archiver.
929  @return number of active clients */
930  uint detach(lsn_t stop_lsn, Arch_Page_Pos *stop_pos) {
931  ut_ad(m_num_active > 0);
932  ut_ad(mutex_own(m_arch_mutex));
933  --m_num_active;
934 
935  if (m_num_active == 0) {
936  m_end_lsn = stop_lsn;
937  if (stop_pos != nullptr) {
938  m_stop_pos = *stop_pos;
939  }
940  }
941 
942  return (m_num_active);
943  }
944 
945  /** Release the archive group from a client.
946  Reduce the reference count. When all clients release the group,
947  the reference count falls down to zero. The function would then
948  return zero and the caller can remove the group.
949  @param[in] is_durable the client needs durable archiving */
950  void release(bool is_durable) {
951  ut_ad(mutex_own(m_arch_mutex));
952  ut_a(!is_durable);
953 
954  ut_ad(m_ref_count > 0);
955  --m_ref_count;
956  }
957 
958  /** Construct file name for the active file which indicates whether a group
959  is active or not.
960  @note Used only by the page archiver.
961  @return error code. */
962  dberr_t build_active_file_name();
963 
964  /** Construct file name for the durable file which indicates whether a group
965  was made durable or not.
966  @note Used only by the page archiver.
967  @return error code. */
968  dberr_t build_durable_file_name();
969 
970  /** Mark the group active by creating a file in the respective group
971  directory. This is required at the time of recovery to know whether a group
972  was active or not in case of a crash.
973  @note Used only by the page archiver.
974  @return error code. */
975  int mark_active();
976 
977  /** Mark the group durable by creating a file in the respective group
978  directory. This is required at the time of recovery to differentiate durable
979  group from group left over by crash during clone operation.
980  @note Used only by the page archiver.
981  @return error code. */
982  int mark_durable();
983 
984  /** Mark the group inactive by deleting the 'active' file. This is required
985  at the time of crash recovery to know whether a group was active or not in
986  case of a crash.
987  @note Used only by the page archiver.
988  @return error code */
989  int mark_inactive();
990 
991  /** Check if archiving is going on for this group
992  @return true, if the group is active */
993  bool is_active() const { return (m_is_active); }
994 
995  /** Write the header (RESET page) to an archived file.
996  @note Used only by the Page Archiver and not by the Redo Log Archiver.
997  @param[in] from_buffer buffer to copy data
998  @param[in] length size of data to copy in bytes
999  @note Used only by the Page Archiver.
1000  @return error code */
1001  dberr_t write_file_header(byte *from_buffer, uint length);
1002 
1003  /** Write to the doublewrite buffer before writing archived data to a file.
1004  The source is either a file context or buffer. Caller must ensure that data
1005  is in single file in source file context.
1006  @param[in] from_file file context to copy data from
1007  @param[in] from_buffer buffer to copy data or NULL
1008  @param[in] write_size size of data to write in bytes
1009  @param[in] offset offset from where to write
1010  @note Used only by the Page Archiver.
1011  @return error code */
1012  static dberr_t write_to_doublewrite_file(Arch_File_Ctx *from_file,
1013  byte *from_buffer, uint write_size,
1014  Arch_Page_Dblwr_Offset offset);
1015 
1016  /** Archive data to one or more files.
1017  The source is either a file context or buffer. Caller must ensure that data
1018  is in single file in source file context.
1019  @param[in] from_file file context to copy data from
1020  @param[in] from_buffer buffer to copy data or NULL
1021  @param[in] length size of data to copy in bytes
1022  @param[in] partial_write true if the operation is part of partial flush
1023  @param[in] do_persist doublewrite to ensure persistence
1024  @return error code */
1025  dberr_t write_to_file(Arch_File_Ctx *from_file, byte *from_buffer,
1026  uint length, bool partial_write, bool do_persist);
1027 
1028  /** Find the appropriate reset LSN that is less than or equal to the
1029  given lsn and fetch the reset point.
1030  @param[in] check_lsn LSN to be searched against
1031  @param[out] reset_point reset position of the fetched reset point
1032  @return true if the search was successful. */
1034  return (m_file_ctx.find_reset_point(check_lsn, reset_point));
1035  }
1036 
1037  /** Find the first stop LSN that is greater than the given LSN and fetch
1038  the stop point.
1039  @param[in] check_lsn LSN to be searched against
1040  @param[out] stop_point stop point
1041  @param[in] write_pos latest write_pos
1042  @return true if the search was successful. */
1044  Arch_Page_Pos write_pos) {
1045  ut_ad(validate_info_in_files());
1046  Arch_Page_Pos last_pos = is_active() ? write_pos : m_stop_pos;
1047  return (m_file_ctx.find_stop_point(this, check_lsn, stop_point, last_pos));
1048  }
1049 
1050 #ifdef UNIV_DEBUG
1051  /** Adjust end LSN to end of file. This is used in debug
1052  mode to test the case when LSN is at file boundary.
1053  @param[in,out] stop_lsn stop lsn for client
1054  @param[out] blk_len last block length */
1055  void adjust_end_lsn(lsn_t &stop_lsn, uint32_t &blk_len);
1056 
1057  /** Adjust redo copy length to end of file. This is used
1058  in debug mode to archive only till end of file.
1059  @param[in,out] length data to copy in bytes */
1060  void adjust_copy_length(uint32_t &length);
1061 
1062  /** Check if the information maintained in the memory is the same
1063  as the information maintained in the files.
1064  @return true if both sets of information are the same */
1065  bool validate_info_in_files();
1066 #endif /* UNIV_DEBUG */
1067 
1068  /** Get the total number of archived files belonging to this group.
1069  @return number of archived files */
1070  uint get_file_count() const { return (m_file_ctx.get_count()); }
1071 
1072  /** Check if any client (durable or not) is attached to the archiver.
1073  @return true if any client is attached, else false */
1074  bool is_referenced() const {
1075  return (m_ref_count > 0) || (m_dur_ref_count > 0);
1076  }
1077 
1078  /** Check if any client requiring durable archiving is active.
1079  @return true if any durable client is still attached, else false */
1081  return (m_num_active != m_ref_count);
1082  }
1083 
1084  /** Check if any client requires durable archiving.
1085  @return true if there is at least 1 client that requires durable archiving*/
1086  bool is_durable() const { return (m_dur_ref_count > 0); }
1087 
1088  /** Attach system client to the archiver during recovery if any group was
1089  active at the time of crash. */
1090  void attach_during_recovery() { ++m_dur_ref_count; }
1091 
1092  /** Purge archived files until the specified purge LSN.
1093  @param[in] purge_lsn LSN until which archived files needs to be
1094  purged
1095  @param[out] purged_lsn LSN until which purging is successfule;
1096  LSN_MAX if there was no purging done
1097  @return error code */
1098  uint purge(lsn_t purge_lsn, lsn_t &purged_lsn);
1099 
1100  /** Operations to be done at the time of shutdown. */
1101  static void shutdown() { s_dblwr_file_ctx.close(); }
1102 
1103  /** Update the reset information in the in-memory structure that we maintain
1104  for faster access.
1105  @param[in] lsn lsn at the time of reset
1106  @param[in] pos pos at the time of reset
1107  @retval true if the reset point was saved
1108  @retval false if the reset point wasn't saved because it was already saved */
1110  m_file_ctx.save_reset_point_in_mem(lsn, pos);
1111  }
1112 
1113  /** Update stop lsn of a file in the group.
1114  @param[in] pos stop position
1115  @param[in] stop_lsn stop point */
1117  m_file_ctx.update_stop_point(Arch_Block::get_file_index(pos.m_block_num),
1118  stop_lsn);
1119  }
1120 
1121  /** Recover the information belonging to this group from the archived files.
1122  @param[in,out] group_info structure containing information of a
1123  group obtained during recovery by scanning files
1124  @param[in,out] new_empty_file true if there is/was an empty archived
1125  file
1126  @param[in] dblwr_ctx file context related to doublewrite
1127  buffer
1128  @param[out] write_pos latest write position at the time of
1129  crash /shutdown that needs to be filled
1130  @param[out] reset_pos latest reset position at the time crash
1131  /shutdown that needs to be filled
1132  @return error code */
1133  dberr_t recover(Arch_Recv_Group_Info *group_info, bool &new_empty_file,
1134  Arch_Dblwr_Ctx *dblwr_ctx, Arch_Page_Pos &write_pos,
1135  Arch_Page_Pos &reset_pos);
1136 
1137  /** Reads the latest data block and reset block.
1138  This would be required in case of active group to start page archiving after
1139  recovery, and in case of inactive group to fetch stop lsn. So we perform this
1140  operation regardless of whether it's an active or inactive group.
1141  @param[in] buf buffer to read the blocks into
1142  @param[in] offset offset from where to read
1143  @param[in] type block type
1144  @return error code */
1145  dberr_t recovery_read_latest_blocks(byte *buf, uint64_t offset,
1147 
1148  /** Fetch the last reset file and last stop point info during recovery
1149  @param[out] reset_file last reset file to be updated
1150  @param[out] stop_lsn last stop lsn to be updated */
1151  void recovery_fetch_info(Arch_Reset_File &reset_file, lsn_t &stop_lsn) {
1152  m_file_ctx.recovery_fetch_info(reset_file, stop_lsn);
1153  }
1154 
1155 #ifdef UNIV_DEBUG
1156  /** Print recovery related data.
1157  @param[in] file_start_index file index from where to begin */
1158  void recovery_reset_print(uint file_start_index) {
1159  DBUG_PRINT("page_archiver", ("Group : %" PRIu64 "", m_begin_lsn));
1160  m_file_ctx.recovery_reset_print(file_start_index);
1161  DBUG_PRINT("page_archiver", ("End lsn: %" PRIu64 "", m_end_lsn));
1162  }
1163 #endif
1164 
1165  /** Parse block for block info (header/data).
1166  @param[in] cur_pos position to read
1167  @param[in,out] buff buffer into which to write the parsed data
1168  @param[in] buff_len length of the buffer
1169  @return error code */
1170  int read_data(Arch_Page_Pos cur_pos, byte *buff, uint buff_len);
1171 
1172  /** Get archived file name at specific index in this group.
1173  Caller would use it to open and copy data from archived files.
1174  @param[in] idx file index in the group
1175  @param[out] name_buf file name and path. Caller must
1176  allocate the buffer.
1177  @param[in] buf_len allocated buffer length */
1178  void get_file_name(uint idx, char *name_buf, uint buf_len) {
1179  ut_ad(name_buf != nullptr);
1180 
1181  /* Build name from the file context. */
1182  m_file_ctx.build_name(idx, m_begin_lsn, name_buf, buf_len);
1183  }
1184 
1185  /** Get file size for this group.
1186  Fixed size files are used for archiving data in a group.
1187  @return file size in bytes */
1188  uint64_t get_file_size() const { return (m_file_ctx.get_size()); }
1189 
1190  /** Get start LSN for this group
1191  @return start LSN */
1192  lsn_t get_begin_lsn() const { return (m_begin_lsn); }
1193 
1194  /** @return stop LSN for this group */
1195  lsn_t get_end_lsn() const { return (m_end_lsn); }
1196 
1197  /** @return stop block position of the group. */
1198  Arch_Page_Pos get_stop_pos() const { return (m_stop_pos); }
1199 
1200  /** Fetch the status of the page tracking system.
1201  @param[out] status vector of a pair of (ID, bool) where ID is the
1202  start/stop point and bool is true if the ID is a start point else false */
1203  void get_status(std::vector<std::pair<lsn_t, bool>> &status) {
1204  m_file_ctx.get_status(status);
1205 
1206  if (!is_active()) {
1207  status.push_back(std::make_pair(m_end_lsn, false));
1208  }
1209  }
1210 
1211  /** Disable copy construction */
1212  Arch_Group(Arch_Group const &) = delete;
1213 
1214  /** Disable assignment */
1215  Arch_Group &operator=(Arch_Group const &) = delete;
1216 
1217  private:
1218  /** Get page IDs from archived file
1219  @param[in] read_pos position to read from
1220  @param[in] read_len length of data to read
1221  @param[in] read_buff buffer to read page IDs
1222  @return error code */
1223  int read_from_file(Arch_Page_Pos *read_pos, uint read_len, byte *read_buff);
1224 
1225  /** Get the directory name for this archive group.
1226  It is used for cleaning up the archive directory.
1227  @param[out] name_buf directory name and path. Caller must
1228  allocate the buffer.
1229  @param[in] buf_len buffer length */
1230  void get_dir_name(char *name_buf, uint buf_len) {
1231  m_file_ctx.build_dir_name(m_begin_lsn, name_buf, buf_len);
1232  }
1233 
1234  /** Check and replace blocks in archived files belonging to a group
1235  from the doublewrite buffer if required.
1236  @param[in] dblwr_ctx Doublewrite context which has the doublewrite
1237  buffer blocks
1238  @return error code */
1239  dberr_t recovery_replace_pages_from_dblwr(Arch_Dblwr_Ctx *dblwr_ctx);
1240 
1241  /** Delete the last file if there are no blocks flushed to it.
1242  @param[out] num_files number of files present in the group
1243  @param[in] start_index file index from where the files are present
1244  If this is not 0 then the files with file index less that this might have
1245  been purged.
1246  @param[in] durable true if the group is durable
1247  @param[out] empty_file true if there is/was an empty archived file
1248  @return error code. */
1249  dberr_t recovery_cleanup_if_required(uint &num_files, uint start_index,
1250  bool durable, bool &empty_file);
1251 
1252  /** Start parsing the archive file for archive group information.
1253  @param[out] write_pos latest write position at the time of
1254  crash /shutdown that needs to be filled
1255  @param[out] reset_pos latest reset position at the time crash
1256  /shutdown that needs to be filled
1257  @param[in] start_index file index from where the files are present
1258  If this is not 0 then the files with file index less that this might have
1259  been purged.
1260  @return error code */
1261  dberr_t recovery_parse(Arch_Page_Pos &write_pos, Arch_Page_Pos &reset_pos,
1262  size_t start_index);
1263 
1264  /** Open the file which was open at the time of a crash, during crash
1265  recovery, and set the file offset to the last written offset.
1266  @param[in] write_pos block position from where page IDs will be
1267  tracked
1268  @param[in] empty_file true if an empty archived file was present at
1269  the time of crash. We delete this file as part of crash recovery process so
1270  this needs to be handled here.
1271  @return error code. */
1272  dberr_t open_file_during_recovery(Arch_Page_Pos write_pos, bool empty_file);
1273 
1274  private:
1275  /** If the group is active */
1276  bool m_is_active{true};
1277 
1278  /** To know which group was active at the time of a crash/shutdown during
1279  recovery we create an empty file in the group directory. This holds the name
1280  of the file. */
1281  char *m_active_file_name{nullptr};
1282 
1283  /** File descriptor for a file required to indicate that the group was
1284  active at the time of crash during recovery . */
1286 
1287  /** File name for the durable file which indicates whether a group was made
1288  durable or not. Required to differentiate durable group from group left over
1289  by crash during clone operation. */
1290  char *m_durable_file_name{nullptr};
1291 
1292  /** File descriptor for a file to indicate that the group was made durable or
1293  not. Required to differentiate durable group from group left over by crash
1294  during clone operation. */
1296 
1297  /** Number of clients referencing the group */
1298  uint m_ref_count{};
1299 
1300  /** Number of clients referencing for durable archiving */
1301  uint m_dur_ref_count{};
1302 
1303  /** Number of clients for which archiving is in progress */
1304  uint m_num_active{};
1305 
1306  /** Start LSN for the archive group */
1307  lsn_t m_begin_lsn{LSN_MAX};
1308 
1309  /** End lsn for this archive group */
1310  lsn_t m_end_lsn{LSN_MAX};
1311 
1312  /** Stop position of the group, if it's not active. */
1313  Arch_Page_Pos m_stop_pos{};
1314 
1315  /** Header length for the archived files */
1316  uint m_header_len{};
1317 
1318  /** Archive file context */
1320 
1321  /** Doublewrite buffer file context.
1322  Note - Used only in the case of page archiver. */
1324 
1325 #ifdef UNIV_DEBUG
1326  /** Mutex protecting concurrent operations by multiple clients.
1327  This is either the redo log or page archive system mutex. Currently
1328  used for assert checks. */
1329  ib_mutex_t *m_arch_mutex;
1330 #endif /* UNIV_DEBUG */
1331 };
1332 
1333 /** A list of archive groups */
1334 using Arch_Grp_List = std::list<Arch_Group *, ut_allocator<Arch_Group *>>;
1335 
1336 /** An iterator for archive group */
1337 using Arch_Grp_List_Iter = Arch_Grp_List::iterator;
1338 
1339 /** Redo log archiving system */
1341  public:
1342  /** Constructor: Initialize members */
1344  : m_state(ARCH_STATE_INIT),
1345  m_archived_lsn(LSN_MAX),
1346  m_group_list(),
1347  m_current_group() {
1348  mutex_create(LATCH_ID_LOG_ARCH, &m_mutex);
1349  }
1350 
1351  /** Destructor: Free mutex */
1353  ut_ad(m_state == ARCH_STATE_INIT || m_state == ARCH_STATE_ABORT);
1354  ut_ad(m_current_group == nullptr);
1355  ut_ad(m_group_list.empty());
1356 
1357  mutex_free(&m_mutex);
1358  }
1359 
1360  /** Check if archiving is in progress.
1361  In #ARCH_STATE_PREPARE_IDLE state, all clients have already detached
1362  but archiver background task is yet to finish.
1363  @return true, if archiving is active */
1364  bool is_active() {
1365  return (m_state == ARCH_STATE_ACTIVE || m_state == ARCH_STATE_PREPARE_IDLE);
1366  }
1367 
1368  /** Check if archiver system is in initial state
1369  @return true, if redo log archiver state is #ARCH_STATE_INIT */
1370  bool is_init() { return (m_state == ARCH_STATE_INIT); }
1371 
1372  /** Get LSN up to which redo is archived
1373  @return last archived redo LSN */
1374  lsn_t get_archived_lsn() { return (m_archived_lsn.load()); }
1375 
1376  /** Get current redo log archive group
1377  @return current archive group */
1378  Arch_Group *get_arch_group() { return (m_current_group); }
1379 
1380  /** Start redo log archiving.
1381  If archiving is already in progress, the client
1382  is attached to current group.
1383  @param[out] group log archive group
1384  @param[out] start_lsn start lsn for client
1385  @param[out] header redo log header
1386  @param[in] is_durable if client needs durable archiving
1387  @return error code */
1388  int start(Arch_Group *&group, lsn_t &start_lsn, byte *header,
1389  bool is_durable);
1390 
1391  /** Stop redo log archiving.
1392  If other clients are there, the client is detached from
1393  the current group.
1394  @param[out] group log archive group
1395  @param[out] stop_lsn stop lsn for client
1396  @param[out] log_blk redo log trailer block
1397  @param[in,out] blk_len length in bytes
1398  @return error code */
1399  int stop(Arch_Group *group, lsn_t &stop_lsn, byte *log_blk,
1400  uint32_t &blk_len);
1401 
1402  /** Force to abort the archiver (state becomes ARCH_STATE_ABORT). */
1403  void force_abort();
1404 
1405  /** Release the current group from client.
1406  @param[in] group group the client is attached to
1407  @param[in] is_durable if client needs durable archiving */
1408  void release(Arch_Group *group, bool is_durable);
1409 
1410  /** Archive accumulated redo log in current group.
1411  This interface is for archiver background task to archive redo log
1412  data by calling it repeatedly over time.
1413  @param[in] init true when called for first time; it will then
1414  be set to false
1415  @param[in] curr_ctx system redo logs to copy data from
1416  @param[out] arch_lsn LSN up to which archiving is completed
1417  @param[out] wait true, if no more redo to archive
1418  @return true, if archiving is aborted */
1419  bool archive(bool init, Arch_File_Ctx *curr_ctx, lsn_t *arch_lsn, bool *wait);
1420 
1421  /** Acquire redo log archiver mutex.
1422  It synchronizes concurrent start and stop operations by
1423  multiple clients. */
1424  void arch_mutex_enter() { mutex_enter(&m_mutex); }
1425 
1426  /** Release redo log archiver mutex */
1427  void arch_mutex_exit() { mutex_exit(&m_mutex); }
1428 
1429  /** Disable copy construction */
1430  Arch_Log_Sys(Arch_Log_Sys const &) = delete;
1431 
1432  /** Disable assignment */
1433  Arch_Log_Sys &operator=(Arch_Log_Sys const &) = delete;
1434 
1435  private:
1436  /** Wait for archive system to come out of #ARCH_STATE_PREPARE_IDLE.
1437  If the system is preparing to idle, #start needs to wait
1438  for it to come to idle state.
1439  @return true, if successful
1440  false, if needs to abort */
1441  bool wait_idle();
1442 
1443  /** Wait for redo log archive up to the target LSN.
1444  We need to wait till current log sys LSN during archive stop.
1445  @param[in] target_lsn target archive LSN to wait for
1446  @return error code */
1447  int wait_archive_complete(lsn_t target_lsn);
1448 
1449  /** Update checkpoint LSN and related information in redo
1450  log header block.
1451  @param[in,out] header redo log header buffer
1452  @param[in] checkpoint_lsn checkpoint LSN for recovery */
1453  void update_header(byte *header, lsn_t checkpoint_lsn);
1454 
1455  /** Check and set log archive system state and output the
1456  amount of redo log available for archiving.
1457  @param[in] is_abort need to abort
1458  @param[in,out] archived_lsn LSN up to which redo log is archived
1459  @param[out] to_archive amount of redo log to be archived */
1460  Arch_State check_set_state(bool is_abort, lsn_t *archived_lsn,
1461  uint *to_archive);
1462 
1463  /** Copy redo log from file context to archiver files.
1464  @param[in] file_ctx file context for system redo logs
1465  @param[in] length data to copy in bytes
1466  @return error code */
1467  dberr_t copy_log(Arch_File_Ctx *file_ctx, uint length);
1468 
1469  private:
1470  /** Mutex to protect concurrent start, stop operations */
1471  ib_mutex_t m_mutex;
1472 
1473  /** Archiver system state.
1474  #m_state is protected by #m_mutex and #log_t::writer_mutex. For changing
1475  the state both needs to be acquired. For reading, hold any of the two
1476  mutexes. Same is true for #m_archived_lsn. */
1478 
1479  /** System has archived log up to this LSN */
1481 
1482  /** List of log archive groups */
1484 
1485  /** Current archive group */
1487 
1488  /** Chunk size to copy redo data */
1490 
1491  /** System log file number where the archiving started */
1493 
1494  /** System log file offset where the archiving started */
1495  ib_uint64_t m_start_log_offset;
1496 };
1497 
1498 /** Vector of page archive in memory blocks */
1499 using Arch_Block_Vec = std::vector<Arch_Block *, ut_allocator<Arch_Block *>>;
1500 
1501 /** Page archiver in memory data */
1503  /** Constructor */
1505 
1506  /** Allocate buffer and initialize blocks
1507  @return true, if successful */
1508  bool init();
1509 
1510  /** Delete blocks and buffer */
1511  void clean();
1512 
1513  /** Get the block for a position
1514  @param[in] pos position in page archive sys
1515  @param[in] type block type
1516  @return page archive in memory block */
1518 
1519  /** @return temporary block used to copy active block for partial flush. */
1521  return (m_partial_flush_block);
1522  }
1523 
1524  /** Vector of data blocks */
1525  Arch_Block_Vec m_data_blocks{};
1526 
1527  /** Reset block */
1528  Arch_Block *m_reset_block{nullptr};
1529 
1530  /** Temporary block used to copy active block for partial flush. */
1531  Arch_Block *m_partial_flush_block{nullptr};
1532 
1533  /** Block size in bytes */
1534  uint m_block_size{};
1535 
1536  /** Total number of blocks */
1537  uint m_num_data_blocks{};
1538 
1539  /** In memory buffer */
1540  byte *m_buffer{nullptr};
1541 };
1542 
1543 /** Forward declaration. */
1544 class Page_Arch_Client_Ctx;
1545 
1546 /** Dirty page archive system */
1548  public:
1549  /** Constructor: Initialize elements and create mutex */
1550  Arch_Page_Sys();
1551 
1552  /** Destructor: Free memory buffer and mutexes */
1553  ~Arch_Page_Sys();
1554 
1555  /** Start dirty page ID archiving.
1556  If archiving is already in progress, the client is attached to current group.
1557  @param[out] group page archive group the client gets attached to
1558  @param[out] start_lsn start lsn for client in archived data
1559  @param[out] start_pos start position for client in archived data
1560  @param[in] is_durable true if client needs durable archiving
1561  @param[in] restart true if client is already attached to current group
1562  @param[in] recovery true if archiving is being started during
1563  recovery
1564  @return error code */
1565  int start(Arch_Group **group, lsn_t *start_lsn, Arch_Page_Pos *start_pos,
1566  bool is_durable, bool restart, bool recovery);
1567 
1568  /** Stop dirty page ID archiving.
1569  If other clients are there, the client is detached from the current group.
1570  @param[in] group page archive group the client is attached to
1571  @param[out] stop_lsn stop lsn for client
1572  @param[out] stop_pos stop position in archived data
1573  @param[in] is_durable true if client needs durable archiving
1574  @return error code */
1575  int stop(Arch_Group *group, lsn_t *stop_lsn, Arch_Page_Pos *stop_pos,
1576  bool is_durable);
1577 
1578  /** Start dirty page ID archiving during recovery.
1579  @param[in] group Group which needs to be attached to the archiver
1580  @param[in] new_empty_file true if there was a empty file created
1581  @return error code */
1582  int start_during_recovery(Arch_Group *group, bool new_empty_file);
1583 
1584  /** Release the current group from client.
1585  @param[in] group group the client is attached to
1586  @param[in] is_durable if client needs durable archiving
1587  @param[in] start_pos start position when the client calling the
1588  release was started */
1589  void release(Arch_Group *group, bool is_durable, Arch_Page_Pos start_pos);
1590 
1591  /** Check and add page ID to archived data.
1592  Check for duplicate page.
1593  @param[in] bpage page to track
1594  @param[in] track_lsn LSN when tracking started
1595  @param[in] frame_lsn current LSN of the page
1596  @param[in] force if true, add page ID without check */
1597  void track_page(buf_page_t *bpage, lsn_t track_lsn, lsn_t frame_lsn,
1598  bool force);
1599 
1600  /** Flush all the unflushed inactive blocks and flush the active block if
1601  required.
1602  @note Used only during the checkpointing process.
1603  @param[in] checkpoint_lsn next checkpoint LSN */
1604  void flush_at_checkpoint(lsn_t checkpoint_lsn);
1605 
1606  /** Archive dirty page IDs in current group.
1607  This interface is for archiver background task to flush page archive
1608  data to disk by calling it repeatedly over time.
1609  @param[out] wait true, if no more data to archive
1610  @return true, if archiving is aborted */
1611  bool archive(bool *wait);
1612 
1613  /** Acquire dirty page ID archiver mutex.
1614  It synchronizes concurrent start and stop operations by multiple clients. */
1615  void arch_mutex_enter() { mutex_enter(&m_mutex); }
1616 
1617  /** Release page ID archiver mutex */
1618  void arch_mutex_exit() { mutex_exit(&m_mutex); }
1619 
1620  /** Acquire dirty page ID archive operation mutex.
1621  It synchronizes concurrent page ID write to memory buffer. */
1622  void arch_oper_mutex_enter() { mutex_enter(&m_oper_mutex); }
1623 
1624  /** Release page ID archiver operatiion mutex */
1625  void arch_oper_mutex_exit() { mutex_exit(&m_oper_mutex); }
1626 
1627  /* Save information at the time of a reset considered as the reset point.
1628  @param[in] is_durable true if it's durable page tracking
1629  @return true if the reset point information stored in the data block needs to
1630  be flushed to disk before returning to the caller, else false */
1631  bool save_reset_point(bool is_durable);
1632 
1633  /** Wait for reset info to be flushed to disk.
1634  @param[in] request_block block number until which blocks need to be
1635  flushed
1636  @return true if flushed, else false */
1637  bool wait_for_reset_info_flush(uint64_t request_block);
1638 
1639  /** Get the group which has tracked pages between the start_id and stop_id.
1640  @param[in,out] start_id start LSN from which tracked pages are
1641  required; updated to the actual start LSN used for the search
1642  @param[in,out] stop_id stop_lsn until when tracked pages are
1643  required; updated to the actual stop LSN used for the search
1644  @param[out] group group which has the required tracked
1645  pages, else nullptr.
1646  @return error */
1647  int fetch_group_within_lsn_range(lsn_t &start_id, lsn_t &stop_id,
1648  Arch_Group **group);
1649 
1650  /** Purge the archived files until the specified purge LSN.
1651  @param[in] purge_lsn purge lsn until where files needs to be purged
1652  @return error code
1653  @retval 0 if purge was successful */
1654  uint purge(lsn_t *purge_lsn);
1655 
1656  /** Update the stop point in all the required structures.
1657  @param[in] cur_blk block which needs to be updated with the stop info */
1658  void update_stop_info(Arch_Block *cur_blk);
1659 
1660  /** Fetch the status of the page tracking system.
1661  @param[out] status vector of a pair of (ID, bool) where ID is the
1662  start/stop point and bool is true if the ID is a start point else false */
1663  void get_status(std::vector<std::pair<lsn_t, bool>> &status) {
1664  for (auto group : m_group_list) {
1665  group->get_status(status);
1666  }
1667  }
1668 
1669  /** Given start and stop position find number of pages tracked between them
1670  @param[in] start_pos start position
1671  @param[in] stop_pos stop position
1672  @param[out] num_pages number of pages tracked between start and stop
1673  position
1674  @return false if start_pos and stop_pos are invalid else true */
1675  bool get_num_pages(Arch_Page_Pos start_pos, Arch_Page_Pos stop_pos,
1676  uint64_t &num_pages);
1677 
1678  /** Get approximate number of tracked pages between two given LSN values.
1679  @param[in,out] start_id fetch archived page Ids from this LSN
1680  @param[in,out] stop_id fetch archived page Ids until this LSN
1681  @param[out] num_pages number of pages tracked between specified
1682  LSN range
1683  @return error code */
1684  int get_num_pages(lsn_t &start_id, lsn_t &stop_id, uint64_t *num_pages);
1685 
1686  /** Get page IDs from a specific position.
1687  Caller must ensure that read_len doesn't exceed the block.
1688  @param[in] group group whose pages we're interested in
1689  @param[in] read_pos position in archived data
1690  @param[in] read_len amount of data to read
1691  @param[out] read_buff buffer to return the page IDs.
1692  @note Caller must allocate the buffer.
1693  @return true if we could successfully read the block. */
1694  bool get_pages(Arch_Group *group, Arch_Page_Pos *read_pos, uint read_len,
1695  byte *read_buff);
1696 
1697  /** Get archived page Ids between two given LSN values.
1698  Attempt to read blocks directly from in memory buffer. If overwritten,
1699  copy from archived files.
1700  @param[in] thd thread handle
1701  @param[in] cbk_func called repeatedly with page ID buffer
1702  @param[in] cbk_ctx callback function context
1703  @param[in,out] start_id fetch archived page Ids from this LSN
1704  @param[in,out] stop_id fetch archived page Ids until this LSN
1705  @param[in] buf buffer to fill page IDs
1706  @param[in] buf_len buffer length in bytes
1707  @return error code */
1708  int get_pages(MYSQL_THD thd, Page_Track_Callback cbk_func, void *cbk_ctx,
1709  lsn_t &start_id, lsn_t &stop_id, byte *buf, uint buf_len);
1710 
1711  /** Set the latest stop LSN to the checkpoint LSN at the time it's called. */
1712  void post_recovery_init();
1713 
1714  /** Recover the archiver system at the time of startup. Recover information
1715  related to all the durable groups and start archiving if any group was active
1716  at the time of crash/shutdown.
1717  @return error code */
1718  dberr_t recover();
1719 
1720 #ifdef UNIV_DEBUG
1721  /** Print information related to the archiver for debugging purposes. */
1722  void print();
1723 #endif
1724 
1725  /** Set the state of the archiver system to read only. */
1727 
1728  /** Check if archiver system is in initial state
1729  @return true, if page ID archiver state is #ARCH_STATE_INIT */
1730  bool is_init() const { return (m_state == ARCH_STATE_INIT); }
1731 
1732  /** Check if archiver system is active
1733  @return true, if page ID archiver state is #ARCH_STATE_ACTIVE or
1734  #ARCH_STATE_PREPARE_IDLE. */
1735  bool is_active() const {
1736  return (m_state == ARCH_STATE_ACTIVE || m_state == ARCH_STATE_PREPARE_IDLE);
1737  }
1738 
1739  /** @return true if in abort state */
1740  bool is_abort() const { return (m_state == ARCH_STATE_ABORT); }
1741 
1742  /** Get the mutex protecting concurrent start, stop operations required
1743  for initialising group during recovery.
1744  @return mutex */
1745  ib_mutex_t *get_mutex() { return (&m_mutex); }
1746 
1747  /** @return operation mutex */
1748  ib_mutex_t *get_oper_mutex() { return (&m_oper_mutex); }
1749 
1750  /** Fetch the system client context.
1751  @return system client context. */
1752  Page_Arch_Client_Ctx *get_sys_client() const { return (m_ctx); }
1753 
1754  /** @return the latest stop LSN */
1755  lsn_t get_latest_stop_lsn() const { return (m_latest_stop_lsn); }
1756 
1757  /** Disable copy construction */
1758  Arch_Page_Sys(Arch_Page_Sys const &) = delete;
1759 
1760  /** Disable assignment */
1761  Arch_Page_Sys &operator=(Arch_Page_Sys const &) = delete;
1762 
1763  class Recv;
1764 
1765  private:
1766  /** Wait for archive system to come out of #ARCH_STATE_PREPARE_IDLE.
1767  If the system is preparing to idle, #start needs to wait
1768  for it to come to idle state.
1769  @return true, if successful
1770  false, if needs to abort */
1771  bool wait_idle();
1772 
1773  /** Check if the gap from last reset is short.
1774  If not many page IDs are added till last reset, we avoid
1775  taking a new reset point
1776  @return true, if the gap is small. */
1777  bool is_gap_small();
1778 
1779  /** Enable tracking pages in all buffer pools.
1780  @param[in] tracking_lsn track pages from this LSN */
1781  void set_tracking_buf_pool(lsn_t tracking_lsn);
1782 
1783  /** Track pages for which IO is already started. */
1784  void track_initial_pages();
1785 
1786  /** Flush the blocks to disk.
1787  @param[out] wait true, if no more data to archive
1788  @return error code */
1789  dberr_t flush_blocks(bool *wait);
1790 
1791  /** Flush all the blocks which are ready to be flushed but not flushed.
1792  @param[out] cur_pos position of block which needs to be flushed
1793  @param[in] end_pos position of block until which the blocks need to
1794  be flushed
1795  @return error code */
1796  dberr_t flush_inactive_blocks(Arch_Page_Pos &cur_pos, Arch_Page_Pos end_pos);
1797 
1798  /** Do a partial flush of the current active block
1799  @param[in] cur_pos position of block which needs to be flushed
1800  @param[in] partial_reset_block_flush true if reset block needs to be
1801  flushed
1802  @return error code */
1803  dberr_t flush_active_block(Arch_Page_Pos cur_pos,
1804  bool partial_reset_block_flush);
1805 
1806  private:
1807  /** Mutex protecting concurrent start, stop operations */
1808  ib_mutex_t m_mutex;
1809 
1810  /** Archiver system state. */
1812 
1813  /** List of log archive groups */
1814  Arch_Grp_List m_group_list{};
1815 
1816  /** Position where last client started archiving */
1817  Arch_Page_Pos m_last_pos{};
1818 
1819  /** LSN when last client started archiving */
1820  lsn_t m_last_lsn{LSN_MAX};
1821 
1822  /** Latest LSN until where the tracked pages have been flushed. */
1823  lsn_t m_latest_stop_lsn{LSN_MAX};
1824 
1825  /** LSN until where the groups are purged. */
1826  lsn_t m_latest_purged_lsn{LSN_MAX};
1827 
1828  /** Mutex protecting concurrent operation on data */
1829  ib_mutex_t m_oper_mutex;
1830 
1831  /** Current archive group */
1832  Arch_Group *m_current_group{nullptr};
1833 
1834  /** In memory data buffer */
1835  ArchPageData m_data{};
1836 
1837  /** Position to add new page ID */
1838  Arch_Page_Pos m_write_pos{};
1839 
1840  /** Position to add new reset element */
1841  Arch_Page_Pos m_reset_pos{};
1842 
1843  /** Position set to explicitly request the flush archiver to flush until
1844  this position.
1845  @note this is always increasing and is only updated by the requester thread
1846  like checkpoint */
1847  Arch_Page_Pos m_request_flush_pos{};
1848 
1849  /** Block number set to explicitly request the flush archiver to partially
1850  flush the current active block with reset LSN.
1851  @note this is always increasing and is only updated by the requester thread
1852  like checkpoint */
1853  uint64_t m_request_blk_num_with_lsn{std::numeric_limits<uint64_t>::max()};
1854 
1855  /** Block number set once the flush archiver partially flushes the current
1856  active block with reset LSN.
1857  @note this is always increasing and is only updated by the requester thread
1858  like checkpoint */
1859  uint64_t m_flush_blk_num_with_lsn{std::numeric_limits<uint64_t>::max()};
1860 
1861  /** Position for start flushing
1862  @note this is always increasing and is only updated by the page archiver
1863  thread */
1864  Arch_Page_Pos m_flush_pos{};
1865 
1866  /** The index of the file the last reset belonged to. */
1867  uint m_last_reset_file_index{0};
1868 
1869  /** System client. */
1871 };
1872 
1873 /** Redo log archiver system global */
1874 extern Arch_Log_Sys *arch_log_sys;
1875 
1876 /** Dirty page ID archiver system global */
1878 
1879 #endif /* ARCH_ARCH_INCLUDE */
void arch_oper_mutex_enter()
Acquire dirty page ID archive operation mutex.
Definition: arch0arch.h:1622
static uint recover
Definition: myisamlog.cc:91
uint64_t get_phy_size() const
Get the physical size of a file that is open in this context.
Definition: arch0arch.h:640
bool read(T *ap, const GV &gv, const char *key)
Definition: sdi_impl.h:339
ib_mutex_t * get_oper_mutex()
Definition: arch0arch.h:1748
static void print(const char *key, int keylen, const char *val, int vallen)
Print the key value pair.
Definition: mcstat.c:106
Doublewrite buffer context.
Definition: arch0recv.h:82
const uint MAX_LSN_DECIMAL_DIGIT
Byte length for printing LSN.
Definition: arch0arch.h:67
Arch_State m_state
Archiver system state.
Definition: arch0arch.h:1477
bool is_durable() const
Check if any client requires durable archiving.
Definition: arch0arch.h:1086
const uint MAX_ARCH_DIR_NAME_LEN
Max string length for archive group directory name.
Definition: arch0arch.h:80
uint get_file_count() const
Get the total number of archived files belonging to this group.
Definition: arch0arch.h:1070
Arch_Blk_Flush_Type
Archiver block flush type.
Definition: arch0arch.h:220
Arch_Page_Sys * arch_page_sys
Dirty page ID archiver system global.
Definition: arch0arch.cc:39
uint64_t lsn_t
Type used for all log sequence number storage and arithmetics.
Definition: log0types.h:60
Arch_Blk_Type m_type
Type of block.
Definition: arch0arch.h:492
Archiving stopped by client.
Definition: arch0arch.h:121
atomic_lsn_t m_archived_lsn
System has archived log up to this LSN.
Definition: arch0arch.h:1480
int start_page_archiver_background()
Start page archiver background thread.
Definition: arch0arch.cc:530
bool is_closed() const
Check if file is closed.
Definition: arch0arch.h:602
void arch_mutex_exit()
Release redo log archiver mutex.
Definition: arch0arch.h:1427
Dirty page archiver client context.
Definition: arch0page.h:166
char buffer[STRING_BUFFER]
Definition: test_sql_9_sessions.cc:57
HARNESS_EXPORT int delete_file(const std::string &path) noexcept
Removes a file.
Definition: filesystem-posix.cc:292
std::vector< T, ut_allocator< T > > vector
Specialization of vector which uses ut_allocator.
Definition: ut0new.h:1307
lsn_t get_end_lsn() const
Definition: arch0arch.h:1195
lsn_t get_last_stop_point() const
Definition: arch0arch.h:735
Definition: arch0arch.h:216
os_event_t page_archiver_thread_event
Archiver thread event to signal that data is available.
Definition: arch0page.cc:48
Sparse file size information.
Definition: os0file.h:570
byte * m_data
Block data buffer.
Definition: arch0arch.h:477
const char ARCH_LOG_FILE[]
Archive log file prefix.
Definition: arch0arch.h:53
constexpr char ARCH_PAGE_GROUP_DURABLE_FILE_NAME[]
File name for the durable file which indicates whether a group was made durable or not...
Definition: arch0arch.h:63
static int check_lsn(app_data_ptr a)
Definition: xcom_base.cc:1688
Arch_Log_Sys * arch_log_sys
Redo log archiver system global.
Definition: arch0arch.cc:36
int start_log_archiver_background()
Start log archiver background thread.
Definition: arch0arch.cc:508
pthread_mutex_t mutex
Definition: memcached.c:384
Archiving started by client.
Definition: arch0arch.h:118
ArchPageData()
Constructor.
Definition: arch0arch.h:1504
uint64_t get_size() const
Get the logical size of a file.
Definition: arch0arch.h:628
bool operator<(Arch_Page_Pos pos)
Definition: arch0arch.h:286
Arch_Grp_List::iterator Arch_Grp_List_Iter
An iterator for archive group.
Definition: arch0arch.h:1337
lsn_t get_oldest_lsn() const
Get oldest LSN among the pages that are added to this block.
Definition: arch0arch.h:416
static Arch_File_Ctx s_dblwr_file_ctx
Doublewrite buffer file context.
Definition: arch0arch.h:1323
ib_mutex_t m_mutex
Mutex to protect concurrent start, stop operations.
Definition: arch0arch.h:1471
lsn_t get_stop_lsn() const
Definition: arch0arch.h:412
const uint MAX_ARCH_LOG_FILE_NAME_LEN
Max string length for archive log file name.
Definition: arch0arch.h:70
lsn_t get_latest_stop_lsn() const
Definition: arch0arch.h:1755
void arch_mutex_enter()
Acquire redo log archiver mutex.
Definition: arch0arch.h:1424
bool is_init() const
Check if block is initialised or not.
Definition: arch0arch.h:348
pfs_os_file_t m_active_file
File descriptor for a file required to indicate that the group was active at the time of crash during...
Definition: arch0arch.h:1285
Position in page ID archiving system.
Definition: arch0arch.h:273
Archiver is active and archiving data.
Definition: arch0arch.h:162
lsn_t lsn
LSN of the point.
Definition: arch0arch.h:298
const char ARCH_PAGE_DIR[]
Archive Page group directory prefix.
Definition: arch0arch.h:50
Information of a group required for parsing information from the archived file.
Definition: arch0recv.h:41
bool is_active(space_id_t space_id, bool get_latch=true)
Definition: trx0purge.cc:1134
uint detach(lsn_t stop_lsn, Arch_Page_Pos *stop_pos)
Detach a client when archiving is stopped by the client.
Definition: arch0arch.h:930
Arch_File_Ctx()
Constructor: Initialize members.
Definition: arch0arch.h:511
ib_uint64_t m_start_log_offset
System log file offset where the archiving started.
Definition: arch0arch.h:1495
uint get_data_len() const
Definition: arch0arch.h:406
void attach(bool is_durable)
Attach a client to the archive group.
Definition: arch0arch.h:911
#define close(s)
Definition: win32.h:103
#define MYSQL_THD
Definition: backup_page_tracker.h:35
void arch_remove_file(const char *file_path, const char *file_name)
Remove files related to page and log archiving.
Definition: arch0arch.cc:59
Redo log constants and functions.
void write(W *w, const T &t, const char *key, size_t key_sz)
Definition: sdi_impl.h:333
bool is_active()
Check if archiving is in progress.
Definition: arch0arch.h:1364
uint64_t m_block_num
Unique block number.
Definition: arch0arch.h:281
Definition: arch0arch.h:235
uint64_t get_file_size() const
Get file size for this group.
Definition: arch0arch.h:1188
std::deque< Arch_Reset_File > Arch_Reset
Definition: arch0arch.h:321
Definition: sync0types.h:393
static uint64_t lsn
Definition: xcom_base.cc:426
Arch_Group * m_current_group
Current archive group.
Definition: arch0arch.h:1486
#define DBUG_PRINT(keyword, arglist)
Definition: my_dbug.h:181
void close()
Close file, if open.
Definition: arch0arch.h:593
~Arch_File_Ctx()
Destructor: Close open file and free resources.
Definition: arch0arch.h:514
Arch_Page_Pos pos
Position of the point.
Definition: arch0arch.h:301
void recovery_fetch_info(Arch_Reset_File &reset_file, lsn_t &stop_lsn)
Fetch the last reset file and last stop point info during recovery.
Definition: arch0arch.h:715
pfs_os_file_t m_file
Current file descriptor.
Definition: arch0arch.h:809
Server is in read only mode, and hence the archiver.
Definition: arch0arch.h:171
static const os_file_t OS_FILE_CLOSED
Definition: os0file.h:165
void save_reset_point_in_mem(lsn_t lsn, Arch_Page_Pos pos)
Update the reset information in the in-memory structure that we maintain for faster access...
Definition: arch0arch.h:1109
bool read_only
Definition: mysqld.cc:1303
void set_next()
Position in the beginning of next block.
Definition: arch0page.cc:1465
int page
Definition: ctype-mb.cc:1234
dberr_t init_file_ctx(const char *path, const char *base_dir, const char *base_file, uint num_files, uint64_t file_size)
Initialize the file context for the archive group.
Definition: arch0arch.h:881
Arch_Page_Dblwr_Offset
Page Archive doublewrite buffer block offsets.
Definition: arch0arch.h:230
char * pos
Definition: do_ctype.cc:76
int flush_blocks(MI_CHECK *param, KEY_CACHE *key_cache, File file)
Definition: mi_check.cc:1739
static uint get_file_index(uint64_t block_num)
Get file index of the file the block belongs to.
Definition: arch0page.cc:1185
lsn_t get_begin_lsn() const
Get start LSN for this group.
Definition: arch0arch.h:1192
constexpr lsn_t LSN_MAX
Maximum possible lsn value is slightly higher than the maximum sn value, because lsn sequence enumera...
Definition: log0log.h:157
void arch_free()
Free Page and Log archiver system.
Definition: arch0arch.cc:149
Definition: arch0arch.h:238
Archiver is idle.
Definition: arch0arch.h:168
uint m_offset
Offset within a block.
Definition: arch0arch.h:284
Archiver is processing last data chunks before idle state.
Definition: arch0arch.h:165
Archiver is aborted.
Definition: arch0arch.h:174
#define mutex_free(M)
Definition: ut0mutex.h:124
bool is_abort() const
Definition: arch0arch.h:1740
bool is_flushable() const
Check if the block can be flushed or not.
Definition: arch0arch.h:353
static void start(mysql_harness::PluginFuncEnv *env)
Definition: http_auth_backend_plugin.cc:161
Dirty page archive system.
Definition: arch0arch.h:1547
bool is_durable_client_active() const
Check if any client requiring durable archiving is active.
Definition: arch0arch.h:1080
ib_mutex_t * get_mutex()
Get the mutex protecting concurrent start, stop operations required for initialising group during rec...
Definition: arch0arch.h:1745
os_offset_t m_total_size
Total size of file in bytes.
Definition: os0file.h:572
bool is_active() const
Definition: arch0arch.h:350
void recovery_fetch_info(Arch_Reset_File &reset_file, lsn_t &stop_lsn)
Fetch the last reset file and last stop point info during recovery.
Definition: arch0arch.h:1151
void arch_mutex_enter()
Acquire dirty page ID archiver mutex.
Definition: arch0arch.h:1615
dberr_t
Definition: db0err.h:38
Data block is active and having data.
Definition: arch0arch.h:201
void arch_oper_mutex_exit()
Release page ID archiver operatiion mutex.
Definition: arch0arch.h:1625
#define os_file_close(file)
Definition: os0file.h:1325
static char * path
Definition: mysqldump.cc:131
void get_status(std::vector< std::pair< lsn_t, bool >> &status)
Fetch the status of the page tracking system.
Definition: arch0arch.h:726
const char ARCH_PAGE_FILE[]
Archive page file prefix.
Definition: arch0arch.h:56
uint64_t bytes_left() const
Check how much is left in current file.
Definition: arch0arch.h:606
constexpr uint ARCH_PAGE_BLK_SIZE
Memory block size.
Definition: arch0arch.h:90
Archive doublewrite buffer page offset for RESET page.
Definition: arch0arch.h:232
unsigned int uint
Definition: uca-dump.cc:29
Arch_Blk_State
Archived data block state.
Definition: arch0arch.h:196
Arch_File_Ctx m_file_ctx
Archive file context.
Definition: arch0arch.h:1319
void get_file_name(uint idx, char *name_buf, uint buf_len)
Get archived file name at specific index in this group.
Definition: arch0arch.h:1178
Arch_Reset m_reset
Queue of file structure holding reset information pertaining to their respective files in a group...
Definition: arch0arch.h:827
#define ut_a(EXPR)
Abort execution if EXPR does not evaluate to nonzero.
Definition: ut0dbg.h:53
void recovery_reset_print(uint file_start_index)
Print recovery related data.
Definition: arch0arch.h:1158
Data block is initialized.
Definition: arch0arch.h:198
Page_Arch_Client_Ctx * get_sys_client() const
Fetch the system client context.
Definition: arch0arch.h:1752
Arch_Client_State
Archiver client state.
Definition: arch0arch.h:113
void set_flushed()
Set current block flushed.
Definition: arch0arch.h:357
bool is_referenced() const
Check if any client (durable or not) is attached to the archiver.
Definition: arch0arch.h:1074
void update_stop_point(Arch_Page_Pos pos, lsn_t stop_lsn)
Update stop lsn of a file in the group.
Definition: arch0arch.h:1116
dberr_t arch_init()
Initialize Page and Log archiver system.
Definition: arch0arch.cc:117
const char ARCH_LOG_DIR[]
Archive Log group directory prefix.
Definition: arch0arch.h:47
Page archiver in memory data.
Definition: arch0arch.h:1502
Policy based mutexes.
std::vector< Arch_Point > m_start_point
Definition: arch0arch.h:317
uint m_start_log_index
System log file number where the archiving started.
Definition: arch0arch.h:1492
void arch_mutex_exit()
Release page ID archiver mutex.
Definition: arch0arch.h:1618
Client is initialized.
Definition: arch0arch.h:115
void release(bool is_durable)
Release the archive group from a client.
Definition: arch0arch.h:950
lsn_t get_archived_lsn()
Get LSN up to which redo is archived.
Definition: arch0arch.h:1374
Definition: buf0buf.h:1165
ib_mutex_t * m_arch_mutex
Mutex protecting concurrent operations by multiple clients.
Definition: arch0arch.h:1329
Archiver file context.
Definition: arch0arch.h:508
ib_mutex_t m_oper_mutex
Mutex protecting concurrent operation on data.
Definition: arch0arch.h:1829
#define os_file_flush(file)
Definition: os0file.h:1347
void set_read_only_mode()
Set the state of the archiver system to read only.
Definition: arch0arch.h:1726
Arch_Group * get_arch_group()
Get current redo log archive group.
Definition: arch0arch.h:1378
bool is_active() const
Check if archiver system is active.
Definition: arch0arch.h:1735
pfs_os_file_t m_durable_file
File descriptor for a file to indicate that the group was made durable or not.
Definition: arch0arch.h:1295
std::vector< Arch_Block *, ut_allocator< Arch_Block * > > Arch_Block_Vec
Vector of page archive in memory blocks.
Definition: arch0arch.h:1499
#define ut_ad(EXPR)
Debug assertion.
Definition: ut0dbg.h:65
In memory data block in Page ID archiving system.
Definition: arch0arch.h:324
void flush()
Flush file.
Definition: arch0arch.h:586
os_event_t log_archiver_thread_event
Archiver thread event to signal that data is available.
Definition: arch0arch.cc:42
static void stop(mysql_harness::PluginFuncEnv *)
Definition: rest_mock_server.cc:274
stdx::expected< int, std::error_code > open(const char *fname, int flags, mode_t mode) noexcept
Definition: file_handle.cc:81
Structure which represents a point in a file.
Definition: arch0arch.h:296
static size_t file_size
Definition: mysql_config_editor.cc:70
void flush(ENGINE_HANDLE *h, ENGINE_HANDLE_V1 *h1)
Definition: suite_stubs.c:97
#define mutex_enter(M)
Definition: ut0mutex.h:115
Definition: arch0arch.h:305
Flush partial block.
Definition: arch0arch.h:226
uint64_t get_offset() const
Definition: arch0arch.h:632
Arch_State
Archiver system state.
Definition: arch0arch.h:157
Arch_Block(byte *blk_buf, uint size, Arch_Blk_Type type)
Constructor: Initialize elements.
Definition: arch0arch.h:330
Page_Arch_Client_Ctx * m_ctx
System client.
Definition: arch0arch.h:1870
std::vector< lsn_t > m_stop_points
Vector of stop points corresponding to a file.
Definition: arch0arch.h:835
Data block is flushed and can be reused.
Definition: arch0arch.h:207
Arch_Block * get_partial_flush_block() const
Definition: arch0arch.h:1520
void init()
Initialize a position.
Definition: arch0page.cc:1459
Common file descriptor for file IO instrumentation with PFS on windows and other platforms.
Definition: os0file.h:152
uint m_size
Total block size in bytes.
Definition: arch0arch.h:483
bool is_init() const
Check if archiver system is in initial state.
Definition: arch0arch.h:1730
void arch_remove_dir(const char *dir_path, const char *dir_name)
Remove group directory and the files related to page and log archiving.
Definition: arch0arch.cc:89
#define mutex_exit(M)
Definition: ut0mutex.h:122
void page_archiver_thread()
Page archiver background thread.
Definition: arch0page.cc:51
Arch_Grp_List m_group_list
List of log archive groups.
Definition: arch0arch.h:1483
static void shutdown()
Operations to be done at the time of shutdown.
Definition: arch0arch.h:1101
#define ut_free(ptr)
Definition: ut0new.h:1123
InnoDB condition variable.
Definition: os0event.cc:66
Archiver is initialized.
Definition: arch0arch.h:159
void get_status(std::vector< std::pair< lsn_t, bool >> &status)
Fetch the status of the page tracking system.
Definition: arch0arch.h:1203
bool find_reset_point(lsn_t check_lsn, Arch_Point &reset_point)
Find the appropriate reset LSN that is less than or equal to the given lsn and fetch the reset point...
Definition: arch0arch.h:1033
void get_dir_name(char *name_buf, uint buf_len)
Get the directory name for this archive group.
Definition: arch0arch.h:1230
uint64_t get_number() const
Definition: arch0arch.h:409
bool arch_wake_threads()
Wakes up archiver threads.
Definition: arch0arch.cc:46
Flush when block is full.
Definition: arch0arch.h:222
void set_data_len(uint data_len)
Set the data length of the block.
Definition: arch0arch.h:403
void attach_during_recovery()
Attach system client to the archiver during recovery if any group was active at the time of crash...
Definition: arch0arch.h:1090
Arch_Blk_Type
Archiver block type.
Definition: arch0arch.h:211
Recovery system data structure for the archiver.
Definition: arch0recv.h:131
void log_archiver_thread()
Log archiver background thread.
Definition: arch0arch.cc:553
bool is_active() const
Check if archiving is going on for this group.
Definition: arch0arch.h:993
Arch_Blk_State get_state() const
Get current state of the block.
Definition: arch0arch.h:420
#define OS_FILE_PREFIX
Prefix all files and directory created under data directory with special string so that it never conf...
Definition: os0file.h:67
Definition: os0file.h:85
Data block is full but not flushed to disk.
Definition: arch0arch.h:204
Arch_Page_Pos get_stop_pos() const
Definition: arch0arch.h:1198
static STATUS status
Definition: mysql.cc:198
ib_mutex_t m_mutex
Mutex protecting concurrent start, stop operations.
Definition: arch0arch.h:1808
~Arch_Log_Sys()
Destructor: Free mutex.
Definition: arch0arch.h:1352
stdx::expected< void, error_type > wait(native_handle_type fd, wait_type wt)
Definition: socket.h:517
#define mutex_create(I, M)
Definition: ut0mutex.h:113
os_file_size_t os_file_get_size(const char *filename)
Gets a file size.
Definition: os0file.cc:3464
Contiguous archived data for redo log or page tracking.
Definition: arch0arch.h:841
const uint MAX_ARCH_PAGE_FILE_NAME_LEN
Max string length for archive page file name.
Definition: arch0arch.h:75
#define mutex_own(M)
Checks that the current thread owns the mutex.
Definition: ut0mutex.h:156
void purge(lob::DeleteContext *ctx, dict_index_t *index, trx_id_t trxid, undo_no_t undo_no, ulint rec_type, const upd_field_t *uf)
Purge an LOB (either of compressed or uncompressed).
Definition: lob0purge.cc:370
unsigned char byte
Blob class.
Definition: common.h:159
const char ARCH_DIR[]
Archive directory prefix.
Definition: arch0arch.h:44
bool find_stop_point(lsn_t check_lsn, Arch_Point &stop_point, Arch_Page_Pos write_pos)
Find the first stop LSN that is greater than the given LSN and fetch the stop point.
Definition: arch0arch.h:1043
Redo log archiving system.
Definition: arch0arch.h:1340
uint m_chunk_size
Chunk size to copy redo data.
Definition: arch0arch.h:1489
std::atomic< lsn_t > atomic_lsn_t
Alias for atomic based on lsn_t.
Definition: log0types.h:66
void close_file_ctxs()
Definition: arch0arch.h:888
Arch_Log_Sys()
Constructor: Initialize members.
Definition: arch0arch.h:1343
void disable(lsn_t end_lsn)
Mark archive group inactive.
Definition: arch0arch.h:901
bool restart(THD *thd)
Initialize the dictionary while restarting the server.
Definition: bootstrapper.cc:902
bool length(const dd::Spatial_reference_system *srs, const Geometry *g1, double *length, bool *null) noexcept
Computes the length of linestrings and multilinestrings.
Definition: length.cc:75
std::list< Arch_Group *, ut_allocator< Arch_Group * > > Arch_Grp_List
A list of archive groups.
Definition: arch0arch.h:1334
bool is_init()
Check if archiver system is in initial state.
Definition: arch0arch.h:1370
uint get_count() const
Get number of files.
Definition: arch0arch.h:636
Arch_Group(lsn_t start_lsn, uint header_len, ib_mutex_t *mutex)
Constructor: Initialize members.
Definition: arch0arch.h:847
void get_status(std::vector< std::pair< lsn_t, bool >> &status)
Fetch the status of the page tracking system.
Definition: arch0arch.h:1663
const char * get_type(TYPELIB *typelib, unsigned int nr)
Get type.
Definition: typelib.cc:140
int(* Page_Track_Callback)(MYSQL_THD thd, const unsigned char *buffer, size_t buf_len, int num_pages, void *user_ctx)
Page tracking callback function.
Definition: page_track_service.h:57
Definition: arch0arch.h:213