MySQL 9.1.0
Source Code Documentation
buf0flu.cc File Reference

The database buffer buf_pool flush algorithm. More...

#include <math.h>
#include <my_dbug.h>
#include <mysql/service_thd_wait.h>
#include <sys/types.h>
#include <time.h>
#include "buf0buf.h"
#include "buf0checksum.h"
#include "buf0flu.h"
#include "ha_prototypes.h"
#include "my_inttypes.h"
#include "sql_thd_internal_api.h"
#include "page0zip.h"
#include "arch0arch.h"
#include "buf0lru.h"
#include "buf0rea.h"
#include "fil0fil.h"
#include "fsp0sysspace.h"
#include "ibuf0ibuf.h"
#include "log0buf.h"
#include "log0chkp.h"
#include "log0write.h"
#include "my_compiler.h"
#include "os0file.h"
#include "os0thread-create.h"
#include "page0page.h"
#include "srv0mon.h"
#include "srv0srv.h"
#include "srv0start.h"
#include "trx0sys.h"
#include "ut0byte.h"
#include "ut0math.h"
#include "ut0stage.h"

Classes

struct  page_cleaner_slot_t
 Page cleaner request state for each buffer pool instance. More...
 
struct  page_cleaner_t
 Page cleaner structure common for all threads. More...
 
struct  Check
 Functor to validate the flush list. More...
 

Namespaces

namespace  Adaptive_flush
 

Enumerations

enum  page_cleaner_state_t { PAGE_CLEANER_STATE_NONE = 0 , PAGE_CLEANER_STATE_REQUESTED , PAGE_CLEANER_STATE_FLUSHING , PAGE_CLEANER_STATE_FINISHED }
 State for page cleaner array slot. More...
 

Functions

lsn_t get_flush_sync_lsn () noexcept
 Get the lsn up to which data pages are to be synchronously flushed. More...
 
static void buf_flush_sync_datafiles ()
 Flush a batch of writes to the datafiles that have already been written to the dblwr buffer on disk. More...
 
static void buf_flush_page_coordinator_thread ()
 Thread tasked with flushing dirty pages from the buffer pools. More...
 
static void buf_flush_page_cleaner_thread ()
 Worker thread of page_cleaner. More...
 
static void incr_flush_list_size_in_bytes (buf_block_t *block, buf_pool_t *buf_pool)
 Increases flush_list size in bytes with the page size in inline function. More...
 
static bool buf_flush_validate_low (const buf_pool_t *buf_pool)
 Validate a buffer pool instance flush list. More...
 
static bool buf_flush_validate_skip (buf_pool_t *buf_pool)
 Validates the flush list some of the time. More...
 
static buf_page_tbuf_flush_insert_in_flush_rbt (buf_page_t *bpage)
 Insert a block in the flush_rbt and returns a pointer to its predecessor or NULL if no predecessor. More...
 
static void buf_flush_delete_from_flush_rbt (buf_page_t *bpage)
 Delete a bpage from the flush_rbt. More...
 
static int buf_flush_block_cmp (const void *p1, const void *p2)
 Compare two modified blocks in the buffer pool. More...
 
void buf_flush_init_flush_rbt (void)
 Initialize the red-black tree to speed up insertions into the flush_list during recovery process. More...
 
void buf_flush_free_flush_rbt (void)
 Frees up the red-black tree. More...
 
bool buf_are_flush_lists_empty_validate (void)
 Checks if all flush lists are empty. More...
 
static bool buf_flush_list_order_validate (lsn_t earlier_added_lsn, lsn_t new_added_lsn)
 Checks that order of two consecutive pages in flush list would be valid, according to their oldest_modification values. More...
 
static lsn_t buf_flush_borrow_lsn (const buf_pool_t *buf_pool)
 Borrows LSN from the recent added dirty page to the flush list. More...
 
void buf_flush_insert_into_flush_list (buf_pool_t *buf_pool, buf_block_t *block, lsn_t lsn)
 Inserts a modified block into the flush list. More...
 
void buf_flush_insert_sorted_into_flush_list (buf_pool_t *buf_pool, buf_block_t *block, lsn_t lsn)
 Inserts a modified block into the flush list in the right sorted position. More...
 
bool buf_flush_ready_for_replace (const buf_page_t *bpage)
 Returns true if the file page block is immediately suitable for replacement, i.e., the transition FILE_PAGE => NOT_USED allowed. More...
 
static bool buf_flush_ready_for_flush_gen (buf_page_t *bpage, buf_flush_t flush_type, bool atomic)
 Check if the block was modified and was ready for flushing. More...
 
static bool buf_flush_was_ready_for_flush (buf_page_t *bpage, buf_flush_t flush_type)
 Check if the block was modified and was ready for flushing at some point in time during the call. More...
 
bool buf_flush_ready_for_flush (buf_page_t *bpage, buf_flush_t flush_type)
 Check if the block is modified and ready for flushing. More...
 
void buf_flush_remove (buf_page_t *bpage)
 Remove a block from the flush list of modified blocks. More...
 
void buf_flush_relocate_on_flush_list (buf_page_t *bpage, buf_page_t *dpage)
 Relocates a buffer control block on the flush_list. More...
 
void buf_flush_write_complete (buf_page_t *bpage)
 Updates the flush system data structures when a write is completed. More...
 
void buf_flush_update_zip_checksum (buf_frame_t *page, ulint size, lsn_t lsn, bool skip_lsn_check)
 Calculate the checksum of a page from compressed table and update the page. More...
 
bool page_is_uncompressed_type (const byte *page)
 Check if page type is uncompressed. More...
 
void buf_flush_init_for_writing (const buf_block_t *block, byte *page, void *page_zip_, lsn_t newest_lsn, bool skip_checksum, bool skip_lsn_check)
 Initialize a page for writing to the tablespace. More...
 
static void buf_flush_write_block_low (buf_page_t *bpage, buf_flush_t flush_type, bool sync)
 Does an asynchronous write of a buffer page. More...
 
bool buf_flush_page (buf_pool_t *buf_pool, buf_page_t *bpage, buf_flush_t flush_type, bool sync)
 Writes a flushable page asynchronously from the buffer pool to a file. More...
 
bool buf_flush_page_try (buf_pool_t *buf_pool, buf_block_t *block)
 Writes a flushable page asynchronously from the buffer pool to a file. More...
 
static bool buf_flush_check_neighbor (const page_id_t &page_id, buf_flush_t flush_type)
 Check if the page is in buffer pool and can be flushed. More...
 
static ulint buf_flush_try_neighbors (const page_id_t &page_id, buf_flush_t flush_type, ulint n_flushed, ulint n_to_flush)
 Flushes to disk all flushable pages within the flush area. More...
 
static bool buf_flush_page_and_try_neighbors (buf_page_t *bpage, buf_flush_t flush_type, ulint n_to_flush, ulint *count)
 Check if the block is modified and ready for flushing. More...
 
static ulint buf_free_from_unzip_LRU_list_batch (buf_pool_t *buf_pool, ulint max)
 This utility moves the uncompressed frames of pages to the free list. More...
 
static ulint buf_flush_LRU_list_batch (buf_pool_t *buf_pool, ulint max)
 This utility flushes dirty blocks from the end of the LRU list. More...
 
static ulint buf_do_LRU_batch (buf_pool_t *buf_pool, ulint max)
 Flush and move pages from LRU or unzip_LRU list to the free list. More...
 
static ulint buf_do_flush_list_batch (buf_pool_t *buf_pool, ulint min_n, lsn_t lsn_limit)
 This utility flushes dirty blocks from the end of the flush_list. More...
 
static ulint buf_flush_batch (buf_pool_t *buf_pool, buf_flush_t flush_type, ulint min_n, lsn_t lsn_limit)
 This utility flushes dirty blocks from the end of the LRU list or flush_list. More...
 
static void buf_flush_stats (ulint page_count_flush, ulint page_count_LRU)
 Gather the aggregated stats for both flush list and LRU list flushing. More...
 
static bool buf_flush_start (buf_pool_t *buf_pool, buf_flush_t flush_type)
 Start a buffer flush batch for LRU or flush list. More...
 
static void buf_flush_end (buf_pool_t *buf_pool, buf_flush_t flush_type)
 End a buffer flush batch for LRU or flush list. More...
 
void buf_flush_await_no_flushing (buf_pool_t *buf_pool, buf_flush_t flush_type)
 Waits until there's no flush of the given type from given BP instance. More...
 
bool buf_flush_do_batch (buf_pool_t *buf_pool, buf_flush_t type, ulint min_n, lsn_t lsn_limit, ulint *n_processed)
 Do flushing batch of a given type. More...
 
bool buf_flush_lists (ulint min_n, lsn_t lsn_limit, ulint *n_processed)
 This utility flushes dirty blocks from the end of the flush list of all buffer pool instances. More...
 
bool buf_flush_single_page_from_LRU (buf_pool_t *buf_pool)
 This function picks up a single page from the tail of the LRU list, flushes it (if it is dirty), removes it from page_hash and LRU list and puts it on the free list. More...
 
static ulint buf_flush_LRU_list (buf_pool_t *buf_pool)
 Clears up tail of the LRU list of a given buffer pool instance: Put replaceable pages at the tail of LRU to the free list Flush dirty pages at the tail of LRU to the disk The depth to which we scan each buffer pool is controlled by dynamic config parameter innodb_LRU_scan_depth. More...
 
bool Adaptive_flush::initialize (ulint n_pages_last)
 Initialize flush parameters for current iteration. More...
 
void Adaptive_flush::set_average ()
 Set average LSN and page flush speed across multiple iterations. More...
 
ulint Adaptive_flush::get_pct_for_dirty ()
 Calculates if flushing is required based on number of dirty pages in the buffer pool. More...
 
ulint Adaptive_flush::get_pct_for_lsn (lsn_t age)
 Calculates if flushing is required based on redo generation rate. More...
 
ulint Adaptive_flush::set_flush_target_by_lsn (bool sync_flush, lsn_t sync_flush_limit_lsn)
 Set page flush target based on LSN change and checkpoint age. More...
 
ulint Adaptive_flush::set_flush_target_by_page (ulint n_pages_lsn)
 Set page flush target based on dirty pages in buffer pool. More...
 
ulint Adaptive_flush::page_recommendation (ulint last_pages_in, bool is_sync_flush, lsn_t sync_flush_limit_lsn)
 This function is called approximately once every second by the page_cleaner thread, unless it is sync flushing mode, in which case it is called every small round. More...
 
static ulint pc_sleep_if_needed (std::chrono::steady_clock::time_point next_loop_time, int64_t sig_count)
 Puts the page_cleaner thread to sleep if it has finished work in less than a second. More...
 
bool buf_flush_page_cleaner_is_active ()
 Checks if page cleaners are active. More...
 
void buf_flush_page_cleaner_init ()
 Initialize page_cleaner. More...
 
static void buf_flush_page_cleaner_close (void)
 Close page_cleaner. More...
 
static void pc_request (ulint min_n, lsn_t lsn_limit)
 Requests for all slots to flush all buffer pool instances. More...
 
static ulint pc_flush_slot (void)
 Do flush for one slot. More...
 
static bool pc_wait_finished (ulint *n_flushed_lru, ulint *n_flushed_list)
 Wait until all flush requests are finished. More...
 
static void buf_flush_page_cleaner_disabled_loop (void)
 Loop used to disable page cleaner threads. More...
 
void buf_flush_page_cleaner_disabled_debug_update (THD *, SYS_VAR *, void *, const void *save)
 Disables page cleaner threads (coordinator and workers). More...
 
void buf_flush_fsync ()
 Executes fsync for all tablespaces, to fsync all pages written to disk. More...
 
void buf_flush_sync_all_buf_pools ()
 Synchronously flush dirty blocks from the end of the flush list of all buffer pool instances. More...
 
bool buf_flush_validate (buf_pool_t *buf_pool)
 Validates the flush list. More...
 
ulint buf_pool_get_dirty_pages_count (buf_pool_t *buf_pool, space_id_t id, Flush_observer *observer)
 Check if there are any dirty pages that belong to a space id in the flush list in a particular buffer pool. More...
 
static ulint buf_flush_get_dirty_pages_count (space_id_t id, Flush_observer *observer)
 Check if there are any dirty pages that belong to a space id in the flush list. More...
 

Variables

static uint buf_flush_lsn_scan_factor = 3
 Factor for scan length to determine n_pages for intended oldest LSN progress. More...
 
static lsn_t buf_flush_sync_lsn = 0
 Target oldest LSN for the requested flush_sync. More...
 
os_event_t buf_flush_event
 Event to synchronise with the flushing. More...
 
os_event_t buf_flush_tick_event
 Event to wait for one flushing step. More...
 
static ut::unique_ptr< page_cleaner_tpage_cleaner
 
bool innodb_page_cleaner_disabled_debug
 Value of MySQL global variable used to disable page cleaner. More...
 
constexpr uint32_t BUF_LRU_MIN_LEN = 256
 If LRU list of a buf_pool is less than this size then LRU eviction should not happen. More...
 
std::chrono::steady_clock::time_point Adaptive_flush::cur_iter_time
 Time stamp of current iteration. More...
 
lsn_t Adaptive_flush::cur_iter_lsn = 0
 LSN at current iteration. More...
 
ulint Adaptive_flush::cur_iter_pages_dirty = 0
 Number of dirty pages in flush list in current iteration. More...
 
ulint Adaptive_flush::cur_iter_dirty_pct = 0
 Dirty page percentage in buffer pool. More...
 
std::chrono::steady_clock::time_point Adaptive_flush::prev_iter_time
 Time stamp of previous iteration. More...
 
ulint Adaptive_flush::prev_iter_pages_dirty = 0
 Number of dirty pages in flush list at previous iteration. More...
 
ulint Adaptive_flush::prev_iter_pages_flushed = 0
 Actual number of pages flushed by last iteration. More...
 
lsn_t Adaptive_flush::lsn_avg_rate = 0
 Average redo generation rate. More...
 
ulint Adaptive_flush::page_avg_rate = 0
 Average page flush rate. More...
 
lsn_t Adaptive_flush::prev_lsn = 0
 LSN when last average rates are computed. More...
 
std::chrono::steady_clock::time_point Adaptive_flush::prev_time
 Time stamp when last average rates are computed. More...
 
ulint Adaptive_flush::n_iterations = 0
 Number of iteration till average rates are computed. More...
 
ulint Adaptive_flush::sum_pages = 0
 Pages flushed till last average rates are computed. More...
 

Detailed Description

The database buffer buf_pool flush algorithm.

Created 11/11/1995 Heikki Tuuri

Enumeration Type Documentation

◆ page_cleaner_state_t

State for page cleaner array slot.

Enumerator
PAGE_CLEANER_STATE_NONE 

Not requested any yet.

Moved from FINISHED by the coordinator.

PAGE_CLEANER_STATE_REQUESTED 

Requested but not started flushing.

Moved from NONE by the coordinator.

PAGE_CLEANER_STATE_FLUSHING 

Flushing is on going.

Moved from REQUESTED by the worker.

PAGE_CLEANER_STATE_FINISHED 

Flushing was finished.

Moved from FLUSHING by the worker.

Function Documentation

◆ buf_are_flush_lists_empty_validate()

bool buf_are_flush_lists_empty_validate ( )

Checks if all flush lists are empty.

It is supposed to be used in single thread, during startup or shutdown. Hence it does not acquire lock and it is caller's responsibility to guarantee that flush lists are not changed in background.

Returns
true if all flush lists were empty.

◆ buf_do_flush_list_batch()

static ulint buf_do_flush_list_batch ( buf_pool_t buf_pool,
ulint  min_n,
lsn_t  lsn_limit 
)
static

This utility flushes dirty blocks from the end of the flush_list.

The calling thread is not allowed to own any latches on pages!

Parameters
[in]buf_poolbuffer pool instance
[in]min_nwished minimum number of blocks flushed (it is not guaranteed that the actual number is that big, though)
[in]lsn_limitall blocks whose oldest_modification is smaller than this should be flushed (if their number does not exceed min_n)
Returns
number of blocks for which the write request was queued; ULINT_UNDEFINED if there was a flush of the same type already running

◆ buf_do_LRU_batch()

static ulint buf_do_LRU_batch ( buf_pool_t buf_pool,
ulint  max 
)
static

Flush and move pages from LRU or unzip_LRU list to the free list.

Whether LRU or unzip_LRU is used depends on the state of the system.

Parameters
[in]buf_poolbuffer pool instance
[in]maxdesired number of blocks in the free_list
Returns
number of blocks for which either the write request was queued or in case of unzip_LRU the number of blocks actually moved to the free list

◆ buf_flush_await_no_flushing()

void buf_flush_await_no_flushing ( buf_pool_t buf_pool,
buf_flush_t  flush_type 
)

Waits until there's no flush of the given type from given BP instance.

Note that in case of BUF_FLUSH_LIST and BUF_FLUSH_LRU we also make sure there's no ongoing batch initialization (which could lead to flushes). The BUF_FLUSH_SINGLE_PAGE does not have batch initialization. Note, that we return as soon as there is no flush, but in general a new one could start right after we've returned (it's up to the caller to prevent this). If buf_pool is nullptr, then it will await a moment with no flushes for each BP instance in turn, which in general doesn't imply there was a single moment when all instances were quiescent - it's up to the caller to ensure that.

Parameters
[in]buf_poolThe specific buffer pool instance to check. Can be null, if we want to wait for each buf_pool in turn.
[in]flush_typeFlush type.

◆ buf_flush_batch()

static ulint buf_flush_batch ( buf_pool_t buf_pool,
buf_flush_t  flush_type,
ulint  min_n,
lsn_t  lsn_limit 
)
static

This utility flushes dirty blocks from the end of the LRU list or flush_list.

NOTE 1: in the case of an LRU flush the calling thread may own latches to pages: to avoid deadlocks, this function must be written so that it cannot end up waiting for these latches! NOTE 2: in the case of a flush list flush, the calling thread is not allowed to own any latches on pages!

Parameters
[in]buf_poolbuffer pool instance
[in]flush_typeBUF_FLUSH_LRU or BUF_FLUSH_LIST; if BUF_FLUSH_LIST, then the caller must not own any latches on pages
[in]min_nwished minimum number of blocks flushed (it is not guaranteed that the actual number is that big, though)
[in]lsn_limitin the case of BUF_FLUSH_LIST all blocks whose oldest_modification is smaller than this should be flushed (if their number does not exceed min_n), otherwise ignored
Returns
number of blocks for which the write request was queued

◆ buf_flush_block_cmp()

static int buf_flush_block_cmp ( const void *  p1,
const void *  p2 
)
static

Compare two modified blocks in the buffer pool.

The key for comparison is: key = <oldest_modification, space, offset> This comparison is used to maintian ordering of blocks in the buf_pool->flush_rbt. Note that for the purpose of flush_rbt, we only need to order blocks on the oldest_modification. The other two fields are used to uniquely identify the blocks.

Returns
< 0 if b2 < b1, 0 if b2 == b1, > 0 if b2 > b1
Parameters
p1in: block1
p2in: block2

◆ buf_flush_borrow_lsn()

static lsn_t buf_flush_borrow_lsn ( const buf_pool_t buf_pool)
inlinestatic

Borrows LSN from the recent added dirty page to the flush list.

This should be the lsn which we may use to mark pages dirtied without underlying redo records, when we add them to the flush list.

The lsn should be chosen in a way which will guarantee that we will not destroy checkpoint calculations if we inserted a new dirty page with such lsn to the flush list. This is strictly related to the limitations we put on the relaxed order in flush lists, which have direct impact on computation of lsn available for next checkpoint.

Therefore when the flush list is empty, the lsn is chosen as the maximum lsn up to which we know, that all dirty pages with smaller oldest_modification were added to the flush list.

This guarantees that the limitations put on the relaxed order are hold and lsn available for next checkpoint is not miscalculated.

Parameters
[in]buf_poolbuffer pool instance
Returns
the borrowed newest_modification of the page or lsn up to which all dirty pages were added to the flush list if the flush list is empty

◆ buf_flush_check_neighbor()

static bool buf_flush_check_neighbor ( const page_id_t page_id,
buf_flush_t  flush_type 
)
static

Check if the page is in buffer pool and can be flushed.

Parameters
[in]page_idpage id
[in]flush_typeBUF_FLUSH_LRU or BUF_FLUSH_LIST
Returns
true if the page can be flushed.

◆ buf_flush_delete_from_flush_rbt()

static void buf_flush_delete_from_flush_rbt ( buf_page_t bpage)
static

Delete a bpage from the flush_rbt.

Parameters
bpagein: bpage to be removed.

◆ buf_flush_do_batch()

bool buf_flush_do_batch ( buf_pool_t buf_pool,
buf_flush_t  type,
ulint  min_n,
lsn_t  lsn_limit,
ulint n_processed 
)

Do flushing batch of a given type.

NOTE: The calling thread is not allowed to own any latches on pages!

Parameters
[in,out]buf_poolbuffer pool instance
[in]typeflush type
[in]min_nwished minimum number of blocks flushed (it is not guaranteed that the actual number is that big, though)
[in]lsn_limitin the case BUF_FLUSH_LIST all blocks whose oldest_modification is smaller than this should be flushed (if their number does not exceed min_n), otherwise ignored
[out]n_processedthe number of pages which were processed is passed back to caller. Ignored if NULL
Return values
trueif a batch was queued successfully.
falseif another batch of same type was already running.

◆ buf_flush_end()

static void buf_flush_end ( buf_pool_t buf_pool,
buf_flush_t  flush_type 
)
static

End a buffer flush batch for LRU or flush list.

Parameters
[in]buf_poolbuffer pool instance
[in]flush_typeBUF_FLUSH_LRU or BUF_FLUSH_LIST

◆ buf_flush_free_flush_rbt()

void buf_flush_free_flush_rbt ( void  )

Frees up the red-black tree.

◆ buf_flush_fsync()

void buf_flush_fsync ( )

Executes fsync for all tablespaces, to fsync all pages written to disk.

◆ buf_flush_get_dirty_pages_count()

static ulint buf_flush_get_dirty_pages_count ( space_id_t  id,
Flush_observer observer 
)
static

Check if there are any dirty pages that belong to a space id in the flush list.

Returns
number of dirty pages present in all the buffer pools
Parameters
idin: space id to check
observerin: flush observer to check

◆ buf_flush_init_flush_rbt()

void buf_flush_init_flush_rbt ( void  )

Initialize the red-black tree to speed up insertions into the flush_list during recovery process.

Should be called at the start of recovery process before any page has been read/written.

◆ buf_flush_init_for_writing()

void buf_flush_init_for_writing ( const buf_block_t block,
byte page,
void *  page_zip_,
lsn_t  newest_lsn,
bool  skip_checksum,
bool  skip_lsn_check 
)

Initialize a page for writing to the tablespace.

Parameters
[in]blockbuffer block; NULL if bypassing the buffer pool
[in,out]pagepage frame
[in,out]page_zip_compressed page, or NULL if uncompressed
[in]newest_lsnnewest modification LSN to the page
[in]skip_checksumwhether to disable the page checksum
[in]skip_lsn_checktrue to skip check for LSN (in DEBUG)

◆ buf_flush_insert_in_flush_rbt()

static buf_page_t * buf_flush_insert_in_flush_rbt ( buf_page_t bpage)
static

Insert a block in the flush_rbt and returns a pointer to its predecessor or NULL if no predecessor.

The ordering is maintained on the basis of the <oldest_modification, space, offset> key.

Returns
pointer to the predecessor or NULL if no predecessor.
Parameters
bpagein: bpage to be inserted.

◆ buf_flush_insert_into_flush_list()

void buf_flush_insert_into_flush_list ( buf_pool_t buf_pool,
buf_block_t block,
lsn_t  lsn 
)

Inserts a modified block into the flush list.

in: oldest modification

Parameters
buf_poolbuffer pool instance
blockin/out: block which is modified
lsnin: oldest modification

◆ buf_flush_insert_sorted_into_flush_list()

void buf_flush_insert_sorted_into_flush_list ( buf_pool_t buf_pool,
buf_block_t block,
lsn_t  lsn 
)

Inserts a modified block into the flush list in the right sorted position.

This function is used by recovery, because there the modifications do not necessarily come in the order of lsn's.

Parameters
buf_poolin: buffer pool instance
blockin/out: block which is modified
lsnin: oldest modification

◆ buf_flush_list_order_validate()

static bool buf_flush_list_order_validate ( lsn_t  earlier_added_lsn,
lsn_t  new_added_lsn 
)
inlinestatic

Checks that order of two consecutive pages in flush list would be valid, according to their oldest_modification values.

Remarks
We have a relaxed order in flush list, but still we have guarantee, that the earliest added page has oldest_modification not greater than minimum oldest_modification of all dirty pages by more than number of slots in the buf_flush_list_added->order_lag().

This is used by assertions only.

Parameters
[in]earlier_added_lsnoldest_modification of page which was added to flush list earlier
[in]new_added_lsnoldest_modification of page which is being added to flush list
Return values
trueif the order is valid
See also
Reclaiming space in redo log
Adding dirty pages to flush lists

◆ buf_flush_lists()

bool buf_flush_lists ( ulint  min_n,
lsn_t  lsn_limit,
ulint n_processed 
)

This utility flushes dirty blocks from the end of the flush list of all buffer pool instances.

NOTE: The calling thread is not allowed to own any latches on pages!

Parameters
[in]min_nwished minimum number of blocks flushed (it is not guaranteed that the actual number is that big, though)
[in]lsn_limitin the case BUF_FLUSH_LIST all blocks whose oldest_modification is smaller than this should be flushed (if their number does not exceed min_n), otherwise ignored
[out]n_processedthe number of pages which were processed is passed back to caller. Ignored if NULL.
Returns
true if a batch was queued successfully for each buffer pool instance. false if another batch of same type was already running in at least one of the buffer pool instance

◆ buf_flush_LRU_list()

static ulint buf_flush_LRU_list ( buf_pool_t buf_pool)
static

Clears up tail of the LRU list of a given buffer pool instance: Put replaceable pages at the tail of LRU to the free list Flush dirty pages at the tail of LRU to the disk The depth to which we scan each buffer pool is controlled by dynamic config parameter innodb_LRU_scan_depth.

Parameters
buf_poolbuffer pool instance
Returns
total pages flushed

◆ buf_flush_LRU_list_batch()

static ulint buf_flush_LRU_list_batch ( buf_pool_t buf_pool,
ulint  max 
)
static

This utility flushes dirty blocks from the end of the LRU list.

The calling thread is not allowed to own any latches on pages! It attempts to make 'max' blocks available in the free list. Note that it is a best effort attempt and it is not guaranteed that after a call to this function there will be 'max' blocks in the free list.

Parameters
[in]buf_poolbuffer pool instance
[in]maxdesired number for blocks in the free_list
Returns
number of blocks for which the write request was queued.

◆ buf_flush_page()

bool buf_flush_page ( buf_pool_t buf_pool,
buf_page_t bpage,
buf_flush_t  flush_type,
bool  sync 
)

Writes a flushable page asynchronously from the buffer pool to a file.

NOTE: 1. in simulated aio we must call os_aio_simulated_wake_handler_threads after we have posted a batch of writes! 2. buf_page_get_mutex(bpage) must be held upon entering this function. The LRU list mutex must be held if flush_type == BUF_FLUSH_SINGLE_PAGE. Both mutexes will be released by this function if it returns true.

Parameters
[in]buf_poolbuffer pool instance
[in]bpagebuffer control block
[in]flush_typetype of flush
[in]synctrue if sync IO request
Returns
true if page was flushed

◆ buf_flush_page_and_try_neighbors()

static bool buf_flush_page_and_try_neighbors ( buf_page_t bpage,
buf_flush_t  flush_type,
ulint  n_to_flush,
ulint count 
)
static

Check if the block is modified and ready for flushing.

If ready to flush then flush the page and try to flush its neighbors. The caller must hold the buffer pool list mutex corresponding to the type of flush.

Parameters
[in]bpagebuffer control block, must be buf_page_in_file(bpage)
[in]flush_typeBUF_FLUSH_LRU or BUF_FLUSH_LIST
[in]n_to_flushnumber of pages to flush
[in,out]countnumber of pages flushed
Returns
true if the list mutex was released during this function. This does not guarantee that some pages were written as well.

◆ buf_flush_page_cleaner_close()

static void buf_flush_page_cleaner_close ( void  )
static

Close page_cleaner.

◆ buf_flush_page_cleaner_disabled_debug_update()

void buf_flush_page_cleaner_disabled_debug_update ( THD thd,
SYS_VAR var,
void *  var_ptr,
const void *  save 
)

Disables page cleaner threads (coordinator and workers).

It's used by: SET GLOBAL innodb_page_cleaner_disabled_debug = 1 (0).

Parameters
[in]thdthread handle
[in]varpointer to system variable
[out]var_ptrwhere the formal string goes
[in]saveimmediate result from check function

◆ buf_flush_page_cleaner_disabled_loop()

static void buf_flush_page_cleaner_disabled_loop ( void  )
static

Loop used to disable page cleaner threads.

◆ buf_flush_page_cleaner_init()

void buf_flush_page_cleaner_init ( )

Initialize page_cleaner.


◆ buf_flush_page_cleaner_is_active()

bool buf_flush_page_cleaner_is_active ( )

Checks if page cleaners are active.

Checks if the page_cleaner is in active state.

◆ buf_flush_page_cleaner_thread()

static void buf_flush_page_cleaner_thread ( )
static

Worker thread of page_cleaner.

◆ buf_flush_page_coordinator_thread()

static void buf_flush_page_coordinator_thread ( )
static

Thread tasked with flushing dirty pages from the buffer pools.

As of now we'll have only one coordinator.

◆ buf_flush_page_try()

bool buf_flush_page_try ( buf_pool_t buf_pool,
buf_block_t block 
)

Writes a flushable page asynchronously from the buffer pool to a file.

NOTE: block and LRU list mutexes must be held upon entering this function, and they will be released by this function after flushing. This is loosely based on buf_flush_batch() and buf_flush_page().

Parameters
[in,out]buf_poolbuffer pool instance
[in,out]blockbuffer control block
Returns
true if the page was flushed and the mutex released

◆ buf_flush_ready_for_flush()

bool buf_flush_ready_for_flush ( buf_page_t bpage,
buf_flush_t  flush_type 
)

Check if the block is modified and ready for flushing.

Requires buf_page_get_mutex(bpage).

Parameters
[in]bpagebuffer control block, must be buf_page_in_file()
[in]flush_typetype of flush
Returns
true if can flush immediately

◆ buf_flush_ready_for_flush_gen()

static bool buf_flush_ready_for_flush_gen ( buf_page_t bpage,
buf_flush_t  flush_type,
bool  atomic 
)
static

Check if the block was modified and was ready for flushing.

This is a common part of the logic of buf_flush_was_ready_for_flush() and buf_flush_ready_for_flush() which differ by the tolerance for stale result.

Parameters
[in]bpagebuffer control block, must be buf_page_in_file()
[in]flush_typetype of flush
[in]atomicfalse if the caller can tolerate stale data, true if the caller needs accurate answer, which requires the caller to hold buf_page_get_mutex.
Returns
true if page seems ready for flush

◆ buf_flush_ready_for_replace()

bool buf_flush_ready_for_replace ( const buf_page_t bpage)

Returns true if the file page block is immediately suitable for replacement, i.e., the transition FILE_PAGE => NOT_USED allowed.

The caller must hold the LRU list and block mutexes.

Parameters
[in]bpagebuffer control block, must be buf_page_in_file() and in the LRU list
Returns
true if can replace immediately

◆ buf_flush_relocate_on_flush_list()

void buf_flush_relocate_on_flush_list ( buf_page_t bpage,
buf_page_t dpage 
)

Relocates a buffer control block on the flush_list.

Note that it is assumed that the contents of bpage have already been copied to dpage. IMPORTANT: When this function is called bpage and dpage are not exact copies of each other. For example, they both will have different "::state". Also the "::list" pointers in dpage may be stale. We need to use the current list node (bpage) to do the list manipulation because the list pointers could have changed between the time that we copied the contents of bpage to the dpage and the flush list manipulation below.

Parameters
bpagein/out: control block being moved
dpagein/out: destination block

◆ buf_flush_remove()

void buf_flush_remove ( buf_page_t bpage)

Remove a block from the flush list of modified blocks.

Parameters
[in]bpagepointer to the block in question

◆ buf_flush_single_page_from_LRU()

bool buf_flush_single_page_from_LRU ( buf_pool_t buf_pool)

This function picks up a single page from the tail of the LRU list, flushes it (if it is dirty), removes it from page_hash and LRU list and puts it on the free list.

It is called from user threads when they are unable to find a replaceable page at the tail of the LRU list i.e.: when the background LRU flushing in the page_cleaner thread is not fast enough to keep pace with the workload.

Parameters
[in,out]buf_poolbuffer pool instance
Returns
true if success.

◆ buf_flush_start()

static bool buf_flush_start ( buf_pool_t buf_pool,
buf_flush_t  flush_type 
)
static

Start a buffer flush batch for LRU or flush list.

Parameters
[in]buf_poolbuffer pool instance
[in]flush_typeBUF_FLUSH_LRU or BUF_FLUSH_LIST

◆ buf_flush_stats()

static void buf_flush_stats ( ulint  page_count_flush,
ulint  page_count_LRU 
)
static

Gather the aggregated stats for both flush list and LRU list flushing.

Parameters
page_count_flushnumber of pages flushed from the end of the flush_list
page_count_LRUnumber of pages flushed from the end of the LRU list

◆ buf_flush_sync_all_buf_pools()

void buf_flush_sync_all_buf_pools ( )

Synchronously flush dirty blocks from the end of the flush list of all buffer pool instances.

NOTE: The calling thread is not allowed to own any latches on pages!

◆ buf_flush_sync_datafiles()

static void buf_flush_sync_datafiles ( )
static

Flush a batch of writes to the datafiles that have already been written to the dblwr buffer on disk.

◆ buf_flush_try_neighbors()

static ulint buf_flush_try_neighbors ( const page_id_t page_id,
buf_flush_t  flush_type,
ulint  n_flushed,
ulint  n_to_flush 
)
static

Flushes to disk all flushable pages within the flush area.

Parameters
[in]page_idpage id
[in]flush_typeBUF_FLUSH_LRU or BUF_FLUSH_LIST
[in]n_flushednumber of pages flushed so far in this batch
[in]n_to_flushmaximum number of pages we are allowed to flush
Returns
number of pages flushed

◆ buf_flush_update_zip_checksum()

void buf_flush_update_zip_checksum ( buf_frame_t page,
ulint  size,
lsn_t  lsn,
bool  skip_lsn_check 
)

Calculate the checksum of a page from compressed table and update the page.

Parameters
[in,out]pagepage to update
[in]sizecompressed page size
[in]lsnLSN to stamp on the page
[in]skip_lsn_checktrue to skip check for lsn (in DEBUG)

◆ buf_flush_validate()

bool buf_flush_validate ( buf_pool_t buf_pool)

Validates the flush list.

Returns
true if ok

◆ buf_flush_validate_low()

static bool buf_flush_validate_low ( const buf_pool_t buf_pool)
static

Validate a buffer pool instance flush list.

Parameters
[in]buf_poolInstance to validate
Returns
true on success.

◆ buf_flush_validate_skip()

static bool buf_flush_validate_skip ( buf_pool_t buf_pool)
static

Validates the flush list some of the time.

Returns
true if ok or the check was skipped

Try buf_flush_validate_low() every this many times

The buf_flush_validate_low() call skip counter. Use a signed type because of the race condition below.

Parameters
buf_poolin: Buffer pool instance

◆ buf_flush_was_ready_for_flush()

static bool buf_flush_was_ready_for_flush ( buf_page_t bpage,
buf_flush_t  flush_type 
)
static

Check if the block was modified and was ready for flushing at some point in time during the call.

Result might be obsolete.

Parameters
[in]bpagebuffer control block, must be buf_page_in_file()
[in]flush_typetype of flush
Returns
true if can flush immediately

◆ buf_flush_write_block_low()

static void buf_flush_write_block_low ( buf_page_t bpage,
buf_flush_t  flush_type,
bool  sync 
)
static

Does an asynchronous write of a buffer page.

Parameters
[in]bpagebuffer block to write
[in]flush_typetype of flush
[in]synctrue if sync IO request

◆ buf_flush_write_complete()

void buf_flush_write_complete ( buf_page_t bpage)

Updates the flush system data structures when a write is completed.

Parameters
[in]bpagepointer to the block in question

◆ buf_free_from_unzip_LRU_list_batch()

static ulint buf_free_from_unzip_LRU_list_batch ( buf_pool_t buf_pool,
ulint  max 
)
static

This utility moves the uncompressed frames of pages to the free list.

Note that this function does not actually flush any data to disk. It just detaches the uncompressed frames from the compressed pages at the tail of the unzip_LRU and puts those freed frames in the free list. Note that it is a best effort attempt and it is not guaranteed that after a call to this function there will be 'max' blocks in the free list. The caller must hold the LRU list mutex.

Parameters
[in]buf_poolbuffer pool instance
[in]maxdesired number of blocks in the free_list
Returns
number of blocks moved to the free list.

◆ buf_pool_get_dirty_pages_count()

ulint buf_pool_get_dirty_pages_count ( buf_pool_t buf_pool,
space_id_t  id,
Flush_observer observer 
)

Check if there are any dirty pages that belong to a space id in the flush list in a particular buffer pool.

Returns
number of dirty pages present in a single buffer pool
Parameters
buf_poolin: buffer pool
idin: space id to check
observerin: flush observer to check

◆ get_flush_sync_lsn()

lsn_t get_flush_sync_lsn ( )
noexcept

Get the lsn up to which data pages are to be synchronously flushed.

Returns
target lsn for the requested flush_sync

◆ incr_flush_list_size_in_bytes()

static void incr_flush_list_size_in_bytes ( buf_block_t block,
buf_pool_t buf_pool 
)
inlinestatic

Increases flush_list size in bytes with the page size in inline function.

Parameters
blockin: control block
buf_poolin: buffer pool instance

◆ page_is_uncompressed_type()

bool page_is_uncompressed_type ( const byte page)

Check if page type is uncompressed.

Parameters
[in]pagepage frame
Returns
true if uncompressed page type.

◆ pc_flush_slot()

static ulint pc_flush_slot ( void  )
static

Do flush for one slot.

Returns
the number of the slots which has not been treated yet.

◆ pc_request()

static void pc_request ( ulint  min_n,
lsn_t  lsn_limit 
)
static

Requests for all slots to flush all buffer pool instances.

Parameters
min_nwished minimum number of blocks flushed (it is not guaranteed that the actual number is that big)
lsn_limitin the case BUF_FLUSH_LIST all blocks whose oldest_modification is smaller than this should be flushed (if their number does not exceed min_n), otherwise ignored

◆ pc_sleep_if_needed()

static ulint pc_sleep_if_needed ( std::chrono::steady_clock::time_point  next_loop_time,
int64_t  sig_count 
)
static

Puts the page_cleaner thread to sleep if it has finished work in less than a second.

Return values
0wake up by event set,
OS_SYNC_TIME_EXCEEDEDif timeout was exceeded
Parameters
next_loop_timetime when next loop iteration should start
sig_countzero or the value returned by previous call of os_event_reset()

◆ pc_wait_finished()

static bool pc_wait_finished ( ulint n_flushed_lru,
ulint n_flushed_list 
)
static

Wait until all flush requests are finished.

Parameters
n_flushed_lrunumber of pages flushed from the end of the LRU list.
n_flushed_listnumber of pages flushed from the end of the flush_list.
Returns
true if all flush_list flushing batch were success.

Variable Documentation

◆ buf_flush_event

os_event_t buf_flush_event

Event to synchronise with the flushing.

◆ buf_flush_lsn_scan_factor

uint buf_flush_lsn_scan_factor = 3
static

Factor for scan length to determine n_pages for intended oldest LSN progress.

◆ buf_flush_sync_lsn

lsn_t buf_flush_sync_lsn = 0
static

Target oldest LSN for the requested flush_sync.

◆ buf_flush_tick_event

os_event_t buf_flush_tick_event

Event to wait for one flushing step.

◆ BUF_LRU_MIN_LEN

constexpr uint32_t BUF_LRU_MIN_LEN = 256
constexpr

If LRU list of a buf_pool is less than this size then LRU eviction should not happen.

This is because when we do LRU flushing we also put the blocks on free list. If LRU list is very small then we can end up in thrashing.

◆ innodb_page_cleaner_disabled_debug

bool innodb_page_cleaner_disabled_debug

Value of MySQL global variable used to disable page cleaner.

◆ page_cleaner

ut::unique_ptr<page_cleaner_t> page_cleaner
static