Innodb Clone Interface. More...

#include <cstdio>
#include <fstream>
#include <iostream>
#include "clone0api.h"
#include "clone0clone.h"
#include "os0thread-create.h"
#include "sql/clone_handler.h"
#include "sql/mysqld.h"
#include "sql/sql_backup_lock.h"
#include "sql/sql_class.h"
#include "sql/sql_prepare.h"
#include "sql/sql_table.h"
#include "sql/sql_thd_internal_api.h"
#include "sql/strfunc.h"
#include "dict0dd.h"
#include "ha_innodb.h"
#include "log0files_io.h"
#include "sql/dd/cache/dictionary_client.h"
#include "sql/dd/dictionary.h"
#include "sql/dd/impl/dictionary_impl.h"
#include "sql/dd/impl/sdi.h"
#include "sql/dd/impl/utils.h"
#include "sql/dd/types/schema.h"
#include "sql/dd/types/table.h"
#include "sql/rpl_msr.h"

Classes
class	anonymous_namespace{clone0api.cc}::Fixup_data
	Fix schema, table and tablespace. More...

Namespaces
namespace	anonymous_namespace{clone0api.cc}

Typedefs
typedef std::function< void(const std::string &)>	Clone_file_handler
	Type of function which is supposed to handle a single file during Clone operations, accepting the file's name (string). More...

template<typename T >
using	DD_Objs = std::vector< const T * >

template<typename T >
using	DD_Objs_Iter = typename DD_Objs< T >::const_iterator

using	Releaser = dd::cache::Dictionary_client::Auto_releaser

Functions
THD *	thd_get_current_thd ()
	Get current THD object from thread local data. More...

static bool	file_exists (const std::string &file_name)
	Check if clone status file exists. More...

static void	rename_file (const std::string &from_file, const std::string &to_file)
	Rename clone status file. More...

static void	create_file (std::string &file_name)
	Create clone status file. More...

static void	remove_file (const std::string &file)
	Delete clone status file or directory. More...

static void	create_status_file (const Clone_Handle *clone)
	Create clone in progress file and error file. More...

static void	drop_status_file (const Clone_Handle *clone)
	Drop clone in progress file and error file. More...

void	clone_init_list_files ()
	Check and delete any old list files. More...

void	clone_remove_list_file (const char *file_name)
	Remove one of the clone list files. More...

int	clone_add_to_list_file (const char list_file_name, const char file_name)
	Add file name to clone list file for future replacement or rollback. More...

static void	track_redo_files ()
	Add redo log directory to the old file list. More...

static bool	clone_execute_query (THD thd, const char sql_stmt, size_t thread_number, bool skip_error)
	Execute sql statement. More...

static int	clone_drop_binary_logs (THD *thd)
	Delete all binary logs before clone. More...

static int	clone_drop_user_data (THD *thd, bool allow_threads)
	Drop all user data before starting clone. More...

static void	clone_init_compression (THD *thd)
	Initialize transparent page compression in innodb space by checking all innodb tables in DD. More...

static int	clone_init_tablespaces (THD *thd)
	Open all Innodb tablespaces. More...

static void	skip_grants (THD *thd, Security_context &sctx)
	Set security context to skip privilege check. More...

void	innodb_clone_get_capability (Ha_clone_flagset &flags)
	Get capability flags for clone operation. More...

static int	clone_begin_check (THD *thd)
	Check if clone can be started. More...

static bool	get_clone_timeout_config (THD *thd, const std::string &config_name, int &timeout)
	Get clone timeout configuration value. More...

static int	get_ddl_timeout (THD *thd)
	Timeout while waiting for DDL commands. More...

int	innodb_clone_begin (handlerton , THD thd, const byte *&loc, uint &loc_len, uint &task_id, Ha_clone_type type, Ha_clone_mode mode)
	Begin copy from source database. More...

int	innodb_clone_copy (handlerton hton, THD thd, const byte loc, uint loc_len, uint task_id, Ha_clone_cbk cbk)
	Copy data from source database in chunks via callback. More...

int	innodb_clone_ack (handlerton hton, THD thd, const byte loc, uint loc_len, uint task_id, int in_err, Ha_clone_cbk cbk)
	Acknowledge data to source database. More...

static Clone_Min	get_donor_timeout (THD *thd)
	Timeout while waiting for recipient after network failure. More...

int	innodb_clone_end (handlerton , THD thd, const byte *loc, uint loc_len, uint task_id, int in_err)
	End copy from source database. More...

int	innodb_clone_apply_begin (handlerton , THD thd, const byte &loc, uint &loc_len, uint &task_id, Ha_clone_mode mode, const char data_dir)
	Begin apply to destination database. More...

int	innodb_clone_apply (handlerton hton, THD thd, const byte loc, uint loc_len, uint task_id, int in_err, Ha_clone_cbk cbk)
	Apply data to destination database in chunks via callback. More...

int	innodb_clone_apply_end (handlerton hton, THD thd, const byte *loc, uint loc_len, uint task_id, int in_err)
	End apply to destination database. More...

static int	get_file_state (const std::string &data_file)
	Get current state of a clone file. More...

static int	file_roll_forward (const std::string &data_file, int final_state)
	Roll forward clone file state till final state. More...

static void	file_rollback (const std::string &data_file)
	Roll back clone file state to normal state. More...

static void	old_file_roll_forward (const std::string &data_file, int final_state)
	Roll forward old data file state till final state. More...

static void	old_file_rollback (const std::string &data_file)
	Roll back old data file state to normal state. More...

static void	clone_files_fatal_error ()
	Fatal error callback function. More...

static void	clone_update_recovery_status (bool finished, bool is_error, bool is_replace)
	Update recovery status file at end of clone recovery. More...

static void	clone_init_recovery_status (bool replace)
	Initialize recovery status for cloned recovery. More...

void	clone_update_gtid_status (std::string &gtids)
	Update cloned GTIDs to recovery status file. More...

static bool	clone_files_for_each_file (const char *status_file_name, const Clone_file_handler &process)
	Processes each file name listed in the given status file, executing a given function for each of them. More...

static void	process_remove_file (const char *file_name, const Clone_file_handler &process)
	Process all entries and remove status file. More...

void	clone_files_error ()
	Revert back clone changes in case of an error. More...

bool	clone_check_recovery_crashpoint (bool is_cloned_db)
	Debug function to check and crash during recovery. More...

void	clone_files_recovery (bool finished)
	Change cloned file states during recovery. More...

dberr_t	clone_init ()
	Initialize Clone system. More...

void	clone_free ()
	Uninitialize Clone system. More...

bool	clone_check_provisioning ()

bool	clone_check_active ()
	Check if active clone is running. More...

bool	fix_cloned_tables (THD *thd)
	Fix cloned non-Innodb tables during recovery. More...

Variables
const int	FILE_DATA = 1
	Data file is found. More...

const int	FILE_SAVED = 10
	Saved data file is found. More...

const int	FILE_CLONED = 100
	Cloned data file is found. More...

const int	FILE_STATE_NONE = 0
	NONE state: file not present. More...

const int	FILE_STATE_NORMAL = FILE_DATA
	Normal state: only data file is present. More...

const int	FILE_STATE_SAVED = FILE_SAVED
	Saved state: only saved data file is present. More...

const int	FILE_STATE_CLONED = FILE_DATA + FILE_CLONED
	Cloned state: data file and cloned data file are present. More...

const int	FILE_STATE_CLONE_SAVED = FILE_SAVED + FILE_CLONED
	Saved clone state: saved data file and cloned data file are present. More...

const int	FILE_STATE_REPLACED = FILE_SAVED + FILE_DATA
	Replaced state: saved data file and data file are present. More...

Detailed Description

Innodb Clone Interface.

Typedef Documentation

◆ Clone_file_handler

typedef std::function<void(const std::string &)> Clone_file_handler

Type of function which is supposed to handle a single file during Clone operations, accepting the file's name (string).

See also: clone_files_for_each_file

◆ DD_Objs

template<typename T >

using DD_Objs = std::vector<const T *>

◆ DD_Objs_Iter

template<typename T >

using DD_Objs_Iter = typename DD_Objs<T>::const_iterator

◆ Releaser

using Releaser = dd::cache::Dictionary_client::Auto_releaser

Function Documentation

◆ clone_add_to_list_file()

int clone_add_to_list_file	(	const char *	list_file_name,
		const char *	file_name
	)

Add file name to clone list file for future replacement or rollback.

Parameters

[in]	list_file_name	list file name where to add the file
[in]	file_name	file name to add to the list

Returns: error code

◆ clone_begin_check()

static int clone_begin_check ( THD * thd )

static

Check if clone can be started.

Parameters

[in,out] thd session THD

Returns: error code.

◆ clone_check_active()

bool clone_check_active ( )

Check if active clone is running.

Returns: true, if any active clone is found.

◆ clone_check_provisioning()

bool clone_check_provisioning ( )

Returns: true, if clone provisioning in progress.

◆ clone_check_recovery_crashpoint()

bool clone_check_recovery_crashpoint ( bool is_cloned_db )

Debug function to check and crash during recovery.

Parameters

[in] is_cloned_db if cloned database recovery

◆ clone_drop_binary_logs()

static int clone_drop_binary_logs ( THD * thd )

static

Delete all binary logs before clone.

Parameters

[in] thd current THD

Returns: error code

◆ clone_drop_user_data()

static int clone_drop_user_data	(	THD *	thd,
		bool	allow_threads
	)

static

Drop all user data before starting clone.

Parameters

[in,out]	thd	current THD
[in]	allow_threads	allow multiple threads

Returns: error code

◆ clone_execute_query()

static bool clone_execute_query	(	THD *	thd,
		const char *	sql_stmt,
		size_t	thread_number,
		bool	skip_error
	)

static

Execute sql statement.

Parameters

[in,out]	thd	current THD
[in]	sql_stmt	SQL statement
[in]	thread_number	executing thread number
[in]	skip_error	skip statement on error

Returns: false, if successful.

◆ clone_files_error()

void clone_files_error ( )

Revert back clone changes in case of an error.

◆ clone_files_fatal_error()

static void clone_files_fatal_error ( )

static

Fatal error callback function.

Don't call other functions from here. Don't use ut_a, ut_ad asserts or ib::fatal to avoid recursive invocation.

◆ clone_files_for_each_file()

static bool clone_files_for_each_file	(	const char *	status_file_name,
		const Clone_file_handler &	process
	)

static

Processes each file name listed in the given status file, executing a given function for each of them.

Parameters

[in]	status_file_name	status file name
[in]	process	the given function, accepting file name string

Returns: true iff status file was successfully opened

◆ clone_files_recovery()

void clone_files_recovery ( bool finished )

Change cloned file states during recovery.

Parameters

[in] finished if recovery is finishing

◆ clone_free()

void clone_free ( )

Uninitialize Clone system.

◆ clone_init()

dberr_t clone_init ( )

Initialize Clone system.

Returns: inndodb error code

◆ clone_init_compression()

static void clone_init_compression ( THD * thd )

static

Initialize transparent page compression in innodb space by checking all innodb tables in DD.

Usually this initialization is done later when user opens a table. Clone needs to read this from innodb space object.

Parameters

[in,out] thd session THD

◆ clone_init_list_files()

void clone_init_list_files ( )

Check and delete any old list files.

◆ clone_init_recovery_status()

static void clone_init_recovery_status ( bool replace )

static

Initialize recovery status for cloned recovery.

Parameters

[in] replace we are replacing current directory.

◆ clone_init_tablespaces()

static int clone_init_tablespaces ( THD * thd )

static

Open all Innodb tablespaces.

Parameters

[in,out] thd session THD

Returns: error code.

◆ clone_remove_list_file()

void clone_remove_list_file ( const char * file_name )

Remove one of the clone list files.

Parameters

[in] file_name list file name to delete

◆ clone_update_gtid_status()

void clone_update_gtid_status ( std::string & gtids )

Update cloned GTIDs to recovery status file.

Parameters

[in] gtids cloned GTIDs

◆ clone_update_recovery_status()

static void clone_update_recovery_status	(	bool	finished,
		bool	is_error,
		bool	is_replace
	)

static

Update recovery status file at end of clone recovery.

Parameters

[in]	finished	true if finishing clone recovery
[in]	is_error	if recovery error
[in]	is_replace	true, if replacing current directory

◆ create_file()

static void create_file ( std::string & file_name )

static

Create clone status file.

Parameters

[in] file_name file name

◆ create_status_file()

static void create_status_file ( const Clone_Handle * clone )

static

Create clone in progress file and error file.

Parameters

[in] clone clone handle

◆ drop_status_file()

static void drop_status_file ( const Clone_Handle * clone )

static

Drop clone in progress file and error file.

Parameters

[in] clone clone handle

◆ file_exists()

static bool file_exists ( const std::string & file_name )

static

Check if clone status file exists.

Parameters

[in] file_name file name

Returns: true if file exists.

◆ file_roll_forward()

static int file_roll_forward	(	const std::string &	data_file,
		int	final_state
	)

static

Roll forward clone file state till final state.

Parameters

[in]	data_file	data file name
[in]	final_state	data file state to forward to

Returns: previous file state before roll forward.

◆ file_rollback()

static void file_rollback ( const std::string & data_file )

static

Roll back clone file state to normal state.

Parameters

[in] data_file data file name

◆ fix_cloned_tables()

bool fix_cloned_tables ( THD * thd )

Fix cloned non-Innodb tables during recovery.

Parameters

[in,out] thd current THD

Returns: true if error

◆ get_clone_timeout_config()

static bool get_clone_timeout_config	(	THD *	thd,
		const std::string &	config_name,
		int &	timeout
	)

static

Get clone timeout configuration value.

Parameters

[in,out]	thd	server thread handle
[in]	config_name	timeout configuration name
[out]	timeout	timeout value

Returns: true iff successful.

◆ get_ddl_timeout()

static int get_ddl_timeout ( THD * thd )

static

Timeout while waiting for DDL commands.

Parameters

[in,out] thd server thread handle

Returns: donor timeout in seconds.

◆ get_donor_timeout()

static Clone_Min get_donor_timeout ( THD * thd )

static

Timeout while waiting for recipient after network failure.

Parameters

[in,out] thd server thread handle

Returns: donor timeout in minutes.

◆ get_file_state()

static int get_file_state ( const std::string & data_file )

static

Get current state of a clone file.

Parameters

[in] data_file data file name

Returns: current file state.

◆ innodb_clone_ack()

int innodb_clone_ack	(	handlerton *	hton,
		THD *	thd,
		const byte *	loc,
		uint	loc_len,
		uint	task_id,
		int	in_err,
		Ha_clone_cbk *	cbk
	)

Acknowledge data to source database.

Parameters

[in]	hton	handlerton for SE
[in]	thd	server thread handle
[in]	loc	locator
[in]	loc_len	locator length in bytes
[in]	task_id	task identifier
[in]	in_err	inform any error occurred
[in]	cbk	callback interface for receiving data

Returns: error code

◆ innodb_clone_apply()

int innodb_clone_apply	(	handlerton *	hton,
		THD *	thd,
		const byte *	loc,
		uint	loc_len,
		uint	task_id,
		int	in_err,
		Ha_clone_cbk *	cbk
	)

Apply data to destination database in chunks via callback.

Parameters

[in]	hton	handlerton for SE
[in]	thd	server thread handle
[in]	loc	locator
[in]	loc_len	locator length in bytes
[in]	task_id	task identifier
[in]	in_err	inform any error occurred
[in]	cbk	callback interface for receiving data

Returns: error code

◆ innodb_clone_apply_begin()

int innodb_clone_apply_begin	(	handlerton *	hton,
		THD *	thd,
		const byte *&	loc,
		uint &	loc_len,
		uint &	task_id,
		Ha_clone_mode	mode,
		const char *	data_dir
	)

Begin apply to destination database.

Parameters

[in]	hton	handlerton for SE
[in]	thd	server thread handle
[in,out]	loc	locator
[in,out]	loc_len	locator length
[out]	task_id	task identifier
[in]	mode	mode for starting clone
[in]	data_dir	target data directory

Returns: error code

◆ innodb_clone_apply_end()

int innodb_clone_apply_end	(	handlerton *	hton,
		THD *	thd,
		const byte *	loc,
		uint	loc_len,
		uint	task_id,
		int	in_err
	)

End apply to destination database.

Parameters

[in]	hton	handlerton for SE
[in]	thd	server thread handle
[in]	loc	locator
[in]	loc_len	locator length in bytes
[in]	task_id	task identifier
[in]	in_err	error code when ending after error

Returns: error code

◆ innodb_clone_begin()

int innodb_clone_begin	(	handlerton *	hton,
		THD *	thd,
		const byte *&	loc,
		uint &	loc_len,
		uint &	task_id,
		Ha_clone_type	type,
		Ha_clone_mode	mode
	)

Begin copy from source database.

Parameters

[in]	hton	handlerton for SE
[in]	thd	server thread handle
[in,out]	loc	locator
[in,out]	loc_len	locator length
[out]	task_id	task identifier
[in]	type	clone type
[in]	mode	mode for starting clone

Returns: error code

◆ innodb_clone_copy()

int innodb_clone_copy	(	handlerton *	hton,
		THD *	thd,
		const byte *	loc,
		uint	loc_len,
		uint	task_id,
		Ha_clone_cbk *	cbk
	)

Copy data from source database in chunks via callback.

Parameters

[in]	hton	handlerton for SE
[in]	thd	server thread handle
[in]	loc	locator
[in]	loc_len	locator length in bytes
[in]	task_id	task identifier
[in]	cbk	callback interface for sending data

Returns: error code

◆ innodb_clone_end()

int innodb_clone_end	(	handlerton *	hton,
		THD *	thd,
		const byte *	loc,
		uint	loc_len,
		uint	task_id,
		int	in_err
	)

End copy from source database.

Parameters

[in]	hton	handlerton for SE
[in]	thd	server thread handle
[in]	loc	locator
[in]	loc_len	locator length in bytes
[in]	task_id	task identifier
[in]	in_err	error code when ending after error

Returns: error code

◆ innodb_clone_get_capability()

void innodb_clone_get_capability ( Ha_clone_flagset & flags )

Get capability flags for clone operation.

Parameters

[out] flags capability flag

◆ old_file_roll_forward()

static void old_file_roll_forward	(	const std::string &	data_file,
		int	final_state
	)

static

Roll forward old data file state till final state.

Parameters

[in]	data_file	data file name
[in]	final_state	data file state to forward to

◆ old_file_rollback()

static void old_file_rollback ( const std::string & data_file )

static

Roll back old data file state to normal state.

Parameters

[in] data_file data file name

◆ process_remove_file()

static void process_remove_file	(	const char *	file_name,
		const Clone_file_handler &	process
	)

static

Process all entries and remove status file.

Parameters

[in]	file_name	status file name
[in]	process	callback to process entries

◆ remove_file()

static void remove_file ( const std::string & file )

static

Delete clone status file or directory.

Parameters

[in] file name of file

◆ rename_file()

static void rename_file	(	const std::string &	from_file,
		const std::string &	to_file
	)

static

Rename clone status file.

The operation is expected to be atomic when the files belong to same directory.

Parameters

[in]	from_file	name of current file
[in]	to_file	name of new file

◆ skip_grants()

static void skip_grants	(	THD *	thd,
		Security_context &	sctx
	)

static

Set security context to skip privilege check.

Parameters

[in,out]	thd	session THD
[in,out]	sctx	security context

◆ thd_get_current_thd()

THD * thd_get_current_thd ( )

Get current THD object from thread local data.

Return values

The	THD object for the thread, NULL if not connection thread

◆ track_redo_files()

static void track_redo_files ( )

static

Add redo log directory to the old file list.

Variable Documentation

◆ FILE_CLONED

const int FILE_CLONED = 100

Cloned data file is found.

◆ FILE_DATA

const int FILE_DATA = 1

Data file is found.

◆ FILE_SAVED

const int FILE_SAVED = 10

Saved data file is found.

◆ FILE_STATE_CLONE_SAVED

const int FILE_STATE_CLONE_SAVED = FILE_SAVED + FILE_CLONED

Saved clone state: saved data file and cloned data file are present.

◆ FILE_STATE_CLONED

const int FILE_STATE_CLONED = FILE_DATA + FILE_CLONED

Cloned state: data file and cloned data file are present.

◆ FILE_STATE_NONE

const int FILE_STATE_NONE = 0

NONE state: file not present.

◆ FILE_STATE_NORMAL

const int FILE_STATE_NORMAL = FILE_DATA

Normal state: only data file is present.

◆ FILE_STATE_REPLACED

const int FILE_STATE_REPLACED = FILE_SAVED + FILE_DATA

Replaced state: saved data file and data file are present.

◆ FILE_STATE_SAVED

const int FILE_STATE_SAVED = FILE_SAVED

Saved state: only saved data file is present.

Classes

Namespaces

Typedefs

Functions

Variables

Detailed Description

Typedef Documentation

◆ Clone_file_handler

◆ DD_Objs

◆ DD_Objs_Iter

◆ Releaser

Function Documentation

◆ clone_add_to_list_file()

◆ clone_begin_check()

◆ clone_check_active()

◆ clone_check_provisioning()

◆ clone_check_recovery_crashpoint()

◆ clone_drop_binary_logs()

◆ clone_drop_user_data()

◆ clone_execute_query()

◆ clone_files_error()

◆ clone_files_fatal_error()

◆ clone_files_for_each_file()

◆ clone_files_recovery()

◆ clone_free()

◆ clone_init()

◆ clone_init_compression()

◆ clone_init_list_files()

◆ clone_init_recovery_status()

◆ clone_init_tablespaces()

◆ clone_remove_list_file()

◆ clone_update_gtid_status()

◆ clone_update_recovery_status()

◆ create_file()

◆ create_status_file()

◆ drop_status_file()

◆ file_exists()

◆ file_roll_forward()

◆ file_rollback()

◆ fix_cloned_tables()

◆ get_clone_timeout_config()

◆ get_ddl_timeout()

◆ get_donor_timeout()

◆ get_file_state()

◆ innodb_clone_ack()

◆ innodb_clone_apply()

◆ innodb_clone_apply_begin()

◆ innodb_clone_apply_end()

◆ innodb_clone_begin()

◆ innodb_clone_copy()

◆ innodb_clone_end()

◆ innodb_clone_get_capability()

◆ old_file_roll_forward()

◆ old_file_rollback()

◆ process_remove_file()

◆ remove_file()

◆ rename_file()

◆ skip_grants()

◆ thd_get_current_thd()

◆ track_redo_files()

Variable Documentation

◆ FILE_CLONED

◆ FILE_DATA

◆ FILE_SAVED

◆ FILE_STATE_CLONE_SAVED

◆ FILE_STATE_CLONED

◆ FILE_STATE_NONE

◆ FILE_STATE_NORMAL

◆ FILE_STATE_REPLACED

◆ FILE_STATE_SAVED