rpl_record.cc File Reference

#include "sql/rpl_record.h"
#include <stddef.h>
#include <algorithm>
#include "field_types.h"
#include "lex_string.h"
#include "m_ctype.h"
#include "m_string.h"
#include "my_bitmap.h"
#include "my_compiler.h"
#include "my_dbug.h"
#include "my_sys.h"
#include "mysql_com.h"
#include "mysqld_error.h"
#include "sql/changestreams/misc/replicated_columns_view_factory.h"
#include "sql/current_thd.h"
#include "sql/debug_sync.h"
#include "sql/derror.h"
#include "sql/field.h"
#include "sql/log_event.h"
#include "sql/rpl_rli.h"
#include "sql/rpl_utility.h"
#include "sql/sql_const.h"
#include "sql/sql_error.h"
#include "sql/sql_gipk.h"
#include "sql/system_variables.h"
#include "sql/table.h"
#include "sql_string.h"
#include "template_utils.h"

Functions
static void	pack_field (uchar **pack_ptr, Field *field, size_t rec_offset, enum_row_image_type row_image_type, ulonglong value_options, bool *is_partial_format)
	Write a single field (column) of a row in a binary log row event to the output. More...
static bool	unpack_field (const uchar **pack_ptr, Field *field, uint metadata, enum_row_image_type row_image_type, bool is_partial_column)
	Read a single field (column) of a row from a binary log row event. More...
size_t	pack_row (TABLE *table, MY_BITMAP const *columns_in_image, uchar *row_data, const uchar *record, enum_row_image_type row_image_type, ulonglong value_options)
	Pack a record of data for a table into a format suitable for the binary log. More...
static const uchar *	start_partial_bit_reader (const uchar *pack_ptr, size_t length, const table_def *tabledef, Bit_reader *partial_bits, ulonglong *event_value_options)
	Read the value_options from a Partial_update_rows_log_event, and if value_options has any bit set, also read partial_bits. More...
bool	unpack_row (Relay_log_info const *rli, TABLE *table, uint const source_column_count, uchar const *const row_data, MY_BITMAP const *column_image, uchar const **const row_image_end_p, uchar const *const event_end, enum_row_image_type row_image_type, bool event_has_value_options, bool only_seek)
	Unpack a row image (either before-image or after-image) into table->record[0]. More...
const uchar *	translate_beginning_of_raw_data (const uchar *raw_data, MY_BITMAP const *column_image, size_t column_count, Bit_reader &null_bits, table_def *tabledef, bool source_has_gipk, bool replica_has_gipk)
	Return a pointer within a row event's row data, to the data of the first column that exists on the replica. More...
int	prepare_record (TABLE *const table, const MY_BITMAP *cols, const bool check)
	Fills table->record[0] with default values. More...

Function Documentation

pack_field()

static void pack_field ( uchar **  pack_ptr, Field *  field, size_t  rec_offset, enum_row_image_type  row_image_type, ulonglong  value_options, bool *  is_partial_format  )

static

Write a single field (column) of a row in a binary log row event to the output.

Parameters

[in,out]	pack_ptr	Pointer to buffer where the field will be written. It is the caller's responsibility to have allocated enough memory for this buffer. The pointer will be updated to point to the next byte after the last byte that was written.
	field	The field to write.
	rec_offset	Offset to the record that will be written. This is defined by the Field interface: it should be the offset from table->record[0], of the record passed to ha_[write|update|delete]_row. In other words, 0 if this is a before-image and the size of the before-image record if this is an after-image.
	row_image_type	The type of image: before-image or after-image, for a Write/Update/Delete event.
	value_options	The value of @session.binlog_row_value_options
[out]	is_partial_format	Will be set to true if this field was written in partial format, otherwise will not be modified.

pack_row()

size_t pack_row	(	TABLE *	table,
		MY_BITMAP const *	columns_in_image,
		uchar *	row_data,
		const uchar *	record,
		enum_row_image_type	row_image_type,
		ulonglong	value_options
	)

Pack a record of data for a table into a format suitable for the binary log.

The format for a row where N columns are included in the image is the following:

+-----------+----------+----------+     +----------+
| null_bits | column_1 | column_2 | ... | column_N |
+-----------+----------+----------+     +----------+

Where

• null_bits is a bitmap using ceil(N/8) bytes. There is one bit for every column included in the image, regardless of whether it can be null or not. The number of null bits is equal to the number of bits set in the columns_in_image bitmap.
• column_i: Each of the N columns is stored in a format that depends on the type of the column.

Parameters

table	Table describing the format of the record
columns_in_image	Bitmap with a set bit for each column that should be stored in the row.
row_data	Pointer to memory where row will be written
record	Pointer to record retrieved from the engine.
row_image_type	The type of image: before-image or after-image, for a Write/Update/Delete event.
value_options	The value of @session.binlog_row_value_options

Returns

The number of bytes written at row_data.

prepare_record()

int prepare_record	(	TABLE *const	table,
		const MY_BITMAP *	cols,
		const bool	check
	)

Fills table->record[0] with default values.

First restore_record() is called to restore the default values for the record concerning the given table. Then, if check is true, a check is performed to see if fields have the default value or can be NULL. Otherwise an error is reported.

Parameters

table	Table whose record[0] buffer is prepared.
cols	bitmap with a set bit for each column that should be stored in a row.
check	Specifies if lack of default error needs checking.

Returns

0 on success or a handler level error code

Save a reference to the original write set bitmaps. We will need this to restore the bitmaps at the end.

Just to be sure that tmp_set is currently not in use as the write_set already.

Set table->write_set bits for all the columns as they will be checked in set_default() function.

start_partial_bit_reader()

static const uchar * start_partial_bit_reader ( const uchar *  pack_ptr, size_t  length, const table_def *  tabledef, Bit_reader *  partial_bits, ulonglong *  event_value_options  )

static

Read the value_options from a Partial_update_rows_log_event, and if value_options has any bit set, also read partial_bits.

Parameters

[in]	pack_ptr	Read position before the value_options.
[in]	length	Number of bytes between pack_ptr and the end of the event.
[in]	tabledef	Table definition according to previous Table_map_log_event.
[out]	partial_bits	If the event has partial_bits, initialize the read position of this Bit_reader to the position of the partial_bits.
[out]	event_value_options	The value of the value_options field found in the event.

Returns

The read position after value_options and partial_bits (if partial_bits is present).

translate_beginning_of_raw_data()

const uchar * translate_beginning_of_raw_data	(	const uchar *	raw_data,
		MY_BITMAP const *	column_image,
		size_t	column_count,
		Bit_reader &	null_bits,
		table_def *	tabledef,
		bool	source_has_gipk,
		bool	replica_has_gipk
	)

Return a pointer within a row event's row data, to the data of the first column that exists on the replica.

This skips the 'null bits' field, which precedes the column definitions in the row image. In case a GIPK exists in the event but not in this replica's table definition, it skips the GIPK too.

Parameters

raw_data	The data received from the source
column_image	The column bitmap
column_count	The number of column in the image bitmap
null_bits	The bits that are null
tabledef	The source table definition structure
source_has_gipk	If the source table has a GIPK
replica_has_gipk	If the replica table has a GIPK

Returns

const uchar* The adjusted point to the source data

unpack_field()

static bool unpack_field ( const uchar **  pack_ptr, Field *  field, uint  metadata, enum_row_image_type  row_image_type, bool  is_partial_column  )

static

Read a single field (column) of a row from a binary log row event.

Parameters

[in,out]	pack_ptr	Pointer to buffer where the field is stored. The pointer will be updated to point to the next byte after the last byte that was read.
	field	The field to read.
	metadata	The so-called 'metadata' for the field. The meaning of this may differ depending on the SQL type. But typically, it is the length of the field that holds the length of the value: e.g. it is 1 for TINYBLOB, 2 for BLOB, 3 for MEDIUMBLOB, and 4 for LARGEBLOB.
	row_image_type	The type of image: before-image or after-image, for a Write/Update/Delete event.
	is_partial_column	true if this column is in partial format, false otherwise. (This should be determined by the caller from the event_type (PARTIAL_UPDATE_ROWS_EVENT), row_image_type (UPDATE_AI), value_options (PARTIAL_JSON), and partial_bits (1 for this column)).

Return values

false	Success.
true	Error. Error can happen when reading in partial format and it fails to apply the diff. The error has already been reported through my_error.

unpack_row()

bool unpack_row	(	Relay_log_info const *	rli,
		TABLE *	table,
		uint const	source_column_count,
		uchar const *const	row_data,
		MY_BITMAP const *	column_image,
		uchar const **const	row_image_end_p,
		uchar const *const	event_end,
		enum_row_image_type	row_image_type,
		bool	event_has_value_options,
		bool	only_seek
	)

Unpack a row image (either before-image or after-image) into table->record[0].

The row is assumed to only consist of the fields for which the corresponding bit in bitset column_image is set; the other parts of the record are left alone.

If the replica table has more columns than the source table, then the extra columns are not touched by this function. If the source table has more columns than the replica table, then the position is moved to after the extra columns, but the values are not used.

If the replica has a GIPK and the source does not, then the extra column is not touched by this function. If the source table has a GIPK and the replica does not, then the position is shifted forward by 1.

• The layout of a row is:

  For WRITE_ROWS_EVENT: +-----------—+ | after-image | +-----------—+

  For DELETE_ROWS_EVENT: +-----------—+ | before-image | +-----------—+

  For UPDATE_ROWS_EVENT: +-----------—+----------—+ | before-image | after-image | +-----------—+----------—+

  For PARTIAL_UPDATE_ROWS_EVENT: +-----------—+-----------—+----------—+ | before-image | shared-image | after-image | +-----------—+-----------—+----------—+

• Each of before-image and after-image has the following format: +-----—+----—+----—+ +----—+ | length | col_1 | col_2 | ... | col_N | +-----—+----—+----—+ +----—+ length is a 4-byte integer in little-endian format, equal to the total length in bytes of col_1, col_2, ..., col_N.
• The shared-image has one of the following formats: +--------------—+ | value_options=0 | +--------------—+ or +--------------—+-----------—+ | value_options=1 | partial_bits | +--------------—+-----------—+ where:
  • value_options is a bitmap, stored as an integer, in the format of net_field_length. Currently only one bit is allowed: 1=PARTIAL_JSON_UPDATES (so therefore the integer is always 0 or 1, so in reality value_options is only one byte). When PARTIAL_JSON_UPDATES=0, there is nothing else in the shared-image. When PARTIAL_JSON_UPDATES=1, there is a partial_bits field.
  • partial_bits has one bit for each JSON column in the table (regardless of whether it is included in the before-image and/or after-image). The bit is 0 if the JSON update is stored as a full document in the after-image, and 1 if the JSON update in partial form in the after-image.

  • Both when reading the before-image and when reading the after-image it is necessary to know the partialness of JSON columns: when reading the before-image, before looking up the row in the table, we need to set the column in the table's 'read_set' (even if the column was not in the before-image), in order to guarantee that the storage engine reads that column, so that there is any base document that the diff can be applied on. When reading the after-image, we need to know which columns are partial so that we can correctly parse the data for that column.

    Therefore, when this function parses the before-image of a PARTIAL_UPDATE_ROWS_LOG_EVENT, it reads both the before-image and the shared-image, but leaves the read position after the before-image. So when it parses the after-image of a PARTIAL_UPDATE_ROWS_LOG_EVENT, the read position is at the beginning of the shared-image, so it parses both the shared-image and the after-image.

Parameters

[in]	rli	Applier execution context
[in,out]	table	Table to unpack into
[in]	source_column_count	Number of columns that the source had in its table
[in]	row_data	Beginning of the row image
[in]	column_image	Pointer to a bit vector where the N'th bit is 0 for columns that are not included in the event, and 1 for columns that are included in the event.
[out]	row_image_end_p	If this function returns successfully, it sets row_image_end to point to the next byte after the row image that it has read.
[in]	event_end	Pointer to the end of the event.
[in]	row_image_type	The type of row image that we are going to read: WRITE_AI, UPDATE_BI, UPDATE_AI, or DELETE_BI.
[in]	event_has_value_options	true for PARTIAL_UPDATE_ROWS_EVENT, false for UPDATE_ROWS_EVENT.
	only_seek	If true, this is a seek operation rather than a read operation. It will only compute the row_image_end_p pointer, and not read anything into the table and not apply any JSON diffs. (This is used in slave_rows_search_algorithms=HASH_SCAN, which (1) unpacks and hashes the before-image for all rows in the event, (2) scans the table, and for each matching row it (3) unpacks the after-image and applies on the table. In step (1) it needs to unpack the after-image too, in order to move the read position forwards, and then it should use only_seek=true. This is an optimization, but more importantly, when the after-image contains partial JSON, the partial JSON cannot be applied in step (1) since there is no JSON document to apply it on.)

Returns

false on success, true on error.

Calling reset just in case one is unpacking on top a record with data.

This could probably go into set_null() but doing so, (i) triggers assertion in other parts of the code at the moment; (ii) it would make us reset the field, always when setting null, which right now doesn't seem needed anywhere else except here.

TODO: maybe in the future we should consider moving the reset to make it part of set_null. But then the assertions triggered need to be addressed/revisited.