page0zip.h File Reference

Compressed page interface. More...

#include <sys/types.h>
#include <zlib.h>
#include "buf0buf.h"
#include "buf0checksum.h"
#include "buf0types.h"
#include "data0type.h"
#include "dict0types.h"
#include "mach0data.h"
#include "mem0mem.h"
#include "mtr0types.h"
#include "page/zipdecompress.h"
#include "page0types.h"
#include "srv0srv.h"
#include "trx0types.h"
#include "univ.i"
#include "ut0crc32.h"
#include "page0zip.ic"

Go to the source code of this file.

Macros
#define	PAGE_ZIP_START   PAGE_NEW_SUPREMUM_END
	Start offset of the area that will be compressed. More...

Functions
static void	page_zip_set_size (page_zip_des_t *page_zip, ulint size)
	Set the size of a compressed page in bytes. More...
static bool	page_zip_rec_needs_ext (ulint rec_size, ulint comp, ulint n_fields, const page_size_t &page_size)
	Determine if a record is so big that it needs to be stored externally. More...
ulint	page_zip_empty_size (ulint n_fields, ulint zip_size)
	Determine the guaranteed free space on an empty page. More...
bool	page_zip_is_too_big (const dict_index_t *index, const dtuple_t *entry)
	Check whether a tuple is too big for compressed table. More...
static void	page_zip_des_init (page_zip_des_t *page_zip)
	Initialize a compressed page descriptor. More...
void	page_zip_set_alloc (void *stream, mem_heap_t *heap)
	Configure the zlib allocator to use the given memory heap. More...
bool	page_zip_compress (page_zip_des_t *page_zip, const page_t *page, dict_index_t *index, ulint level, mtr_t *mtr)
	Compress a page. More...
ulint	page_zip_fields_encode (ulint n, const dict_index_t *index, ulint trx_id_pos, byte *buf)
	Write the index information for the compressed page. More...
bool	page_zip_decompress (page_zip_des_t *page_zip, page_t *page, bool all)
	Decompress a page. More...
static lint	page_zip_max_ins_size (const page_zip_des_t *page_zip, bool is_clust)
	Determine how big record can be inserted without re-compressing the page. More...
static bool	page_zip_available (const page_zip_des_t *page_zip, bool is_clust, ulint length, ulint create)
	Determine if enough space is available in the modification log. More...
static void	page_zip_write_header (page_zip_des_t *page_zip, const byte *str, ulint length, mtr_t *mtr)
	Write data to the uncompressed header portion of a page. More...
void	page_zip_write_rec (page_zip_des_t *page_zip, const byte *rec, const dict_index_t *index, const ulint *offsets, ulint create)
	Write an entire record on the compressed page. More...
const byte *	page_zip_parse_write_blob_ptr (const byte *ptr, const byte *end_ptr, page_t *page, page_zip_des_t *page_zip)
	Parses a log record of writing a BLOB pointer of a record. More...
void	page_zip_write_blob_ptr (page_zip_des_t *page_zip, const byte *rec, const dict_index_t *index, const ulint *offsets, ulint n, mtr_t *mtr)
	Write a BLOB pointer of a record on the leaf page of a clustered index. More...
const byte *	page_zip_parse_write_node_ptr (const byte *ptr, const byte *end_ptr, page_t *page, page_zip_des_t *page_zip)
	Parses a log record of writing the node pointer of a record. More...
void	page_zip_write_node_ptr (page_zip_des_t *page_zip, byte *rec, ulint size, ulint ptr, mtr_t *mtr)
	Write the node pointer of a record on a non-leaf compressed page. More...
void	page_zip_write_trx_id_and_roll_ptr (page_zip_des_t *page_zip, byte *rec, const ulint *offsets, ulint trx_id_col, trx_id_t trx_id, roll_ptr_t roll_ptr)
	Write the trx_id and roll_ptr of a record on a B-tree leaf node page. More...
void	page_zip_rec_set_deleted (page_zip_des_t *page_zip, const byte *rec, bool flag)
	Write the "deleted" flag of a record on a compressed page. More...
void	page_zip_rec_set_owned (page_zip_des_t *page_zip, const byte *rec, ulint flag)
	Write the "owned" flag of a record on a compressed page. More...
void	page_zip_dir_insert (page_zip_des_t *page_zip, const byte *prev_rec, const byte *free_rec, byte *rec)
	Insert a record to the dense page directory. More...
void	page_zip_dir_delete (page_zip_des_t *page_zip, byte *rec, const dict_index_t *index, const ulint *offsets, const byte *free)
	Shift the dense page directory and the array of BLOB pointers when a record is deleted. More...
void	page_zip_dir_add_slot (page_zip_des_t *page_zip, bool is_clustered)
	Add a slot to the dense page directory. More...
const byte *	page_zip_parse_write_header (const byte *ptr, const byte *end_ptr, page_t *page, page_zip_des_t *page_zip)
	Parses a log record of writing to the header of a page. More...
bool	page_zip_reorganize (buf_block_t *block, dict_index_t *index, mtr_t *mtr)
	Reorganize and compress a page. More...
void	page_zip_copy_recs (page_zip_des_t *page_zip, page_t *page, const page_zip_des_t *src_zip, const page_t *src, dict_index_t *index, mtr_t *mtr)
	Copy the records of a page byte for byte. More...
const byte *	page_zip_parse_compress (const byte *ptr, const byte *end_ptr, page_t *page, page_zip_des_t *page_zip)
	Parses a log record of compressing an index page. More...
static void	page_zip_compress_write_log_no_data (ulint level, const page_t *page, dict_index_t *index, mtr_t *mtr)
	Write a log record of compressing an index page without the data on the page. More...
static const byte *	page_zip_parse_compress_no_data (const byte *ptr, const byte *end_ptr, page_t *page, page_zip_des_t *page_zip, dict_index_t *index)
	Parses a log record of compressing an index page without the data. More...
static void	page_zip_reset_stat_per_index ()
	Reset the counters used for filling INFORMATION_SCHEMA.innodb_cmp_per_index. More...

Variables
uint	page_zip_level
constexpr uint32_t	DEFAULT_COMPRESSION_LEVEL = 6
constexpr uint32_t	PAGE_ZIP_CLUST_LEAF_SLOT_SIZE
	Predefine the sum of DIR_SLOT, TRX_ID & ROLL_PTR. More...
constexpr uint32_t	PAGE_ZIP_DIR_SLOT_MASK = 0x3fff
	Mask of record offsets. More...
constexpr uint32_t	PAGE_ZIP_DIR_SLOT_OWNED = 0x4000
	'owned' flag More...
constexpr uint32_t	PAGE_ZIP_DIR_SLOT_DEL = 0x8000
	'deleted' flag More...
bool	page_zip_log_pages

Detailed Description

Compressed page interface.

Created June 2005 by Marko Makela

Macro Definition Documentation

PAGE_ZIP_START

#define PAGE_ZIP_START   PAGE_NEW_SUPREMUM_END

Start offset of the area that will be compressed.

Function Documentation

page_zip_available()

static bool page_zip_available ( const page_zip_des_t *  page_zip, bool  is_clust, ulint  length, ulint  create  )

inlinestatic

Determine if enough space is available in the modification log.

Parameters

[in]	page_zip	Compressed page.
[in]	is_clust	True if clustered index.
[in]	length	Combined size of the record.
[in]	create	Nonzero=add the record to the heap.

Returns

true if page_zip_write_rec() will succeed

page_zip_compress()

bool page_zip_compress	(	page_zip_des_t *	page_zip,
		const page_t *	page,
		dict_index_t *	index,
		ulint	level,
		mtr_t *	mtr
	)

Compress a page.

Returns

true on success, false on failure; page_zip will be left intact on failure. in/out: mini-transaction, or NULL

true on success, false on failure; page_zip will be left intact on failure.

< index field information

< compressed payload of the page

< dense page directory, sorted by address

Parameters

page_zip	in: size; out: data, n_blobs, m_start, m_end, m_nonempty
page	in: uncompressed page
index	in: index tree
level	in: compression level
mtr	in/out: mini-transaction, or NULL

page_zip_compress_write_log_no_data()

static void page_zip_compress_write_log_no_data ( ulint  level, const page_t *  page, dict_index_t *  index, mtr_t *  mtr  )

inlinestatic

Write a log record of compressing an index page without the data on the page.

Parameters

[in]	level	compression level
[in]	page	page that is compressed
[in]	index	index
[in]	mtr	mtr

page_zip_copy_recs()

void page_zip_copy_recs	(	page_zip_des_t *	page_zip,
		page_t *	page,
		const page_zip_des_t *	src_zip,
		const page_t *	src,
		dict_index_t *	index,
		mtr_t *	mtr
	)

Copy the records of a page byte for byte.

Do not copy the page header or trailer, except those B-tree header fields that are directly related to the storage of records. Also copy PAGE_MAX_TRX_ID. NOTE: The caller must update the lock table and the adaptive hash index. in: mini-transaction

Do not copy the page header or trailer, except those B-tree header fields that are directly related to the storage of records. Also copy PAGE_MAX_TRX_ID. NOTE: The caller must update the lock table and the adaptive hash index.

Parameters

page_zip	out: copy of src_zip (n_blobs, m_start, m_end, m_nonempty, data[0..size-1])
page	out: copy of src
src_zip	in: compressed page
src	in: page
index	in: index of the B-tree
mtr	in: mini-transaction

page_zip_decompress()

bool page_zip_decompress	(	page_zip_des_t *	page_zip,
		page_t *	page,
		bool	all
	)

Decompress a page.

This function should tolerate errors on the compressed page. Instead of letting assertions fail, it will return false if an inconsistency is detected.

Returns

true on success, false on failure in: true=decompress the whole page; false=verify but do not copy some page header fields that should not change after page creation

This function should tolerate errors on the compressed page. Instead of letting assertions fail, it will return false if an inconsistency is detected.

Returns

true on success, false on failure

Parameters

page_zip	in: data, ssize; out: m_start, m_end, m_nonempty, n_blobs
page	out: uncompressed page, may be trashed
all	in: true=decompress the whole page; false=verify but do not copy some page header fields that should not change after page creation

page_zip_des_init()

static void page_zip_des_init ( page_zip_des_t *  page_zip )

inlinestatic

Initialize a compressed page descriptor.

in/out: compressed page descriptor

page_zip_dir_add_slot()

void page_zip_dir_add_slot	(	page_zip_des_t *	page_zip,
		bool	is_clustered
	)

Add a slot to the dense page directory.

Parameters

[in,out]	page_zip	Compressed page
[in]	is_clustered	Nonzero for clustered index, zero for others

page_zip_dir_delete()

void page_zip_dir_delete	(	page_zip_des_t *	page_zip,
		byte *	rec,
		const dict_index_t *	index,
		const ulint *	offsets,
		const byte *	free
	)

Shift the dense page directory and the array of BLOB pointers when a record is deleted.

Parameters

[in,out]	page_zip	compressed page
[in]	rec	deleted record
[in]	index	index of rec
[in]	offsets	rec_get_offsets(rec)
[in]	free	previous start of the free list

page_zip_dir_insert()

void page_zip_dir_insert	(	page_zip_des_t *	page_zip,
		const byte *	prev_rec,
		const byte *	free_rec,
		byte *	rec
	)

Insert a record to the dense page directory.

Parameters

[in,out]	page_zip	Compressed page
[in]	prev_rec	Record after which to insert
[in]	free_rec	Record from which rec was allocated, or null
[in]	rec	Record to insert

page_zip_empty_size()

ulint page_zip_empty_size	(	ulint	n_fields,
		ulint	zip_size
	)

Determine the guaranteed free space on an empty page.

Parameters

[in]	n_fields	number of columns in the index
[in]	zip_size	compressed page size in bytes

Returns

minimum payload size on the page

minimum payload size on the page

Parameters

n_fields	in: number of columns in the index
zip_size	in: compressed page size in bytes

page_zip_fields_encode()

ulint page_zip_fields_encode	(	ulint	n,
		const dict_index_t *	index,
		ulint	trx_id_pos,
		byte *	buf
	)

Write the index information for the compressed page.

Returns

used size of buf out: buffer of (n + 1) * 2 bytes

used size of buf

Parameters

n	in: number of fields to compress
index	in: index comprising at least n fields
trx_id_pos	in: position of the trx_id column in the index, or ULINT_UNDEFINED if this is a non-leaf page
buf	out: buffer of (n + 1) * 2 bytes

page_zip_is_too_big()

bool page_zip_is_too_big	(	const dict_index_t *	index,
		const dtuple_t *	entry
	)

Check whether a tuple is too big for compressed table.

Parameters

[in]	index	dict index object
[in]	entry	entry for the index

Returns

true if it's too big, otherwise false

page_zip_max_ins_size()

static lint page_zip_max_ins_size ( const page_zip_des_t *  page_zip, bool  is_clust  )

inlinestatic

Determine how big record can be inserted without re-compressing the page.

Parameters

[in]	page_zip	Compressed page.
[in]	is_clust	True if clustered index.

Returns

a positive number indicating the maximum size of a record whose insertion is guaranteed to succeed, or zero or negative

page_zip_parse_compress()

const byte * page_zip_parse_compress	(	const byte *	ptr,
		const byte *	end_ptr,
		page_t *	page,
		page_zip_des_t *	page_zip
	)

Parses a log record of compressing an index page.

Returns

end of log record or NULL out: compressed page

end of log record or NULL

Parameters

ptr	in: buffer
end_ptr	in: buffer end
page	out: uncompressed page
page_zip	out: compressed page

page_zip_parse_compress_no_data()

static const byte * page_zip_parse_compress_no_data ( const byte *  ptr, const byte *  end_ptr, page_t *  page, page_zip_des_t *  page_zip, dict_index_t *  index  )

inlinestatic

Parses a log record of compressing an index page without the data.

Parameters

[in]	ptr	buffer
[in]	end_ptr	buffer end
[in]	page	uncompressed page
[out]	page_zip	compressed page
[in]	index	index

Returns

end of log record or NULL

page_zip_parse_write_blob_ptr()

const byte * page_zip_parse_write_blob_ptr	(	const byte *	ptr,
		const byte *	end_ptr,
		page_t *	page,
		page_zip_des_t *	page_zip
	)

Parses a log record of writing a BLOB pointer of a record.

Returns

end of log record or NULL in/out: compressed page

end of log record or NULL

Parameters

ptr	in: redo log buffer
end_ptr	in: redo log buffer end
page	in/out: uncompressed page
page_zip	in/out: compressed page

page_zip_parse_write_header()

const byte * page_zip_parse_write_header	(	const byte *	ptr,
		const byte *	end_ptr,
		page_t *	page,
		page_zip_des_t *	page_zip
	)

Parses a log record of writing to the header of a page.

Returns

end of log record or NULL in/out: compressed page

end of log record or NULL

Parameters

ptr	in: redo log buffer
end_ptr	in: redo log buffer end
page	in/out: uncompressed page
page_zip	in/out: compressed page

page_zip_parse_write_node_ptr()

const byte * page_zip_parse_write_node_ptr	(	const byte *	ptr,
		const byte *	end_ptr,
		page_t *	page,
		page_zip_des_t *	page_zip
	)

Parses a log record of writing the node pointer of a record.

Returns

end of log record or NULL in/out: compressed page

end of log record or NULL

Parameters

ptr	in: redo log buffer
end_ptr	in: redo log buffer end
page	in/out: uncompressed page
page_zip	in/out: compressed page

page_zip_rec_needs_ext()

static bool page_zip_rec_needs_ext ( ulint  rec_size, ulint  comp, ulint  n_fields, const page_size_t &  page_size  )

inlinestatic

Determine if a record is so big that it needs to be stored externally.

Parameters

[in]	rec_size	length of the record in bytes
[in]	comp	nonzero=compact format
[in]	n_fields	number of fields in the record; ignored if tablespace is not compressed
[in]	page_size	page size

Returns

false if the entire record can be stored locally on the page

page_zip_rec_set_deleted()

void page_zip_rec_set_deleted	(	page_zip_des_t *	page_zip,
		const byte *	rec,
		bool	flag
	)

Write the "deleted" flag of a record on a compressed page.

The flag must already have been written on the uncompressed page. in: the deleted flag (nonzero=true)

The flag must already have been written on the uncompressed page.

Parameters

page_zip	in/out: compressed page
rec	in: record on the uncompressed page
flag	in: the deleted flag (nonzero=true)

page_zip_rec_set_owned()

void page_zip_rec_set_owned	(	page_zip_des_t *	page_zip,
		const byte *	rec,
		ulint	flag
	)

Write the "owned" flag of a record on a compressed page.

The n_owned field must already have been written on the uncompressed page.

Parameters

[in,out]	page_zip	Compressed page
[in]	rec	Record on the uncompressed page
[in]	flag	The owned flag (nonzero=true)

page_zip_reorganize()

bool page_zip_reorganize	(	buf_block_t *	block,
		dict_index_t *	index,
		mtr_t *	mtr
	)

Reorganize and compress a page.

This is a low-level operation for compressed pages, to be used when page_zip_compress() fails. On success, a redo log entry MLOG_ZIP_PAGE_COMPRESS will be written. The function btr_page_reorganize() should be preferred whenever possible. IMPORTANT: if page_zip_reorganize() is invoked on a leaf page of a non-clustered index, the caller must update the insert buffer free bits in the same mini-transaction in such a way that the modification will be redo-logged.

Returns

true on success, false on failure; page_zip will be left intact on failure, but page will be overwritten. in: mini-transaction

This is a low-level operation for compressed pages, to be used when page_zip_compress() fails. On success, a redo log entry MLOG_ZIP_PAGE_COMPRESS will be written. The function btr_page_reorganize() should be preferred whenever possible. IMPORTANT: if page_zip_reorganize() is invoked on a leaf page of a non-clustered index, the caller must update the insert buffer free bits in the same mini-transaction in such a way that the modification will be redo-logged.

Returns

true on success, false on failure; page_zip will be left intact on failure, but page will be overwritten.

Parameters

block	in/out: page with compressed page; on the compressed page, in: size; out: data, n_blobs, m_start, m_end, m_nonempty
index	in: index of the B-tree node
mtr	in: mini-transaction

page_zip_reset_stat_per_index()

static void page_zip_reset_stat_per_index ( )

inlinestatic

Reset the counters used for filling INFORMATION_SCHEMA.innodb_cmp_per_index.

page_zip_set_alloc()

void page_zip_set_alloc	(	void *	stream,
		mem_heap_t *	heap
	)

Configure the zlib allocator to use the given memory heap.

Parameters

[in,out]	stream	zlib stream
[in]	heap	Memory heap to use

page_zip_set_size()

static void page_zip_set_size ( page_zip_des_t *  page_zip, ulint  size  )

inlinestatic

Set the size of a compressed page in bytes.

Parameters

[in,out]	page_zip	compressed page
[in]	size	size in bytes

page_zip_write_blob_ptr()

void page_zip_write_blob_ptr	(	page_zip_des_t *	page_zip,
		const byte *	rec,
		const dict_index_t *	index,
		const ulint *	offsets,
		ulint	n,
		mtr_t *	mtr
	)

Write a BLOB pointer of a record on the leaf page of a clustered index.

The information must already have been updated on the uncompressed page. in: mini-transaction handle, or NULL if no logging is needed

The information must already have been updated on the uncompressed page.

Parameters

page_zip	in/out: compressed page
rec	in/out: record whose data is being written
index	in: index of the page
offsets	in: rec_get_offsets(rec, index)
n	in: column index
mtr	in: mini-transaction handle, or NULL if no logging is needed

page_zip_write_header()

static void page_zip_write_header ( page_zip_des_t *  page_zip, const byte *  str, ulint  length, mtr_t *  mtr  )

inlinestatic

Write data to the uncompressed header portion of a page.

The data must already have been written to the uncompressed page. However, the data portion of the uncompressed page may differ from the compressed page when a record is being inserted in page_cur_insert_rec_low().

Parameters

[in,out]	page_zip	Compressed page
[in]	str	Address on the uncompressed page
[in]	length	Length of the data
[in]	mtr	Mini-transaction, or NULL

page_zip_write_node_ptr()

void page_zip_write_node_ptr	(	page_zip_des_t *	page_zip,
		byte *	rec,
		ulint	size,
		ulint	ptr,
		mtr_t *	mtr
	)

Write the node pointer of a record on a non-leaf compressed page.

Parameters

[in,out]	page_zip	Compressed page
[in,out]	rec	Record
[in]	size	Data size of rec
[in]	ptr	Node pointer
[in]	mtr	Mini-transaction, or null

page_zip_write_rec()

void page_zip_write_rec	(	page_zip_des_t *	page_zip,
		const byte *	rec,
		const dict_index_t *	index,
		const ulint *	offsets,
		ulint	create
	)

Write an entire record on the compressed page.

The data must already have been written to the uncompressed page. in: nonzero=insert, zero=update

The data must already have been written to the uncompressed page.

Parameters

page_zip	in/out: compressed page
rec	in: record being written
index	in: the index the record belongs to
offsets	in: rec_get_offsets(rec, index)
create	in: nonzero=insert, zero=update

page_zip_write_trx_id_and_roll_ptr()

void page_zip_write_trx_id_and_roll_ptr	(	page_zip_des_t *	page_zip,
		byte *	rec,
		const ulint *	offsets,
		ulint	trx_id_col,
		trx_id_t	trx_id,
		roll_ptr_t	roll_ptr
	)

Write the trx_id and roll_ptr of a record on a B-tree leaf node page.

Parameters

[in,out]	page_zip	Compressed page
[in,out]	rec	Record
[in]	offsets	Rec_get_offsets(rec, index)
[in]	trx_id_col	Column number of trx_id in rec
[in]	trx_id	Transaction identifier
[in]	roll_ptr	Roll_ptr

Variable Documentation

DEFAULT_COMPRESSION_LEVEL

constexpr uint32_t DEFAULT_COMPRESSION_LEVEL = 6

constexpr

PAGE_ZIP_CLUST_LEAF_SLOT_SIZE

constexpr uint32_t PAGE_ZIP_CLUST_LEAF_SLOT_SIZE

constexpr

Initial value:

=
    PAGE_ZIP_DIR_SLOT_SIZE + DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN

Predefine the sum of DIR_SLOT, TRX_ID & ROLL_PTR.

PAGE_ZIP_DIR_SLOT_DEL

constexpr uint32_t PAGE_ZIP_DIR_SLOT_DEL = 0x8000

constexpr

'deleted' flag

PAGE_ZIP_DIR_SLOT_MASK

constexpr uint32_t PAGE_ZIP_DIR_SLOT_MASK = 0x3fff

constexpr

Mask of record offsets.

PAGE_ZIP_DIR_SLOT_OWNED

constexpr uint32_t PAGE_ZIP_DIR_SLOT_OWNED = 0x4000

constexpr

'owned' flag

page_zip_level

uint page_zip_level

extern

page_zip_log_pages

bool page_zip_log_pages

extern