MySQL 8.1.0
Source Code Documentation
page0zip.h
Go to the documentation of this file.
1/*****************************************************************************
2
3Copyright (c) 2005, 2023, Oracle and/or its affiliates.
4Copyright (c) 2012, Facebook Inc.
5
6This program is free software; you can redistribute it and/or modify it under
7the terms of the GNU General Public License, version 2.0, as published by the
8Free Software Foundation.
9
10This program is also distributed with certain software (including but not
11limited to OpenSSL) that is licensed under separate terms, as designated in a
12particular file or component or in included license documentation. The authors
13of MySQL hereby grant you an additional permission to link the program and
14your derivative works with the separately licensed software that they have
15included with MySQL.
16
17This program is distributed in the hope that it will be useful, but WITHOUT
18ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
19FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0,
20for more details.
21
22You should have received a copy of the GNU General Public License along with
23this program; if not, write to the Free Software Foundation, Inc.,
2451 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
25
26*****************************************************************************/
27
28/** @file include/page0zip.h
29 Compressed page interface
30
31 Created June 2005 by Marko Makela
32 *******************************************************/
33
34#ifndef page0zip_h
35#define page0zip_h
36
37#include <sys/types.h>
38#include <zlib.h>
39
40#include "buf0buf.h"
41#include "buf0checksum.h"
42#include "buf0types.h"
43#include "data0type.h"
44#include "dict0types.h"
45#include "mach0data.h"
46#include "mem0mem.h"
47#include "mtr0types.h"
48#include "page/zipdecompress.h"
49#include "page0types.h"
50#include "srv0srv.h"
51#include "trx0types.h"
52#include "univ.i"
53#include "ut0crc32.h"
54
55/* Compression level to be used by zlib. Settable by user. */
56extern uint page_zip_level;
57
58/* Default compression level. */
59constexpr uint32_t DEFAULT_COMPRESSION_LEVEL = 6;
60/** Start offset of the area that will be compressed */
61#define PAGE_ZIP_START PAGE_NEW_SUPREMUM_END
62/** Predefine the sum of DIR_SLOT, TRX_ID & ROLL_PTR */
65/** Mask of record offsets */
66constexpr uint32_t PAGE_ZIP_DIR_SLOT_MASK = 0x3fff;
67/** 'owned' flag */
68constexpr uint32_t PAGE_ZIP_DIR_SLOT_OWNED = 0x4000;
69/** 'deleted' flag */
70constexpr uint32_t PAGE_ZIP_DIR_SLOT_DEL = 0x8000;
71
72/* Whether or not to log compressed page images to avoid possible
73compression algorithm changes in zlib. */
74extern bool page_zip_log_pages;
75
76/** Set the size of a compressed page in bytes.
77@param[in,out] page_zip compressed page
78@param[in] size size in bytes */
79static inline void page_zip_set_size(page_zip_des_t *page_zip, ulint size);
80
81#ifndef UNIV_HOTBACKUP
82/** Determine if a record is so big that it needs to be stored externally.
83@param[in] rec_size length of the record in bytes
84@param[in] comp nonzero=compact format
85@param[in] n_fields number of fields in the record; ignored if
86tablespace is not compressed
87@param[in] page_size page size
88@return false if the entire record can be stored locally on the page */
89[[nodiscard]] static inline bool page_zip_rec_needs_ext(
90 ulint rec_size, ulint comp, ulint n_fields, const page_size_t &page_size);
91#endif /* !UNIV_HOTBACKUP */
92
93/** Determine the guaranteed free space on an empty page.
94@param[in] n_fields number of columns in the index
95@param[in] zip_size compressed page size in bytes
96@return minimum payload size on the page */
97ulint page_zip_empty_size(ulint n_fields, ulint zip_size);
98
99#ifndef UNIV_HOTBACKUP
100/** Check whether a tuple is too big for compressed table
101@param[in] index dict index object
102@param[in] entry entry for the index
103@return true if it's too big, otherwise false */
104bool page_zip_is_too_big(const dict_index_t *index, const dtuple_t *entry);
105#endif /* !UNIV_HOTBACKUP */
106
107/** Initialize a compressed page descriptor. */
108static inline void page_zip_des_init(
109 page_zip_des_t *page_zip); /*!< in/out: compressed page
110 descriptor */
111
112/** Configure the zlib allocator to use the given memory heap.
113@param[in,out] stream zlib stream
114@param[in] heap Memory heap to use */
115void page_zip_set_alloc(void *stream, mem_heap_t *heap);
116
117/** Compress a page.
118 @return true on success, false on failure; page_zip will be left
119 intact on failure. */
120bool page_zip_compress(page_zip_des_t *page_zip, /*!< in: size; out: data,
121 n_blobs, m_start, m_end,
122 m_nonempty */
123 const page_t *page, /*!< in: uncompressed page */
124 dict_index_t *index, /*!< in: index tree */
125 ulint level, /*!< in: compression level */
126 mtr_t *mtr); /*!< in/out: mini-transaction,
127 or NULL */
128
129/** Write the index information for the compressed page.
130 @return used size of buf */
132 ulint n, /*!< in: number of fields
133 to compress */
134 const dict_index_t *index, /*!< in: index comprising
135 at least n fields */
136 ulint trx_id_pos,
137 /*!< in: position of the trx_id column
138 in the index, or ULINT_UNDEFINED if
139 this is a non-leaf page */
140 byte *buf); /*!< out: buffer of (n + 1) * 2 bytes */
141
142/** Decompress a page. This function should tolerate errors on the compressed
143 page. Instead of letting assertions fail, it will return false if an
144 inconsistency is detected.
145 @return true on success, false on failure */
147 page_zip_des_t *page_zip, /*!< in: data, ssize;
148 out: m_start, m_end, m_nonempty, n_blobs */
149 page_t *page, /*!< out: uncompressed page, may be trashed */
150 bool all); /*!< in: true=decompress the whole page;
151 false=verify but do not copy some
152 page header fields that should not change
153 after page creation */
154
155#ifdef UNIV_ZIP_DEBUG
156/** Check that the compressed and decompressed pages match.
157 @return true if valid, false if not */
158bool page_zip_validate_low(
159 const page_zip_des_t *page_zip, /*!< in: compressed page */
160 const page_t *page, /*!< in: uncompressed page */
161 const dict_index_t *index, /*!< in: index of the page, if known */
162 bool sloppy); /*!< in: false=strict,
163 true=ignore the MIN_REC_FLAG */
164/** Check that the compressed and decompressed pages match. */
165bool page_zip_validate(
166 const page_zip_des_t *page_zip, /*!< in: compressed page */
167 const page_t *page, /*!< in: uncompressed page */
168 const dict_index_t *index); /*!< in: index of the page, if known */
169#endif /* UNIV_ZIP_DEBUG */
170
171/** Determine how big record can be inserted without re-compressing the page.
172@param[in] page_zip Compressed page.
173@param[in] is_clust True if clustered index.
174@return a positive number indicating the maximum size of a record
175whose insertion is guaranteed to succeed, or zero or negative */
176[[nodiscard]] static inline lint page_zip_max_ins_size(
177 const page_zip_des_t *page_zip, bool is_clust);
178
179/** Determine if enough space is available in the modification log.
180@param[in] page_zip Compressed page.
181@param[in] is_clust True if clustered index.
182@param[in] length Combined size of the record.
183@param[in] create Nonzero=add the record to the heap.
184@return true if page_zip_write_rec() will succeed */
185[[nodiscard]] static inline bool page_zip_available(
186 const page_zip_des_t *page_zip, bool is_clust, ulint length, ulint create);
187
188/** Write data to the uncompressed header portion of a page. The data must
189already have been written to the uncompressed page.
190However, the data portion of the uncompressed page may differ from the
191compressed page when a record is being inserted in page_cur_insert_rec_low().
192@param[in,out] page_zip Compressed page
193@param[in] str Address on the uncompressed page
194@param[in] length Length of the data
195@param[in] mtr Mini-transaction, or NULL */
196static inline void page_zip_write_header(page_zip_des_t *page_zip,
197 const byte *str, ulint length,
198 mtr_t *mtr);
199
200/** Write an entire record on the compressed page. The data must already
201 have been written to the uncompressed page. */
203 page_zip_des_t *page_zip, /*!< in/out: compressed page */
204 const byte *rec, /*!< in: record being written */
205 const dict_index_t *index, /*!< in: the index the record belongs to */
206 const ulint *offsets, /*!< in: rec_get_offsets(rec, index) */
207 ulint create); /*!< in: nonzero=insert, zero=update */
208
209/** Parses a log record of writing a BLOB pointer of a record.
210 @return end of log record or NULL */
212 byte *ptr, /*!< in: redo log buffer */
213 byte *end_ptr, /*!< in: redo log buffer end */
214 page_t *page, /*!< in/out: uncompressed page */
215 page_zip_des_t *page_zip); /*!< in/out: compressed page */
216
217/** Write a BLOB pointer of a record on the leaf page of a clustered index.
218 The information must already have been updated on the uncompressed page. */
220 page_zip_des_t *page_zip, /*!< in/out: compressed page */
221 const byte *rec, /*!< in/out: record whose data is being
222 written */
223 const dict_index_t *index, /*!< in: index of the page */
224 const ulint *offsets, /*!< in: rec_get_offsets(rec, index) */
225 ulint n, /*!< in: column index */
226 mtr_t *mtr); /*!< in: mini-transaction handle,
227 or NULL if no logging is needed */
228
229/** Parses a log record of writing the node pointer of a record.
230 @return end of log record or NULL */
232 byte *ptr, /*!< in: redo log buffer */
233 byte *end_ptr, /*!< in: redo log buffer end */
234 page_t *page, /*!< in/out: uncompressed page */
235 page_zip_des_t *page_zip); /*!< in/out: compressed page */
236
237/** Write the node pointer of a record on a non-leaf compressed page.
238@param[in,out] page_zip Compressed page
239@param[in,out] rec Record
240@param[in] size Data size of rec
241@param[in] ptr Node pointer
242@param[in] mtr Mini-transaction, or null */
243void page_zip_write_node_ptr(page_zip_des_t *page_zip, byte *rec, ulint size,
244 ulint ptr, mtr_t *mtr);
245
246/** Write the trx_id and roll_ptr of a record on a B-tree leaf node page.
247@param[in,out] page_zip Compressed page
248@param[in,out] rec Record
249@param[in] offsets Rec_get_offsets(rec, index)
250@param[in] trx_id_col Column number of trx_id in rec
251@param[in] trx_id Transaction identifier
252@param[in] roll_ptr Roll_ptr */
253void page_zip_write_trx_id_and_roll_ptr(page_zip_des_t *page_zip, byte *rec,
254 const ulint *offsets, ulint trx_id_col,
255 trx_id_t trx_id, roll_ptr_t roll_ptr);
256
257/** Write the "deleted" flag of a record on a compressed page. The flag must
258 already have been written on the uncompressed page. */
260 page_zip_des_t *page_zip, /*!< in/out: compressed page */
261 const byte *rec, /*!< in: record on the uncompressed page */
262 bool flag); /*!< in: the deleted flag (nonzero=true) */
263
264/** Write the "owned" flag of a record on a compressed page. The n_owned field
265must already have been written on the uncompressed page.
266@param[in,out] page_zip Compressed page
267@param[in] rec Record on the uncompressed page
268@param[in] flag The owned flag (nonzero=true) */
269void page_zip_rec_set_owned(page_zip_des_t *page_zip, const byte *rec,
270 ulint flag);
271
272/** Insert a record to the dense page directory.
273@param[in,out] page_zip Compressed page
274@param[in] prev_rec Record after which to insert
275@param[in] free_rec Record from which rec was allocated, or null
276@param[in] rec Record to insert */
277void page_zip_dir_insert(page_zip_des_t *page_zip, const byte *prev_rec,
278 const byte *free_rec, byte *rec);
279
280/** Shift the dense page directory and the array of BLOB pointers when a record
281is deleted.
282@param[in,out] page_zip compressed page
283@param[in] rec deleted record
284@param[in] index index of rec
285@param[in] offsets rec_get_offsets(rec)
286@param[in] free previous start of the free list */
287void page_zip_dir_delete(page_zip_des_t *page_zip, byte *rec,
288 const dict_index_t *index, const ulint *offsets,
289 const byte *free);
290
291/** Add a slot to the dense page directory.
292@param[in,out] page_zip Compressed page
293@param[in] is_clustered Nonzero for clustered index, zero for others */
294void page_zip_dir_add_slot(page_zip_des_t *page_zip, bool is_clustered);
295
296/** Parses a log record of writing to the header of a page.
297 @return end of log record or NULL */
299 byte *ptr, /*!< in: redo log buffer */
300 byte *end_ptr, /*!< in: redo log buffer end */
301 page_t *page, /*!< in/out: uncompressed page */
302 page_zip_des_t *page_zip); /*!< in/out: compressed page */
303
304/** Reorganize and compress a page. This is a low-level operation for
305 compressed pages, to be used when page_zip_compress() fails.
306 On success, a redo log entry MLOG_ZIP_PAGE_COMPRESS will be written.
307 The function btr_page_reorganize() should be preferred whenever possible.
308 IMPORTANT: if page_zip_reorganize() is invoked on a leaf page of a
309 non-clustered index, the caller must update the insert buffer free
310 bits in the same mini-transaction in such a way that the modification
311 will be redo-logged.
312 @return true on success, false on failure; page_zip will be left
313 intact on failure, but page will be overwritten. */
315 buf_block_t *block, /*!< in/out: page with compressed page;
316 on the compressed page, in: size;
317 out: data, n_blobs,
318 m_start, m_end, m_nonempty */
319 dict_index_t *index, /*!< in: index of the B-tree node */
320 mtr_t *mtr); /*!< in: mini-transaction */
321/** Copy the records of a page byte for byte. Do not copy the page header
322 or trailer, except those B-tree header fields that are directly
323 related to the storage of records. Also copy PAGE_MAX_TRX_ID.
324 NOTE: The caller must update the lock table and the adaptive hash index. */
326 page_zip_des_t *page_zip, /*!< out: copy of src_zip
327 (n_blobs, m_start, m_end,
328 m_nonempty, data[0..size-1]) */
329 page_t *page, /*!< out: copy of src */
330 const page_zip_des_t *src_zip, /*!< in: compressed page */
331 const page_t *src, /*!< in: page */
332 dict_index_t *index, /*!< in: index of the B-tree */
333 mtr_t *mtr); /*!< in: mini-transaction */
334#ifndef UNIV_HOTBACKUP
335#endif /* !UNIV_HOTBACKUP */
336
337/** Parses a log record of compressing an index page.
338 @return end of log record or NULL */
340 byte *ptr, /*!< in: buffer */
341 byte *end_ptr, /*!< in: buffer end */
342 page_t *page, /*!< out: uncompressed page */
343 page_zip_des_t *page_zip); /*!< out: compressed page */
344
345/** Write a log record of compressing an index page without the data on the
346page.
347@param[in] level compression level
348@param[in] page page that is compressed
349@param[in] index index
350@param[in] mtr mtr */
352 const page_t *page,
353 dict_index_t *index,
354 mtr_t *mtr);
355
356/** Parses a log record of compressing an index page without the data.
357@param[in] ptr buffer
358@param[in] end_ptr buffer end
359@param[in] page uncompressed page
360@param[out] page_zip compressed page
361@param[in] index index
362@return end of log record or NULL */
363static inline byte *page_zip_parse_compress_no_data(byte *ptr, byte *end_ptr,
364 page_t *page,
365 page_zip_des_t *page_zip,
366 dict_index_t *index);
367
368#ifndef UNIV_HOTBACKUP
369/** Reset the counters used for filling
370 INFORMATION_SCHEMA.innodb_cmp_per_index. */
371static inline void page_zip_reset_stat_per_index();
372
373#endif /* !UNIV_HOTBACKUP */
374
375#include "page0zip.ic"
376
377#endif /* page0zip_h */
The database buffer pool high-level routines.
Buffer pool checksum functions, also linked from /extra/innochecksum.cc.
The database buffer pool global types for the directory.
Page size descriptor.
Definition: page0size.h:49
int page
Definition: ctype-mb.cc:1233
Data types.
constexpr size_t DATA_ROLL_PTR_LEN
Rollback data pointer type size in bytes.
Definition: data0type.h:190
constexpr size_t DATA_TRX_ID_LEN
Transaction ID type size in bytes.
Definition: data0type.h:184
Data dictionary global types.
static int flag
Definition: hp_test1.cc:39
#define free(A)
Definition: lexyy.cc:915
Utilities for converting data from the database file to the machine format.
The memory management.
Mini-transaction buffer global types.
std::string str(const mysqlrouter::ConfigGenerator::Options::Endpoint &ep)
Definition: config_generator.cc:1063
Definition: buf0block_hint.cc:29
bool length(const dd::Spatial_reference_system *srs, const Geometry *g1, double *length, bool *null) noexcept
Computes the length of linestrings and multilinestrings.
Definition: length.cc:75
static mysql_service_status_t create(const char *service_names[], reference_caching_channel *out_channel) noexcept
Definition: component.cc:44
Index page routines.
byte page_t
Type of the index page.
Definition: page0types.h:151
void page_zip_rec_set_owned(page_zip_des_t *page_zip, const byte *rec, ulint flag)
Write the "owned" flag of a record on a compressed page.
Definition: page0zip.cc:2242
bool page_zip_reorganize(buf_block_t *block, dict_index_t *index, mtr_t *mtr)
Reorganize and compress a page.
Definition: page0zip.cc:2547
void page_zip_dir_delete(page_zip_des_t *page_zip, byte *rec, const dict_index_t *index, const ulint *offsets, const byte *free)
Shift the dense page directory and the array of BLOB pointers when a record is deleted.
Definition: page0zip.cc:2329
static void page_zip_write_header(page_zip_des_t *page_zip, const byte *str, ulint length, mtr_t *mtr)
Write data to the uncompressed header portion of a page.
void page_zip_write_trx_id_and_roll_ptr(page_zip_des_t *page_zip, byte *rec, const ulint *offsets, ulint trx_id_col, trx_id_t trx_id, roll_ptr_t roll_ptr)
Write the trx_id and roll_ptr of a record on a B-tree leaf node page.
Definition: page0zip.cc:2092
void page_zip_set_alloc(void *stream, mem_heap_t *heap)
Configure the zlib allocator to use the given memory heap.
Definition: zipdecompress.cc:116
void page_zip_write_blob_ptr(page_zip_des_t *page_zip, const byte *rec, const dict_index_t *index, const ulint *offsets, ulint n, mtr_t *mtr)
Write a BLOB pointer of a record on the leaf page of a clustered index.
Definition: page0zip.cc:1877
constexpr uint32_t PAGE_ZIP_CLUST_LEAF_SLOT_SIZE
Predefine the sum of DIR_SLOT, TRX_ID & ROLL_PTR.
Definition: page0zip.h:63
byte * page_zip_parse_compress(byte *ptr, byte *end_ptr, page_t *page, page_zip_des_t *page_zip)
Parses a log record of compressing an index page.
Definition: page0zip.cc:2715
static byte * page_zip_parse_compress_no_data(byte *ptr, byte *end_ptr, page_t *page, page_zip_des_t *page_zip, dict_index_t *index)
Parses a log record of compressing an index page without the data.
byte * page_zip_parse_write_blob_ptr(byte *ptr, byte *end_ptr, page_t *page, page_zip_des_t *page_zip)
Parses a log record of writing a BLOB pointer of a record.
Definition: page0zip.cc:1827
byte * page_zip_parse_write_node_ptr(byte *ptr, byte *end_ptr, page_t *page, page_zip_des_t *page_zip)
Parses a log record of writing the node pointer of a record.
Definition: page0zip.cc:1958
static void page_zip_compress_write_log_no_data(ulint level, const page_t *page, dict_index_t *index, mtr_t *mtr)
Write a log record of compressing an index page without the data on the page.
constexpr uint32_t PAGE_ZIP_DIR_SLOT_OWNED
'owned' flag
Definition: page0zip.h:68
static bool page_zip_available(const page_zip_des_t *page_zip, bool is_clust, ulint length, ulint create)
Determine if enough space is available in the modification log.
static bool page_zip_rec_needs_ext(ulint rec_size, ulint comp, ulint n_fields, const page_size_t &page_size)
Determine if a record is so big that it needs to be stored externally.
constexpr uint32_t DEFAULT_COMPRESSION_LEVEL
Definition: page0zip.h:59
static void page_zip_reset_stat_per_index()
Reset the counters used for filling INFORMATION_SCHEMA.innodb_cmp_per_index.
ulint page_zip_fields_encode(ulint n, const dict_index_t *index, ulint trx_id_pos, byte *buf)
Write the index information for the compressed page.
Definition: page0zip.cc:311
void page_zip_rec_set_deleted(page_zip_des_t *page_zip, const byte *rec, bool flag)
Write the "deleted" flag of a record on a compressed page.
Definition: page0zip.cc:2219
void page_zip_write_node_ptr(page_zip_des_t *page_zip, byte *rec, ulint size, ulint ptr, mtr_t *mtr)
Write the node pointer of a record on a non-leaf compressed page.
Definition: page0zip.cc:2030
ulint page_zip_empty_size(ulint n_fields, ulint zip_size)
Determine the guaranteed free space on an empty page.
Definition: page0zip.cc:109
void page_zip_copy_recs(page_zip_des_t *page_zip, page_t *page, const page_zip_des_t *src_zip, const page_t *src, dict_index_t *index, mtr_t *mtr)
Copy the records of a page byte for byte.
Definition: page0zip.cc:2633
static void page_zip_des_init(page_zip_des_t *page_zip)
Initialize a compressed page descriptor.
bool page_zip_is_too_big(const dict_index_t *index, const dtuple_t *entry)
Check whether a tuple is too big for compressed table.
Definition: page0zip.cc:129
void page_zip_dir_add_slot(page_zip_des_t *page_zip, bool is_clustered)
Add a slot to the dense page directory.
Definition: page0zip.cc:2414
static void page_zip_set_size(page_zip_des_t *page_zip, ulint size)
Set the size of a compressed page in bytes.
static lint page_zip_max_ins_size(const page_zip_des_t *page_zip, bool is_clust)
Determine how big record can be inserted without re-compressing the page.
bool page_zip_compress(page_zip_des_t *page_zip, const page_t *page, dict_index_t *index, ulint level, mtr_t *mtr)
Compress a page.
Definition: page0zip.cc:920
constexpr uint32_t PAGE_ZIP_DIR_SLOT_MASK
Mask of record offsets.
Definition: page0zip.h:66
constexpr uint32_t PAGE_ZIP_DIR_SLOT_DEL
'deleted' flag
Definition: page0zip.h:70
bool page_zip_decompress(page_zip_des_t *page_zip, page_t *page, bool all)
Decompress a page.
Definition: page0zip.cc:1265
uint page_zip_level
Definition: page0zip.cc:86
void page_zip_dir_insert(page_zip_des_t *page_zip, const byte *prev_rec, const byte *free_rec, byte *rec)
Insert a record to the dense page directory.
Definition: page0zip.cc:2259
bool page_zip_log_pages
Definition: page0zip.cc:90
byte * page_zip_parse_write_header(byte *ptr, byte *end_ptr, page_t *page, page_zip_des_t *page_zip)
Parses a log record of writing to the header of a page.
Definition: page0zip.cc:2454
void page_zip_write_rec(page_zip_des_t *page_zip, const byte *rec, const dict_index_t *index, const ulint *offsets, ulint create)
Write an entire record on the compressed page.
Definition: page0zip.cc:1658
Compressed page interface.
The server main program.
Definition: completion_hash.h:34
The buffer control block structure.
Definition: buf0buf.h:1689
Data structure for an index.
Definition: dict0mem.h:1045
Structure for an SQL data tuple of fields (logical record)
Definition: data0data.h:681
The info structure stored at the beginning of a heap block.
Definition: mem0mem.h:301
Mini-transaction handle and buffer.
Definition: mtr0mtr.h:176
Compressed page descriptor.
Definition: page0types.h:199
Transaction system global type definitions.
ib_id_t trx_id_t
Transaction identifier (DB_TRX_ID, DATA_TRX_ID)
Definition: trx0types.h:137
ib_id_t roll_ptr_t
Rollback pointer (DB_ROLL_PTR, DATA_ROLL_PTR)
Definition: trx0types.h:139
Version control for database, common definitions, and include files.
unsigned long int ulint
Definition: univ.i:405
long int lint
Definition: univ.i:406
CRC32 implementation.
int n
Definition: xcom_base.cc:508
Compressed page interface.
constexpr uint32_t PAGE_ZIP_DIR_SLOT_SIZE
Size of an compressed page directory entry.
Definition: zipdecompress.ic:47