MySQL  8.0.22
Source Code Documentation
lob0lob.h
Go to the documentation of this file.
1 /*****************************************************************************
2 
3 Copyright (c) 2015, 2020, Oracle and/or its affiliates. All Rights Reserved.
4 
5 This program is free software; you can redistribute it and/or modify it under
6 the terms of the GNU General Public License, version 2.0, as published by the
7 Free Software Foundation.
8 
9 This program is also distributed with certain software (including but not
10 limited to OpenSSL) that is licensed under separate terms, as designated in a
11 particular file or component or in included license documentation. The authors
12 of MySQL hereby grant you an additional permission to link the program and
13 your derivative works with the separately licensed software that they have
14 included with MySQL.
15 
16 This program is distributed in the hope that it will be useful, but WITHOUT
17 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18 FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0,
19 for more details.
20 
21 You should have received a copy of the GNU General Public License along with
22 this program; if not, write to the Free Software Foundation, Inc.,
23 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
24 
25 *****************************************************************************/
26 #ifndef lob0lob_h
27 #define lob0lob_h
28 
29 #include <my_dbug.h>
30 #include "btr0pcur.h"
31 #include "dict0mem.h"
32 #include "page0page.h"
33 #include "row0log.h"
34 #include "univ.i"
35 
36 /* Uncomment the following line to generate debug data, useful to analyze
37 LOB issues. */
38 /* #define LOB_DEBUG */
39 /* #define ZLOB_DEBUG */
40 
41 struct upd_t;
42 typedef std::map<page_no_t, buf_block_t *> BlockCache;
43 
44 /**
45 @file
46 @brief Implements the large objects (LOB) module.
47 
48 InnoDB supports large objects (LOB). Previously, the LOB was called as
49 externally stored fields. A large object contains a singly linked list of
50 database pages, aka LOB pages. A reference to the first LOB page is stored
51 along with the clustered index record. This reference is called the LOB
52 reference (lob::ref_t). A single clustered index record can have many LOB
53 references. Secondary indexes cannot have LOB references.
54 
55 There are two types of LOB - compressed and uncompressed.
56 
57 The main operations implemented for LOB are - INSERT, DELETE and FETCH. To
58 carry out these main operations the following classes are provided.
59 
60 Inserter - for inserting uncompressed LOB data.
61 zInserter - for inserting compressed LOB data.
62 BaseInserter - a base class containing common state and functions useful for
63  both Inserter and zInserter. Inserter and zInserter derives
64  from this base class.
65 Reader - for reading uncompressed LOB data.
66 zReader - for reading compressed LOB data.
67 Deleter - for deleting both compressed and uncompressed LOB data.
68 
69 For each main operation, the context information is identified separately.
70 They are as follows:
71 
72 InsertContext - context information for doing insert of LOB. `
73 DeleteContext - context information for doing delete of LOB. `
74 ReadContext - context information for doing fetch of LOB. `
75 
76 */
77 
78 /** Provides the large objects (LOB) module. Previously, the LOB was called as
79 externally stored fields. */
80 namespace lob {
81 
82 /** The maximum size possible for an LOB */
83 const ulint MAX_SIZE = UINT32_MAX;
84 
85 /** The compressed LOB is stored as a collection of zlib streams. The
86  * uncompressed LOB is divided into chunks of size Z_CHUNK_SIZE and each of
87  * these chunks are compressed individually and stored as compressed LOB.
88 data. */
89 #define KB128 (128 * 1024)
90 #define Z_CHUNK_SIZE KB128
91 
92 /** The reference in a field for which data is stored on a different page.
93 The reference is at the end of the 'locally' stored part of the field.
94 'Locally' means storage in the index record.
95 We store locally a long enough prefix of each column so that we can determine
96 the ordering parts of each index record without looking into the externally
97 stored part. */
98 /*-------------------------------------- @{ */
99 
100 /** Space identifier where stored. */
101 const ulint BTR_EXTERN_SPACE_ID = 0;
102 
103 /** page number where stored */
104 const ulint BTR_EXTERN_PAGE_NO = 4;
105 
106 /** offset of BLOB header on that page */
107 const ulint BTR_EXTERN_OFFSET = 8;
108 
109 /** Version number of LOB (LOB in new format)*/
111 
112 /** 8 bytes containing the length of the externally stored part of the LOB.
113 The 2 highest bits are reserved to the flags below. */
114 const ulint BTR_EXTERN_LEN = 12;
115 
116 /*-------------------------------------- @} */
117 
118 /** The most significant bit of BTR_EXTERN_LEN (i.e., the most
119 significant bit of the byte at smallest address) is set to 1 if this
120 field does not 'own' the externally stored field; only the owner field
121 is allowed to free the field in purge! */
122 const ulint BTR_EXTERN_OWNER_FLAG = 128UL;
123 
124 /** If the second most significant bit of BTR_EXTERN_LEN (i.e., the
125 second most significant bit of the byte at smallest address) is 1 then
126 it means that the externally stored field was inherited from an
127 earlier version of the row. In rollback we are not allowed to free an
128 inherited external field. */
129 const ulint BTR_EXTERN_INHERITED_FLAG = 64UL;
130 
131 /** If the 3rd most significant bit of BTR_EXTERN_LEN is 1, then it
132 means that the externally stored field is currently being modified.
133 This is mainly used by the READ UNCOMMITTED transaction to avoid returning
134 inconsistent blob data. */
136 
137 /** The structure of uncompressed LOB page header */
138 
139 /** Offset within header of LOB length on this page. */
140 const ulint LOB_HDR_PART_LEN = 0;
141 
142 /** Offset within header of next BLOB part page no.
143 FIL_NULL if none */
144 const ulint LOB_HDR_NEXT_PAGE_NO = 4;
145 
146 /** Size of an uncompressed LOB page header, in bytes */
147 const ulint LOB_HDR_SIZE = 8;
148 
149 /** Start of the data on an LOB page */
151 
152 /** In memory representation of the LOB reference. */
153 struct ref_mem_t {
154  /** Space Identifier of the clustered index. */
156 
157  /** Page number of first LOB page. */
159 
160  /** Offset within m_page_no where LOB begins. */
161  ulint m_offset;
162 
163  /** Length of LOB */
164  ulint m_length;
165 
166  /** Whether the LOB is null. */
167  bool m_null;
168 
169  /** Whether the clustered index record owns this LOB. */
170  bool m_owner;
171 
172  /** Whether the clustered index record inherited this LOB from
173  another clustered index record. */
174  bool m_inherit;
175 
176  /** Whether the LOB is partially updated. */
177  bool m_partial;
178 
179  /** Whether the blob is being modified. */
181 
182  /** Check if the LOB has already been purged.
183  @return true if LOB has been purged, false otherwise. */
184  bool is_purged() const {
185  return ((m_page_no == FIL_NULL) && (m_length == 0));
186  }
187 };
188 
190 
191 /** The struct 'lob::ref_t' represents an external field reference. The
192 reference in a field for which data is stored on a different page. The
193 reference is at the end of the 'locally' stored part of the field. 'Locally'
194 means storage in the index record. We store locally a long enough prefix of
195 each column so that we can determine the ordering parts of each index record
196 without looking into the externally stored part. */
197 struct ref_t {
198  private:
199  /** If the LOB size is equal to or above this limit (in physical page
200  size terms), then the LOB is big enough to be partially updated. Only
201  in this case LOB index needs to be built. */
202  static const ulint LOB_BIG_THRESHOLD_SIZE = 2;
203 
204  public:
205  /** If the total number of bytes modified in an LOB, in an update
206  operation, is less than or equal to this threshold LOB_SMALL_CHANGE_THRESHOLD,
207  then it is considered as a small change. For small changes to LOB,
208  the changes are undo logged like any other update operation. */
209  static const ulint LOB_SMALL_CHANGE_THRESHOLD = 100;
210 
211  /** Constructor.
212  @param[in] ptr Pointer to the external field reference. */
213  explicit ref_t(byte *ptr) : m_ref(ptr) {}
214 
215  /** For compressed LOB, if the length is less than or equal to Z_CHUNK_SIZE
216  then use the older single z stream format to store the LOB. */
217  bool use_single_z_stream() const { return (length() <= Z_CHUNK_SIZE); }
218 
219  /** For compressed LOB, if the length is less than or equal to Z_CHUNK_SIZE
220  then use the older single z stream format to store the LOB. */
221  static bool use_single_z_stream(ulint len) { return (len <= Z_CHUNK_SIZE); }
222 
223  /** Check if this LOB is big enough to do partial update.
224  @param[in] page_size the page size
225  @param[in] lob_length the size of BLOB in bytes.
226  @return true if LOB is big enough, false otherwise. */
227  static bool is_big(const page_size_t &page_size, const ulint lob_length) {
228  /* Disable a performance optimization */
229  return (true);
230 
231  const ulint limit = page_size.physical() * LOB_BIG_THRESHOLD_SIZE;
232  return (lob_length >= limit);
233  }
234 
235  /** Check if this LOB is big enough to do partial update.
236  @param[in] page_size the page size
237  @return true if LOB is big enough, false otherwise. */
238  bool is_big(const page_size_t &page_size) const {
239  /* Disable a performance optimization */
240  return (true);
241 
242  const ulint limit = page_size.physical() * LOB_BIG_THRESHOLD_SIZE;
243  const ulint lob_length = length();
244  return (lob_length >= limit);
245  }
246 
247  /** Parse the LOB reference object and copy data into the given
248  ref_mem_t object.
249  @param[out] obj LOB reference memory object. */
250  void parse(ref_mem_t &obj) const {
251  obj.m_space_id = space_id();
252  obj.m_page_no = page_no();
253  obj.m_offset = offset();
254  obj.m_length = length();
255  obj.m_null = is_null();
256  obj.m_owner = is_owner();
257  obj.m_inherit = is_inherited();
258  obj.m_being_modified = is_being_modified();
259  }
260 
261  /** Copy the LOB reference into the given memory location.
262  @param[out] field_ref write LOB reference in this
263  location.*/
264  void copy(byte *field_ref) const { memcpy(field_ref, m_ref, SIZE); }
265 
266  /** Check whether the stored external field reference is equal to the
267  given field reference.
268  @param[in] ptr supplied external field reference. */
269  bool is_equal(const byte *ptr) const { return (m_ref == ptr); }
270 
271  /** Set the external field reference to the given memory location.
272  @param[in] ptr the new external field reference. */
273  void set_ref(byte *ptr) { m_ref = ptr; }
274 
275  /** Set the external field reference to null.
276  @param[in,out] mtr Mini-transaction. */
277  void set_null(mtr_t *mtr) {
279  }
280 
281  /** Check if the field reference is made of zeroes except the being_modified
282  bit.
283  @return true if field reference is made of zeroes, false otherwise. */
284  bool is_null_relaxed() const {
285  return (is_null() || memcmp(field_ref_almost_zero, m_ref, SIZE) == 0);
286  }
287 
288  /** Check if the field reference is made of zeroes.
289  @return true if field reference is made of zeroes, false otherwise. */
290  bool is_null() const { return (memcmp(field_ref_zero, m_ref, SIZE) == 0); }
291 
292 #ifdef UNIV_DEBUG
293  /** Check if the LOB reference is null (all zeroes) except the "is being
294  modified" bit.
295  @param[in] ref the LOB reference.
296  @return true if the LOB reference is null (all zeros) except the "is being
297  modified" bit, false otherwise. */
298  static bool is_null_relaxed(const byte *ref) {
299  return (is_null(ref) || memcmp(field_ref_almost_zero, ref, SIZE) == 0);
300  }
301 
302  /** Check if the LOB reference is null (all zeroes).
303  @param[in] ref the LOB reference.
304  @return true if the LOB reference is null (all zeros), false otherwise. */
305  static bool is_null(const byte *ref) {
306  return (memcmp(field_ref_zero, ref, SIZE) == 0);
307  }
308 #endif /* UNIV_DEBUG */
309 
310  /** Set the ownership flag in the blob reference.
311  @param[in] owner Whether to own or disown. If owner, unset
312  the owner flag.
313  @param[in] mtr Mini-transaction or NULL.*/
314  void set_owner(bool owner, mtr_t *mtr) {
315  ulint byte_val = mach_read_from_1(m_ref + BTR_EXTERN_LEN);
316 
317  if (owner) {
318  /* owns the blob */
319  byte_val &= ~BTR_EXTERN_OWNER_FLAG;
320  } else {
321  byte_val |= BTR_EXTERN_OWNER_FLAG;
322  }
323 
324  mlog_write_ulint(m_ref + BTR_EXTERN_LEN, byte_val, MLOG_1BYTE, mtr);
325  }
326 
327  /** Set the being_modified flag in the field reference.
328  @param[in,out] ref The LOB reference
329  @param[in] modifying true, if blob is being modified.
330  @param[in] mtr Mini-transaction context.*/
331  static void set_being_modified(byte *ref, bool modifying, mtr_t *mtr) {
332  ulint byte_val = mach_read_from_1(ref + BTR_EXTERN_LEN);
333 
334  if (modifying) {
335  byte_val |= BTR_EXTERN_BEING_MODIFIED_FLAG;
336  } else {
337  byte_val &= ~BTR_EXTERN_BEING_MODIFIED_FLAG;
338  }
339 
340  mlog_write_ulint(ref + BTR_EXTERN_LEN, byte_val, MLOG_1BYTE, mtr);
341  }
342 
343  /** Set the being_modified flag in the field reference.
344  @param[in] modifying true, if blob is being modified.
345  @param[in] mtr Mini-transaction context.*/
346  void set_being_modified(bool modifying, mtr_t *mtr) {
347  set_being_modified(m_ref, modifying, mtr);
348  }
349 
350  /** Check if the current blob is being modified
351  @param[in] field_ref blob field reference
352  @return true if it is being modified, false otherwise. */
353  bool static is_being_modified(const byte *field_ref) {
354  const ulint byte_val = mach_read_from_1(field_ref + BTR_EXTERN_LEN);
355  return (byte_val & BTR_EXTERN_BEING_MODIFIED_FLAG);
356  }
357 
358  /** Check if the current blob is being modified
359  @return true if it is being modified, false otherwise. */
360  bool is_being_modified() const { return (is_being_modified(m_ref)); }
361 
362  /** Set the inherited flag in the field reference.
363  @param[in] inherited true, if inherited.
364  @param[in] mtr Mini-transaction context.*/
365  void set_inherited(bool inherited, mtr_t *mtr) {
366  ulint byte_val = mach_read_from_1(m_ref + BTR_EXTERN_LEN);
367 
368  if (inherited) {
369  byte_val |= BTR_EXTERN_INHERITED_FLAG;
370  } else {
371  byte_val &= ~BTR_EXTERN_INHERITED_FLAG;
372  }
373 
374  mlog_write_ulint(m_ref + BTR_EXTERN_LEN, byte_val, MLOG_1BYTE, mtr);
375  }
376 
377  /** Check if the current row is the owner of the blob.
378  @return true if owner, false otherwise. */
379  bool is_owner() const {
380  ulint byte_val = mach_read_from_1(m_ref + BTR_EXTERN_LEN);
381  return (!(byte_val & BTR_EXTERN_OWNER_FLAG));
382  }
383 
384  /** Check if the current row inherited the blob from parent row.
385  @return true if inherited, false otherwise. */
386  bool is_inherited() const {
387  const ulint byte_val = mach_read_from_1(m_ref + BTR_EXTERN_LEN);
388  return (byte_val & BTR_EXTERN_INHERITED_FLAG);
389  }
390 
391 #ifdef UNIV_DEBUG
392  /** Read the space id from the given blob reference.
393  @param[in] ref the blob reference.
394  @return the space id */
395  static space_id_t space_id(const byte *ref) {
396  return (mach_read_from_4(ref));
397  }
398 
399  /** Read the page no from the blob reference.
400  @return the page no */
401  static page_no_t page_no(const byte *ref) {
402  return (mach_read_from_4(ref + BTR_EXTERN_PAGE_NO));
403  }
404 #endif /* UNIV_DEBUG */
405 
406  /** Read the space id from the blob reference.
407  @return the space id */
408  space_id_t space_id() const { return (mach_read_from_4(m_ref)); }
409 
410  /** Read the page number from the blob reference.
411  @return the page number */
412  page_no_t page_no() const {
413  return (mach_read_from_4(m_ref + BTR_EXTERN_PAGE_NO));
414  }
415 
416  /** Read the offset of blob header from the blob reference.
417  @return the offset of the blob header */
418  ulint offset() const { return (mach_read_from_4(m_ref + BTR_EXTERN_OFFSET)); }
419 
420  /** Read the LOB version from the blob reference.
421  @return the LOB version number. */
422  uint32_t version() const {
423  return (mach_read_from_4(m_ref + BTR_EXTERN_VERSION));
424  }
425 
426  /** Read the length from the blob reference.
427  @return length of the blob */
428  ulint length() const {
429  return (mach_read_from_4(m_ref + BTR_EXTERN_LEN + 4));
430  }
431 
432  /** Update the information stored in the external field reference.
433  @param[in] space_id the space identifier.
434  @param[in] page_no the page number.
435  @param[in] offset the offset within the page_no
436  @param[in] mtr the mini trx or NULL. */
437  void update(space_id_t space_id, ulint page_no, ulint offset, mtr_t *mtr) {
438  set_space_id(space_id, mtr);
439  set_page_no(page_no, mtr);
440  set_offset(offset, mtr);
441  }
442 
443  /** Set the space_id in the external field reference.
444  @param[in] space_id the space identifier.
445  @param[in] mtr mini-trx or NULL. */
446  void set_space_id(const space_id_t space_id, mtr_t *mtr) {
447  mlog_write_ulint(m_ref + BTR_EXTERN_SPACE_ID, space_id, MLOG_4BYTES, mtr);
448  }
449 
450  /** Set the page number in the external field reference.
451  @param[in] page_no the page number.
452  @param[in] mtr mini-trx or NULL. */
453  void set_page_no(const ulint page_no, mtr_t *mtr) {
454  mlog_write_ulint(m_ref + BTR_EXTERN_PAGE_NO, page_no, MLOG_4BYTES, mtr);
455  }
456 
457  /** Set the offset information in the external field reference.
458  @param[in] offset the offset.
459  @param[in] mtr mini-trx or NULL. */
460  void set_offset(const ulint offset, mtr_t *mtr) {
461  mlog_write_ulint(m_ref + BTR_EXTERN_OFFSET, offset, MLOG_4BYTES, mtr);
462  }
463 
464  /** Set the length of blob in the external field reference.
465  @param[in] len the blob length .
466  @param[in] mtr mini-trx or NULL. */
467  void set_length(const ulint len, mtr_t *mtr) {
468  ut_ad(len <= MAX_SIZE);
469  mlog_write_ulint(m_ref + BTR_EXTERN_LEN + 4, len, MLOG_4BYTES, mtr);
470  }
471 
472  /** Get the start of a page containing this blob reference.
473  @return start of the page */
474  page_t *page_align() const { return (::page_align(m_ref)); }
475 
476 #ifdef UNIV_DEBUG
477  /** Check if the given mtr has necessary latches to update this LOB
478  reference.
479  @param[in] mtr Mini-transaction that needs to
480  be checked.
481  @return true if valid, false otherwise. */
482  bool validate(mtr_t *mtr) {
483  ut_ad(m_ref != nullptr);
484  ut_ad(mtr != nullptr);
485 
486  if (mtr->get_log_mode() == MTR_LOG_NO_REDO) {
487  return (true);
488  }
489 
492  ut_ad(block != nullptr);
493  return (true);
494  }
495 
496  /** Check if the space_id in the LOB reference is equal to the
497  space_id of the index to which it belongs.
498  @param[in] index the index to which LOB belongs.
499  @return true if space is valid in LOB reference, false otherwise. */
500  bool check_space_id(dict_index_t *index) const;
501 #endif /* UNIV_DEBUG */
502 
503  /** Check if the LOB can be partially updated. This is done by loading
504  the first page of LOB and looking at the flags.
505  @param[in] index the index to which LOB belongs.
506  @return true if LOB is partially updatable, false otherwise.*/
507  bool is_lob_partially_updatable(const dict_index_t *index) const;
508 
509  /** Load the first page of the LOB and mark it as not partially
510  updatable anymore.
511  @param[in] trx Current transaction
512  @param[in] mtr Mini-transaction context.
513  @param[in] index Index dictionary object.
514  @param[in] page_size Page size information. */
515  void mark_not_partially_updatable(trx_t *trx, mtr_t *mtr, dict_index_t *index,
516  const page_size_t &page_size);
517 
518  /** Load the first page of LOB and read its page type.
519  @param[in] index the index object.
520  @param[in] page_size the page size of LOB.
521  @param[out] is_partially_updatable is the LOB partially updatable.
522  @return the page type of first page of LOB.*/
523  ulint get_lob_page_info(const dict_index_t *index,
524  const page_size_t &page_size,
525  bool &is_partially_updatable) const;
526 
527  /** Print this LOB reference into the given output stream.
528  @param[in] out the output stream.
529  @return the output stream. */
530  std::ostream &print(std::ostream &out) const;
531 
532  /** The size of an LOB reference object (in bytes) */
533  static const uint SIZE = BTR_EXTERN_FIELD_REF_SIZE;
534 
535  private:
536  /** Pointing to a memory of size BTR_EXTERN_FIELD_REF_SIZE */
538 };
539 
540 /** Overload the global output stream operator to easily print the
541 lob::ref_t object into the output stream.
542 @param[in,out] out the output stream.
543 @param[in] obj the lob::ref_t object to be printed
544 @return the output stream. */
545 inline std::ostream &operator<<(std::ostream &out, const ref_t &obj) {
546  return (obj.print(out));
547 }
548 
549 /** LOB operation code for btr_store_big_rec_extern_fields(). */
550 enum opcode {
551 
552  /** Store off-page columns for a freshly inserted record */
554 
555  /** Store off-page columns for an insert by update */
557 
558  /** Store off-page columns for an update */
560 
561  /** Store off-page columns for a freshly inserted record by bulk */
563 
564  /** The operation code is unknown or not important. */
566 };
567 
568 /** Stores the fields in big_rec_vec to the tablespace and puts pointers to
569 them in rec. The extern flags in rec will have to be set beforehand. The
570 fields are stored on pages allocated from leaf node file segment of the index
571 tree.
572 
573 TODO: If the allocation extends the tablespace, it will not be redo logged, in
574 any mini-transaction. Tablespace extension should be redo-logged, so that
575 recovery will not fail when the big_rec was written to the extended portion of
576 the file, in case the file was somehow truncated in the crash.
577 @param[in] trx the trx doing LOB store. If unavailable it
578  could be nullptr.
579 @param[in,out] pcur a persistent cursor. if btr_mtr is restarted,
580  then this can be repositioned.
581 @param[in] upd update vector
582 @param[in,out] offsets rec_get_offsets() on pcur. the "external in
583  offsets will correctly correspond storage"
584  flagsin offsets will correctly correspond to
585  rec when this function returns
586 @param[in] big_rec_vec vector containing fields to be stored
587  externally
588 @param[in,out] btr_mtr mtr containing the latches to the clustered
589  index. can be committed and restarted.
590 @param[in] op operation code
591 @return DB_SUCCESS or DB_OUT_OF_FILE_SPACE */
593  const upd_t *upd, ulint *offsets,
594  const big_rec_t *big_rec_vec,
595  mtr_t *btr_mtr, opcode op)
596  MY_ATTRIBUTE((warn_unused_result));
597 
598 /** Copies an externally stored field of a record to mem heap.
599 @param[in] trx the current transaction.
600 @param[in] index the clustered index
601 @param[in] rec record in a clustered index; must be
602  protected by a lock or a page latch
603 @param[in] offsets array returned by rec_get_offsets()
604 @param[in] page_size BLOB page size
605 @param[in] no field number
606 @param[out] len length of the field
607 @param[out] lob_version version of lob that has been copied */
608 #ifdef UNIV_DEBUG
609 /**
610 @param[in] is_sdi true for SDI Indexes */
611 #endif /* UNIV_DEBUG */
612 /**
613 @param[in,out] heap mem heap
614 @return the field copied to heap, or NULL if the field is incomplete */
616  trx_t *trx, const dict_index_t *index, const rec_t *rec,
617  const ulint *offsets, const page_size_t &page_size, ulint no, ulint *len,
618  size_t *lob_version,
619 #ifdef UNIV_DEBUG
620  bool is_sdi,
621 #endif /* UNIV_DEBUG */
622  mem_heap_t *heap);
623 
624 #ifdef UNIV_DEBUG
625 #define btr_rec_copy_externally_stored_field( \
626  trx, index, rec, offsets, page_size, no, len, ver, is_sdi, heap) \
627  btr_rec_copy_externally_stored_field_func( \
628  trx, index, rec, offsets, page_size, no, len, ver, is_sdi, heap)
629 #else /* UNIV_DEBUG */
630 #define btr_rec_copy_externally_stored_field( \
631  trx, index, rec, offsets, page_size, no, len, ver, is_sdi, heap) \
632  btr_rec_copy_externally_stored_field_func(trx, index, rec, offsets, \
633  page_size, no, len, ver, heap)
634 #endif /* UNIV_DEBUG */
635 
636 /** Gets the offset of the pointer to the externally stored part of a field.
637 @param[in] offsets array returned by rec_get_offsets()
638 @param[in] n index of the external field
639 @return offset of the pointer to the externally stored part */
640 ulint btr_rec_get_field_ref_offs(const ulint *offsets, ulint n);
641 
642 /** Gets a pointer to the externally stored part of a field.
643 @param rec record
644 @param offsets rec_get_offsets(rec)
645 @param n index of the externally stored field
646 @return pointer to the externally stored part */
647 #define btr_rec_get_field_ref(rec, offsets, n) \
648  ((rec) + lob::btr_rec_get_field_ref_offs(offsets, n))
649 
650 /** Deallocate a buffer block that was reserved for a BLOB part.
651 @param[in] index Index
652 @param[in] block Buffer block
653 @param[in] all TRUE=remove also the compressed page
654  if there is one
655 @param[in] mtr Mini-transaction to commit */
656 void blob_free(dict_index_t *index, buf_block_t *block, bool all, mtr_t *mtr);
657 
658 /** The B-tree context under which the LOB operation is done. */
659 class BtrContext {
660  public:
661  /** Default Constructor */
663  : m_mtr(nullptr),
664  m_pcur(nullptr),
665  m_index(nullptr),
666  m_rec(nullptr),
667  m_offsets(nullptr),
668  m_block(nullptr),
669  m_op(OPCODE_UNKNOWN),
670  m_btr_page_no(FIL_NULL) {}
671 
672  /** Constructor **/
673  BtrContext(mtr_t *mtr, btr_pcur_t *pcur, dict_index_t *index, rec_t *rec,
674  ulint *offsets, buf_block_t *block)
675  : m_mtr(mtr),
676  m_pcur(pcur),
677  m_index(index),
678  m_rec(rec),
679  m_offsets(offsets),
680  m_block(block),
681  m_op(OPCODE_UNKNOWN),
682  m_btr_page_no(FIL_NULL) {
683  ut_ad(m_pcur == nullptr || rec_offs_validate());
684  ut_ad(m_block == nullptr || m_rec == nullptr ||
685  m_block->frame == page_align(m_rec));
686  ut_ad(m_pcur == nullptr || m_rec == btr_pcur_get_rec(m_pcur));
687  }
688 
689  /** Constructor **/
690  BtrContext(mtr_t *mtr, btr_pcur_t *pcur, dict_index_t *index, rec_t *rec,
691  ulint *offsets, buf_block_t *block, opcode op)
692  : m_mtr(mtr),
693  m_pcur(pcur),
694  m_index(index),
695  m_rec(rec),
696  m_offsets(offsets),
697  m_block(block),
698  m_op(op),
699  m_btr_page_no(FIL_NULL) {
700  ut_ad(m_pcur == nullptr || rec_offs_validate());
701  ut_ad(m_block->frame == page_align(m_rec));
702  ut_ad(m_pcur == nullptr || m_rec == btr_pcur_get_rec(m_pcur));
703  }
704 
705  /** Copy Constructor **/
706  BtrContext(const BtrContext &other)
707  : m_mtr(other.m_mtr),
708  m_pcur(other.m_pcur),
709  m_index(other.m_index),
710  m_rec(other.m_rec),
711  m_offsets(other.m_offsets),
712  m_block(other.m_block),
713  m_op(other.m_op),
714  m_btr_page_no(other.m_btr_page_no) {}
715 
716  /** Marks non-updated off-page fields as disowned by this record.
717  The ownership must be transferred to the updated record which is
718  inserted elsewhere in the index tree. In purge only the owner of
719  externally stored field is allowed to free the field.
720  @param[in] update update vector. */
721  void disown_inherited_fields(const upd_t *update);
722 
723  /** Sets the ownership bit of an externally stored field in a record.
724  @param[in] i field number
725  @param[in] val value to set */
726  void set_ownership_of_extern_field(ulint i, ibool val) {
727  byte *data;
728  ulint local_len;
729 
730  data =
731  const_cast<byte *>(rec_get_nth_field(m_rec, m_offsets, i, &local_len));
732  ut_ad(rec_offs_nth_extern(m_offsets, i));
733  ut_a(local_len >= BTR_EXTERN_FIELD_REF_SIZE);
734 
735  local_len -= BTR_EXTERN_FIELD_REF_SIZE;
736 
737  ref_t ref(data + local_len);
738 
739  ut_a(val || ref.is_owner());
740 
741  page_zip_des_t *page_zip = get_page_zip();
742 
743  if (page_zip) {
744  ref.set_owner(val, nullptr);
745  page_zip_write_blob_ptr(page_zip, m_rec, m_index, m_offsets, i, m_mtr);
746  } else {
747  ref.set_owner(val, m_mtr);
748  }
749  }
750 
751  /** Marks all extern fields in a record as owned by the record.
752  This function should be called if the delete mark of a record is
753  removed: a not delete marked record always owns all its extern
754  fields.*/
756  ut_ad(!rec_offs_comp(m_offsets) || !rec_get_node_ptr_flag(m_rec));
757 
758  ulint n = rec_offs_n_fields(m_offsets);
759 
760  if (!rec_offs_any_extern(m_offsets)) {
761  return;
762  }
763 
764  for (ulint i = 0; i < n; i++) {
765  if (rec_offs_nth_extern(m_offsets, i)) {
766  set_ownership_of_extern_field(i, TRUE);
767  }
768  }
769  }
770 
771  /** Frees the externally stored fields for a record.
772  @param[in] trx_id transaction identifier whose LOB is
773  being freed.
774  @param[in] undo_no undo number within a transaction whose
775  LOB is being freed.
776  @param[in] rollback performing rollback?
777  @param[in] rec_type undo record type.*/
778  void free_externally_stored_fields(trx_id_t trx_id, undo_no_t undo_no,
779  bool rollback, ulint rec_type);
780 
781  /** Frees the externally stored fields for a record, if the field
782  is mentioned in the update vector.
783  @param[in] trx_id the transaction identifier.
784  @param[in] undo_no undo number within a transaction whose
785  LOB is being freed.
786  @param[in] update update vector
787  @param[in] rollback performing rollback? */
788  void free_updated_extern_fields(trx_id_t trx_id, undo_no_t undo_no,
789  const upd_t *update, bool rollback);
790 
791  /** Gets the compressed page descriptor
792  @return the compressed page descriptor. */
794  return (buf_block_get_page_zip(m_block));
795  }
796 
797  /** Get the page number of clustered index block.
798  @return the page number. */
800  return (page_get_page_no(buf_block_get_frame(m_block)));
801  }
802 
803  /** Get the record offset within page of the clustered index record.
804  @return the record offset. */
805  ulint get_rec_offset() const { return (page_offset(m_rec)); }
806 
807  /** Check if there is a need to recalculate the context information.
808  @return true if there is a need to recalculate, false otherwise. */
809  bool need_recalc() const {
810  return ((m_pcur != nullptr) && (m_rec != btr_pcur_get_rec(m_pcur)));
811  }
812 
813  /** Get the clustered index record pointer.
814  @return clustered index record pointer. */
815  rec_t *rec() const {
816  ut_ad(m_pcur == nullptr || m_rec == btr_pcur_get_rec(m_pcur));
817  return (m_rec);
818  }
819 
820  /** Get the LOB reference for the given field number.
821  @param[in] field_no field number.
822  @return LOB reference (aka external field reference).*/
823  byte *get_field_ref(ulint field_no) const {
824  return (btr_rec_get_field_ref(m_rec, get_offsets(), field_no));
825  }
826 
827 #ifdef UNIV_DEBUG
828  /** Validate the current BLOB context object. The BLOB context object
829  is valid if the necessary latches are being held by the
830  mini-transaction of the B-tree (btr mtr). Does not return if the
831  validation fails.
832  @return true if valid */
833  bool validate() const {
834  rec_offs_make_valid(rec(), index(), m_offsets);
835 
836  ut_ad(m_mtr->memo_contains_page_flagged(
838  table()->is_intrinsic());
839 
842  table()->is_intrinsic());
843 
844  return (true);
845  }
846 
847  /** Check to see if all pointers to externally stored columns in
848  the record must be valid.
849  @return true if all blob references are valid.
850  @return will not return if any blob reference is invalid. */
851  bool are_all_blobrefs_valid() const {
852  for (ulint i = 0; i < rec_offs_n_fields(m_offsets); i++) {
853  if (!rec_offs_nth_extern(m_offsets, i)) {
854  continue;
855  }
856 
857  byte *field_ref = btr_rec_get_field_ref(rec(), m_offsets, i);
858 
859  ref_t blobref(field_ref);
860 
861  /* The pointer must not be zero if the operation
862  succeeded. */
863  ut_a(!blobref.is_null());
864 
865  /* The column must not be disowned by this record. */
866  ut_a(blobref.is_owner());
867  }
868 
869  return (true);
870  }
871 #endif /* UNIV_DEBUG */
872 
873  /** Determine whether current operation is a bulk insert operation.
874  @return true, if bulk insert operation, false otherwise. */
875  bool is_bulk() const { return (m_op == OPCODE_INSERT_BULK); }
876 
877  /** Get the beginning of the B-tree clustered index page frame
878  that contains the current clustered index record (m_rec).
879  @return the page frame containing the clust rec. */
880  const page_t *rec_frame() const {
881  ut_ad(m_block->frame == page_align(m_rec));
882  return (m_block->frame);
883  }
884 
885  /** Commit the mini-transaction that is holding the latches
886  of the clustered index record block. */
887  void commit_btr_mtr() { m_mtr->commit(); }
888 
889  /** Start the mini-transaction that will be holding the latches
890  of the clustered index record block. */
891  void start_btr_mtr() {
892  mtr_log_t log_mode = m_mtr->get_log_mode();
893  m_mtr->start();
894  m_mtr->set_log_mode(log_mode);
895  }
896 
897  /** Get the page number of clustered index record.
898  @return page number of clustered index record. */
900  return (page_get_page_no(buf_block_get_frame(m_block)));
901  }
902 
903 #ifndef UNIV_HOTBACKUP
904 
905  /** Increment the buffer fix count of the clustered index record block.
906  This is to be called before commit_btr_mtr() which decrements the count when
907  you want to prevent the block from being freed:
908  rec_block_fix(); // buf_fix_count++
909  commit_btr_mtr(); // releasing mtr internally does buf_fix_count--
910  start_btr_mtr();
911  rec_block_unfix(); // calls btr_block_get() which does buf_fix_count++ and
912  // then does buf_fix_count--
913  */
914  void rec_block_fix() {
915  m_rec_offset = page_offset(m_rec);
916  m_btr_page_no = page_get_page_no(buf_block_get_frame(m_block));
917  buf_block_buf_fix_inc(m_block, __FILE__, __LINE__);
918  }
919 
920  /** Decrement the buffer fix count of the clustered index record block,
921  X-latching it before, so that the overall buffer_fix_count doesn't change.
922  This is done to restore X-latch on the page after mtr restart:
923  rec_block_fix(); // buf_fix_count++
924  commit_btr_mtr(); // releasing mtr internally does buf_fix_count--
925  start_btr_mtr();
926  rec_block_unfix(); // calls btr_block_get() which does buf_fix_count++ and
927  // then does buf_fix_count--
928  */
930  space_id_t space_id = space();
931  page_id_t page_id(space_id, m_btr_page_no);
932  page_size_t page_size(dict_table_page_size(table()));
933  page_cur_t *page_cur = &m_pcur->m_btr_cur.page_cur;
934 
935  mtr_x_lock(dict_index_get_lock(index()), m_mtr);
936 
937  page_cur->block =
938  btr_block_get(page_id, page_size, RW_X_LATCH, index(), m_mtr);
939 
940  page_cur->rec = buf_block_get_frame(page_cur->block) + m_rec_offset;
941 
942  buf_block_buf_fix_dec(page_cur->block);
943  /* This decrement above is paired with increment in rec_block_fix(), and
944  there is another increment done within btr_block_get(), so overall the block
945  should be buffer-fixed and thus safe to be used. */
946  ut_ad(page_cur->block->page.buf_fix_count > 0);
947  recalc();
948  }
949 #endif /* !UNIV_HOTBACKUP */
950 
951  /** Restore the position of the persistent cursor. */
953  ut_ad(m_pcur->m_rel_pos == BTR_PCUR_ON);
954  bool ret = btr_pcur_restore_position(m_pcur->m_latch_mode, m_pcur, m_mtr);
955 
956  ut_a(ret);
957 
958  recalc();
959  }
960 
961  /** Get the index object.
962  @return index object */
963  dict_index_t *index() const { return (m_index); }
964 
965  /** Get the table object.
966  @return table object or NULL. */
967  dict_table_t *table() const {
968  dict_table_t *result = nullptr;
969 
970  if (m_pcur != nullptr && m_pcur->index() != nullptr) {
971  result = m_pcur->index()->table;
972  }
973 
974  return (result);
975  }
976 
977  /** Get the space id.
978  @return space id. */
979  space_id_t space() const { return (index()->space); }
980 
981  /** Obtain the page size of the underlying table.
982  @return page size of the underlying table. */
983  const page_size_t page_size() const {
984  return (dict_table_page_size(table()));
985  }
986 
987  /** Determine the extent size (in pages) for the underlying table
988  @return extent size in pages */
990  return (dict_table_extent_size(table()));
991  }
992 
993  /** Check if there is enough space in the redo log file. The btr
994  mini-transaction will be restarted. */
995  void check_redolog() {
996  is_bulk() ? check_redolog_bulk() : check_redolog_normal();
997  }
998 
999  /** Mark the nth field as externally stored.
1000  @param[in] field_no the field number. */
1001  void make_nth_extern(ulint field_no) {
1002  rec_offs_make_nth_extern(m_offsets, field_no);
1003  }
1004 
1005  /** Get the log mode of the btr mtr.
1006  @return the log mode. */
1007  mtr_log_t get_log_mode() { return (m_mtr->get_log_mode()); }
1008 
1009  /** Get flush observer
1010  @return flush observer */
1012  return (m_mtr->get_flush_observer());
1013  }
1014 
1015  /** Get the record offsets array.
1016  @return the record offsets array. */
1017  ulint *get_offsets() const { return (m_offsets); }
1018 
1019  /** Validate the record offsets array.
1020  @return true if validation succeeds, false otherwise. */
1021  bool rec_offs_validate() const {
1022  if (m_rec != nullptr) {
1023  ut_ad(::rec_offs_validate(m_rec, m_index, m_offsets));
1024  }
1025  return (true);
1026  }
1027 
1028  /** Get the associated mini-transaction.
1029  @return the mini-transaction. */
1030  mtr_t *get_mtr() { return (m_mtr); }
1031 
1032  /** Get the pointer to the clustered record block.
1033  @return pointer to the clustered rec block. */
1034  buf_block_t *block() const { return (m_block); }
1035 
1036  /** Save the position of the persistent cursor. */
1037  void store_position() { btr_pcur_store_position(m_pcur, m_mtr); }
1038 
1039  /** Check if there is enough space in log file. Commit and re-start the
1040  mini-transaction. */
1041  void check_redolog_normal();
1042 
1043  /** When bulk load is being done, check if there is enough space in redo
1044  log file. */
1045  void check_redolog_bulk();
1046 
1047  /** Recalculate some of the members after restoring the persistent
1048  cursor. */
1049  void recalc() {
1050  m_block = btr_pcur_get_block(m_pcur);
1051  m_rec = btr_pcur_get_rec(m_pcur);
1052  m_btr_page_no = page_get_page_no(buf_block_get_frame(m_block));
1053  m_rec_offset = page_offset(m_rec);
1054 
1055  rec_offs_make_valid(rec(), index(), m_offsets);
1056  }
1057 
1058  /** Write a blob reference of a field into a clustered index record
1059  in a compressed leaf page. The information must already have been
1060  updated on the uncompressed page.
1061  @param[in] field_no BLOB field number
1062  @param[in] mtr Mini-transaction to update blob page. */
1063  void zblob_write_blobref(ulint field_no, mtr_t *mtr) {
1064  page_zip_write_blob_ptr(get_page_zip(), m_rec, index(), m_offsets, field_no,
1065  mtr);
1066  }
1067 
1072  ulint *m_offsets;
1075 
1076  /** Record offset within the page. */
1078 
1079  /** Page number where the clust rec is present. */
1081 };
1082 
1083 /** The context for a LOB operation. It contains the necessary information
1084 to carry out a LOB operation. */
1085 struct InsertContext : public BtrContext {
1086  /** Constructor
1087  @param[in] btr_ctx b-tree context for lob operation.
1088  @param[in] big_rec_vec array of blobs */
1089  InsertContext(const BtrContext &btr_ctx, const big_rec_t *big_rec_vec)
1090  : BtrContext(btr_ctx), m_big_rec_vec(big_rec_vec) {}
1091 
1092  /** Get the vector containing fields to be stored externally.
1093  @return the big record vector */
1094  const big_rec_t *get_big_rec_vec() { return (m_big_rec_vec); }
1095 
1096  /** Get the size of vector containing fields to be stored externally.
1097  @return the big record vector size */
1098  ulint get_big_rec_vec_size() { return (m_big_rec_vec->n_fields); }
1099 
1100  /** The B-tree Context */
1101  // const BtrContext m_btr_ctx;
1102 
1103  /** vector containing fields to be stored externally */
1105 };
1106 
1107 /** Information about data stored in one BLOB page. */
1109  /** Constructor.
1110  @param[in] page_no the BLOB page number.
1111  @param[in] bytes amount of uncompressed BLOB data
1112  in BLOB page in bytes.
1113  @param[in] zbytes amount of compressed BLOB data
1114  in BLOB page in bytes. */
1115  blob_page_info_t(page_no_t page_no, uint bytes, uint zbytes)
1116  : m_page_no(page_no), m_bytes(bytes), m_zbytes(zbytes) {}
1117 
1118  /** Re-initialize the current object. */
1119  void reset() {
1120  m_page_no = 0;
1121  m_bytes = 0;
1122  m_zbytes = 0;
1123  }
1124 
1125  /** Print this blob_page_into_t object into the given output stream.
1126  @param[in] out the output stream.
1127  @return the output stream. */
1128  std::ostream &print(std::ostream &out) const;
1129 
1130  /** Set the compressed data size in bytes.
1131  @param[in] bytes the new compressed data size. */
1132  void set_compressed_size(uint bytes) { m_zbytes = bytes; }
1133 
1134  /** Set the uncompressed data size in bytes.
1135  @param[in] bytes the new uncompressed data size. */
1136  void set_uncompressed_size(uint bytes) { m_bytes = bytes; }
1137 
1138  /** Set the page number.
1139  @param[in] page_no the page number */
1140  void set_page_no(page_no_t page_no) { m_page_no = page_no; }
1141 
1142  private:
1143  /** The BLOB page number */
1145 
1146  /** Amount of uncompressed data (in bytes) in the BLOB page. */
1148 
1149  /** Amount of compressed data (in bytes) in the BLOB page. */
1151 };
1152 
1153 inline std::ostream &operator<<(std::ostream &out,
1154  const blob_page_info_t &obj) {
1155  return (obj.print(out));
1156 }
1157 
1158 /** The in-memory blob directory. Each blob contains a sequence of pages.
1159 This directory contains a list of those pages along with their metadata. */
1160 struct blob_dir_t {
1161  typedef std::vector<blob_page_info_t>::const_iterator const_iterator;
1162 
1163  /** Print this blob directory into the given output stream.
1164  @param[in] out the output stream.
1165  @return the output stream. */
1166  std::ostream &print(std::ostream &out) const;
1167 
1168  /** Clear the contents of this blob directory. */
1169  void clear() { m_pages.clear(); }
1170 
1171  /** Append the given blob page information.
1172  @param[in] page the blob page information to be added.
1173  @return DB_SUCCESS on success, error code on failure. */
1175  m_pages.push_back(page);
1176  return (DB_SUCCESS);
1177  }
1178 
1179  /** A vector of blob pages along with its metadata. */
1180  std::vector<blob_page_info_t> m_pages;
1181 };
1182 
1183 /** Overloading the global output operator to print the blob_dir_t
1184 object into an output stream.
1185 @param[in,out] out the output stream.
1186 @param[in] obj the object to be printed.
1187 @return the output stream. */
1188 inline std::ostream &operator<<(std::ostream &out, const blob_dir_t &obj) {
1189  return (obj.print(out));
1190 }
1191 
1192 /** The context information for reading a single BLOB */
1193 struct ReadContext {
1194  /** Constructor
1195  @param[in] page_size page size information.
1196  @param[in] data 'internally' stored part of the field
1197  containing also the reference to the
1198  external part; must be protected by
1199  a lock or a page latch.
1200  @param[in] prefix_len length of BLOB data stored inline in
1201  the clustered index record, including
1202  the blob reference.
1203  @param[out] buf the output buffer.
1204  @param[in] len the output buffer length. */
1205 #ifdef UNIV_DEBUG
1206  /**
1207  @param[in] is_sdi true for SDI Indexes. */
1208 #endif /* UNIV_DEBUG */
1209  ReadContext(const page_size_t &page_size, const byte *data, ulint prefix_len,
1210  byte *buf, ulint len
1211 #ifdef UNIV_DEBUG
1212  ,
1213  bool is_sdi
1214 #endif /* UNIV_DEBUG */
1215  )
1216  : m_page_size(page_size),
1217  m_data(data),
1218  m_local_len(prefix_len),
1219  m_blobref(const_cast<byte *>(data) + prefix_len -
1221  m_buf(buf),
1222  m_len(len),
1223  m_lob_version(0)
1224 #ifdef UNIV_DEBUG
1225  ,
1226  m_is_sdi(is_sdi)
1227 #endif /* UNIV_DEBUG */
1228  {
1229  read_blobref();
1230  }
1231 
1232  /** Read the space_id, page_no and offset information from the BLOB
1233  reference object and update the member variables. */
1234  void read_blobref() {
1235  m_space_id = m_blobref.space_id();
1236  m_page_no = m_blobref.page_no();
1237  m_offset = m_blobref.offset();
1238  }
1239 
1240  /** Check if the BLOB reference is valid. For this particular check,
1241  if the length of the BLOB is greater than 0, it is considered
1242  valid.
1243  @return true if valid. */
1244  bool is_valid_blob() const { return (m_blobref.length() > 0); }
1245 
1246  dict_index_t *index() { return (m_index); }
1247 
1248  /** The page size information. */
1250 
1251  /** The 'internally' stored part of the field containing also the
1252  reference to the external part; must be protected by a lock or a page
1253  latch */
1254  const byte *m_data;
1255 
1256  /** Length (in bytes) of BLOB prefix stored inline in clustered
1257  index record. */
1259 
1260  /** The blob reference of the blob that is being read. */
1262 
1263  /** Buffer into which data is read. */
1265 
1266  /** Length of the buffer m_buf. */
1267  ulint m_len;
1268 
1269  /** The identifier of the space in which blob is available. */
1271 
1272  /** The page number obtained from the blob reference. */
1274 
1275  /** The offset information obtained from the blob reference. */
1276  ulint m_offset;
1277 
1279 
1281 
1282 #ifdef UNIV_DEBUG
1283  /** Is it a space dictionary index (SDI)?
1284  @return true if SDI, false otherwise. */
1285  bool is_sdi() const { return (m_is_sdi); }
1286 
1287  /** Is it a tablespace dictionary index (SDI)? */
1288  const bool m_is_sdi;
1289 
1290  /** Assert that current trx is using isolation level read uncommitted.
1291  @return true if transaction is using read uncommitted, false otherwise. */
1292  bool assert_read_uncommitted() const;
1293 #endif /* UNIV_DEBUG */
1294 
1295  /** The transaction that is reading. */
1296  trx_t *m_trx = nullptr;
1297 };
1298 
1299 /** Fetch compressed BLOB */
1300 struct zReader {
1301  /** Constructor. */
1302  explicit zReader(const ReadContext &ctx) : m_rctx(ctx) {}
1303 
1304  /** Fetch the BLOB.
1305  @return DB_SUCCESS on success. */
1306  dberr_t fetch();
1307 
1308  /** Fetch one BLOB page.
1309  @return DB_SUCCESS on success. */
1310  dberr_t fetch_page();
1311 
1312  /** Get the length of data that has been read.
1313  @return the length of data that has been read. */
1314  ulint length() const { return (m_stream.total_out); }
1315 
1316  private:
1317  /** Do setup of the zlib stream.
1318  @return code returned by zlib. */
1319  int setup_zstream();
1320 
1321 #ifdef UNIV_DEBUG
1322  /** Assert that the local prefix is empty. For compressed row format,
1323  there is no local prefix stored. This function doesn't return if the
1324  local prefix is non-empty.
1325  @return true if local prefix is empty*/
1326  bool assert_empty_local_prefix();
1327 #endif /* UNIV_DEBUG */
1328 
1330 
1331  /** Bytes yet to be read. */
1333 
1334  /** The zlib stream used to uncompress while fetching blob. */
1335  z_stream m_stream;
1336 
1337  /** The memory heap that will be used by zlib allocator. */
1339 
1340  /* There is no latch on m_bpage directly. Instead,
1341  m_bpage is protected by the B-tree page latch that
1342  is being held on the clustered index record, or,
1343  in row_merge_copy_blobs(), by an exclusive table lock. */
1345 
1346 #ifdef UNIV_DEBUG
1347  /** The expected page type. */
1349 #endif /* UNIV_DEBUG */
1350 };
1351 
1352 /** Fetch uncompressed BLOB */
1353 struct Reader {
1354  /** Constructor. */
1355  Reader(const ReadContext &ctx)
1356  : m_rctx(ctx), m_cur_block(nullptr), m_copied_len(0) {}
1357 
1358  /** Fetch the complete or prefix of the uncompressed LOB data.
1359  @return bytes of LOB data fetched. */
1360  ulint fetch();
1361 
1362  /** Fetch one BLOB page. */
1363  void fetch_page();
1364 
1366 
1367  /** Buffer block of the current BLOB page */
1369 
1370  /** Total bytes of LOB data that has been copied from multiple
1371  LOB pages. This is a cumulative value. When this value reaches
1372  m_rctx.m_len, then the read operation is completed. */
1374 };
1375 
1376 /** The context information when the delete operation on LOB is
1377 taking place. */
1378 struct DeleteContext : public BtrContext {
1379  /** Constructor. */
1380  DeleteContext(const BtrContext &btr, byte *field_ref, ulint field_no,
1381  bool rollback)
1382  : BtrContext(btr),
1383  m_blobref(field_ref),
1384  m_field_no(field_no),
1385  m_rollback(rollback),
1386  m_page_size(table() == nullptr ? get_page_size()
1387  : dict_table_page_size(table())) {
1388  m_blobref.parse(m_blobref_mem);
1389  }
1390 
1391  bool is_ref_valid() const {
1392  return (m_blobref_mem.m_page_no == m_blobref.page_no());
1393  }
1394 
1395  /** Determine if it is compressed page format.
1396  @return true if compressed. */
1397  bool is_compressed() const { return (m_page_size.is_compressed()); }
1398 
1399  /** Check if tablespace supports atomic blobs.
1400  @return true if tablespace has atomic blobs. */
1401  bool has_atomic_blobs() const {
1402  space_id_t space_id = m_blobref.space_id();
1403  uint32_t flags = fil_space_get_flags(space_id);
1404  return (DICT_TF_HAS_ATOMIC_BLOBS(flags));
1405  }
1406 
1407  bool is_delete_marked() const {
1408  rec_t *clust_rec = rec();
1409  if (clust_rec == nullptr) {
1410  return (true);
1411  }
1412  return (rec_get_deleted_flag(clust_rec, page_rec_is_comp(clust_rec)));
1413  }
1414 
1415 #ifdef UNIV_DEBUG
1416  /** Validate the LOB reference object.
1417  @return true if valid, false otherwise. */
1418  bool validate_blobref() const {
1419  rec_t *clust_rec = rec();
1420  if (clust_rec != nullptr) {
1421  const byte *v2 =
1422  btr_rec_get_field_ref(clust_rec, get_offsets(), m_field_no);
1423 
1424  ut_ad(m_blobref.is_equal(v2));
1425  }
1426  return (true);
1427  }
1428 #endif /* UNIV_DEBUG */
1429 
1430  /** Acquire an x-latch on the index page containing the clustered
1431  index record, in the given mini-transaction context.
1432  @param[in] mtr Mini-transaction context. */
1433  void x_latch_rec_page(mtr_t *mtr);
1434 
1435  /** the BLOB reference or external field reference. */
1437 
1438  /** field number of externally stored column; ignored if rec == NULL */
1439  ulint m_field_no;
1440 
1441  /** Is this operation part of rollback? */
1443 
1445 
1446  private:
1447  /** Memory copy of the original LOB reference. */
1449 
1450  /** Obtain the page size from the tablespace flags.
1451  @return the page size. */
1453  bool found;
1454  space_id_t space_id = m_blobref.space_id();
1455  const page_size_t &tmp = fil_space_get_page_size(space_id, &found);
1456  ut_ad(found);
1457  return (tmp);
1458  }
1459 };
1460 
1461 /** Determine if an operation on off-page columns is an update.
1462 @param[in] op type of BLOB operation.
1463 @return true if op != OPCODE_INSERT */
1464 inline bool btr_lob_op_is_update(opcode op) {
1465  switch (op) {
1466  case OPCODE_INSERT:
1467  case OPCODE_INSERT_BULK:
1468  return (false);
1469  case OPCODE_INSERT_UPDATE:
1470  case OPCODE_UPDATE:
1471  return (true);
1472  case OPCODE_UNKNOWN:
1473  break;
1474  }
1475 
1476  ut_ad(0);
1477  return (FALSE);
1478 }
1479 
1480 #ifdef UNIV_DEBUG
1481 #define btr_copy_externally_stored_field_prefix( \
1482  trx, index, buf, len, page_size, data, is_sdi, local_len) \
1483  btr_copy_externally_stored_field_prefix_func( \
1484  trx, index, buf, len, page_size, data, is_sdi, local_len)
1485 
1486 #define btr_copy_externally_stored_field(trx, index, len, ver, data, \
1487  page_size, local_len, is_sdi, heap) \
1488  btr_copy_externally_stored_field_func(trx, index, len, ver, data, page_size, \
1489  local_len, is_sdi, heap)
1490 
1491 #else /* UNIV_DEBUG */
1492 #define btr_copy_externally_stored_field_prefix( \
1493  trx, index, buf, len, page_size, data, is_sdi, local_len) \
1494  btr_copy_externally_stored_field_prefix_func(trx, index, buf, len, \
1495  page_size, data, local_len)
1496 
1497 #define btr_copy_externally_stored_field(trx, index, len, ver, data, \
1498  page_size, local_len, is_sdi, heap) \
1499  btr_copy_externally_stored_field_func(trx, index, len, ver, data, page_size, \
1500  local_len, heap)
1501 #endif /* UNIV_DEBUG */
1502 
1503 /** Copies the prefix of an externally stored field of a record.
1504 The clustered index record must be protected by a lock or a page latch.
1505 @param[in] trx the current transaction object if available
1506 or nullptr.
1507 @param[in] index the clust index in which lob is read.
1508 @param[out] buf the field, or a prefix of it
1509 @param[in] len length of buf, in bytes
1510 @param[in] page_size BLOB page size
1511 @param[in] data 'internally' stored part of the field
1512  containing also the reference to the external
1513  part; must be protected by a lock or a page
1514  latch. */
1515 #ifdef UNIV_DEBUG
1516 /**
1517 @param[in] is_sdi true for SDI indexes */
1518 #endif /* UNIV_DEBUG */
1519 /**
1520 @param[in] local_len length of data, in bytes
1521 @return the length of the copied field, or 0 if the column was being
1522 or has been deleted */
1524  const dict_index_t *index,
1525  byte *buf, ulint len,
1526  const page_size_t &page_size,
1527  const byte *data,
1528 #ifdef UNIV_DEBUG
1529  bool is_sdi,
1530 #endif /* UNIV_DEBUG */
1531  ulint local_len);
1532 
1533 /** Copies an externally stored field of a record to mem heap.
1534 The clustered index record must be protected by a lock or a page latch.
1535 @param[in] trx the current trx object or nullptr
1536 @param[in] index the clust index in which lob is read.
1537 @param[out] len length of the whole field
1538 @param[out] lob_version LOB version number.
1539 @param[in] data 'internally' stored part of the field
1540  containing also the reference to the external
1541  part; must be protected by a lock or a page
1542  latch.
1543 @param[in] page_size BLOB page size
1544 @param[in] local_len length of data */
1545 #ifdef UNIV_DEBUG
1546 /**
1547 @param[in] is_sdi true for SDI Indexes */
1548 #endif /* UNIV_DEBUG */
1549 /**
1550 @param[in,out] heap mem heap
1551 @return the whole field copied to heap */
1553  trx_t *trx, const dict_index_t *index, ulint *len, size_t *lob_version,
1554  const byte *data, const page_size_t &page_size, ulint local_len,
1555 #ifdef UNIV_DEBUG
1556  bool is_sdi,
1557 #endif /* UNIV_DEBUG */
1558  mem_heap_t *heap);
1559 
1560 /** Gets the externally stored size of a record, in units of a database page.
1561 @param[in] rec record
1562 @param[in] offsets array returned by rec_get_offsets()
1563 @return externally stored part, in units of a database page */
1564 ulint btr_rec_get_externally_stored_len(const rec_t *rec, const ulint *offsets);
1565 
1566 /** Purge an LOB (either of compressed or uncompressed).
1567 @param[in] ctx the delete operation context information.
1568 @param[in] index clustered index in which LOB is present
1569 @param[in] trxid the transaction that is being purged.
1570 @param[in] undo_no during rollback to savepoint, purge only upto
1571  this undo number.
1572 @param[in] rec_type undo record type.
1573 @param[in] uf the update vector for the field. */
1574 void purge(lob::DeleteContext *ctx, dict_index_t *index, trx_id_t trxid,
1575  undo_no_t undo_no, ulint rec_type, const upd_field_t *uf);
1576 
1577 /** Update a portion of the given LOB.
1578 @param[in] ctx update operation context information.
1579 @param[in] trx the transaction that is doing the modification.
1580 @param[in] index the clustered index containing the LOB.
1581 @param[in] upd update vector
1582 @param[in] field_no the LOB field number
1583 @param[in] blobref LOB reference stored in clust record.
1584 @return DB_SUCCESS on success, error code on failure. */
1585 dberr_t update(InsertContext &ctx, trx_t *trx, dict_index_t *index,
1586  const upd_t *upd, ulint field_no, ref_t blobref);
1587 
1588 /** Update a portion of the given LOB.
1589 @param[in] ctx update operation context information.
1590 @param[in] trx the transaction that is doing the modification.
1591 @param[in] index the clustered index containing the LOB.
1592 @param[in] upd update vector
1593 @param[in] field_no the LOB field number
1594 @param[in] blobref LOB reference stored in clust record.
1595 @return DB_SUCCESS on success, error code on failure. */
1596 dberr_t z_update(InsertContext &ctx, trx_t *trx, dict_index_t *index,
1597  const upd_t *upd, ulint field_no, ref_t blobref);
1598 
1599 /** Print information about the given LOB.
1600 @param[in] trx the current transaction.
1601 @param[in] index the clust index that contains the LOB.
1602 @param[in] out the output stream into which LOB info is printed.
1603 @param[in] ref the LOB reference
1604 @param[in] fatal if true assert at end of function. */
1605 void print(trx_t *trx, dict_index_t *index, std::ostream &out, ref_t ref,
1606  bool fatal);
1607 
1608 /** Import the given LOB. Update the creator trx id and the modifier trx
1609 id to the given import trx id.
1610 @param[in] index clustered index containing the lob.
1611 @param[in] field_ref the lob reference.
1612 @param[in] trx_id the import trx id. */
1613 void z_import(const dict_index_t *index, byte *field_ref, trx_id_t trx_id);
1614 
1615 /** Import the given LOB. Update the creator trx id and the modifier trx
1616 id to the given import trx id.
1617 @param[in] index clustered index containing the lob.
1618 @param[in] field_ref the lob reference.
1619 @param[in] trx_id the import trx id. */
1620 void import(const dict_index_t *index, byte *field_ref, trx_id_t trx_id);
1621 
1622 #ifdef UNIV_DEBUG
1623 /** Check if all the LOB references in the given clustered index record has
1624 valid space_id in it.
1625 @param[in] index the index to which the LOB belongs.
1626 @param[in] rec the clust_rec in which the LOB references are checked.
1627 @param[in] offsets the field offets of the given rec.
1628 @return true if LOB references have valid space_id, false otherwise. */
1629 bool rec_check_lobref_space_id(dict_index_t *index, const rec_t *rec,
1630  const ulint *offsets);
1631 #endif /* UNIV_DEBUG */
1632 
1633 /** Mark an LOB that it is not partially updatable anymore.
1634 @param[in] trx Current transaction.
1635 @param[in] index Clustered index to which the LOB belongs.
1636 @param[in] update Update vector.
1637 @param[in] btr_mtr Mini-transaction context holding latches on the B-tree.
1638 This function does not generate redo log using this btr_mtr. It only obtains
1639 the log mode.
1640 @return DB_SUCCESS on success, error code on failure. */
1642  const upd_t *update, const mtr_t *btr_mtr);
1643 
1644 } // namespace lob
1645 
1646 #endif /* lob0lob_h */
bool rec_check_lobref_space_id(dict_index_t *index, const rec_t *rec, const ulint *offsets)
Check if all the LOB references in the given clustered index record has valid space_id in it...
Definition: lob0lob.cc:1312
Definition: db0err.h:42
#define Z_CHUNK_SIZE
Definition: lob0lob.h:90
buf_block_t * block() const
Get the pointer to the clustered record block.
Definition: lob0lob.h:1034
Definition: result.h:29
page_no_t get_page_no() const
Get the page number of clustered index block.
Definition: lob0lob.h:799
Reader(const ReadContext &ctx)
Constructor.
Definition: lob0lob.h:1355
uint m_zbytes
Amount of compressed data (in bytes) in the BLOB page.
Definition: lob0lob.h:1150
UNIV_INLINE bool rec_get_node_ptr_flag(const rec_t *rec)
The following function tells if a new-style record is a node pointer.
Data dictionary memory object creation.
Compressed page descriptor.
Definition: page0types.h:197
buf_page_t page
page information; this must be the first field, so that buf_pool->page_hash can point to buf_page_t o...
Definition: buf0buf.h:1330
const ulint LOB_HDR_PART_LEN
The structure of uncompressed LOB page header.
Definition: lob0lob.h:140
const ulint BTR_EXTERN_INHERITED_FLAG
If the second most significant bit of BTR_EXTERN_LEN (i.e., the second most significant bit of the by...
Definition: lob0lob.h:129
dberr_t z_update(trx_t *trx, dict_index_t *index, const upd_t *upd, ulint field_no)
Update a portion of the given LOB.
buf_block_t * block
pointer to the block containing rec
Definition: page0cur.h:318
std::ostream & operator<<(std::ostream &out, const plist_node_t &obj)
Definition: lob0impl.h:238
ulint m_field_no
field number of externally stored column; ignored if rec == NULL
Definition: lob0lob.h:1439
void start_btr_mtr()
Start the mini-transaction that will be holding the latches of the clustered index record block...
Definition: lob0lob.h:891
uint32 page_no_t
Page number.
Definition: api0api.h:57
ulint btr_rec_get_externally_stored_len(const rec_t *rec, const ulint *offsets)
Gets the externally stored size of a record, in units of a database page.
Definition: lob0lob.cc:1105
void rec_block_unfix()
Decrement the buffer fix count of the clustered index record block, X-latching it before...
Definition: lob0lob.h:929
buf_block_t * m_cur_block
Buffer block of the current BLOB page.
Definition: lob0lob.h:1368
bool is_valid_blob() const
Check if the BLOB reference is valid.
Definition: lob0lob.h:1244
Fetch compressed BLOB.
Definition: lob0lob.h:1300
Definition: sync0rw.h:137
bool is_being_modified() const
Check if the current blob is being modified.
Definition: lob0lob.h:360
page_no_t m_btr_page_no
Page number where the clust rec is present.
Definition: lob0lob.h:1080
#define buf_block_buf_fix_inc(b, f, l)
Increments the bufferfix count.
Definition: buf0buf.h:622
Store off-page columns for an insert by update.
Definition: lob0lob.h:556
std::ostream & print(std::ostream &out) const
Print this LOB reference into the given output stream.
Definition: lob0lob.cc:1271
bool use_single_z_stream() const
For compressed LOB, if the length is less than or equal to Z_CHUNK_SIZE then use the older single z s...
Definition: lob0lob.h:217
std::ostream & print(std::ostream &out) const
Print this blob_page_into_t object into the given output stream.
Definition: lob0lob.cc:152
bool is_big(const page_size_t &page_size) const
Check if this LOB is big enough to do partial update.
Definition: lob0lob.h:238
page_no_t page_no() const
Read the page number from the blob reference.
Definition: lob0lob.h:412
space_id_t m_space_id
Space Identifier of the clustered index.
Definition: lob0lob.h:155
const ulint LOB_HDR_NEXT_PAGE_NO
Offset within header of next BLOB part page no.
Definition: lob0lob.h:144
bool is_compressed() const
Determine if it is compressed page format.
Definition: lob0lob.h:1397
Definition: trx0trx.h:833
The buffer control block structure.
Definition: buf0buf.h:1324
Page identifier.
Definition: buf0types.h:158
The B-tree context under which the LOB operation is done.
Definition: lob0lob.h:659
page_zip_des_t * get_page_zip() const
Gets the compressed page descriptor.
Definition: lob0lob.h:793
void rec_block_fix()
Increment the buffer fix count of the clustered index record block.
Definition: lob0lob.h:914
const ref_t m_blobref
The blob reference of the blob that is being read.
Definition: lob0lob.h:1261
ulint m_length
Length of LOB.
Definition: lob0lob.h:164
const big_rec_t * m_big_rec_vec
The B-tree Context.
Definition: lob0lob.h:1104
void copy(byte *field_ref) const
Copy the LOB reference into the given memory location.
Definition: lob0lob.h:264
UNIV_INLINE page_t * page_align(const void *ptr)
Gets the start of a page.
UNIV_INLINE ulint rec_offs_any_extern(const ulint *offsets)
Determine if the offsets are for a record containing externally stored columns.
void check_redolog()
Check if there is enough space in the redo log file.
Definition: lob0lob.h:995
void set_uncompressed_size(uint bytes)
Set the uncompressed data size in bytes.
Definition: lob0lob.h:1136
Store off-page columns for an update.
Definition: lob0lob.h:559
const byte field_ref_almost_zero[FIELD_REF_SIZE]
A BLOB field reference has all the bits set to zero, except the "being modified" bit.
Definition: lob0lob.cc:44
DeleteContext(const BtrContext &btr, byte *field_ref, ulint field_no, bool rollback)
Constructor.
Definition: lob0lob.h:1380
void zblob_write_blobref(ulint field_no, mtr_t *mtr)
Write a blob reference of a field into a clustered index record in a compressed leaf page...
Definition: lob0lob.h:1063
zReader(const ReadContext &ctx)
Constructor.
Definition: lob0lob.h:1302
ref_mem_t m_blobref_mem
Memory copy of the original LOB reference.
Definition: lob0lob.h:1448
Definition: mtr0types.h:261
const page_size_t page_size() const
Obtain the page size of the underlying table.
Definition: lob0lob.h:983
ulint offset() const
Read the offset of blob header from the blob reference.
Definition: lob0lob.h:418
page_no_t m_page_no
Page number of first LOB page.
Definition: lob0lob.h:158
BtrContext(mtr_t *mtr, btr_pcur_t *pcur, dict_index_t *index, rec_t *rec, ulint *offsets, buf_block_t *block)
Constructor.
Definition: lob0lob.h:673
void set_ownership_of_extern_field(ulint i, ibool val)
Sets the ownership bit of an externally stored field in a record.
Definition: lob0lob.h:726
mtr_log_t get_log_mode() const
Get the logging mode.
The in-memory blob directory.
Definition: lob0lob.h:1160
UNIV_INLINE ulint rec_offs_nth_extern(const ulint *offsets, ulint n)
Returns nonzero if the extern bit is set in nth field of rec.
Data structure for a database table.
Definition: dict0mem.h:1532
void set_compressed_size(uint bytes)
Set the compressed data size in bytes.
Definition: lob0lob.h:1132
bool is_inherited() const
Check if the current row inherited the blob from parent row.
Definition: lob0lob.h:386
ref_t m_blobref
the BLOB reference or external field reference.
Definition: lob0lob.h:1436
bool rec_offs_validate() const
Validate the record offsets array.
Definition: lob0lob.h:1021
mem_heap_t * m_heap
The memory heap that will be used by zlib allocator.
Definition: lob0lob.h:1338
UNIV_INLINE uint8_t mach_read_from_1(const byte *b)
The following function is used to fetch data from one byte.
static void rollback(DeleteContext *ctx, dict_index_t *index, trx_id_t trxid, undo_no_t undo_no, ulint rec_type, const upd_field_t *uf)
Rollback modification of a uncompressed LOB.
Definition: lob0purge.cc:63
ulint length() const
Get the length of data that has been read.
Definition: lob0lob.h:1314
FlushObserver * get_flush_observer() const
Get flush observer.
Definition: lob0lob.h:1011
BtrContext()
Default Constructor.
Definition: lob0lob.h:662
void set_page_no(const ulint page_no, mtr_t *mtr)
Set the page number in the external field reference.
Definition: lob0lob.h:453
void set_offset(const ulint offset, mtr_t *mtr)
Set the offset information in the external field reference.
Definition: lob0lob.h:460
void z_import(const dict_index_t *index, byte *field_ref, trx_id_t trx_id)
Import the given LOB.
Definition: buf0block_hint.cc:29
BtrContext(const BtrContext &other)
Copy Constructor.
Definition: lob0lob.h:706
blob_page_info_t(page_no_t page_no, uint bytes, uint zbytes)
Constructor.
Definition: lob0lob.h:1115
bool is_sdi() const
Is it a space dictionary index (SDI)?
Definition: lob0lob.h:1285
ReadContext m_rctx
Definition: lob0lob.h:1329
void mlog_write_string(byte *ptr, const byte *str, ulint len, mtr_t *mtr)
Writes a string to a file page buffered in the buffer pool.
Definition: mtr0log.cc:318
The info structure stored at the beginning of a heap block.
Definition: mem0mem.h:349
dberr_t add(const blob_page_info_t &page)
Append the given blob page information.
Definition: lob0lob.h:1174
Definition: btr0pcur.h:178
space_id_t space() const
Get the space id.
Definition: lob0lob.h:979
void rec_offs_make_nth_extern(ulint *offsets, const ulint n)
Mark the nth field as externally stored.
Definition: rem0rec.cc:1746
UNIV_INLINE const page_size_t dict_table_page_size(const dict_table_t *table)
Get the table page size.
In memory representation of the LOB reference.
Definition: lob0lob.h:153
Fetch uncompressed BLOB.
Definition: lob0lob.h:1353
const ulint BTR_EXTERN_OFFSET
offset of BLOB header on that page
Definition: lob0lob.h:107
const ulint BTR_EXTERN_VERSION
Version number of LOB (LOB in new format)
Definition: lob0lob.h:110
btr_pcur_t * m_pcur
Definition: lob0lob.h:1069
mtr_log_t
Logging modes for a mini-transaction.
Definition: mtr0types.h:41
ulint btr_rec_get_field_ref_offs(const ulint *offsets, ulint n)
Gets the offset of the pointer to the externally stored part of a field.
Definition: lob0lob.cc:59
mtr_t * get_mtr()
Get the associated mini-transaction.
Definition: lob0lob.h:1030
UNIV_INLINE ulint page_offset(const void *ptr)
Gets the offset within a page.
buf_block_t * m_block
Definition: lob0lob.h:1073
page_no_t pages_in_extent() const
Determine the extent size (in pages) for the underlying table.
Definition: lob0lob.h:989
Definition: row0upd.h:486
void blob_free(dict_index_t *index, buf_block_t *block, bool all, mtr_t *mtr)
Deallocate a buffer block that was reserved for a BLOB part.
Definition: lob0lob.cc:1066
page_no_t dict_table_extent_size(const dict_table_t *table)
Determine the extent size (in pages) for the given table.
Definition: dict0dict.cc:4943
space_id_t m_space_id
The identifier of the space in which blob is available.
Definition: lob0lob.h:1270
page_size_t m_page_size
Definition: lob0lob.h:1444
bool validate() const
Validate the current BLOB context object.
Definition: lob0lob.h:833
static bool is_null(const byte *ref)
Check if the LOB reference is null (all zeroes).
Definition: lob0lob.h:305
#define FALSE
Definition: types.h:46
ulint * get_offsets() const
Get the record offsets array.
Definition: lob0lob.h:1017
void make_nth_extern(ulint field_no)
Mark the nth field as externally stored.
Definition: lob0lob.h:1001
Definition: mtr0types.h:263
std::vector< blob_page_info_t > m_pages
A vector of blob pages along with its metadata.
Definition: lob0lob.h:1180
The operation code is unknown or not important.
Definition: lob0lob.h:565
const ulint MAX_SIZE
The maximum size possible for an LOB.
Definition: lob0lob.h:83
const ulint BTR_EXTERN_SPACE_ID
The reference in a field for which data is stored on a different page.
Definition: lob0lob.h:101
bool need_recalc() const
Check if there is a need to recalculate the context information.
Definition: lob0lob.h:809
void set_space_id(const space_id_t space_id, mtr_t *mtr)
Set the space_id in the external field reference.
Definition: lob0lob.h:446
int page
Definition: ctype-mb.cc:1234
UNIV_INLINE ulint rec_get_deleted_flag(const rec_t *rec, ulint comp)
The following function tells if record is delete marked.
The context information when the delete operation on LOB is taking place.
Definition: lob0lob.h:1378
const byte * m_data
The &#39;internally&#39; stored part of the field containing also the reference to the external part; must be...
Definition: lob0lob.h:1254
#define btr_pcur_restore_position(l, p, mtr)
Definition: btr0pcur.h:94
page_no_t get_btr_page_no() const
Get the page number of clustered index record.
Definition: lob0lob.h:899
ulint m_local_len
Length (in bytes) of BLOB prefix stored inline in clustered index record.
Definition: lob0lob.h:1258
static void set_being_modified(byte *ref, bool modifying, mtr_t *mtr)
Set the being_modified flag in the field reference.
Definition: lob0lob.h:331
#define btr_rec_get_field_ref(rec, offsets, n)
Gets a pointer to the externally stored part of a field.
Definition: lob0lob.h:647
ulint get_rec_offset() const
Get the record offset within page of the clustered index record.
Definition: lob0lob.h:805
void recalc()
Recalculate some of the members after restoring the persistent cursor.
Definition: lob0lob.h:1049
void set_being_modified(bool modifying, mtr_t *mtr)
Set the being_modified flag in the field reference.
Definition: lob0lob.h:346
void set_page_no(page_no_t page_no)
Set the page number.
Definition: lob0lob.h:1140
bool m_being_modified
Whether the blob is being modified.
Definition: lob0lob.h:180
BtrContext(mtr_t *mtr, btr_pcur_t *pcur, dict_index_t *index, rec_t *rec, ulint *offsets, buf_block_t *block, opcode op)
Constructor.
Definition: lob0lob.h:690
void set_ref(byte *ptr)
Set the external field reference to the given memory location.
Definition: lob0lob.h:273
Index page routines.
#define mtr_x_lock(l, m)
Lock an rw-lock in x-mode.
Definition: mtr0mtr.h:129
byte rec_t
Definition: rem0types.h:39
#define UINT32_MAX
Definition: lexyy.cc:86
opcode
LOB operation code for btr_store_big_rec_extern_fields().
Definition: lob0lob.h:550
UNIV_INLINE ulint page_rec_is_comp(const rec_t *rec)
TRUE if the record is on a page in compact format.
UNIV_INLINE ulint rec_offs_comp(const ulint *offsets)
Determine if the offsets are for a record in the new compact format.
void read_blobref()
Read the space_id, page_no and offset information from the BLOB reference object and update the membe...
Definition: lob0lob.h:1234
bool is_owner() const
Check if the current row is the owner of the blob.
Definition: lob0lob.h:379
ReadContext m_rctx
Definition: lob0lob.h:1365
UNIV_INLINE void rec_offs_make_valid(const rec_t *rec, const dict_index_t *index, ulint *offsets)
Updates debug data in offsets, in order to avoid bogus rec_offs_validate() failures.
Definition: rec.h:553
byte * rec_get_nth_field(const rec_t *rec, const ulint *offsets, ulint n, ulint *len)
Gets the value of the specified field in the record.
Definition: rem0rec.h:317
dberr_t
Definition: db0err.h:38
uint m_bytes
Amount of uncompressed data (in bytes) in the BLOB page.
Definition: lob0lob.h:1147
void restore_position()
Restore the position of the persistent cursor.
Definition: lob0lob.h:952
void mlog_write_ulint(byte *ptr, ulint val, mlog_id_t type, mtr_t *mtr)
Writes 1, 2 or 4 bytes to a file page.
Definition: mtr0log.cc:247
ulint * m_offsets
Definition: lob0lob.h:1072
std::map< page_no_t, buf_block_t * > BlockCache
Definition: lob0lob.h:41
const ulint BTR_EXTERN_LEN
8 bytes containing the length of the externally stored part of the LOB.
Definition: lob0lob.h:114
We use FlushObserver to track flushing of non-redo logged pages in bulk create index(BtrBulk.cc).Since we disable redo logging during a index build, we need to make sure that all dirty pages modifed by the index build are flushed to disk before any redo logged operations go to the index.
Definition: buf0flu.h:275
void update(space_id_t space_id, ulint page_no, ulint offset, mtr_t *mtr)
Update the information stored in the external field reference.
Definition: lob0lob.h:437
std::ostream & print(std::ostream &out) const
Print this blob directory into the given output stream.
Definition: lob0lob.cc:140
unsigned int uint
Definition: uca-dump.cc:29
ref_t(byte *ptr)
Constructor.
Definition: lob0lob.h:213
UNIV_INLINE rw_lock_t * dict_index_get_lock(dict_index_t *index)
Gets the read-write lock of the index tree.
ulint m_page_type_ex
The expected page type.
Definition: lob0lob.h:1348
opcode m_op
Definition: lob0lob.h:1074
bool m_partial
Whether the LOB is partially updated.
Definition: lob0lob.h:177
#define buf_block_get_page_zip(block)
Gets the compressed page descriptor corresponding to an uncompressed page if applicable.
Definition: buf0buf.h:911
const byte field_ref_zero[FIELD_REF_SIZE]
A BLOB field reference full of zero, for use in assertions and tests.Initially, BLOB field references...
Definition: page0zip.cc:41
#define ut_a(EXPR)
Abort execution if EXPR does not evaluate to nonzero.
Definition: ut0dbg.h:54
Don&#39;t generate REDO log but add dirty pages to flush list.
Definition: mtr0types.h:49
dict_index_t * m_index
Definition: lob0lob.h:1070
const ulint BTR_EXTERN_BEING_MODIFIED_FLAG
If the 3rd most significant bit of BTR_EXTERN_LEN is 1, then it means that the externally stored fiel...
Definition: lob0lob.h:135
#define btr_block_get(page_id, page_size, mode, index, mtr)
Gets a buffer page and declares its latching order level.
Definition: btr0btr.h:206
size_t physical() const
Retrieve the physical page size (on-disk).
Definition: page0size.h:120
constexpr ulint FIL_PAGE_DATA
start of the data on the page
Definition: fil0types.h:110
mtr_t * m_mtr
Definition: lob0lob.h:1068
#define btr_pcur_get_rec(p)
Definition: btr0pcur.h:121
ulint m_offset
Offset within m_page_no where LOB begins.
Definition: lob0lob.h:161
The index tree persistent cursor.
Definition: btr0pcur.h:52
Store off-page columns for a freshly inserted record by bulk.
Definition: lob0lob.h:562
ulint m_offset
The offset information obtained from the blob reference.
Definition: lob0lob.h:1276
const bool m_is_sdi
Is it a tablespace dictionary index (SDI)?
Definition: lob0lob.h:1288
dict_table_t * table() const
Get the table object.
Definition: lob0lob.h:967
ulint m_copied_len
Total bytes of LOB data that has been copied from multiple LOB pages.
Definition: lob0lob.h:1373
page_no_t m_page_no
The BLOB page number.
Definition: lob0lob.h:1144
UNIV_INLINE ulint rec_offs_n_fields(const ulint *offsets)
The following function returns the number of fields in a record.
Definition: rec.h:443
dberr_t mark_not_partially_updatable(trx_t *trx, dict_index_t *index, const upd_t *update, const mtr_t *btr_mtr)
Mark an LOB that it is not partially updatable anymore.
Definition: lob0lob.cc:1352
static bool is_null_relaxed(const byte *ref)
Check if the LOB reference is null (all zeroes) except the "is being modified" bit.
Definition: lob0lob.h:298
static bool is_being_modified(const byte *field_ref)
Check if the current blob is being modified.
Definition: lob0lob.h:353
dict_index_t * index() const
Get the index object.
Definition: lob0lob.h:963
bool m_inherit
Whether the clustered index record inherited this LOB from another clustered index record...
Definition: lob0lob.h:174
ulint length() const
Read the length from the blob reference.
Definition: lob0lob.h:428
#define DICT_TF_HAS_ATOMIC_BLOBS(flags)
Return the value of the ATOMIC_BLOBS field.
Definition: dict0mem.h:230
bool is_delete_marked() const
Definition: lob0lob.h:1407
static bool use_single_z_stream(ulint len)
For compressed LOB, if the length is less than or equal to Z_CHUNK_SIZE then use the older single z s...
Definition: lob0lob.h:221
const big_rec_t * get_big_rec_vec()
Get the vector containing fields to be stored externally.
Definition: lob0lob.h:1094
ulint m_remaining
Bytes yet to be read.
Definition: lob0lob.h:1332
Definition: buf0buf.h:1164
const page_size_t fil_space_get_page_size(space_id_t space_id, bool *found)
Returns the page size of the space and whether it is compressed or not.
Definition: fil0fil.cc:3630
Definition: mtr0types.h:275
Storage format for overflow data in a big record, that is, a clustered index record which needs exter...
Definition: data0data.h:859
uint32 space_id_t
Tablespace identifier.
Definition: api0api.h:59
rec_t * rec
pointer to a record on page
Definition: page0cur.h:316
rec_t * m_rec
Definition: lob0lob.h:1071
#define ut_ad(EXPR)
Debug assertion.
Definition: ut0dbg.h:66
byte * btr_rec_copy_externally_stored_field_func(trx_t *trx, const dict_index_t *index, const rec_t *rec, const ulint *offsets, const page_size_t &page_size, ulint no, ulint *len, size_t *lob_version, bool is_sdi, mem_heap_t *heap)
Copies an externally stored field of a record to mem heap.
Definition: lob0lob.cc:653
Information about data stored in one BLOB page.
Definition: lob0lob.h:1108
rec_t * rec() const
Get the clustered index record pointer.
Definition: lob0lob.h:815
void parse(ref_mem_t &obj) const
Parse the LOB reference object and copy data into the given ref_mem_t object.
Definition: lob0lob.h:250
#define TRUE
Definition: types.h:50
bool m_rollback
Is this operation part of rollback?
Definition: lob0lob.h:1442
void commit_btr_mtr()
Commit the mini-transaction that is holding the latches of the clustered index record block...
Definition: lob0lob.h:887
dberr_t btr_store_big_rec_extern_fields(trx_t *trx, btr_pcur_t *pcur, const upd_t *upd, ulint *offsets, const big_rec_t *big_rec_vec, mtr_t *btr_mtr, opcode op)
Stores the fields in big_rec_vec to the tablespace and puts pointers to them in rec.
Definition: lob0lob.cc:410
InsertContext(const BtrContext &btr_ctx, const big_rec_t *big_rec_vec)
Constructor.
Definition: lob0lob.h:1089
void page_zip_write_blob_ptr(page_zip_des_t *page_zip, const byte *rec, const dict_index_t *index, const ulint *offsets, ulint n, mtr_t *mtr)
Write a BLOB pointer of a record on the leaf page of a clustered index.
Definition: page0zip.cc:1886
Page size descriptor.
Definition: page0size.h:49
ib_id_t trx_id_t
Transaction identifier (DB_TRX_ID, DATA_TRX_ID)
Definition: trx0types.h:144
space_id_t space_id() const
Read the space id from the blob reference.
Definition: lob0lob.h:408
page_size_t get_page_size() const
Obtain the page size from the tablespace flags.
Definition: lob0lob.h:1452
bool is_null() const
Check if the field reference is made of zeroes.
Definition: lob0lob.h:290
UNIV_INLINE buf_frame_t * buf_block_get_frame(const buf_block_t *block)
Gets a pointer to the memory frame of a block.
bool is_bulk() const
Determine whether current operation is a bulk insert operation.
Definition: lob0lob.h:875
page_no_t m_page_no
The page number obtained from the blob reference.
Definition: lob0lob.h:1273
UNIV_INLINE ibool rec_offs_validate(const rec_t *rec, const dict_index_t *index, const ulint *offsets)
Validates offsets returned by rec_get_offsets().
Definition: rec.h:497
void set_null(mtr_t *mtr)
Set the external field reference to null.
Definition: lob0lob.h:277
void store_position()
Save the position of the persistent cursor.
Definition: lob0lob.h:1037
#define BTR_EXTERN_FIELD_REF_SIZE
The size of a reference to data stored on a different page.
Definition: btr0types.h:60
void clear()
Clear the contents of this blob directory.
Definition: lob0lob.h:1169
static space_id_t space_id(const byte *ref)
Read the space id from the given blob reference.
Definition: lob0lob.h:395
bool btr_lob_op_is_update(opcode op)
Determine if an operation on off-page columns is an update.
Definition: lob0lob.h:1464
The context information for reading a single BLOB.
Definition: lob0lob.h:1193
bool m_owner
Whether the clustered index record owns this LOB.
Definition: lob0lob.h:170
byte * m_ref
Pointing to a memory of size BTR_EXTERN_FIELD_REF_SIZE.
Definition: lob0lob.h:537
bool has_atomic_blobs() const
Check if tablespace supports atomic blobs.
Definition: lob0lob.h:1401
#define btr_pcur_store_position(p, m)
Definition: btr0pcur.h:97
static page_no_t page_no(const byte *ref)
Read the page no from the blob reference.
Definition: lob0lob.h:401
void set_owner(bool owner, mtr_t *mtr)
Set the ownership flag in the blob reference.
Definition: lob0lob.h:314
ulint m_rec_offset
Record offset within the page.
Definition: lob0lob.h:1077
uint32_t fil_space_get_flags(space_id_t space_id)
Returns the flags of the space.
Definition: fil0fil.cc:3563
bool m_null
Whether the LOB is null.
Definition: lob0lob.h:167
Modification log for online index creation and online table rebuild.
int n
Definition: xcom_base.cc:442
bool validate(mtr_t *mtr)
Check if the given mtr has necessary latches to update this LOB reference.
Definition: lob0lob.h:482
dict_index_t * m_index
Definition: lob0lob.h:1278
UNIV_INLINE page_no_t page_get_page_no(const page_t *page)
Gets the page number.
UNIV_INLINE uint32_t mach_read_from_4(const byte *b)
The following function is used to fetch data from 4 consecutive bytes.
void set_inherited(bool inherited, mtr_t *mtr)
Set the inherited flag in the field reference.
Definition: lob0lob.h:365
byte page_t
Type of the index page.
Definition: page0types.h:148
ulint btr_copy_externally_stored_field_prefix_func(trx_t *trx, const dict_index_t *index, byte *buf, ulint len, const page_size_t &page_size, const byte *data, bool is_sdi, ulint local_len)
Copies the prefix of an externally stored field of a record.
Definition: lob0lob.cc:831
ulint m_len
Length of the buffer m_buf.
Definition: lob0lob.h:1267
byte * m_buf
Buffer into which data is read.
Definition: lob0lob.h:1264
bool is_ref_valid() const
Definition: lob0lob.h:1391
buf_block_t * memo_contains_page_flagged(const byte *ptr, ulint flags) const
Check if memo contains the given page.
Definition: mtr0mtr.cc:993
ulint get_big_rec_vec_size()
Get the size of vector containing fields to be stored externally.
Definition: lob0lob.h:1098
const ulint LOB_HDR_SIZE
Size of an uncompressed LOB page header, in bytes.
Definition: lob0lob.h:147
Definition: mtr0types.h:273
bool is_null_relaxed() const
Check if the field reference is made of zeroes except the being_modified bit.
Definition: lob0lob.h:284
const ulint BTR_EXTERN_PAGE_NO
page number where stored
Definition: lob0lob.h:104
constexpr page_no_t FIL_NULL
&#39;null&#39; (undefined) page offset in the context of file spaces
Definition: fil0fil.h:928
bool is_purged() const
Check if the LOB has already been purged.
Definition: lob0lob.h:184
The context for a LOB operation.
Definition: lob0lob.h:1085
Provides the large objects (LOB) module.
Definition: lob0del.h:31
byte * get_field_ref(ulint field_no) const
Get the LOB reference for the given field number.
Definition: lob0lob.h:823
void set_length(const ulint len, mtr_t *mtr)
Set the length of blob in the external field reference.
Definition: lob0lob.h:467
std::vector< blob_page_info_t >::const_iterator const_iterator
Definition: lob0lob.h:1161
void purge(lob::DeleteContext *ctx, dict_index_t *index, trx_id_t trxid, undo_no_t undo_no, ulint rec_type, const upd_field_t *uf)
Purge an LOB (either of compressed or uncompressed).
Definition: lob0purge.cc:380
buf_page_t * m_bpage
Definition: lob0lob.h:1344
one byte is written
Definition: mtr0types.h:69
mtr_log_t get_log_mode()
Get the log mode of the btr mtr.
Definition: lob0lob.h:1007
static int flags[50]
Definition: hp_test1.cc:39
Index page cursor.
Definition: page0cur.h:314
#define btr_pcur_get_block(p)
Definition: btr0pcur.h:119
unsigned char byte
Blob class.
Definition: common.h:159
const page_t * rec_frame() const
Get the beginning of the B-tree clustered index page frame that contains the current clustered index ...
Definition: lob0lob.h:880
bool are_all_blobrefs_valid() const
Check to see if all pointers to externally stored columns in the record must be valid.
Definition: lob0lob.h:851
4 bytes ...
Definition: mtr0types.h:75
void reset()
Re-initialize the current object.
Definition: lob0lob.h:1119
dict_index_t * index()
Definition: lob0lob.h:1246
Mini-transaction handle and buffer.
Definition: mtr0mtr.h:169
dberr_t update(trx_t *trx, dict_index_t *index, const upd_t *upd, ulint field_no)
Update a portion of the given LOB.
Store off-page columns for a freshly inserted record.
Definition: lob0lob.h:553
uint32_t version() const
Read the LOB version from the blob reference.
Definition: lob0lob.h:422
ulint m_lob_version
Definition: lob0lob.h:1280
page_t * page_align() const
Get the start of a page containing this blob reference.
Definition: lob0lob.h:474
#define mtr_memo_contains_flagged(m, p, l)
Definition: mtr0mtr.h:137
uint32_t buf_fix_count
Count of how manyfold this block is currently bufferfixed.
Definition: buf0buf.h:1189
bool is_equal(const byte *ptr) const
Check whether the stored external field reference is equal to the given field reference.
Definition: lob0lob.h:269
void unmark_extern_fields()
Marks all extern fields in a record as owned by the record.
Definition: lob0lob.h:755
byte * btr_copy_externally_stored_field_func(trx_t *trx, const dict_index_t *index, ulint *len, size_t *lob_version, const byte *data, const page_size_t &page_size, ulint local_len, bool is_sdi, mem_heap_t *heap)
Copies an externally stored field of a record to mem heap.
Definition: lob0lob.cc:929
z_stream m_stream
The zlib stream used to uncompress while fetching blob.
Definition: lob0lob.h:1335
ib_id_t undo_no_t
Undo number.
Definition: trx0types.h:148
bool length(const dd::Spatial_reference_system *srs, const Geometry *g1, double *length, bool *null) noexcept
Computes the length of linestrings and multilinestrings.
Definition: length.cc:75
The struct &#39;lob::ref_t&#39; represents an external field reference.
Definition: lob0lob.h:197
void print(trx_t *trx, dict_index_t *index, std::ostream &out, ref_t ref, bool fatal)
Print information about the given LOB.
Definition: lob0impl.cc:1340
constexpr size_t FIELD_REF_SIZE
Definition: page0size.h:38
bool validate_blobref() const
Validate the LOB reference object.
Definition: lob0lob.h:1418
const ulint BTR_EXTERN_OWNER_FLAG
The most significant bit of BTR_EXTERN_LEN (i.e., the most significant bit of the byte at smallest ad...
Definition: lob0lob.h:122
const page_size_t & m_page_size
The page size information.
Definition: lob0lob.h:1249
Dialog Client Authentication nullptr
Definition: dialog.cc:353
ReadContext(const page_size_t &page_size, const byte *data, ulint prefix_len, byte *buf, ulint len, bool is_sdi)
Constructor.
Definition: lob0lob.h:1209
Definition: row0upd.h:580
const uint ZLOB_PAGE_DATA
Start of the data on an LOB page.
Definition: lob0lob.h:150
static bool is_big(const page_size_t &page_size, const ulint lob_length)
Check if this LOB is big enough to do partial update.
Definition: lob0lob.h:227
Data structure for an index.
Definition: dict0mem.h:886