MySQL 9.0.0
Source Code Documentation
ddl0impl-builder.h
Go to the documentation of this file.
1/*****************************************************************************
2
3Copyright (c) 2020, 2024, Oracle and/or its affiliates.
4
5This program is free software; you can redistribute it and/or modify it under
6the terms of the GNU General Public License, version 2.0, as published by the
7Free Software Foundation.
8
9This program is designed to work with certain software (including
10but not limited to OpenSSL) that is licensed under separate terms,
11as designated in a particular file or component or in included license
12documentation. The authors of MySQL hereby grant you an additional
13permission to link the program and your derivative works with the
14separately licensed software that they have either included with
15the program or referenced in the documentation.
16
17This program is distributed in the hope that it will be useful, but WITHOUT
18ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
19FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0,
20for more details.
21
22You should have received a copy of the GNU General Public License along with
23this program; if not, write to the Free Software Foundation, Inc.,
2451 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
25
26*****************************************************************************/
27
28/** @file include/ddl0impl-builder.h
29 DDL index builder data interface.
30 Created 2020-11-01 by Sunny Bains. */
31
32#ifndef ddl0impl_builder_h
33#define ddl0impl_builder_h
34
35#include "btr0load.h"
36#include "ddl0impl-buffer.h"
38#include "row0pread.h"
39
40namespace ddl {
41
42// Forward declaration.
43struct Copy_ctx;
44struct File_cursor;
45class RTree_inserter;
46
47/** For loading indexes. */
48struct Builder {
49 /** Build phase/states. */
50 enum class State : uint8_t {
51 /** Initial phase. */
52 INIT,
53
54 /** Collect the rows for the index to build. */
55 ADD,
56
57 /** Setup the merge sort and add the tasks to the task queue. */
59
60 /** Sort the collected rows, if required. The builder moves to state
61 BTREE_BUILD after all sort tasks are completed successfully or there
62 was an error during the sort phase. */
63 SORT,
64
65 /** Build the btree. */
67
68 /** FTS sort and build, this is done in one "step" */
70
71 /** Finish the loading of the index. */
72 FINISH,
73
74 /** Stop on success. */
75 STOP,
76
77 /** Stop on error. */
78 ERROR
79 };
80
81 /** Constructor.
82 @param[in,out] ctx DDL context.
83 @param[in,out] loader Owner of the instance.
84 @param[in] i Index ordinal value. */
85 Builder(ddl::Context &ctx, Loader &loader, size_t i) noexcept;
86
87 /** Destructor/ */
88 ~Builder() noexcept;
89
90 /** @return the error status. */
91 dberr_t get_error() const noexcept { return m_ctx.get_error(); }
92
93 /** Set the error code.
94 @param[in] err Error code to set. */
95 void set_error(dberr_t err) noexcept { m_ctx.set_error(err, m_id); }
96
97 /** @return the instance ID. */
98 [[nodiscard]] size_t id() const noexcept { return m_id; }
99
100 /** @return the index being built. */
101 [[nodiscard]] dict_index_t *index() noexcept { return m_sort_index; }
102
103 /** @return the DDL context. */
104 Context &ctx() noexcept { return m_ctx; }
105
106 /** Parallel scan thread spawn failed, release the extra thread states. */
107 void fallback_to_single_thread() noexcept;
108
109 /** @return true if the index is a spatial index. */
110 [[nodiscard]] bool is_spatial_index() const noexcept {
112 }
113
114 /** @return true if the index is an FTS index. */
115 [[nodiscard]] bool is_fts_index() const noexcept {
116 return m_index->type & DICT_FTS;
117 }
118
119 /** @return true if the index is a unique index. */
120 [[nodiscard]] bool is_unique_index() const noexcept {
121 ut_a(!is_fts_index());
123 }
124
125 /** @return the current builder state. */
126 [[nodiscard]] State get_state() const noexcept {
127 return m_state.load(std::memory_order_seq_cst);
128 }
129
130 /** Set the next state.
131 @param[in] state State to set. */
132 void set_state(State state) noexcept {
133 m_state.store(state, std::memory_order_seq_cst);
134 }
135
136 /** @return the PFS instance that is used to report progress (or nullptr). */
137 Alter_stage *stage() noexcept { return m_local_stage; }
138
139 /** Set the next state. */
140 void set_next_state() noexcept;
141
142 /** Initialize the cursor.
143 @param[in,out] cursor Cursor to initialize.
144 @param[in] n_threads Number of threads used for reading.
145 @return DB_SUCCESS or error code. */
146 [[nodiscard]] dberr_t init(Cursor &cursor, size_t n_threads) noexcept;
147
148 /** Add a row to the merge buffer.
149 @param[in,out] cursor Current scan cursor.
150 @param[in,out] row Row to add.
151 @param[in] thread_id ID of current thread.
152 @param[in,out] latch_release Called when a log free check is required.
153 @return DB_SUCCESS or error code. */
154 [[nodiscard]] dberr_t add_row(Cursor &cursor, Row &row, size_t thread_id,
155 Latch_release &&latch_release) noexcept;
156
157 /** @return true if file sorting can be skipped. */
158 bool is_skip_file_sort() const noexcept {
160 }
161
162 /** FTS: Sort and insert the rows read.
163 @return DB_SUCCESS or error code. */
164 [[nodiscard]] dberr_t fts_sort_and_build() noexcept;
165
166 /** Non-FTS: Sort the rows read.
167 @return DB_SUCCESS or error code. */
168 [[nodiscard]] dberr_t setup_sort() noexcept;
169
170 /** Non-FTS: Sort the rows read.
171 @param[in] thread_id Thread state ID.
172 @return DB_SUCCESS or error code. */
173 [[nodiscard]] dberr_t merge_sort(size_t thread_id) noexcept;
174
175 /** Load the sorted data into the B+Tree.
176 @return DB_SUCESS or error code. */
177 [[nodiscard]] dberr_t btree_build() noexcept;
178
179 /** Close temporary files, Flush all dirty pages, apply the row log
180 and write the redo log record.
181 @return DB_SUCCESS or error code. */
182 dberr_t finish() noexcept;
183
184 /** Copy blobs to the tuple.
185 @param[out] dtuple Tuple to copy to.
186 @param[in,out] offsets Column offsets in the row.
187 @param[in] mrec Current row.
188 @param[in,out] heap Heap for the allocating tuple memory.
189 @return DB_SUCCESS or error code. */
190 [[nodiscard]] dberr_t dtuple_copy_blobs(dtuple_t *dtuple, ulint *offsets,
191 const mrec_t *mrec,
192 mem_heap_t *heap) noexcept;
193
194 /** Write data to disk - in append mode. Increment the file size.
195 @param[in,out] file File handle.
196 @param[in] file_buffer Write the buffer contents to disk.
197 @return DB_SUCCESS or error code. */
198 [[nodiscard]] dberr_t append(ddl::file_t &file,
199 IO_buffer file_buffer) noexcept;
200
201 /** @return the path for temporary files. */
202 const char *tmpdir() const noexcept { return m_tmpdir; }
203
204 /** Insert cached rows.
205 @param[in] thread_id Insert cached rows for this thread ID.
206 @param[in,out] latch_release Called when a log free check is required.
207 @return DB_SUCCESS or error number */
208 [[nodiscard]] dberr_t batch_insert(size_t thread_id,
209 Latch_release &&latch_release) noexcept;
210
211 /** Note that the latches are going to be released. Do a deep copy of the
212 tuples that are being inserted in batches by batch_insert
213 @param[in] thread_id Deep copy cached rows for this thread ID. */
214 void batch_insert_deep_copy_tuples(size_t thread_id) noexcept;
215
216 /** Check the state of the online build log for the index.
217 @return DB_SUCCESS or error code. */
218 [[nodiscard]] dberr_t check_state_of_online_build_log() noexcept;
219
220 /** Write an MLOG_INDEX_LOAD record to indicate in the redo-log
221 that redo-logging of individual index pages was disabled, and
222 the flushing of such pages to the data files was completed.
223 @param[in] index Index on which redo logging was disabled */
224 static void write_redo(const dict_index_t *index) noexcept;
225
226 private:
227 /** State of a cluster index reader thread. */
228 struct Thread_ctx {
229 /** Constructor.
230 @param[in] id Thread state ID.
231 @param[in,out] key_buffer Buffer for building the target index. Note, the
232 thread state will own the key buffer and is
233 responsible for deleting it. */
234 explicit Thread_ctx(size_t id, Key_sort_buffer *key_buffer) noexcept;
235
236 /** Destructor. */
237 ~Thread_ctx() noexcept;
238
239 /** Thread ID. */
240 size_t m_id{};
241
242 /** Key sort buffer. */
243 Key_sort_buffer *m_key_buffer{};
244
245 /** Total number of records added to the key sort buffer. */
246 size_t m_n_recs{};
247
248 /** Merge file handle. */
249 ddl::file_t m_file{};
250
251 /** Buffer to use for file writes. */
253
254 /** Buffer to use for file writes. */
256
257 /** Record list starting offset in the output file. */
258 Merge_offsets m_offsets{};
259
260 /** For spatial/Rtree rows handling. */
261 RTree_inserter *m_rtree_inserter{};
262 };
263
265 using Thread_ctxs = std::vector<Thread_ctx *, Allocator>;
266
267 /** Create the tasks to merge Sort the file before we load the file into
268 the Btree index.
269 @return DB_SUCCESS or error code. */
270 [[nodiscard]] dberr_t create_merge_sort_tasks() noexcept;
271
272 /** Flush all dirty pages, apply the row log and write the redo log record.
273 @return DB_SUCCESS or error code. */
274 dberr_t finalize() noexcept;
275
276 /** Convert the field data from compact to redundant format.
277 @param[in] clust_index Clustered index being built
278 @param[in] row_field Field to copy from
279 @param[out] field Field to copy to
280 @param[in] len Length of the field data
281 @param[in] page_size Compressed BLOB page size
282 @param[in] is_sdi true for SDI Indexes
283 @param[in,out] heap Memory heap where to allocate
284 data when converting to ROW_FORMAT=REDUNDANT,
285 or nullptr */
286 static void convert(const dict_index_t *clust_index,
287 const dfield_t *row_field, dfield_t *field, ulint len,
288 const page_size_t &page_size,
289 IF_DEBUG(bool is_sdi, ) mem_heap_t *heap) noexcept;
290
291 /** Copy externally stored columns to the data tuple.
292 @param[in] index Index dictionary object.
293 @param[in] mrec Record containing BLOB pointers, or
294 nullptr to use tuple instead.
295 @param[in] offsets Offsets of mrec.
296 @param[in] page_size Compressed page size in bytes, or 0
297 @param[in,out] tuple Data tuple.
298 @param[in] is_sdi True for SDI Indexes
299 @param[in,out] heap Memory heap */
300 static void copy_blobs(const dict_index_t *index, const mrec_t *mrec,
301 const ulint *offsets, const page_size_t &page_size,
302 dtuple_t *tuple,
303 IF_DEBUG(bool is_sdi, ) mem_heap_t *heap) noexcept;
304
305 /** Cache a row for batch inserts. Currently used by spatial indexes.
306 @param[in,out] row Row to add.
307 @param[in] thread_id ID of current thread.
308 @return DB_SUCCESS or error code. */
309 [[nodiscard]] dberr_t batch_add_row(Row &row, size_t thread_id) noexcept;
310
311 /** Add a row to the merge buffer.
312 @param[in,out] cursor Current scan cursor.
313 @param[in,out] row Row to add.
314 @param[in] thread_id ID of current thread.
315 @param[in,out] latch_release Called when a log free check is required.
316 @return DB_SUCCESS or error code. */
317 [[nodiscard]] dberr_t bulk_add_row(Cursor &cursor, Row &row, size_t thread_id,
318 Latch_release &&latch_release) noexcept;
319
320 /** Clear the heap used for virtual columns. */
321 void clear_virtual_heap() noexcept { m_v_heap.clear(); }
322
323 /** Add the FTS document ID to the destination field.
324 @param[in,out] dst Field to write to.
325 @param[in] src Field to copy meta data from.
326 @param[out] write_doc_id Buffer for copying the doc id. */
327 void fts_add_doc_id(dfield_t *dst, const dict_field_t *src,
328 doc_id_t &write_doc_id) noexcept;
329
330 /** Add a row to the write buffer.
331 @param[in,out] ctx Copy context.
332 @param[in,out] mv_rows_added Number of multi-value rows added.
333 @return DB_SUCCESS or error code. */
334 [[nodiscard]] dberr_t copy_row(Copy_ctx &ctx, size_t &mv_rows_added) noexcept;
335
336 /** Setup the virtual column src column.
337 @param[in,out] ctx Copy context.
338 @param[in] ifield Index field.
339 @param[in] col Table column.
340 @param[out] src_field Computed value.
341 @param[in,out] mv_rows_added Number of multi-value rows added.
342 @return DB_SUCCESS or error code. */
343 [[nodiscard]] dberr_t get_virtual_column(Copy_ctx &ctx,
344 const dict_field_t *ifield,
345 dict_col_t *col,
346 dfield_t *&src_field,
347 size_t &mv_rows_added) noexcept;
348
349 /** Copy the FTS columns.
350 @param[in,out] ctx Copy context.
351 @param[in,out] field Field to write to.
352 @return DB_SUCCESS or error code. */
353 [[nodiscard]] dberr_t copy_fts_column(Copy_ctx &ctx,
354 dfield_t *field) noexcept;
355
356 /** Copy the columns to the temporary file buffer.
357 @param[in,out] ctx Copy context.
358 @param[in,out] mv_rows_added Multi value rows added.
359 @param[in,out] write_doc_id Buffer for storing the FTS doc ID.
360 @return DB_SUCCESS or error code. */
361 [[nodiscard]] dberr_t copy_columns(Copy_ctx &ctx, size_t &mv_rows_added,
362 doc_id_t &write_doc_id) noexcept;
363
364 /** Add row to the key buffer.
365 @param[in,out] ctx Copy context.
366 @param[in,out] mv_rows_added Number of multi-value index rows added.
367 @return DB_SUCCESS or error code. */
368 [[nodiscard]] dberr_t add_to_key_buffer(Copy_ctx &ctx,
369 size_t &mv_rows_added) noexcept;
370
371 /** Wait for FTS completion.
372 @param[in] index Index being built. */
374
375 /** Sort the data in the key buffer.
376 @param[in] thread_id Thread ID of current thread.
377 @return DB_SUCCESS or error code. */
378 [[nodiscard]] dberr_t key_buffer_sort(size_t thread_id) noexcept;
379
380 /** Sort the buffer in memory and insert directly in the BTree loader,
381 don't write to a temporary file.
382 @param[in,out] cursor Current scan cursor.
383 @param[in] thread_id ID of current thread.
384 @return DB_SUCCESS or error code. */
385 dberr_t insert_direct(Cursor &cursor, size_t thread_id) noexcept;
386
387 /** Create the merge file, if needed.
388 @param[in,out] file File handle.
389 @return true if file was opened successfully . */
390 [[nodiscard]] bool create_file(ddl::file_t &file) noexcept;
391
392 /** Check for duplicates in the first block
393 @param[in] dupcheck Files to check for duplicates.
394 @param[in,out] dup For collecting duplicate key information.
395 @return DB_SUCCESS or error code. */
396 [[nodiscard]] dberr_t check_duplicates(Thread_ctxs &dupcheck,
397 Dup *dup) noexcept;
398
399 /** Cleanup DDL after error in online build
400 Note: To be called if DDL must cleanup due to error in online build. Pages
401 which are buffer-fixed (in Page_load::release) until the next iteration, must
402 be unfixed (with Page_load::latch) before returning the error.
403 @note: Assumes that either m_btr_load->release is called before or
404 m_n_recs is 0 (no records are inserted yet).
405 @param[in] err Error hit in online build
406 @return the cursor error status. */
407 [[nodiscard]] dberr_t online_build_handle_error(dberr_t err) noexcept;
408
409 private:
410 /** Buffer ID. */
411 size_t m_id{};
412
413 /** Initial phase. */
414 std::atomic<State> m_state{State::INIT};
415
416 /** DDL Context. */
418
419 /** Loader that owns the instance. */
421
422 /** Index to create (if not FTS index). */
424
425 /** Temporary file path. */
426 const char *m_tmpdir{};
427
428 /** Per thread context. */
430
431 /** For tracking duplicates. */
433
434 /** For collecting duplicate entries (error reporting). */
436
437 /** Scoped virtual column heap. */
439
440 /** Scoped conversion heap. */
442
443 /** The index to be built, FTS or non-FTS. */
445
446 /** Number of active sort tasks. */
447 std::atomic<size_t> m_n_sort_tasks{};
448
449 /** Cluster index bulk load instance to use, direct insert without
450 a file sort. */
452
453 /** Stage per builder. */
455};
456
458 /** Default constructor. */
459 explicit Load_cursor(Builder *builder, Dup *dup) noexcept
460 : m_dup(dup), m_builder(builder) {}
461
462 /** Default destructor. */
463 virtual ~Load_cursor() override = default;
464
465 /** @return the cursor error status. */
466 [[nodiscard]] dberr_t get_err() const noexcept { return m_err; }
467
468 /** @return true if duplicates detected. */
469 [[nodiscard]] bool duplicates_detected() const noexcept override;
470
471 /** Duplicate checking and reporting. */
473
474 /** Operation error code. */
476
477 /** Index meta data. */
479
480 /** Heap for the raw row to dtuple_t conversion. */
482};
483
484/** Merge the sorted files. */
485struct Merge_cursor : public Load_cursor {
486 /** File cursors to use for the scan. */
487 using File_readers = std::vector<File_reader *, ut::allocator<File_reader *>>;
488
489 /** Constructor.
490 @param[in,out] builder Index builder.
491 @param[in,out] dup If not nullptr, then report duplicates.
492 @param[in,out] stage PFS stage monitoring. */
493 explicit Merge_cursor(Builder *builder, Dup *dup,
494 Alter_stage *stage) noexcept;
495
496 /** Destructor. */
497 ~Merge_cursor() noexcept override;
498
499 /** Add the cursor to use for merge load.
500 @param[in] file File to merge from.
501 @param[in] buffer_size IO buffer size to use for reading.
502 @return DB_SUCCESS or error code. */
503 [[nodiscard]] dberr_t add_file(const ddl::file_t &file,
504 size_t buffer_size) noexcept;
505
506 /** Add the cursor to use for merge load.
507 @param[in] file File file to read.
508 @param[in] buffer_size IO buffer size to use for reading.
509 @param[in] offset Page to read from.
510 @return DB_SUCCESS or error code. */
511 [[nodiscard]] dberr_t add_file(const ddl::file_t &file, size_t buffer_size,
512 os_offset_t offset) noexcept;
513
514 /** Open the cursor.
515 @return DB_SUCCESS or error code. */
516 [[nodiscard]] dberr_t open() noexcept;
517
518 /** Fetch the current row as a tuple. Note: Tuple columns are shallow copies.
519 @param[out] dtuple Row represented as a tuple.
520 @return DB_SUCCESS, DB_END_OF_INDEX or error code. */
521 [[nodiscard]] dberr_t fetch(dtuple_t *&dtuple) noexcept override;
522
523 /** Fetch the current row.
524 @param[out] mrec Current merge record.
525 @param[out] offsets Columns offsets inside mrec.
526 @return DB_SUCCESS, DB_END_OF_INDEX or error code. */
527 [[nodiscard]] dberr_t fetch(const mrec_t *&mrec, ulint *&offsets) noexcept;
528
529 /** Move to the next record.
530 @return DB_SUCCESS, DB_END_OF_INDEX or error code. */
531 [[nodiscard]] dberr_t next() noexcept override;
532
533 /** @return the file reader instances. */
534 [[nodiscard]] File_readers file_readers() noexcept;
535
536 /** Add the active cursors to the priority queue. */
537 void clear_eof() noexcept;
538
539 /** @return the number of active readers. */
540 [[nodiscard]] size_t size() const noexcept { return m_pq.size(); }
541
542 /** @return the number of rows read from the files. */
543 [[nodiscard]] uint64_t get_n_rows() const noexcept;
544
545 /** @return the number of cursors being merged. */
546 [[nodiscard]] size_t number_of_cursors() const noexcept {
547 return m_cursors.size();
548 }
549
550 private:
551 /** @return the current cursor at the head of the queue. */
552 [[nodiscard]] File_cursor *pop() noexcept;
553
554 private:
555 /** Comparator. */
556 struct Compare {
557 /** Constructor.
558 @param[in] index Index that the rows belong to.
559 @param[in,out] dup For reporting duplicates, can be nullptr. */
560 explicit Compare(const dict_index_t *index, Dup *dup)
561 : m_dup(dup), m_index(index) {}
562
563 /** Destructor. */
564 Compare() = default;
565
566 /** Compare the keys of two cursors.
567 @param[in] lhs Left hand side.
568 @param[in] rhs Right hand side.
569 @return true if lhs strictly less than rhs. */
570 bool operator()(const File_cursor *lhs,
571 const File_cursor *rhs) const noexcept;
572
573 /** For reporting duplicates. */
575
576 /** Index being built. */
577 const dict_index_t *m_index{};
578 };
579
580 /** File cursors to use for the scan. */
581 using File_cursors = std::vector<File_cursor *, ut::allocator<File_cursor *>>;
582
583 /** Priority queue for ordering the rows. */
584 using Queue = std::priority_queue<File_cursor *, File_cursors, Compare>;
585
586 /** Priority queue for merging the file cursors. */
588
589 /** Cursors to use for parallel loading of the index. */
591
592 /** Current cursor. */
594
595 /** PFS stage monitoring. */
597};
598
599} // namespace ddl
600
601#endif /* !ddl0impl_builder_h */
The B-tree bulk load.
static Mysys_charset_loader * loader
Definition: charset.cc:185
Class used to report ALTER TABLE progress via performance_schema.
Definition: ut0stage.h:81
Definition: btr0load.h:51
Implements a persistent FIFO using server List method names.
Definition: sql_profile.h:76
Build indexes on a table by reading a clustered index, creating a temporary file containing index ent...
Definition: ddl0impl-loader.h:44
Class that caches RTree index tuples made from a single cluster index page scan, and then insert into...
Definition: ddl0impl-rtree.h:44
Page size descriptor.
Definition: page0size.h:50
Allocator that allows std::* containers to manage their memory through ut::malloc* and ut::free libra...
Definition: ut0new.h:2182
dberr_t
Definition: db0err.h:39
@ DB_SUCCESS
Definition: db0err.h:43
DDL buffer infrastructure.
For scanning the temporary file.
static ulint dict_index_is_unique(const dict_index_t *index)
Check whether the index is unique.
static ulint dict_index_is_spatial(const dict_index_t *index)
Check whether the index is a Spatial Index.
constexpr uint32_t DICT_FTS
FTS index; can't be combined with the other flags.
Definition: dict0mem.h:104
uint64_t doc_id_t
Document id type.
Definition: fts0fts.h:79
static my_thread_id thread_id
Definition: my_thr_init.cc:63
static struct st_file_buffer file_buffer
Definition: myisampack.cc:209
The general architecture is that the work is done in two phases, roughly the read and write phase.
Definition: btr0load.cc:42
std::deque< os_offset_t, ut::allocator< os_offset_t > > Merge_offsets
Start offsets in the file, from where to merge records.
Definition: ddl0impl.h:64
byte mrec_t
Merge record in Aligned_buffer.
Definition: ddl0ddl.h:78
std::function< dberr_t()> Latch_release
Called when a log free check is required.
Definition: ddl0impl.h:50
std::pair< byte *, os_offset_t > IO_buffer
Block size for DDL I/O operations.
Definition: ddl0impl.h:47
Definition: os0file.h:89
static Value err()
Create a Value object that represents an error condition.
Definition: json_binary.cc:927
size_t buffer_size(const ConstBufferSequence &buffers) noexcept
Definition: buffer.h:313
std::conditional_t< !std::is_array< T >::value, std::unique_ptr< T, detail::Aligned_deleter< T > >, std::conditional_t< detail::is_unbounded_array_v< T >, std::unique_ptr< T, detail::Aligned_array_deleter< std::remove_extent_t< T > > >, void > > unique_ptr_aligned
The following is a common type that is returned by all the ut::make_unique_aligned (non-aligned) spec...
Definition: ut0new.h:2573
uint64_t os_offset_t
File offset in bytes.
Definition: os0file.h:87
Parallel read interface.
Interface to consume from.
Definition: btr0load.h:54
Heap wrapper that destroys the heap instance when it goes out of scope.
Definition: mem0mem.h:439
void clear() noexcept
Empty the heap.
Definition: mem0mem.h:479
State of a cluster index reader thread.
Definition: ddl0impl-builder.h:228
IO_buffer m_io_buffer
Buffer to use for file writes.
Definition: ddl0impl-builder.h:255
For loading indexes.
Definition: ddl0impl-builder.h:48
dberr_t finalize() noexcept
Flush all dirty pages, apply the row log and write the redo log record.
Definition: ddl0builder.cc:1948
dberr_t insert_direct(Cursor &cursor, size_t thread_id) noexcept
Sort the buffer in memory and insert directly in the BTree loader, don't write to a temporary file.
Definition: ddl0builder.cc:1238
dberr_t btree_build() noexcept
Load the sorted data into the B+Tree.
Definition: ddl0builder.cc:1751
void set_state(State state) noexcept
Set the next state.
Definition: ddl0impl-builder.h:132
std::atomic< size_t > m_n_sort_tasks
Number of active sort tasks.
Definition: ddl0impl-builder.h:447
void fts_wait_for_completion(const dict_index_t *index) noexcept
Wait for FTS completion.
dberr_t add_row(Cursor &cursor, Row &row, size_t thread_id, Latch_release &&latch_release) noexcept
Add a row to the merge buffer.
Definition: ddl0builder.cc:1574
void clear_virtual_heap() noexcept
Clear the heap used for virtual columns.
Definition: ddl0impl-builder.h:321
bool is_skip_file_sort() const noexcept
Definition: ddl0impl-builder.h:158
void set_next_state() noexcept
Set the next state.
Definition: ddl0builder.cc:2084
dberr_t fts_sort_and_build() noexcept
FTS: Sort and insert the rows read.
Definition: ddl0builder.cc:1923
static void write_redo(const dict_index_t *index) noexcept
Write an MLOG_INDEX_LOAD record to indicate in the redo-log that redo-logging of individual index pag...
Definition: ddl0builder.cc:1904
static void copy_blobs(const dict_index_t *index, const mrec_t *mrec, const ulint *offsets, const page_size_t &page_size, dtuple_t *tuple, bool is_sdi, mem_heap_t *heap) noexcept
Copy externally stored columns to the data tuple.
Definition: ddl0builder.cc:1597
bool is_spatial_index() const noexcept
Definition: ddl0impl-builder.h:110
bool create_file(ddl::file_t &file) noexcept
Create the merge file, if needed.
Definition: ddl0builder.cc:1160
dberr_t online_build_handle_error(dberr_t err) noexcept
Cleanup DDL after error in online build Note: To be called if DDL must cleanup due to error in online...
Definition: ddl0builder.cc:1221
std::atomic< State > m_state
Initial phase.
Definition: ddl0impl-builder.h:414
dberr_t key_buffer_sort(size_t thread_id) noexcept
Sort the data in the key buffer.
Definition: ddl0builder.cc:1201
static void convert(const dict_index_t *clust_index, const dfield_t *row_field, dfield_t *field, ulint len, const page_size_t &page_size, bool is_sdi, mem_heap_t *heap) noexcept
Convert the field data from compact to redundant format.
Definition: ddl0builder.cc:478
dberr_t create_merge_sort_tasks() noexcept
Create the tasks to merge Sort the file before we load the file into the Btree index.
Definition: ddl0builder.cc:1823
Context & ctx() noexcept
Definition: ddl0impl-builder.h:104
Alter_stage * m_local_stage
Stage per builder.
Definition: ddl0impl-builder.h:454
dberr_t check_state_of_online_build_log() noexcept
Check the state of the online build log for the index.
Definition: ddl0builder.cc:620
dberr_t dtuple_copy_blobs(dtuple_t *dtuple, ulint *offsets, const mrec_t *mrec, mem_heap_t *heap) noexcept
Copy blobs to the tuple.
Definition: ddl0builder.cc:1646
dberr_t init(Cursor &cursor, size_t n_threads) noexcept
Initialize the cursor.
Definition: ddl0builder.cc:630
Scoped_heap m_conv_heap
Scoped conversion heap.
Definition: ddl0impl-builder.h:441
void fallback_to_single_thread() noexcept
Parallel scan thread spawn failed, release the extra thread states.
Definition: ddl0builder.cc:2074
dict_index_t * m_sort_index
The index to be built, FTS or non-FTS.
Definition: ddl0impl-builder.h:444
Loader & m_loader
Loader that owns the instance.
Definition: ddl0impl-builder.h:420
const char * tmpdir() const noexcept
Definition: ddl0impl-builder.h:202
Builder(ddl::Context &ctx, Loader &loader, size_t i) noexcept
Constructor.
Definition: ddl0builder.cc:585
void set_error(dberr_t err) noexcept
Set the error code.
Definition: ddl0impl-builder.h:95
bool is_unique_index() const noexcept
Definition: ddl0impl-builder.h:120
std::vector< Thread_ctx *, Allocator > Thread_ctxs
Definition: ddl0impl-builder.h:265
dberr_t copy_fts_column(Copy_ctx &ctx, dfield_t *field) noexcept
Copy the FTS columns.
Definition: ddl0builder.cc:847
void fts_add_doc_id(dfield_t *dst, const dict_field_t *src, doc_id_t &write_doc_id) noexcept
Add the FTS document ID to the destination field.
Definition: ddl0builder.cc:766
Dup m_clust_dup
For collecting duplicate entries (error reporting).
Definition: ddl0impl-builder.h:435
dberr_t bulk_add_row(Cursor &cursor, Row &row, size_t thread_id, Latch_release &&latch_release) noexcept
Add a row to the merge buffer.
Definition: ddl0builder.cc:1424
dict_index_t * index() noexcept
Definition: ddl0impl-builder.h:101
ddl::Context & m_ctx
DDL Context.
Definition: ddl0impl-builder.h:417
State
Build phase/states.
Definition: ddl0impl-builder.h:50
@ STOP
Stop on success.
@ ADD
Collect the rows for the index to build.
@ SORT
Sort the collected rows, if required.
@ ERROR
Stop on error.
@ BTREE_BUILD
Build the btree.
@ SETUP_SORT
Setup the merge sort and add the tasks to the task queue.
@ FINISH
Finish the loading of the index.
@ FTS_SORT_AND_BUILD
FTS sort and build, this is done in one "step".
@ INIT
Initial phase.
void batch_insert_deep_copy_tuples(size_t thread_id) noexcept
Note that the latches are going to be released.
Definition: ddl0builder.cc:1195
dict_index_t * m_index
Index to create (if not FTS index).
Definition: ddl0impl-builder.h:423
dberr_t check_duplicates(Thread_ctxs &dupcheck, Dup *dup) noexcept
Check for duplicates in the first block.
Definition: ddl0builder.cc:1683
dberr_t merge_sort(size_t thread_id) noexcept
Non-FTS: Sort the rows read.
Definition: ddl0builder.cc:1984
~Builder() noexcept
Destructor/.
Definition: ddl0builder.cc:602
dberr_t append(ddl::file_t &file, IO_buffer file_buffer) noexcept
Write data to disk - in append mode.
Definition: ddl0builder.cc:1172
dberr_t finish() noexcept
Close temporary files, Flush all dirty pages, apply the row log and write the redo log record.
Definition: ddl0builder.cc:2037
dberr_t get_virtual_column(Copy_ctx &ctx, const dict_field_t *ifield, dict_col_t *col, dfield_t *&src_field, size_t &mv_rows_added) noexcept
Setup the virtual column src column.
Definition: ddl0builder.cc:783
dberr_t copy_columns(Copy_ctx &ctx, size_t &mv_rows_added, doc_id_t &write_doc_id) noexcept
Copy the columns to the temporary file buffer.
Definition: ddl0builder.cc:889
dberr_t add_to_key_buffer(Copy_ctx &ctx, size_t &mv_rows_added) noexcept
Add row to the key buffer.
Definition: ddl0builder.cc:1355
State get_state() const noexcept
Definition: ddl0impl-builder.h:126
dberr_t batch_add_row(Row &row, size_t thread_id) noexcept
Cache a row for batch inserts.
Definition: ddl0builder.cc:1328
dberr_t batch_insert(size_t thread_id, Latch_release &&latch_release) noexcept
Insert cached rows.
Definition: ddl0builder.cc:1185
size_t m_id
Buffer ID.
Definition: ddl0impl-builder.h:411
dberr_t copy_row(Copy_ctx &ctx, size_t &mv_rows_added) noexcept
Add a row to the write buffer.
Definition: ddl0builder.cc:1031
dberr_t setup_sort() noexcept
Non-FTS: Sort the rows read.
Definition: ddl0builder.cc:2021
dberr_t get_error() const noexcept
Definition: ddl0impl-builder.h:91
const char * m_tmpdir
Temporary file path.
Definition: ddl0impl-builder.h:426
Btree_load * m_btr_load
Cluster index bulk load instance to use, direct insert without a file sort.
Definition: ddl0impl-builder.h:451
Scoped_heap m_v_heap
Scoped virtual column heap.
Definition: ddl0impl-builder.h:438
dfield_t * m_prev_fields
For tracking duplicates.
Definition: ddl0impl-builder.h:432
size_t id() const noexcept
Definition: ddl0impl-builder.h:98
bool is_fts_index() const noexcept
Definition: ddl0impl-builder.h:115
Alter_stage * stage() noexcept
Definition: ddl0impl-builder.h:137
Thread_ctxs m_thread_ctxs
Per thread context.
Definition: ddl0impl-builder.h:429
DDL context/configuration.
Definition: ddl0ddl.h:321
void set_error(dberr_t err) noexcept
Set the error code, when it's not specific to an index.
Definition: ddl0ddl.h:427
bool m_skip_pk_sort
Skip the sorting phase if true.
Definition: ddl0ddl.h:631
dberr_t get_error() const noexcept
Definition: ddl0ddl.h:423
Context for copying cluster index row for the index to being created.
Definition: ddl0builder.cc:50
Cursor for reading the data.
Definition: ddl0impl-cursor.h:41
Structure for reporting duplicate records.
Definition: ddl0ddl.h:132
For loading a Btree index from a file.
Definition: ddl0builder.cc:177
Buffer for sorting in main memory.
Definition: ddl0impl-buffer.h:41
Definition: ddl0impl-builder.h:457
virtual ~Load_cursor() override=default
Default destructor.
Scoped_heap m_tuple_heap
Heap for the raw row to dtuple_t conversion.
Definition: ddl0impl-builder.h:481
Builder * m_builder
Index meta data.
Definition: ddl0impl-builder.h:478
dberr_t get_err() const noexcept
Definition: ddl0impl-builder.h:466
Load_cursor(Builder *builder, Dup *dup) noexcept
Default constructor.
Definition: ddl0impl-builder.h:459
bool duplicates_detected() const noexcept override
Definition: ddl0builder.cc:231
dberr_t m_err
Operation error code.
Definition: ddl0impl-builder.h:475
Dup * m_dup
Duplicate checking and reporting.
Definition: ddl0impl-builder.h:472
Comparator.
Definition: ddl0impl-builder.h:556
Compare(const dict_index_t *index, Dup *dup)
Constructor.
Definition: ddl0impl-builder.h:560
Compare()=default
Destructor.
Merge the sorted files.
Definition: ddl0impl-builder.h:485
std::vector< File_cursor *, ut::allocator< File_cursor * > > File_cursors
File cursors to use for the scan.
Definition: ddl0impl-builder.h:581
void clear_eof() noexcept
Add the active cursors to the priority queue.
Definition: ddl0builder.cc:381
dberr_t add_file(const ddl::file_t &file, size_t buffer_size) noexcept
Add the cursor to use for merge load.
Definition: ddl0builder.cc:351
std::vector< File_reader *, ut::allocator< File_reader * > > File_readers
File cursors to use for the scan.
Definition: ddl0impl-builder.h:487
size_t number_of_cursors() const noexcept
Definition: ddl0impl-builder.h:546
Merge_cursor(Builder *builder, Dup *dup, Alter_stage *stage) noexcept
Constructor.
Definition: ddl0builder.cc:320
dberr_t fetch(dtuple_t *&dtuple) noexcept override
Fetch the current row as a tuple.
Definition: ddl0builder.cc:429
~Merge_cursor() noexcept override
Destructor.
Definition: ddl0builder.cc:326
File_readers file_readers() noexcept
Definition: ddl0builder.cc:310
Queue m_pq
Priority queue for merging the file cursors.
Definition: ddl0impl-builder.h:587
File_cursor * pop() noexcept
Definition: ddl0builder.cc:417
dberr_t next() noexcept override
Move to the next record.
Definition: ddl0builder.cc:451
size_t size() const noexcept
Definition: ddl0impl-builder.h:540
Alter_stage * m_stage
PFS stage monitoring.
Definition: ddl0impl-builder.h:596
dberr_t open() noexcept
Open the cursor.
Definition: ddl0builder.cc:397
uint64_t get_n_rows() const noexcept
Definition: ddl0builder.cc:468
File_cursors m_cursors
Cursors to use for parallel loading of the index.
Definition: ddl0impl-builder.h:590
File_cursor * m_cursor
Current cursor.
Definition: ddl0impl-builder.h:593
Physical row context.
Definition: ddl0impl.h:121
Information about temporary files used in merge sort.
Definition: ddl0impl.h:67
Structure for an SQL data field.
Definition: data0data.h:605
Data structure for a column in a table.
Definition: dict0mem.h:489
Data structure for a field in an index.
Definition: dict0mem.h:895
Data structure for an index.
Definition: dict0mem.h:1046
unsigned type
index type (DICT_CLUSTERED, DICT_UNIQUE, DICT_IBUF, DICT_CORRUPT)
Definition: dict0mem.h:1073
bool is_clustered() const
Definition: dict0mem.h:1311
Structure for an SQL data tuple of fields (logical record)
Definition: data0data.h:684
The info structure stored at the beginning of a heap block.
Definition: mem0mem.h:302
#define IF_DEBUG(...)
Definition: univ.i:674
unsigned long int ulint
Definition: univ.i:406
#define ut_a(EXPR)
Abort execution if EXPR does not evaluate to nonzero.
Definition: ut0dbg.h:93