MySQL 8.3.0
Source Code Documentation
ddl0impl-builder.h
Go to the documentation of this file.
1/*****************************************************************************
2
3Copyright (c) 2020, 2023, Oracle and/or its affiliates.
4
5This program is free software; you can redistribute it and/or modify it under
6the terms of the GNU General Public License, version 2.0, as published by the
7Free Software Foundation.
8
9This program is also distributed with certain software (including but not
10limited to OpenSSL) that is licensed under separate terms, as designated in a
11particular file or component or in included license documentation. The authors
12of MySQL hereby grant you an additional permission to link the program and
13your derivative works with the separately licensed software that they have
14included with MySQL.
15
16This program is distributed in the hope that it will be useful, but WITHOUT
17ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0,
19for more details.
20
21You should have received a copy of the GNU General Public License along with
22this program; if not, write to the Free Software Foundation, Inc.,
2351 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
24
25*****************************************************************************/
26
27/** @file include/ddl0impl-builder.h
28 DDL index builder data interface.
29 Created 2020-11-01 by Sunny Bains. */
30
31#ifndef ddl0impl_builder_h
32#define ddl0impl_builder_h
33
34#include "btr0load.h"
35#include "ddl0impl-buffer.h"
37#include "row0pread.h"
38
39namespace ddl {
40
41// Forward declaration.
42struct Copy_ctx;
43struct File_cursor;
44class RTree_inserter;
45
46/** For loading indexes. */
47struct Builder {
48 /** Build phase/states. */
49 enum class State : uint8_t {
50 /** Initial phase. */
51 INIT,
52
53 /** Collect the rows for the index to build. */
54 ADD,
55
56 /** Setup the merge sort and add the tasks to the task queue. */
58
59 /** Sort the collected rows, if required. The builder moves to state
60 BTREE_BUILD after all sort tasks are completed successfully or there
61 was an error during the sort phase. */
62 SORT,
63
64 /** Build the btree. */
66
67 /** FTS sort and build, this is done in one "step" */
69
70 /** Finish the loading of the index. */
71 FINISH,
72
73 /** Stop on success. */
74 STOP,
75
76 /** Stop on error. */
77 ERROR
78 };
79
80 /** Constructor.
81 @param[in,out] ctx DDL context.
82 @param[in,out] loader Owner of the instance.
83 @param[in] i Index ordinal value. */
84 Builder(ddl::Context &ctx, Loader &loader, size_t i) noexcept;
85
86 /** Destructor/ */
87 ~Builder() noexcept;
88
89 /** @return the error status. */
90 dberr_t get_error() const noexcept { return m_ctx.get_error(); }
91
92 /** Set the error code.
93 @param[in] err Error code to set. */
94 void set_error(dberr_t err) noexcept { m_ctx.set_error(err, m_id); }
95
96 /** @return the instance ID. */
97 [[nodiscard]] size_t id() const noexcept { return m_id; }
98
99 /** @return the index being built. */
100 [[nodiscard]] dict_index_t *index() noexcept { return m_sort_index; }
101
102 /** @return the DDL context. */
103 Context &ctx() noexcept { return m_ctx; }
104
105 /** Parallel scan thread spawn failed, release the extra thread states. */
106 void fallback_to_single_thread() noexcept;
107
108 /** @return true if the index is a spatial index. */
109 [[nodiscard]] bool is_spatial_index() const noexcept {
111 }
112
113 /** @return true if the index is an FTS index. */
114 [[nodiscard]] bool is_fts_index() const noexcept {
115 return m_index->type & DICT_FTS;
116 }
117
118 /** @return true if the index is a unique index. */
119 [[nodiscard]] bool is_unique_index() const noexcept {
120 ut_a(!is_fts_index());
122 }
123
124 /** @return the current builder state. */
125 [[nodiscard]] State get_state() const noexcept {
126 return m_state.load(std::memory_order_seq_cst);
127 }
128
129 /** Set the next state.
130 @param[in] state State to set. */
131 void set_state(State state) noexcept {
132 m_state.store(state, std::memory_order_seq_cst);
133 }
134
135 /** @return the PFS instance that is used to report progress (or nullptr). */
136 Alter_stage *stage() noexcept { return m_local_stage; }
137
138 /** Set the next state. */
139 void set_next_state() noexcept;
140
141 /** Initialize the cursor.
142 @param[in,out] cursor Cursor to initialize.
143 @param[in] n_threads Number of threads used for reading.
144 @return DB_SUCCESS or error code. */
145 [[nodiscard]] dberr_t init(Cursor &cursor, size_t n_threads) noexcept;
146
147 /** Add a row to the merge buffer.
148 @param[in,out] cursor Current scan cursor.
149 @param[in,out] row Row to add.
150 @param[in] thread_id ID of current thread.
151 @param[in,out] latch_release Called when a log free check is required.
152 @return DB_SUCCESS or error code. */
153 [[nodiscard]] dberr_t add_row(Cursor &cursor, Row &row, size_t thread_id,
154 Latch_release &&latch_release) noexcept;
155
156 /** @return true if file sorting can be skipped. */
157 bool is_skip_file_sort() const noexcept {
159 }
160
161 /** FTS: Sort and insert the rows read.
162 @return DB_SUCCESS or error code. */
163 [[nodiscard]] dberr_t fts_sort_and_build() noexcept;
164
165 /** Non-FTS: Sort the rows read.
166 @return DB_SUCCESS or error code. */
167 [[nodiscard]] dberr_t setup_sort() noexcept;
168
169 /** Non-FTS: Sort the rows read.
170 @param[in] thread_id Thread state ID.
171 @return DB_SUCCESS or error code. */
172 [[nodiscard]] dberr_t merge_sort(size_t thread_id) noexcept;
173
174 /** Load the sorted data into the B+Tree.
175 @return DB_SUCESS or error code. */
176 [[nodiscard]] dberr_t btree_build() noexcept;
177
178 /** Close temporary files, Flush all dirty pages, apply the row log
179 and write the redo log record.
180 @return DB_SUCCESS or error code. */
181 dberr_t finish() noexcept;
182
183 /** Copy blobs to the tuple.
184 @param[out] dtuple Tuple to copy to.
185 @param[in,out] offsets Column offsets in the row.
186 @param[in] mrec Current row.
187 @param[in,out] heap Heap for the allocating tuple memory.
188 @return DB_SUCCESS or error code. */
189 [[nodiscard]] dberr_t dtuple_copy_blobs(dtuple_t *dtuple, ulint *offsets,
190 const mrec_t *mrec,
191 mem_heap_t *heap) noexcept;
192
193 /** Write data to disk - in append mode. Increment the file size.
194 @param[in,out] file File handle.
195 @param[in] file_buffer Write the buffer contents to disk.
196 @return DB_SUCCESS or error code. */
197 [[nodiscard]] dberr_t append(ddl::file_t &file,
198 IO_buffer file_buffer) noexcept;
199
200 /** @return the path for temporary files. */
201 const char *tmpdir() const noexcept { return m_tmpdir; }
202
203 /** Insert cached rows.
204 @param[in] thread_id Insert cached rows for this thread ID.
205 @param[in,out] latch_release Called when a log free check is required.
206 @return DB_SUCCESS or error number */
207 [[nodiscard]] dberr_t batch_insert(size_t thread_id,
208 Latch_release &&latch_release) noexcept;
209
210 /** Note that the latches are going to be released. Do a deep copy of the
211 tuples that are being inserted in batches by batch_insert
212 @param[in] thread_id Deep copy cached rows for this thread ID. */
213 void batch_insert_deep_copy_tuples(size_t thread_id) noexcept;
214
215 /** Check the state of the online build log for the index.
216 @return DB_SUCCESS or error code. */
217 [[nodiscard]] dberr_t check_state_of_online_build_log() noexcept;
218
219 /** Write an MLOG_INDEX_LOAD record to indicate in the redo-log
220 that redo-logging of individual index pages was disabled, and
221 the flushing of such pages to the data files was completed.
222 @param[in] index Index on which redo logging was disabled */
223 static void write_redo(const dict_index_t *index) noexcept;
224
225 private:
226 /** State of a cluster index reader thread. */
227 struct Thread_ctx {
228 /** Constructor.
229 @param[in] id Thread state ID.
230 @param[in,out] key_buffer Buffer for building the target index. Note, the
231 thread state will own the key buffer and is
232 responsible for deleting it. */
233 explicit Thread_ctx(size_t id, Key_sort_buffer *key_buffer) noexcept;
234
235 /** Destructor. */
236 ~Thread_ctx() noexcept;
237
238 /** Thread ID. */
239 size_t m_id{};
240
241 /** Key sort buffer. */
242 Key_sort_buffer *m_key_buffer{};
243
244 /** Total number of records added to the key sort buffer. */
245 size_t m_n_recs{};
246
247 /** Merge file handle. */
248 ddl::file_t m_file{};
249
250 /** Buffer to use for file writes. */
252
253 /** Buffer to use for file writes. */
255
256 /** Record list starting offset in the output file. */
257 Merge_offsets m_offsets{};
258
259 /** For spatial/Rtree rows handling. */
260 RTree_inserter *m_rtree_inserter{};
261 };
262
264 using Thread_ctxs = std::vector<Thread_ctx *, Allocator>;
265
266 /** Create the tasks to merge Sort the file before we load the file into
267 the Btree index.
268 @return DB_SUCCESS or error code. */
269 [[nodiscard]] dberr_t create_merge_sort_tasks() noexcept;
270
271 /** Flush all dirty pages, apply the row log and write the redo log record.
272 @return DB_SUCCESS or error code. */
273 dberr_t finalize() noexcept;
274
275 /** Convert the field data from compact to redundant format.
276 @param[in] clust_index Clustered index being built
277 @param[in] row_field Field to copy from
278 @param[out] field Field to copy to
279 @param[in] len Length of the field data
280 @param[in] page_size Compressed BLOB page size
281 @param[in] is_sdi true for SDI Indexes
282 @param[in,out] heap Memory heap where to allocate
283 data when converting to ROW_FORMAT=REDUNDANT,
284 or nullptr */
285 static void convert(const dict_index_t *clust_index,
286 const dfield_t *row_field, dfield_t *field, ulint len,
287 const page_size_t &page_size,
288 IF_DEBUG(bool is_sdi, ) mem_heap_t *heap) noexcept;
289
290 /** Copy externally stored columns to the data tuple.
291 @param[in] index Index dictionary object.
292 @param[in] mrec Record containing BLOB pointers, or
293 nullptr to use tuple instead.
294 @param[in] offsets Offsets of mrec.
295 @param[in] page_size Compressed page size in bytes, or 0
296 @param[in,out] tuple Data tuple.
297 @param[in] is_sdi True for SDI Indexes
298 @param[in,out] heap Memory heap */
299 static void copy_blobs(const dict_index_t *index, const mrec_t *mrec,
300 const ulint *offsets, const page_size_t &page_size,
301 dtuple_t *tuple,
302 IF_DEBUG(bool is_sdi, ) mem_heap_t *heap) noexcept;
303
304 /** Cache a row for batch inserts. Currently used by spatial indexes.
305 @param[in,out] row Row to add.
306 @param[in] thread_id ID of current thread.
307 @return DB_SUCCESS or error code. */
308 [[nodiscard]] dberr_t batch_add_row(Row &row, size_t thread_id) noexcept;
309
310 /** Add a row to the merge buffer.
311 @param[in,out] cursor Current scan cursor.
312 @param[in,out] row Row to add.
313 @param[in] thread_id ID of current thread.
314 @param[in,out] latch_release Called when a log free check is required.
315 @return DB_SUCCESS or error code. */
316 [[nodiscard]] dberr_t bulk_add_row(Cursor &cursor, Row &row, size_t thread_id,
317 Latch_release &&latch_release) noexcept;
318
319 /** Clear the heap used for virtual columns. */
320 void clear_virtual_heap() noexcept { m_v_heap.clear(); }
321
322 /** Add the FTS document ID to the destination field.
323 @param[in,out] dst Field to write to.
324 @param[in] src Field to copy meta data from.
325 @param[out] write_doc_id Buffer for copying the doc id. */
326 void fts_add_doc_id(dfield_t *dst, const dict_field_t *src,
327 doc_id_t &write_doc_id) noexcept;
328
329 /** Add a row to the write buffer.
330 @param[in,out] ctx Copy context.
331 @param[in,out] mv_rows_added Number of multi-value rows added.
332 @return DB_SUCCESS or error code. */
333 [[nodiscard]] dberr_t copy_row(Copy_ctx &ctx, size_t &mv_rows_added) noexcept;
334
335 /** Setup the virtual column src column.
336 @param[in,out] ctx Copy context.
337 @param[in] ifield Index field.
338 @param[in] col Table column.
339 @param[out] src_field Computed value.
340 @param[in,out] mv_rows_added Number of multi-value rows added.
341 @return DB_SUCCESS or error code. */
342 [[nodiscard]] dberr_t get_virtual_column(Copy_ctx &ctx,
343 const dict_field_t *ifield,
344 dict_col_t *col,
345 dfield_t *&src_field,
346 size_t &mv_rows_added) noexcept;
347
348 /** Copy the FTS columns.
349 @param[in,out] ctx Copy context.
350 @param[in,out] field Field to write to.
351 @return DB_SUCCESS or error code. */
352 [[nodiscard]] dberr_t copy_fts_column(Copy_ctx &ctx,
353 dfield_t *field) noexcept;
354
355 /** Copy the columns to the temporary file buffer.
356 @param[in,out] ctx Copy context.
357 @param[in,out] mv_rows_added Multi value rows added.
358 @param[in,out] write_doc_id Buffer for storing the FTS doc ID.
359 @return DB_SUCCESS or error code. */
360 [[nodiscard]] dberr_t copy_columns(Copy_ctx &ctx, size_t &mv_rows_added,
361 doc_id_t &write_doc_id) noexcept;
362
363 /** Add row to the key buffer.
364 @param[in,out] ctx Copy context.
365 @param[in,out] mv_rows_added Number of multi-value index rows added.
366 @return DB_SUCCESS or error code. */
367 [[nodiscard]] dberr_t add_to_key_buffer(Copy_ctx &ctx,
368 size_t &mv_rows_added) noexcept;
369
370 /** Wait for FTS completion.
371 @param[in] index Index being built. */
373
374 /** Sort the data in the key buffer.
375 @param[in] thread_id Thread ID of current thread.
376 @return DB_SUCCESS or error code. */
377 [[nodiscard]] dberr_t key_buffer_sort(size_t thread_id) noexcept;
378
379 /** Sort the buffer in memory and insert directly in the BTree loader,
380 don't write to a temporary file.
381 @param[in,out] cursor Current scan cursor.
382 @param[in] thread_id ID of current thread.
383 @return DB_SUCCESS or error code. */
384 dberr_t insert_direct(Cursor &cursor, size_t thread_id) noexcept;
385
386 /** Create the merge file, if needed.
387 @param[in,out] file File handle.
388 @return true if file was opened successfully . */
389 [[nodiscard]] bool create_file(ddl::file_t &file) noexcept;
390
391 /** Check for duplicates in the first block
392 @param[in] dupcheck Files to check for duplicates.
393 @param[in,out] dup For collecting duplicate key information.
394 @return DB_SUCCESS or error code. */
395 [[nodiscard]] dberr_t check_duplicates(Thread_ctxs &dupcheck,
396 Dup *dup) noexcept;
397
398 /** Cleanup DDL after error in online build
399 Note: To be called if DDL must cleanup due to error in online build. Pages
400 which are buffer-fixed (in Page_load::release) until the next iteration, must
401 be unfixed (with Page_load::latch) before returning the error.
402 @note: Assumes that either m_btr_load->release is called before or
403 m_n_recs is 0 (no records are inserted yet).
404 @param[in] err Error hit in online build
405 @return the cursor error status. */
406 [[nodiscard]] dberr_t online_build_handle_error(dberr_t err) noexcept;
407
408 private:
409 /** Buffer ID. */
410 size_t m_id{};
411
412 /** Initial phase. */
413 std::atomic<State> m_state{State::INIT};
414
415 /** DDL Context. */
417
418 /** Loader that owns the instance. */
420
421 /** Index to create (if not FTS index). */
423
424 /** Temporary file path. */
425 const char *m_tmpdir{};
426
427 /** Per thread context. */
429
430 /** For tracking duplicates. */
432
433 /** For collecting duplicate entries (error reporting). */
435
436 /** Scoped virtual column heap. */
438
439 /** Scoped conversion heap. */
441
442 /** The index to be built, FTS or non-FTS. */
444
445 /** Number of active sort tasks. */
446 std::atomic<size_t> m_n_sort_tasks{};
447
448 /** Cluster index bulk load instance to use, direct insert without
449 a file sort. */
451
452 /** Stage per builder. */
454};
455
457 /** Default constructor. */
458 explicit Load_cursor(Builder *builder, Dup *dup) noexcept
459 : m_dup(dup), m_builder(builder) {}
460
461 /** Default destructor. */
462 virtual ~Load_cursor() override = default;
463
464 /** @return the cursor error status. */
465 [[nodiscard]] dberr_t get_err() const noexcept { return m_err; }
466
467 /** @return true if duplicates detected. */
468 [[nodiscard]] bool duplicates_detected() const noexcept override;
469
470 /** Duplicate checking and reporting. */
472
473 /** Operation error code. */
475
476 /** Index meta data. */
478
479 /** Heap for the raw row to dtuple_t conversion. */
481};
482
483/** Merge the sorted files. */
484struct Merge_cursor : public Load_cursor {
485 /** File cursors to use for the scan. */
486 using File_readers = std::vector<File_reader *, ut::allocator<File_reader *>>;
487
488 /** Constructor.
489 @param[in,out] builder Index builder.
490 @param[in,out] dup If not nullptr, then report duplicates.
491 @param[in,out] stage PFS stage monitoring. */
492 explicit Merge_cursor(Builder *builder, Dup *dup,
493 Alter_stage *stage) noexcept;
494
495 /** Destructor. */
496 ~Merge_cursor() noexcept override;
497
498 /** Add the cursor to use for merge load.
499 @param[in] file File to merge from.
500 @param[in] buffer_size IO buffer size to use for reading.
501 @return DB_SUCCESS or error code. */
502 [[nodiscard]] dberr_t add_file(const ddl::file_t &file,
503 size_t buffer_size) noexcept;
504
505 /** Add the cursor to use for merge load.
506 @param[in] file File file to read.
507 @param[in] buffer_size IO buffer size to use for reading.
508 @param[in] offset Page to read from.
509 @return DB_SUCCESS or error code. */
510 [[nodiscard]] dberr_t add_file(const ddl::file_t &file, size_t buffer_size,
511 os_offset_t offset) noexcept;
512
513 /** Open the cursor.
514 @return DB_SUCCESS or error code. */
515 [[nodiscard]] dberr_t open() noexcept;
516
517 /** Fetch the current row as a tuple. Note: Tuple columns are shallow copies.
518 @param[out] dtuple Row represented as a tuple.
519 @return DB_SUCCESS, DB_END_OF_INDEX or error code. */
520 [[nodiscard]] dberr_t fetch(dtuple_t *&dtuple) noexcept override;
521
522 /** Fetch the current row.
523 @param[out] mrec Current merge record.
524 @param[out] offsets Columns offsets inside mrec.
525 @return DB_SUCCESS, DB_END_OF_INDEX or error code. */
526 [[nodiscard]] dberr_t fetch(const mrec_t *&mrec, ulint *&offsets) noexcept;
527
528 /** Move to the next record.
529 @return DB_SUCCESS, DB_END_OF_INDEX or error code. */
530 [[nodiscard]] dberr_t next() noexcept override;
531
532 /** @return the file reader instances. */
533 [[nodiscard]] File_readers file_readers() noexcept;
534
535 /** Add the active cursors to the priority queue. */
536 void clear_eof() noexcept;
537
538 /** @return the number of active readers. */
539 [[nodiscard]] size_t size() const noexcept { return m_pq.size(); }
540
541 /** @return the number of rows read from the files. */
542 [[nodiscard]] uint64_t get_n_rows() const noexcept;
543
544 /** @return the number of cursors being merged. */
545 [[nodiscard]] size_t number_of_cursors() const noexcept {
546 return m_cursors.size();
547 }
548
549 private:
550 /** @return the current cursor at the head of the queue. */
551 [[nodiscard]] File_cursor *pop() noexcept;
552
553 private:
554 /** Comparator. */
555 struct Compare {
556 /** Constructor.
557 @param[in] index Index that the rows belong to.
558 @param[in,out] dup For reporting duplicates, can be nullptr. */
559 explicit Compare(const dict_index_t *index, Dup *dup)
560 : m_dup(dup), m_index(index) {}
561
562 /** Destructor. */
563 Compare() = default;
564
565 /** Compare the keys of two cursors.
566 @param[in] lhs Left hand side.
567 @param[in] rhs Right hand side.
568 @return true if lhs strictly less than rhs. */
569 bool operator()(const File_cursor *lhs,
570 const File_cursor *rhs) const noexcept;
571
572 /** For reporting duplicates. */
574
575 /** Index being built. */
576 const dict_index_t *m_index{};
577 };
578
579 /** File cursors to use for the scan. */
580 using File_cursors = std::vector<File_cursor *, ut::allocator<File_cursor *>>;
581
582 /** Priority queue for ordering the rows. */
583 using Queue = std::priority_queue<File_cursor *, File_cursors, Compare>;
584
585 /** Priority queue for merging the file cursors. */
587
588 /** Cursors to use for parallel loading of the index. */
590
591 /** Current cursor. */
593
594 /** PFS stage monitoring. */
596};
597
598} // namespace ddl
599
600#endif /* !ddl0impl_builder_h */
The B-tree bulk load.
static Mysys_charset_loader * loader
Definition: charset.cc:184
Class used to report ALTER TABLE progress via performance_schema.
Definition: ut0stage.h:80
Definition: btr0load.h:50
Implements a persistent FIFO using server List method names.
Definition: sql_profile.h:75
Build indexes on a table by reading a clustered index, creating a temporary file containing index ent...
Definition: ddl0impl-loader.h:43
Class that caches RTree index tuples made from a single cluster index page scan, and then insert into...
Definition: ddl0impl-rtree.h:43
Page size descriptor.
Definition: page0size.h:49
Allocator that allows std::* containers to manage their memory through ut::malloc* and ut::free libra...
Definition: ut0new.h:2180
dberr_t
Definition: db0err.h:38
@ DB_SUCCESS
Definition: db0err.h:42
DDL buffer infrastructure.
For scanning the temporary file.
static ulint dict_index_is_unique(const dict_index_t *index)
Check whether the index is unique.
static ulint dict_index_is_spatial(const dict_index_t *index)
Check whether the index is a Spatial Index.
constexpr uint32_t DICT_FTS
FTS index; can't be combined with the other flags.
Definition: dict0mem.h:103
uint64_t doc_id_t
Document id type.
Definition: fts0fts.h:78
static my_thread_id thread_id
Definition: my_thr_init.cc:62
static struct st_file_buffer file_buffer
Definition: myisampack.cc:208
The general architecture is that the work is done in two phases, roughly the read and write phase.
Definition: btr0load.cc:41
std::deque< os_offset_t, ut::allocator< os_offset_t > > Merge_offsets
Start offsets in the file, from where to merge records.
Definition: ddl0impl.h:63
byte mrec_t
Merge record in Aligned_buffer.
Definition: ddl0ddl.h:77
std::function< dberr_t()> Latch_release
Called when a log free check is required.
Definition: ddl0impl.h:49
std::pair< byte *, os_offset_t > IO_buffer
Block size for DDL I/O operations.
Definition: ddl0impl.h:46
Definition: os0file.h:88
static Value err()
Create a Value object that represents an error condition.
Definition: json_binary.cc:926
size_t buffer_size(const ConstBufferSequence &buffers) noexcept
Definition: buffer.h:312
std::conditional_t< !std::is_array< T >::value, std::unique_ptr< T, detail::Aligned_deleter< T > >, std::conditional_t< detail::is_unbounded_array_v< T >, std::unique_ptr< T, detail::Aligned_array_deleter< std::remove_extent_t< T > > >, void > > unique_ptr_aligned
The following is a common type that is returned by all the ut::make_unique_aligned (non-aligned) spec...
Definition: ut0new.h:2571
uint64_t os_offset_t
File offset in bytes.
Definition: os0file.h:86
Parallel read interface.
Interface to consume from.
Definition: btr0load.h:53
Heap wrapper that destroys the heap instance when it goes out of scope.
Definition: mem0mem.h:438
void clear() noexcept
Empty the heap.
Definition: mem0mem.h:478
State of a cluster index reader thread.
Definition: ddl0impl-builder.h:227
IO_buffer m_io_buffer
Buffer to use for file writes.
Definition: ddl0impl-builder.h:254
For loading indexes.
Definition: ddl0impl-builder.h:47
dberr_t finalize() noexcept
Flush all dirty pages, apply the row log and write the redo log record.
Definition: ddl0builder.cc:1951
dberr_t insert_direct(Cursor &cursor, size_t thread_id) noexcept
Sort the buffer in memory and insert directly in the BTree loader, don't write to a temporary file.
Definition: ddl0builder.cc:1237
dberr_t btree_build() noexcept
Load the sorted data into the B+Tree.
Definition: ddl0builder.cc:1754
void set_state(State state) noexcept
Set the next state.
Definition: ddl0impl-builder.h:131
std::atomic< size_t > m_n_sort_tasks
Number of active sort tasks.
Definition: ddl0impl-builder.h:446
void fts_wait_for_completion(const dict_index_t *index) noexcept
Wait for FTS completion.
dberr_t add_row(Cursor &cursor, Row &row, size_t thread_id, Latch_release &&latch_release) noexcept
Add a row to the merge buffer.
Definition: ddl0builder.cc:1577
void clear_virtual_heap() noexcept
Clear the heap used for virtual columns.
Definition: ddl0impl-builder.h:320
bool is_skip_file_sort() const noexcept
Definition: ddl0impl-builder.h:157
void set_next_state() noexcept
Set the next state.
Definition: ddl0builder.cc:2087
dberr_t fts_sort_and_build() noexcept
FTS: Sort and insert the rows read.
Definition: ddl0builder.cc:1926
static void write_redo(const dict_index_t *index) noexcept
Write an MLOG_INDEX_LOAD record to indicate in the redo-log that redo-logging of individual index pag...
Definition: ddl0builder.cc:1907
static void copy_blobs(const dict_index_t *index, const mrec_t *mrec, const ulint *offsets, const page_size_t &page_size, dtuple_t *tuple, bool is_sdi, mem_heap_t *heap) noexcept
Copy externally stored columns to the data tuple.
Definition: ddl0builder.cc:1600
bool is_spatial_index() const noexcept
Definition: ddl0impl-builder.h:109
bool create_file(ddl::file_t &file) noexcept
Create the merge file, if needed.
Definition: ddl0builder.cc:1159
dberr_t online_build_handle_error(dberr_t err) noexcept
Cleanup DDL after error in online build Note: To be called if DDL must cleanup due to error in online...
Definition: ddl0builder.cc:1220
std::atomic< State > m_state
Initial phase.
Definition: ddl0impl-builder.h:413
dberr_t key_buffer_sort(size_t thread_id) noexcept
Sort the data in the key buffer.
Definition: ddl0builder.cc:1200
static void convert(const dict_index_t *clust_index, const dfield_t *row_field, dfield_t *field, ulint len, const page_size_t &page_size, bool is_sdi, mem_heap_t *heap) noexcept
Convert the field data from compact to redundant format.
Definition: ddl0builder.cc:477
dberr_t create_merge_sort_tasks() noexcept
Create the tasks to merge Sort the file before we load the file into the Btree index.
Definition: ddl0builder.cc:1826
Context & ctx() noexcept
Definition: ddl0impl-builder.h:103
Alter_stage * m_local_stage
Stage per builder.
Definition: ddl0impl-builder.h:453
dberr_t check_state_of_online_build_log() noexcept
Check the state of the online build log for the index.
Definition: ddl0builder.cc:619
dberr_t dtuple_copy_blobs(dtuple_t *dtuple, ulint *offsets, const mrec_t *mrec, mem_heap_t *heap) noexcept
Copy blobs to the tuple.
Definition: ddl0builder.cc:1649
dberr_t init(Cursor &cursor, size_t n_threads) noexcept
Initialize the cursor.
Definition: ddl0builder.cc:629
Scoped_heap m_conv_heap
Scoped conversion heap.
Definition: ddl0impl-builder.h:440
void fallback_to_single_thread() noexcept
Parallel scan thread spawn failed, release the extra thread states.
Definition: ddl0builder.cc:2077
dict_index_t * m_sort_index
The index to be built, FTS or non-FTS.
Definition: ddl0impl-builder.h:443
Loader & m_loader
Loader that owns the instance.
Definition: ddl0impl-builder.h:419
const char * tmpdir() const noexcept
Definition: ddl0impl-builder.h:201
Builder(ddl::Context &ctx, Loader &loader, size_t i) noexcept
Constructor.
Definition: ddl0builder.cc:584
void set_error(dberr_t err) noexcept
Set the error code.
Definition: ddl0impl-builder.h:94
bool is_unique_index() const noexcept
Definition: ddl0impl-builder.h:119
std::vector< Thread_ctx *, Allocator > Thread_ctxs
Definition: ddl0impl-builder.h:264
dberr_t copy_fts_column(Copy_ctx &ctx, dfield_t *field) noexcept
Copy the FTS columns.
Definition: ddl0builder.cc:846
void fts_add_doc_id(dfield_t *dst, const dict_field_t *src, doc_id_t &write_doc_id) noexcept
Add the FTS document ID to the destination field.
Definition: ddl0builder.cc:765
Dup m_clust_dup
For collecting duplicate entries (error reporting).
Definition: ddl0impl-builder.h:434
dberr_t bulk_add_row(Cursor &cursor, Row &row, size_t thread_id, Latch_release &&latch_release) noexcept
Add a row to the merge buffer.
Definition: ddl0builder.cc:1423
dict_index_t * index() noexcept
Definition: ddl0impl-builder.h:100
ddl::Context & m_ctx
DDL Context.
Definition: ddl0impl-builder.h:416
State
Build phase/states.
Definition: ddl0impl-builder.h:49
@ STOP
Stop on success.
@ ADD
Collect the rows for the index to build.
@ SORT
Sort the collected rows, if required.
@ ERROR
Stop on error.
@ BTREE_BUILD
Build the btree.
@ SETUP_SORT
Setup the merge sort and add the tasks to the task queue.
@ FINISH
Finish the loading of the index.
@ FTS_SORT_AND_BUILD
FTS sort and build, this is done in one "step".
@ INIT
Initial phase.
void batch_insert_deep_copy_tuples(size_t thread_id) noexcept
Note that the latches are going to be released.
Definition: ddl0builder.cc:1194
dict_index_t * m_index
Index to create (if not FTS index).
Definition: ddl0impl-builder.h:422
dberr_t check_duplicates(Thread_ctxs &dupcheck, Dup *dup) noexcept
Check for duplicates in the first block.
Definition: ddl0builder.cc:1686
dberr_t merge_sort(size_t thread_id) noexcept
Non-FTS: Sort the rows read.
Definition: ddl0builder.cc:1987
~Builder() noexcept
Destructor/.
Definition: ddl0builder.cc:601
dberr_t append(ddl::file_t &file, IO_buffer file_buffer) noexcept
Write data to disk - in append mode.
Definition: ddl0builder.cc:1171
dberr_t finish() noexcept
Close temporary files, Flush all dirty pages, apply the row log and write the redo log record.
Definition: ddl0builder.cc:2040
dberr_t get_virtual_column(Copy_ctx &ctx, const dict_field_t *ifield, dict_col_t *col, dfield_t *&src_field, size_t &mv_rows_added) noexcept
Setup the virtual column src column.
Definition: ddl0builder.cc:782
dberr_t copy_columns(Copy_ctx &ctx, size_t &mv_rows_added, doc_id_t &write_doc_id) noexcept
Copy the columns to the temporary file buffer.
Definition: ddl0builder.cc:888
dberr_t add_to_key_buffer(Copy_ctx &ctx, size_t &mv_rows_added) noexcept
Add row to the key buffer.
Definition: ddl0builder.cc:1354
State get_state() const noexcept
Definition: ddl0impl-builder.h:125
dberr_t batch_add_row(Row &row, size_t thread_id) noexcept
Cache a row for batch inserts.
Definition: ddl0builder.cc:1327
dberr_t batch_insert(size_t thread_id, Latch_release &&latch_release) noexcept
Insert cached rows.
Definition: ddl0builder.cc:1184
size_t m_id
Buffer ID.
Definition: ddl0impl-builder.h:410
dberr_t copy_row(Copy_ctx &ctx, size_t &mv_rows_added) noexcept
Add a row to the write buffer.
Definition: ddl0builder.cc:1030
dberr_t setup_sort() noexcept
Non-FTS: Sort the rows read.
Definition: ddl0builder.cc:2024
dberr_t get_error() const noexcept
Definition: ddl0impl-builder.h:90
const char * m_tmpdir
Temporary file path.
Definition: ddl0impl-builder.h:425
Btree_load * m_btr_load
Cluster index bulk load instance to use, direct insert without a file sort.
Definition: ddl0impl-builder.h:450
Scoped_heap m_v_heap
Scoped virtual column heap.
Definition: ddl0impl-builder.h:437
dfield_t * m_prev_fields
For tracking duplicates.
Definition: ddl0impl-builder.h:431
size_t id() const noexcept
Definition: ddl0impl-builder.h:97
bool is_fts_index() const noexcept
Definition: ddl0impl-builder.h:114
Alter_stage * stage() noexcept
Definition: ddl0impl-builder.h:136
Thread_ctxs m_thread_ctxs
Per thread context.
Definition: ddl0impl-builder.h:428
DDL context/configuration.
Definition: ddl0ddl.h:320
void set_error(dberr_t err) noexcept
Set the error code, when it's not specific to an index.
Definition: ddl0ddl.h:426
bool m_skip_pk_sort
Skip the sorting phase if true.
Definition: ddl0ddl.h:630
dberr_t get_error() const noexcept
Definition: ddl0ddl.h:422
Context for copying cluster index row for the index to being created.
Definition: ddl0builder.cc:49
Cursor for reading the data.
Definition: ddl0impl-cursor.h:40
Structure for reporting duplicate records.
Definition: ddl0ddl.h:131
For loading a Btree index from a file.
Definition: ddl0builder.cc:176
Buffer for sorting in main memory.
Definition: ddl0impl-buffer.h:40
Definition: ddl0impl-builder.h:456
virtual ~Load_cursor() override=default
Default destructor.
Scoped_heap m_tuple_heap
Heap for the raw row to dtuple_t conversion.
Definition: ddl0impl-builder.h:480
Builder * m_builder
Index meta data.
Definition: ddl0impl-builder.h:477
dberr_t get_err() const noexcept
Definition: ddl0impl-builder.h:465
Load_cursor(Builder *builder, Dup *dup) noexcept
Default constructor.
Definition: ddl0impl-builder.h:458
bool duplicates_detected() const noexcept override
Definition: ddl0builder.cc:230
dberr_t m_err
Operation error code.
Definition: ddl0impl-builder.h:474
Dup * m_dup
Duplicate checking and reporting.
Definition: ddl0impl-builder.h:471
Comparator.
Definition: ddl0impl-builder.h:555
Compare(const dict_index_t *index, Dup *dup)
Constructor.
Definition: ddl0impl-builder.h:559
Compare()=default
Destructor.
Merge the sorted files.
Definition: ddl0impl-builder.h:484
std::vector< File_cursor *, ut::allocator< File_cursor * > > File_cursors
File cursors to use for the scan.
Definition: ddl0impl-builder.h:580
void clear_eof() noexcept
Add the active cursors to the priority queue.
Definition: ddl0builder.cc:380
dberr_t add_file(const ddl::file_t &file, size_t buffer_size) noexcept
Add the cursor to use for merge load.
Definition: ddl0builder.cc:350
std::vector< File_reader *, ut::allocator< File_reader * > > File_readers
File cursors to use for the scan.
Definition: ddl0impl-builder.h:486
size_t number_of_cursors() const noexcept
Definition: ddl0impl-builder.h:545
Merge_cursor(Builder *builder, Dup *dup, Alter_stage *stage) noexcept
Constructor.
Definition: ddl0builder.cc:319
dberr_t fetch(dtuple_t *&dtuple) noexcept override
Fetch the current row as a tuple.
Definition: ddl0builder.cc:428
~Merge_cursor() noexcept override
Destructor.
Definition: ddl0builder.cc:325
File_readers file_readers() noexcept
Definition: ddl0builder.cc:309
Queue m_pq
Priority queue for merging the file cursors.
Definition: ddl0impl-builder.h:586
File_cursor * pop() noexcept
Definition: ddl0builder.cc:416
dberr_t next() noexcept override
Move to the next record.
Definition: ddl0builder.cc:450
size_t size() const noexcept
Definition: ddl0impl-builder.h:539
Alter_stage * m_stage
PFS stage monitoring.
Definition: ddl0impl-builder.h:595
dberr_t open() noexcept
Open the cursor.
Definition: ddl0builder.cc:396
uint64_t get_n_rows() const noexcept
Definition: ddl0builder.cc:467
File_cursors m_cursors
Cursors to use for parallel loading of the index.
Definition: ddl0impl-builder.h:589
File_cursor * m_cursor
Current cursor.
Definition: ddl0impl-builder.h:592
Physical row context.
Definition: ddl0impl.h:120
Information about temporary files used in merge sort.
Definition: ddl0impl.h:66
Structure for an SQL data field.
Definition: data0data.h:604
Data structure for a column in a table.
Definition: dict0mem.h:488
Data structure for a field in an index.
Definition: dict0mem.h:894
Data structure for an index.
Definition: dict0mem.h:1045
unsigned type
index type (DICT_CLUSTERED, DICT_UNIQUE, DICT_IBUF, DICT_CORRUPT)
Definition: dict0mem.h:1072
bool is_clustered() const
Definition: dict0mem.h:1310
Structure for an SQL data tuple of fields (logical record)
Definition: data0data.h:681
The info structure stored at the beginning of a heap block.
Definition: mem0mem.h:301
#define IF_DEBUG(...)
Definition: univ.i:673
unsigned long int ulint
Definition: univ.i:405
#define ut_a(EXPR)
Abort execution if EXPR does not evaluate to nonzero.
Definition: ut0dbg.h:92