MySQL 8.1.0
Source Code Documentation
ddl0impl-builder.h
Go to the documentation of this file.
1/*****************************************************************************
2
3Copyright (c) 2020, 2023, Oracle and/or its affiliates.
4
5This program is free software; you can redistribute it and/or modify it under
6the terms of the GNU General Public License, version 2.0, as published by the
7Free Software Foundation.
8
9This program is also distributed with certain software (including but not
10limited to OpenSSL) that is licensed under separate terms, as designated in a
11particular file or component or in included license documentation. The authors
12of MySQL hereby grant you an additional permission to link the program and
13your derivative works with the separately licensed software that they have
14included with MySQL.
15
16This program is distributed in the hope that it will be useful, but WITHOUT
17ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0,
19for more details.
20
21You should have received a copy of the GNU General Public License along with
22this program; if not, write to the Free Software Foundation, Inc.,
2351 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
24
25*****************************************************************************/
26
27/** @file include/ddl0impl-builder.h
28 DDL index builder data interface.
29 Created 2020-11-01 by Sunny Bains. */
30
31#ifndef ddl0impl_builder_h
32#define ddl0impl_builder_h
33
34#include "btr0load.h"
35#include "ddl0impl-buffer.h"
37#include "row0pread.h"
38
39namespace ddl {
40
41// Forward declaration.
42struct Copy_ctx;
43struct File_cursor;
44class RTree_inserter;
45
46/** For loading indexes. */
47struct Builder {
48 /** Build phase/states. */
49 enum class State : uint8_t {
50 /** Initial phase. */
51 INIT,
52
53 /** Collect the rows for the index to build. */
54 ADD,
55
56 /** Setup the merge sort and add the tasks to the task queue. */
58
59 /** Sort the collected rows, if required. The builder moves to state
60 BTREE_BUILD after all sort tasks are completed successfully or there
61 was an error during the sort phase. */
62 SORT,
63
64 /** Build the btree. */
66
67 /** FTS sort and build, this is done in one "step" */
69
70 /** Finish the loading of the index. */
71 FINISH,
72
73 /** Stop on success. */
74 STOP,
75
76 /** Stop on error. */
77 ERROR
78 };
79
80 /** Constructor.
81 @param[in,out] ctx DDL context.
82 @param[in,out] loader Owner of the instance.
83 @param[in] i Index ordinal value. */
84 Builder(ddl::Context &ctx, Loader &loader, size_t i) noexcept;
85
86 /** Destructor/ */
87 ~Builder() noexcept;
88
89 /** @return the error status. */
90 dberr_t get_error() const noexcept { return m_ctx.get_error(); }
91
92 /** Set the error code.
93 @param[in] err Error code to set. */
94 void set_error(dberr_t err) noexcept { m_ctx.set_error(err, m_id); }
95
96 /** @return the instance ID. */
97 [[nodiscard]] size_t id() const noexcept { return m_id; }
98
99 /** @return the index being built. */
100 [[nodiscard]] dict_index_t *index() noexcept { return m_sort_index; }
101
102 /** @return the DDL context. */
103 Context &ctx() noexcept { return m_ctx; }
104
105 /** Parallel scan thread spawn failed, release the extra thread states. */
106 void fallback_to_single_thread() noexcept;
107
108 /** @return true if the index is a spatial index. */
109 [[nodiscard]] bool is_spatial_index() const noexcept {
111 }
112
113 /** @return true if the index is an FTS index. */
114 [[nodiscard]] bool is_fts_index() const noexcept {
115 return m_index->type & DICT_FTS;
116 }
117
118 /** @return true if the index is a unique index. */
119 [[nodiscard]] bool is_unique_index() const noexcept {
120 ut_a(!is_fts_index());
122 }
123
124 /** @return the current builder state. */
125 [[nodiscard]] State get_state() const noexcept {
126 return m_state.load(std::memory_order_seq_cst);
127 }
128
129 /** Set the next state.
130 @param[in] state State to set. */
131 void set_state(State state) noexcept {
132 m_state.store(state, std::memory_order_seq_cst);
133 }
134
135 /** @return the PFS instance that is used to report progress (or nullptr). */
136 Alter_stage *stage() noexcept { return m_local_stage; }
137
138 /** Set the next state. */
139 void set_next_state() noexcept;
140
141 /** Initialize the cursor.
142 @param[in,out] cursor Cursor to initialize.
143 @param[in] n_threads Number of threads used for reading.
144 @return DB_SUCCESS or error code. */
145 [[nodiscard]] dberr_t init(Cursor &cursor, size_t n_threads) noexcept;
146
147 /** Add a row to the merge buffer.
148 @param[in,out] cursor Current scan cursor.
149 @param[in,out] row Row to add.
150 @param[in] thread_id ID of current thread.
151 @param[in,out] latch_release Called when a log free check is required.
152 @return DB_SUCCESS or error code. */
153 [[nodiscard]] dberr_t add_row(Cursor &cursor, Row &row, size_t thread_id,
154 Latch_release &&latch_release) noexcept;
155
156 /** @return true if file sorting can be skipped. */
157 bool is_skip_file_sort() const noexcept {
159 }
160
161 /** FTS: Sort and insert the rows read.
162 @return DB_SUCCESS or error code. */
163 [[nodiscard]] dberr_t fts_sort_and_build() noexcept;
164
165 /** Non-FTS: Sort the rows read.
166 @return DB_SUCCESS or error code. */
167 [[nodiscard]] dberr_t setup_sort() noexcept;
168
169 /** Non-FTS: Sort the rows read.
170 @param[in] thread_id Thread state ID.
171 @return DB_SUCCESS or error code. */
172 [[nodiscard]] dberr_t merge_sort(size_t thread_id) noexcept;
173
174 /** Load the sorted data into the B+Tree.
175 @return DB_SUCESS or error code. */
176 [[nodiscard]] dberr_t btree_build() noexcept;
177
178 /** Close temporary files, Flush all dirty pages, apply the row log
179 and write the redo log record.
180 @return DB_SUCCESS or error code. */
181 dberr_t finish() noexcept;
182
183 /** Copy blobs to the tuple.
184 @param[out] dtuple Tuple to copy to.
185 @param[in,out] offsets Column offsets in the row.
186 @param[in] mrec Current row.
187 @param[in,out] heap Heap for the allocating tuple memory.
188 @return DB_SUCCESS or error code. */
189 [[nodiscard]] dberr_t dtuple_copy_blobs(dtuple_t *dtuple, ulint *offsets,
190 const mrec_t *mrec,
191 mem_heap_t *heap) noexcept;
192
193 /** Write data to disk - in append mode. Increment the file size.
194 @param[in,out] file File handle.
195 @param[in] file_buffer Write the buffer contents to disk.
196 @return DB_SUCCESS or error code. */
197 [[nodiscard]] dberr_t append(ddl::file_t &file,
198 IO_buffer file_buffer) noexcept;
199
200 /** @return the path for temporary files. */
201 const char *tmpdir() const noexcept { return m_tmpdir; }
202
203 /** Insert cached rows.
204 @param[in] thread_id Insert cached rows for this thread ID.
205 @param[in,out] latch_release Called when a log free check is required.
206 @return DB_SUCCESS or error number */
207 [[nodiscard]] dberr_t batch_insert(size_t thread_id,
208 Latch_release &&latch_release) noexcept;
209
210 /** Note that the latches are going to be released. Do a deep copy of the
211 tuples that are being inserted in batches by batch_insert
212 @param[in] thread_id Deep copy cached rows for this thread ID. */
213 void batch_insert_deep_copy_tuples(size_t thread_id) noexcept;
214
215 /** Check the state of the online build log for the index.
216 @return DB_SUCCESS or error code. */
217 [[nodiscard]] dberr_t check_state_of_online_build_log() noexcept;
218
219 /** Write an MLOG_INDEX_LOAD record to indicate in the redo-log
220 that redo-logging of individual index pages was disabled, and
221 the flushing of such pages to the data files was completed.
222 @param[in] index Index on which redo logging was disabled */
223 static void write_redo(const dict_index_t *index) noexcept;
224
225 private:
226 /** State of a cluster index reader thread. */
227 struct Thread_ctx {
228 /** Constructor.
229 @param[in] id Thread state ID.
230 @param[in,out] key_buffer Buffer for building the target index. Note, the
231 thread state will own the key buffer and is
232 responsible for deleting it. */
233 explicit Thread_ctx(size_t id, Key_sort_buffer *key_buffer) noexcept;
234
235 /** Destructor. */
236 ~Thread_ctx() noexcept;
237
238 /** Thread ID. */
239 size_t m_id{};
240
241 /** Key sort buffer. */
242 Key_sort_buffer *m_key_buffer{};
243
244 /** Total number of records added to the key sort buffer. */
245 size_t m_n_recs{};
246
247 /** Merge file handle. */
248 ddl::file_t m_file{};
249
250 /** Buffer to use for file writes. */
252
253 /** Buffer to use for file writes. */
255
256 /** Record list starting offset in the output file. */
257 Merge_offsets m_offsets{};
258
259 /** For spatial/Rtree rows handling. */
260 RTree_inserter *m_rtree_inserter{};
261 };
262
264 using Thread_ctxs = std::vector<Thread_ctx *, Allocator>;
265
266 /** Create the tasks to merge Sort the file before we load the file into
267 the Btree index.
268 @return DB_SUCCESS or error code. */
269 [[nodiscard]] dberr_t create_merge_sort_tasks() noexcept;
270
271 /** Flush all dirty pages, apply the row log and write the redo log record.
272 @return DB_SUCCESS or error code. */
273 dberr_t finalize() noexcept;
274
275 /** Convert the field data from compact to redundant format.
276 @param[in] clust_index Clustered index being built
277 @param[in] row_field Field to copy from
278 @param[out] field Field to copy to
279 @param[in] len Length of the field data
280 @param[in] page_size Compressed BLOB page size
281 @param[in] is_sdi true for SDI Indexes
282 @param[in,out] heap Memory heap where to allocate
283 data when converting to ROW_FORMAT=REDUNDANT,
284 or nullptr */
285 static void convert(const dict_index_t *clust_index,
286 const dfield_t *row_field, dfield_t *field, ulint len,
287 const page_size_t &page_size,
288 IF_DEBUG(bool is_sdi, ) mem_heap_t *heap) noexcept;
289
290 /** Copy externally stored columns to the data tuple.
291 @param[in] index Index dictionary object.
292 @param[in] mrec Record containing BLOB pointers, or
293 nullptr to use tuple instead.
294 @param[in] offsets Offsets of mrec.
295 @param[in] page_size Compressed page size in bytes, or 0
296 @param[in,out] tuple Data tuple.
297 @param[in] is_sdi True for SDI Indexes
298 @param[in,out] heap Memory heap */
299 static void copy_blobs(const dict_index_t *index, const mrec_t *mrec,
300 const ulint *offsets, const page_size_t &page_size,
301 dtuple_t *tuple,
302 IF_DEBUG(bool is_sdi, ) mem_heap_t *heap) noexcept;
303
304 /** Cache a row for batch inserts. Currently used by spatial indexes.
305 @param[in,out] row Row to add.
306 @param[in] thread_id ID of current thread.
307 @return DB_SUCCESS or error code. */
308 [[nodiscard]] dberr_t batch_add_row(Row &row, size_t thread_id) noexcept;
309
310 /** Add a row to the merge buffer.
311 @param[in,out] cursor Current scan cursor.
312 @param[in,out] row Row to add.
313 @param[in] thread_id ID of current thread.
314 @param[in,out] latch_release Called when a log free check is required.
315 @return DB_SUCCESS or error code. */
316 [[nodiscard]] dberr_t bulk_add_row(Cursor &cursor, Row &row, size_t thread_id,
317 Latch_release &&latch_release) noexcept;
318
319 /** Clear the heap used for virtual columns. */
320 void clear_virtual_heap() noexcept { m_v_heap.clear(); }
321
322 /** Add the FTS document ID to the destination field.
323 @param[in,out] dst Field to write to.
324 @param[in] src Field to copy meta data from.
325 @param[out] write_doc_id Buffer for copying the doc id. */
326 void fts_add_doc_id(dfield_t *dst, const dict_field_t *src,
327 doc_id_t &write_doc_id) noexcept;
328
329 /** Add a row to the write buffer.
330 @param[in,out] ctx Copy context.
331 @param[in,out] mv_rows_added Number of multi-value rows added.
332 @return DB_SUCCESS or error code. */
333 [[nodiscard]] dberr_t copy_row(Copy_ctx &ctx, size_t &mv_rows_added) noexcept;
334
335 /** Setup the virtual column src column.
336 @param[in,out] ctx Copy context.
337 @param[in] ifield Index field.
338 @param[in] col Table column.
339 @param[out] src_field Computed value.
340 @param[in,out] mv_rows_added Number of multi-value rows added.
341 @return DB_SUCCESS or error code. */
342 [[nodiscard]] dberr_t get_virtual_column(Copy_ctx &ctx,
343 const dict_field_t *ifield,
344 dict_col_t *col,
345 dfield_t *&src_field,
346 size_t &mv_rows_added) noexcept;
347
348 /** Copy the FTS columns.
349 @param[in,out] ctx Copy context.
350 @param[in,out] field Field to write to.
351 @return DB_SUCCESS or error code. */
352 [[nodiscard]] dberr_t copy_fts_column(Copy_ctx &ctx,
353 dfield_t *field) noexcept;
354
355 /** Copy the columns to the temporary file buffer.
356 @param[in,out] ctx Copy context.
357 @param[in,out] mv_rows_added Multi value rows added.
358 @param[in,out] write_doc_id Buffer for storing the FTS doc ID.
359 @return DB_SUCCESS or error code. */
360 [[nodiscard]] dberr_t copy_columns(Copy_ctx &ctx, size_t &mv_rows_added,
361 doc_id_t &write_doc_id) noexcept;
362
363 /** Add row to the key buffer.
364 @param[in,out] ctx Copy context.
365 @param[in,out] mv_rows_added Number of multi-value index rows added.
366 @return DB_SUCCESS or error code. */
367 [[nodiscard]] dberr_t add_to_key_buffer(Copy_ctx &ctx,
368 size_t &mv_rows_added) noexcept;
369
370 /** Wait for FTS completion.
371 @param[in] index Index being built. */
373
374 /** Sort the data in the key buffer.
375 @param[in] thread_id Thread ID of current thread.
376 @return DB_SUCCESS or error code. */
377 [[nodiscard]] dberr_t key_buffer_sort(size_t thread_id) noexcept;
378
379 /** Sort the buffer in memory and insert directly in the BTree loader,
380 don't write to a temporary file.
381 @param[in,out] cursor Current scan cursor.
382 @param[in] thread_id ID of current thread.
383 @return DB_SUCCESS or error code. */
384 dberr_t insert_direct(Cursor &cursor, size_t thread_id) noexcept;
385
386 /** Create the merge file, if needed.
387 @param[in,out] file File handle.
388 @return true if file was opened successfully . */
389 [[nodiscard]] bool create_file(ddl::file_t &file) noexcept;
390
391 /** Check for duplicates in the first block
392 @param[in] dupcheck Files to check for duplicates.
393 @param[in,out] dup For collecting duplicate key information.
394 @return DB_SUCCESS or error code. */
395 [[nodiscard]] dberr_t check_duplicates(Thread_ctxs &dupcheck,
396 Dup *dup) noexcept;
397
398 private:
399 /** Buffer ID. */
400 size_t m_id{};
401
402 /** Initial phase. */
403 std::atomic<State> m_state{State::INIT};
404
405 /** DDL Context. */
407
408 /** Loader that owns the instance. */
410
411 /** Index to create (if not FTS index). */
413
414 /** Temporary file path. */
415 const char *m_tmpdir{};
416
417 /** Per thread context. */
419
420 /** For tracking duplicates. */
422
423 /** For collecting duplicate entries (error reporting). */
425
426 /** Scoped virtual column heap. */
428
429 /** Scoped conversion heap. */
431
432 /** The index to be built, FTS or non-FTS. */
434
435 /** Number of active sort tasks. */
436 std::atomic<size_t> m_n_sort_tasks{};
437
438 /** Cluster index bulk load instance to use, direct insert without
439 a file sort. */
441
442 /** Stage per builder. */
444};
445
447 /** Default constructor. */
448 explicit Load_cursor(Builder *builder, Dup *dup) noexcept
449 : m_dup(dup), m_builder(builder) {}
450
451 /** Default destructor. */
452 virtual ~Load_cursor() override = default;
453
454 /** @return the cursor error status. */
455 [[nodiscard]] dberr_t get_err() const noexcept { return m_err; }
456
457 /** @return true if duplicates detected. */
458 [[nodiscard]] bool duplicates_detected() const noexcept override;
459
460 /** Duplicate checking and reporting. */
462
463 /** Operation error code. */
465
466 /** Index meta data. */
468
469 /** Heap for the raw row to dtuple_t conversion. */
471};
472
473/** Merge the sorted files. */
474struct Merge_cursor : public Load_cursor {
475 /** File cursors to use for the scan. */
476 using File_readers = std::vector<File_reader *, ut::allocator<File_reader *>>;
477
478 /** Constructor.
479 @param[in,out] builder Index builder.
480 @param[in,out] dup If not nullptr, then report duplicates.
481 @param[in,out] stage PFS stage monitoring. */
482 explicit Merge_cursor(Builder *builder, Dup *dup,
483 Alter_stage *stage) noexcept;
484
485 /** Destructor. */
486 ~Merge_cursor() noexcept override;
487
488 /** Add the cursor to use for merge load.
489 @param[in] file File to merge from.
490 @param[in] buffer_size IO buffer size to use for reading.
491 @return DB_SUCCESS or error code. */
492 [[nodiscard]] dberr_t add_file(const ddl::file_t &file,
493 size_t buffer_size) noexcept;
494
495 /** Add the cursor to use for merge load.
496 @param[in] file File file to read.
497 @param[in] buffer_size IO buffer size to use for reading.
498 @param[in] offset Page to read from.
499 @return DB_SUCCESS or error code. */
500 [[nodiscard]] dberr_t add_file(const ddl::file_t &file, size_t buffer_size,
501 os_offset_t offset) noexcept;
502
503 /** Open the cursor.
504 @return DB_SUCCESS or error code. */
505 [[nodiscard]] dberr_t open() noexcept;
506
507 /** Fetch the current row as a tuple. Note: Tuple columns are shallow copies.
508 @param[out] dtuple Row represented as a tuple.
509 @return DB_SUCCESS, DB_END_OF_INDEX or error code. */
510 [[nodiscard]] dberr_t fetch(dtuple_t *&dtuple) noexcept override;
511
512 /** Fetch the current row.
513 @param[out] mrec Current merge record.
514 @param[out] offsets Columns offsets inside mrec.
515 @return DB_SUCCESS, DB_END_OF_INDEX or error code. */
516 [[nodiscard]] dberr_t fetch(const mrec_t *&mrec, ulint *&offsets) noexcept;
517
518 /** Move to the next record.
519 @return DB_SUCCESS, DB_END_OF_INDEX or error code. */
520 [[nodiscard]] dberr_t next() noexcept override;
521
522 /** @return the file reader instances. */
523 [[nodiscard]] File_readers file_readers() noexcept;
524
525 /** Add the active cursors to the priority queue. */
526 void clear_eof() noexcept;
527
528 /** @return the number of active readers. */
529 [[nodiscard]] size_t size() const noexcept { return m_pq.size(); }
530
531 /** @return the number of rows read from the files. */
532 [[nodiscard]] uint64_t get_n_rows() const noexcept;
533
534 /** @return the number of cursors being merged. */
535 [[nodiscard]] size_t number_of_cursors() const noexcept {
536 return m_cursors.size();
537 }
538
539 private:
540 /** @return the current cursor at the head of the queue. */
541 [[nodiscard]] File_cursor *pop() noexcept;
542
543 private:
544 /** Comparator. */
545 struct Compare {
546 /** Constructor.
547 @param[in] index Index that the rows belong to.
548 @param[in,out] dup For reporting duplicates, can be nullptr. */
549 explicit Compare(const dict_index_t *index, Dup *dup)
550 : m_dup(dup), m_index(index) {}
551
552 /** Destructor. */
553 Compare() = default;
554
555 /** Compare the keys of two cursors.
556 @param[in] lhs Left hand side.
557 @param[in] rhs Right hand side.
558 @return true if lhs strictly less than rhs. */
559 bool operator()(const File_cursor *lhs,
560 const File_cursor *rhs) const noexcept;
561
562 /** For reporting duplicates. */
564
565 /** Index being built. */
566 const dict_index_t *m_index{};
567 };
568
569 /** File cursors to use for the scan. */
570 using File_cursors = std::vector<File_cursor *, ut::allocator<File_cursor *>>;
571
572 /** Priority queue for ordering the rows. */
573 using Queue = std::priority_queue<File_cursor *, File_cursors, Compare>;
574
575 /** Priority queue for merging the file cursors. */
577
578 /** Cursors to use for parallel loading of the index. */
580
581 /** Current cursor. */
583
584 /** PFS stage monitoring. */
586};
587
588} // namespace ddl
589
590#endif /* !ddl0impl_builder_h */
The B-tree bulk load.
static Mysys_charset_loader * loader
Definition: charset.cc:184
Class used to report ALTER TABLE progress via performance_schema.
Definition: ut0stage.h:80
Definition: btr0load.h:50
Implements a persistent FIFO using server List method names.
Definition: sql_profile.h:75
Build indexes on a table by reading a clustered index, creating a temporary file containing index ent...
Definition: ddl0impl-loader.h:43
Class that caches RTree index tuples made from a single cluster index page scan, and then insert into...
Definition: ddl0impl-rtree.h:43
Page size descriptor.
Definition: page0size.h:49
Allocator that allows std::* containers to manage their memory through ut::malloc* and ut::free libra...
Definition: ut0new.h:2180
dberr_t
Definition: db0err.h:38
@ DB_SUCCESS
Definition: db0err.h:42
DDL buffer infrastructure.
For scanning the temporary file.
static ulint dict_index_is_unique(const dict_index_t *index)
Check whether the index is unique.
static ulint dict_index_is_spatial(const dict_index_t *index)
Check whether the index is a Spatial Index.
constexpr uint32_t DICT_FTS
FTS index; can't be combined with the other flags.
Definition: dict0mem.h:103
uint64_t doc_id_t
Document id type.
Definition: fts0fts.h:78
static my_thread_id thread_id
Definition: my_thr_init.cc:62
static struct st_file_buffer file_buffer
Definition: myisampack.cc:208
The general architecture is that the work is done in two phases, roughly the read and write phase.
Definition: btr0load.cc:41
std::deque< os_offset_t, ut::allocator< os_offset_t > > Merge_offsets
Start offsets in the file, from where to merge records.
Definition: ddl0impl.h:63
byte mrec_t
Merge record in Aligned_buffer.
Definition: ddl0ddl.h:77
std::function< dberr_t()> Latch_release
Called when a log free check is required.
Definition: ddl0impl.h:49
std::pair< byte *, os_offset_t > IO_buffer
Block size for DDL I/O operations.
Definition: ddl0impl.h:46
Definition: os0file.h:85
static Value err()
Create a Value object that represents an error condition.
Definition: json_binary.cc:909
size_t buffer_size(const ConstBufferSequence &buffers) noexcept
Definition: buffer.h:314
std::conditional_t< !std::is_array< T >::value, std::unique_ptr< T, detail::Aligned_deleter< T > >, std::conditional_t< detail::is_unbounded_array_v< T >, std::unique_ptr< T, detail::Aligned_array_deleter< std::remove_extent_t< T > > >, void > > unique_ptr_aligned
The following is a common type that is returned by all the ut::make_unique_aligned (non-aligned) spec...
Definition: ut0new.h:2571
uint64_t os_offset_t
File offset in bytes.
Definition: os0file.h:83
Parallel read interface.
Interface to consume from.
Definition: btr0load.h:53
Heap wrapper that destroys the heap instance when it goes out of scope.
Definition: mem0mem.h:438
void clear() noexcept
Empty the heap.
Definition: mem0mem.h:474
State of a cluster index reader thread.
Definition: ddl0impl-builder.h:227
IO_buffer m_io_buffer
Buffer to use for file writes.
Definition: ddl0impl-builder.h:254
For loading indexes.
Definition: ddl0impl-builder.h:47
dberr_t finalize() noexcept
Flush all dirty pages, apply the row log and write the redo log record.
Definition: ddl0builder.cc:1907
dberr_t insert_direct(Cursor &cursor, size_t thread_id) noexcept
Sort the buffer in memory and insert directly in the BTree loader, don't write to a temporary file.
Definition: ddl0builder.cc:1215
dberr_t btree_build() noexcept
Load the sorted data into the B+Tree.
Definition: ddl0builder.cc:1710
void set_state(State state) noexcept
Set the next state.
Definition: ddl0impl-builder.h:131
std::atomic< size_t > m_n_sort_tasks
Number of active sort tasks.
Definition: ddl0impl-builder.h:436
void fts_wait_for_completion(const dict_index_t *index) noexcept
Wait for FTS completion.
dberr_t add_row(Cursor &cursor, Row &row, size_t thread_id, Latch_release &&latch_release) noexcept
Add a row to the merge buffer.
Definition: ddl0builder.cc:1538
void clear_virtual_heap() noexcept
Clear the heap used for virtual columns.
Definition: ddl0impl-builder.h:320
bool is_skip_file_sort() const noexcept
Definition: ddl0impl-builder.h:157
void set_next_state() noexcept
Set the next state.
Definition: ddl0builder.cc:2043
dberr_t fts_sort_and_build() noexcept
FTS: Sort and insert the rows read.
Definition: ddl0builder.cc:1882
static void write_redo(const dict_index_t *index) noexcept
Write an MLOG_INDEX_LOAD record to indicate in the redo-log that redo-logging of individual index pag...
Definition: ddl0builder.cc:1863
static void copy_blobs(const dict_index_t *index, const mrec_t *mrec, const ulint *offsets, const page_size_t &page_size, dtuple_t *tuple, bool is_sdi, mem_heap_t *heap) noexcept
Copy externally stored columns to the data tuple.
Definition: ddl0builder.cc:1556
bool is_spatial_index() const noexcept
Definition: ddl0impl-builder.h:109
bool create_file(ddl::file_t &file) noexcept
Create the merge file, if needed.
Definition: ddl0builder.cc:1154
std::atomic< State > m_state
Initial phase.
Definition: ddl0impl-builder.h:403
dberr_t key_buffer_sort(size_t thread_id) noexcept
Sort the data in the key buffer.
Definition: ddl0builder.cc:1195
static void convert(const dict_index_t *clust_index, const dfield_t *row_field, dfield_t *field, ulint len, const page_size_t &page_size, bool is_sdi, mem_heap_t *heap) noexcept
Convert the field data from compact to redundant format.
Definition: ddl0builder.cc:477
dberr_t create_merge_sort_tasks() noexcept
Create the tasks to merge Sort the file before we load the file into the Btree index.
Definition: ddl0builder.cc:1782
Context & ctx() noexcept
Definition: ddl0impl-builder.h:103
Alter_stage * m_local_stage
Stage per builder.
Definition: ddl0impl-builder.h:443
dberr_t check_state_of_online_build_log() noexcept
Check the state of the online build log for the index.
Definition: ddl0builder.cc:618
dberr_t dtuple_copy_blobs(dtuple_t *dtuple, ulint *offsets, const mrec_t *mrec, mem_heap_t *heap) noexcept
Copy blobs to the tuple.
Definition: ddl0builder.cc:1605
dberr_t init(Cursor &cursor, size_t n_threads) noexcept
Initialize the cursor.
Definition: ddl0builder.cc:628
Scoped_heap m_conv_heap
Scoped conversion heap.
Definition: ddl0impl-builder.h:430
void fallback_to_single_thread() noexcept
Parallel scan thread spawn failed, release the extra thread states.
Definition: ddl0builder.cc:2033
dict_index_t * m_sort_index
The index to be built, FTS or non-FTS.
Definition: ddl0impl-builder.h:433
Loader & m_loader
Loader that owns the instance.
Definition: ddl0impl-builder.h:409
const char * tmpdir() const noexcept
Definition: ddl0impl-builder.h:201
Builder(ddl::Context &ctx, Loader &loader, size_t i) noexcept
Constructor.
Definition: ddl0builder.cc:584
void set_error(dberr_t err) noexcept
Set the error code.
Definition: ddl0impl-builder.h:94
bool is_unique_index() const noexcept
Definition: ddl0impl-builder.h:119
std::vector< Thread_ctx *, Allocator > Thread_ctxs
Definition: ddl0impl-builder.h:264
dberr_t copy_fts_column(Copy_ctx &ctx, dfield_t *field) noexcept
Copy the FTS columns.
Definition: ddl0builder.cc:845
void fts_add_doc_id(dfield_t *dst, const dict_field_t *src, doc_id_t &write_doc_id) noexcept
Add the FTS document ID to the destination field.
Definition: ddl0builder.cc:764
Dup m_clust_dup
For collecting duplicate entries (error reporting).
Definition: ddl0impl-builder.h:424
dberr_t bulk_add_row(Cursor &cursor, Row &row, size_t thread_id, Latch_release &&latch_release) noexcept
Add a row to the merge buffer.
Definition: ddl0builder.cc:1384
dict_index_t * index() noexcept
Definition: ddl0impl-builder.h:100
ddl::Context & m_ctx
DDL Context.
Definition: ddl0impl-builder.h:406
State
Build phase/states.
Definition: ddl0impl-builder.h:49
@ STOP
Stop on success.
@ ADD
Collect the rows for the index to build.
@ SORT
Sort the collected rows, if required.
@ ERROR
Stop on error.
@ BTREE_BUILD
Build the btree.
@ SETUP_SORT
Setup the merge sort and add the tasks to the task queue.
@ FINISH
Finish the loading of the index.
@ FTS_SORT_AND_BUILD
FTS sort and build, this is done in one "step".
@ INIT
Initial phase.
void batch_insert_deep_copy_tuples(size_t thread_id) noexcept
Note that the latches are going to be released.
Definition: ddl0builder.cc:1189
dict_index_t * m_index
Index to create (if not FTS index).
Definition: ddl0impl-builder.h:412
dberr_t check_duplicates(Thread_ctxs &dupcheck, Dup *dup) noexcept
Check for duplicates in the first block.
Definition: ddl0builder.cc:1642
dberr_t merge_sort(size_t thread_id) noexcept
Non-FTS: Sort the rows read.
Definition: ddl0builder.cc:1943
~Builder() noexcept
Destructor/.
Definition: ddl0builder.cc:600
dberr_t append(ddl::file_t &file, IO_buffer file_buffer) noexcept
Write data to disk - in append mode.
Definition: ddl0builder.cc:1166
dberr_t finish() noexcept
Close temporary files, Flush all dirty pages, apply the row log and write the redo log record.
Definition: ddl0builder.cc:1996
dberr_t get_virtual_column(Copy_ctx &ctx, const dict_field_t *ifield, dict_col_t *col, dfield_t *&src_field, size_t &mv_rows_added) noexcept
Setup the virtual column src column.
Definition: ddl0builder.cc:781
dberr_t copy_columns(Copy_ctx &ctx, size_t &mv_rows_added, doc_id_t &write_doc_id) noexcept
Copy the columns to the temporary file buffer.
Definition: ddl0builder.cc:887
dberr_t add_to_key_buffer(Copy_ctx &ctx, size_t &mv_rows_added) noexcept
Add row to the key buffer.
Definition: ddl0builder.cc:1315
State get_state() const noexcept
Definition: ddl0impl-builder.h:125
dberr_t batch_add_row(Row &row, size_t thread_id) noexcept
Cache a row for batch inserts.
Definition: ddl0builder.cc:1288
dberr_t batch_insert(size_t thread_id, Latch_release &&latch_release) noexcept
Insert cached rows.
Definition: ddl0builder.cc:1179
size_t m_id
Buffer ID.
Definition: ddl0impl-builder.h:400
dberr_t copy_row(Copy_ctx &ctx, size_t &mv_rows_added) noexcept
Add a row to the write buffer.
Definition: ddl0builder.cc:1025
dberr_t setup_sort() noexcept
Non-FTS: Sort the rows read.
Definition: ddl0builder.cc:1980
dberr_t get_error() const noexcept
Definition: ddl0impl-builder.h:90
const char * m_tmpdir
Temporary file path.
Definition: ddl0impl-builder.h:415
Btree_load * m_btr_load
Cluster index bulk load instance to use, direct insert without a file sort.
Definition: ddl0impl-builder.h:440
Scoped_heap m_v_heap
Scoped virtual column heap.
Definition: ddl0impl-builder.h:427
dfield_t * m_prev_fields
For tracking duplicates.
Definition: ddl0impl-builder.h:421
size_t id() const noexcept
Definition: ddl0impl-builder.h:97
bool is_fts_index() const noexcept
Definition: ddl0impl-builder.h:114
Alter_stage * stage() noexcept
Definition: ddl0impl-builder.h:136
Thread_ctxs m_thread_ctxs
Per thread context.
Definition: ddl0impl-builder.h:418
DDL context/configuration.
Definition: ddl0ddl.h:320
void set_error(dberr_t err) noexcept
Set the error code, when it's not specific to an index.
Definition: ddl0ddl.h:426
bool m_skip_pk_sort
Skip the sorting phase if true.
Definition: ddl0ddl.h:630
dberr_t get_error() const noexcept
Definition: ddl0ddl.h:422
Context for copying cluster index row for the index to being created.
Definition: ddl0builder.cc:49
Cursor for reading the data.
Definition: ddl0impl-cursor.h:40
Structure for reporting duplicate records.
Definition: ddl0ddl.h:131
For loading a Btree index from a file.
Definition: ddl0builder.cc:176
Buffer for sorting in main memory.
Definition: ddl0impl-buffer.h:39
Definition: ddl0impl-builder.h:446
virtual ~Load_cursor() override=default
Default destructor.
Scoped_heap m_tuple_heap
Heap for the raw row to dtuple_t conversion.
Definition: ddl0impl-builder.h:470
Builder * m_builder
Index meta data.
Definition: ddl0impl-builder.h:467
dberr_t get_err() const noexcept
Definition: ddl0impl-builder.h:455
Load_cursor(Builder *builder, Dup *dup) noexcept
Default constructor.
Definition: ddl0impl-builder.h:448
bool duplicates_detected() const noexcept override
Definition: ddl0builder.cc:230
dberr_t m_err
Operation error code.
Definition: ddl0impl-builder.h:464
Dup * m_dup
Duplicate checking and reporting.
Definition: ddl0impl-builder.h:461
Comparator.
Definition: ddl0impl-builder.h:545
Compare(const dict_index_t *index, Dup *dup)
Constructor.
Definition: ddl0impl-builder.h:549
Compare()=default
Destructor.
Merge the sorted files.
Definition: ddl0impl-builder.h:474
std::vector< File_cursor *, ut::allocator< File_cursor * > > File_cursors
File cursors to use for the scan.
Definition: ddl0impl-builder.h:570
void clear_eof() noexcept
Add the active cursors to the priority queue.
Definition: ddl0builder.cc:380
dberr_t add_file(const ddl::file_t &file, size_t buffer_size) noexcept
Add the cursor to use for merge load.
Definition: ddl0builder.cc:350
std::vector< File_reader *, ut::allocator< File_reader * > > File_readers
File cursors to use for the scan.
Definition: ddl0impl-builder.h:476
size_t number_of_cursors() const noexcept
Definition: ddl0impl-builder.h:535
Merge_cursor(Builder *builder, Dup *dup, Alter_stage *stage) noexcept
Constructor.
Definition: ddl0builder.cc:319
dberr_t fetch(dtuple_t *&dtuple) noexcept override
Fetch the current row as a tuple.
Definition: ddl0builder.cc:428
~Merge_cursor() noexcept override
Destructor.
Definition: ddl0builder.cc:325
File_readers file_readers() noexcept
Definition: ddl0builder.cc:309
Queue m_pq
Priority queue for merging the file cursors.
Definition: ddl0impl-builder.h:576
File_cursor * pop() noexcept
Definition: ddl0builder.cc:416
dberr_t next() noexcept override
Move to the next record.
Definition: ddl0builder.cc:450
size_t size() const noexcept
Definition: ddl0impl-builder.h:529
Alter_stage * m_stage
PFS stage monitoring.
Definition: ddl0impl-builder.h:585
dberr_t open() noexcept
Open the cursor.
Definition: ddl0builder.cc:396
uint64_t get_n_rows() const noexcept
Definition: ddl0builder.cc:467
File_cursors m_cursors
Cursors to use for parallel loading of the index.
Definition: ddl0impl-builder.h:579
File_cursor * m_cursor
Current cursor.
Definition: ddl0impl-builder.h:582
Physical row context.
Definition: ddl0impl.h:120
Information about temporary files used in merge sort.
Definition: ddl0impl.h:66
Structure for an SQL data field.
Definition: data0data.h:604
Data structure for a column in a table.
Definition: dict0mem.h:488
Data structure for a field in an index.
Definition: dict0mem.h:894
Data structure for an index.
Definition: dict0mem.h:1045
unsigned type
index type (DICT_CLUSTERED, DICT_UNIQUE, DICT_IBUF, DICT_CORRUPT)
Definition: dict0mem.h:1072
bool is_clustered() const
Definition: dict0mem.h:1310
Structure for an SQL data tuple of fields (logical record)
Definition: data0data.h:681
The info structure stored at the beginning of a heap block.
Definition: mem0mem.h:301
#define IF_DEBUG(...)
Definition: univ.i:673
unsigned long int ulint
Definition: univ.i:405
#define ut_a(EXPR)
Abort execution if EXPR does not evaluate to nonzero.
Definition: ut0dbg.h:56