MySQL 8.0.32
Source Code Documentation
ddl0impl-builder.h
Go to the documentation of this file.
1/*****************************************************************************
2
3Copyright (c) 2020, 2022, Oracle and/or its affiliates.
4
5This program is free software; you can redistribute it and/or modify it under
6the terms of the GNU General Public License, version 2.0, as published by the
7Free Software Foundation.
8
9This program is also distributed with certain software (including but not
10limited to OpenSSL) that is licensed under separate terms, as designated in a
11particular file or component or in included license documentation. The authors
12of MySQL hereby grant you an additional permission to link the program and
13your derivative works with the separately licensed software that they have
14included with MySQL.
15
16This program is distributed in the hope that it will be useful, but WITHOUT
17ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0,
19for more details.
20
21You should have received a copy of the GNU General Public License along with
22this program; if not, write to the Free Software Foundation, Inc.,
2351 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
24
25*****************************************************************************/
26
27/** @file include/ddl0impl-builder.h
28 DDL index builder data interface.
29 Created 2020-11-01 by Sunny Bains. */
30
31#ifndef ddl0impl_builder_h
32#define ddl0impl_builder_h
33
34#include "btr0load.h"
35#include "ddl0impl-buffer.h"
37#include "row0pread.h"
38
39namespace ddl {
40
41// Forward declaration.
42struct Copy_ctx;
43struct File_cursor;
44class RTree_inserter;
45
46/** For loading indexes. */
47struct Builder {
48 /** Build phase/states. */
49 enum class State : uint8_t {
50 /** Initial phase. */
51 INIT,
52
53 /** Collect the rows for the index to build. */
54 ADD,
55
56 /** Setup the merge sort and add the tasks to the task queue. */
58
59 /** Sort the collected rows, if required. The builder moves to state
60 BTREE_BUILD after all sort tasks are completed successfully or there
61 was an error during the sort phase. */
62 SORT,
63
64 /** Build the btree. */
66
67 /** FTS sort and build, this is done in one "step" */
69
70 /** Finish the loading of the index. */
71 FINISH,
72
73 /** Stop on success. */
74 STOP,
75
76 /** Stop on error. */
77 ERROR
78 };
79
80 /** Constructor.
81 @param[in,out] ctx DDL context.
82 @param[in,out] loader Owner of the instance.
83 @param[in] i Index ordinal value. */
84 Builder(ddl::Context &ctx, Loader &loader, size_t i) noexcept;
85
86 /** Destructor/ */
87 ~Builder() noexcept;
88
89 /** @return the error status. */
90 dberr_t get_error() const noexcept { return m_ctx.get_error(); }
91
92 /** Set the error code.
93 @param[in] err Error code to set. */
94 void set_error(dberr_t err) noexcept { m_ctx.set_error(err, m_id); }
95
96 /** @return the instance ID. */
97 [[nodiscard]] size_t id() const noexcept { return m_id; }
98
99 /** @return the index being built. */
100 [[nodiscard]] dict_index_t *index() noexcept { return m_sort_index; }
101
102 /** @return the DDL context. */
103 Context &ctx() noexcept { return m_ctx; }
104
105 /** Parallel scan thread spawn failed, release the extra thread states. */
106 void fallback_to_single_thread() noexcept;
107
108 /** @return true if the index is a spatial index. */
109 [[nodiscard]] bool is_spatial_index() const noexcept {
111 }
112
113 /** @return true if the index is an FTS index. */
114 [[nodiscard]] bool is_fts_index() const noexcept {
115 return m_index->type & DICT_FTS;
116 }
117
118 /** @return true if the index is a unique index. */
119 [[nodiscard]] bool is_unique_index() const noexcept {
120 ut_a(!is_fts_index());
122 }
123
124 /** @return the current builder state. */
125 [[nodiscard]] State get_state() const noexcept {
126 return m_state.load(std::memory_order_seq_cst);
127 }
128
129 /** Set the next state.
130 @param[in] state State to set. */
131 void set_state(State state) noexcept {
132 m_state.store(state, std::memory_order_seq_cst);
133 }
134
135 /** @return the PFS instance that is used to report progress (or nullptr). */
136 Alter_stage *stage() noexcept { return m_local_stage; }
137
138 /** Set the next state. */
139 void set_next_state() noexcept;
140
141 /** Initialize the cursor.
142 @param[in,out] cursor Cursor to initialize.
143 @param[in] n_threads Number of threads used for reading.
144 @return DB_SUCCESS or error code. */
145 [[nodiscard]] dberr_t init(Cursor &cursor, size_t n_threads) noexcept;
146
147 /** Add a row to the merge buffer.
148 @param[in,out] cursor Current scan cursor.
149 @param[in,out] row Row to add.
150 @param[in] thread_id ID of current thread.
151 @param[in,out] latch_release Called when a log free check is required.
152 @return DB_SUCCESS or error code. */
153 [[nodiscard]] dberr_t add_row(Cursor &cursor, Row &row, size_t thread_id,
154 Latch_release &&latch_release) noexcept;
155
156 /** @return true if file sorting can be skipped. */
157 bool is_skip_file_sort() const noexcept {
159 }
160
161 /** FTS: Sort and insert the rows read.
162 @return DB_SUCCESS or error code. */
163 [[nodiscard]] dberr_t fts_sort_and_build() noexcept;
164
165 /** Non-FTS: Sort the rows read.
166 @return DB_SUCCESS or error code. */
167 [[nodiscard]] dberr_t setup_sort() noexcept;
168
169 /** Non-FTS: Sort the rows read.
170 @param[in] thread_id Thread state ID.
171 @return DB_SUCCESS or error code. */
172 [[nodiscard]] dberr_t merge_sort(size_t thread_id) noexcept;
173
174 /** Load the sorted data into the B+Tree.
175 @return DB_SUCESS or error code. */
176 [[nodiscard]] dberr_t btree_build() noexcept;
177
178 /** Close temporary files, Flush all dirty pages, apply the row log
179 and write the redo log record.
180 @return DB_SUCCESS or error code. */
181 dberr_t finish() noexcept;
182
183 /** Copy blobs to the tuple.
184 @param[out] dtuple Tuple to copy to.
185 @param[in,out] offsets Column offsets in the row.
186 @param[in] mrec Current row.
187 @param[in,out] heap Heap for the allocating tuple memory.
188 @return DB_SUCCESS or error code. */
189 [[nodiscard]] dberr_t dtuple_copy_blobs(dtuple_t *dtuple, ulint *offsets,
190 const mrec_t *mrec,
191 mem_heap_t *heap) noexcept;
192
193 /** Write data to disk - in append mode. Increment the file size.
194 @param[in,out] file File handle.
195 @param[in] file_buffer Write the buffer contents to disk.
196 @return DB_SUCCESS or error code. */
197 [[nodiscard]] dberr_t append(ddl::file_t &file,
198 IO_buffer file_buffer) noexcept;
199
200 /** @return the path for temporary files. */
201 const char *tmpdir() const noexcept { return m_tmpdir; }
202
203 /** Insert cached rows.
204 @param[in] thread_id Insert cached rows for this thread ID.
205 @param[in,out] latch_release Called when a log free check is required.
206 @return DB_SUCCESS or error number */
207 [[nodiscard]] dberr_t batch_insert(size_t thread_id,
208 Latch_release &&latch_release) noexcept;
209
210 /** Note that the latches are going to be released. Do a deep copy of the
211 tuples that are being inserted in batches by batch_insert
212 @param[in] thread_id Deep copy cached rows for this thread ID. */
213 void batch_insert_deep_copy_tuples(size_t thread_id) noexcept;
214
215 /** Check the state of the online build log for the index.
216 @return DB_SUCCESS or error code. */
217 [[nodiscard]] dberr_t check_state_of_online_build_log() noexcept;
218
219 /** Write an MLOG_INDEX_LOAD record to indicate in the redo-log
220 that redo-logging of individual index pages was disabled, and
221 the flushing of such pages to the data files was completed.
222 @param[in] index Index on which redo logging was disabled */
223 static void write_redo(const dict_index_t *index) noexcept;
224
225 private:
226 /** State of a cluster index reader thread. */
227 struct Thread_ctx {
228 /** Constructor.
229 @param[in] id Thread state ID.
230 @param[in,out] key_buffer Buffer for building the target index. Note, the
231 thread state will own the key buffer and is
232 responsible for deleting it. */
233 explicit Thread_ctx(size_t id, Key_sort_buffer *key_buffer) noexcept;
234
235 /** Destructor. */
236 ~Thread_ctx() noexcept;
237
238 /** Thread ID. */
239 size_t m_id{};
240
241 /** Key sort buffer. */
242 Key_sort_buffer *m_key_buffer{};
243
244 /** Total number of records added to the key sort buffer. */
245 size_t m_n_recs{};
246
247 /** Merge file handle. */
248 ddl::file_t m_file{};
249
250 /** Buffer to use for file writes. */
251 Aligned_buffer m_aligned_buffer{};
252
253 /** Record list starting offset in the output file. */
254 Merge_offsets m_offsets{};
255
256 /** For spatial/Rtree rows handling. */
257 RTree_inserter *m_rtree_inserter{};
258 };
259
261 using Thread_ctxs = std::vector<Thread_ctx *, Allocator>;
262
263 /** Create the tasks to merge Sort the file before we load the file into
264 the Btree index.
265 @return DB_SUCCESS or error code. */
266 [[nodiscard]] dberr_t create_merge_sort_tasks() noexcept;
267
268 /** Flush all dirty pages, apply the row log and write the redo log record.
269 @return DB_SUCCESS or error code. */
270 dberr_t finalize() noexcept;
271
272 /** Convert the field data from compact to redundant format.
273 @param[in] clust_index Clustered index being built
274 @param[in] row_field Field to copy from
275 @param[out] field Field to copy to
276 @param[in] len Length of the field data
277 @param[in] page_size Compressed BLOB page size
278 @param[in] is_sdi true for SDI Indexes
279 @param[in,out] heap Memory heap where to allocate
280 data when converting to ROW_FORMAT=REDUNDANT,
281 or nullptr */
282 static void convert(const dict_index_t *clust_index,
283 const dfield_t *row_field, dfield_t *field, ulint len,
284 const page_size_t &page_size,
285 IF_DEBUG(bool is_sdi, ) mem_heap_t *heap) noexcept;
286
287 /** Copy externally stored columns to the data tuple.
288 @param[in] index Index dictionary object.
289 @param[in] mrec Record containing BLOB pointers, or
290 nullptr to use tuple instead.
291 @param[in] offsets Offsets of mrec.
292 @param[in] page_size Compressed page size in bytes, or 0
293 @param[in,out] tuple Data tuple.
294 @param[in] is_sdi True for SDI Indexes
295 @param[in,out] heap Memory heap */
296 static void copy_blobs(const dict_index_t *index, const mrec_t *mrec,
297 const ulint *offsets, const page_size_t &page_size,
298 dtuple_t *tuple,
299 IF_DEBUG(bool is_sdi, ) mem_heap_t *heap) noexcept;
300
301 /** Cache a row for batch inserts. Currently used by spatial indexes.
302 @param[in,out] row Row to add.
303 @param[in] thread_id ID of current thread.
304 @return DB_SUCCESS or error code. */
305 [[nodiscard]] dberr_t batch_add_row(Row &row, size_t thread_id) noexcept;
306
307 /** Add a row to the merge buffer.
308 @param[in,out] cursor Current scan cursor.
309 @param[in,out] row Row to add.
310 @param[in] thread_id ID of current thread.
311 @param[in,out] latch_release Called when a log free check is required.
312 @return DB_SUCCESS or error code. */
313 [[nodiscard]] dberr_t bulk_add_row(Cursor &cursor, Row &row, size_t thread_id,
314 Latch_release &&latch_release) noexcept;
315
316 /** Clear the heap used for virtual columns. */
317 void clear_virtual_heap() noexcept { m_v_heap.clear(); }
318
319 /** Add the FTS document ID to the destination field.
320 @param[in,out] dst Field to write to.
321 @param[in] src Field to copy meta data from.
322 @param[out] write_doc_id Buffer for copying the doc id. */
323 void fts_add_doc_id(dfield_t *dst, const dict_field_t *src,
324 doc_id_t &write_doc_id) noexcept;
325
326 /** Add a row to the write buffer.
327 @param[in,out] ctx Copy context.
328 @param[in,out] mv_rows_added Number of multi-value rows added.
329 @return DB_SUCCESS or error code. */
330 [[nodiscard]] dberr_t copy_row(Copy_ctx &ctx, size_t &mv_rows_added) noexcept;
331
332 /** Setup the virtual column src column.
333 @param[in,out] ctx Copy context.
334 @param[in] ifield Index field.
335 @param[in] col Table column.
336 @param[out] src_field Computed value.
337 @param[in,out] mv_rows_added Number of multi-value rows added.
338 @return DB_SUCCESS or error code. */
339 [[nodiscard]] dberr_t get_virtual_column(Copy_ctx &ctx,
340 const dict_field_t *ifield,
341 dict_col_t *col,
342 dfield_t *&src_field,
343 size_t &mv_rows_added) noexcept;
344
345 /** Copy the FTS columns.
346 @param[in,out] ctx Copy context.
347 @param[in,out] field Field to write to.
348 @return DB_SUCCESS or error code. */
349 [[nodiscard]] dberr_t copy_fts_column(Copy_ctx &ctx,
350 dfield_t *field) noexcept;
351
352 /** Copy the columns to the temporary file buffer.
353 @param[in,out] ctx Copy context.
354 @param[in,out] mv_rows_added Multi value rows added.
355 @param[in,out] write_doc_id Buffer for storing the FTS doc ID.
356 @return DB_SUCCESS or error code. */
357 [[nodiscard]] dberr_t copy_columns(Copy_ctx &ctx, size_t &mv_rows_added,
358 doc_id_t &write_doc_id) noexcept;
359
360 /** Add row to the key buffer.
361 @param[in,out] ctx Copy context.
362 @param[in,out] mv_rows_added Number of multi-value index rows added.
363 @return DB_SUCCESS or error code. */
364 [[nodiscard]] dberr_t add_to_key_buffer(Copy_ctx &ctx,
365 size_t &mv_rows_added) noexcept;
366
367 /** Wait for FTS completion.
368 @param[in] index Index being built. */
370
371 /** Sort the data in the key buffer.
372 @param[in] thread_id Thread ID of current thread.
373 @return DB_SUCCESS or error code. */
374 [[nodiscard]] dberr_t key_buffer_sort(size_t thread_id) noexcept;
375
376 /** Sort the buffer in memory and insert directly in the BTree loader,
377 don't write to a temporary file.
378 @param[in,out] cursor Current scan cursor.
379 @param[in] thread_id ID of current thread.
380 @return DB_SUCCESS or error code. */
381 dberr_t insert_direct(Cursor &cursor, size_t thread_id) noexcept;
382
383 /** Create the merge file, if needed.
384 @param[in,out] file File handle.
385 @return true if file was opened successfully . */
386 [[nodiscard]] bool create_file(ddl::file_t &file) noexcept;
387
388 /** Check for duplicates in the first block
389 @param[in] dupcheck Files to check for duplicates.
390 @param[in,out] dup For collecting duplicate key information.
391 @return DB_SUCCESS or error code. */
392 [[nodiscard]] dberr_t check_duplicates(Thread_ctxs &dupcheck,
393 Dup *dup) noexcept;
394
395 private:
396 /** Buffer ID. */
397 size_t m_id{};
398
399 /** Initial phase. */
400 std::atomic<State> m_state{State::INIT};
401
402 /** DDL Context. */
404
405 /** Loader that owns the instance. */
407
408 /** Index to create (if not FTS index). */
410
411 /** Temporary file path. */
412 const char *m_tmpdir{};
413
414 /** Per thread context. */
416
417 /** For tracking duplicates. */
419
420 /** For collecting duplicate entries (error reporting). */
422
423 /** Scoped virtual column heap. */
425
426 /** Scoped conversion heap. */
428
429 /** The index to be built, FTS or non-FTS. */
431
432 /** Number of active sort tasks. */
433 std::atomic<size_t> m_n_sort_tasks{};
434
435 /** Cluster index bulk load instance to use, direct insert without
436 a file sort. */
438
439 /** Stage per builder. */
441};
442
444 /** Default constructor. */
445 explicit Load_cursor(Builder *builder, Dup *dup) noexcept
446 : m_dup(dup), m_builder(builder) {}
447
448 /** Default destructor. */
449 virtual ~Load_cursor() override = default;
450
451 /** @return the cursor error status. */
452 [[nodiscard]] dberr_t get_err() const noexcept { return m_err; }
453
454 /** @return true if duplicates detected. */
455 [[nodiscard]] bool duplicates_detected() const noexcept override;
456
457 /** Duplicate checking and reporting. */
459
460 /** Operation error code. */
462
463 /** Index meta data. */
465
466 /** Heap for the raw row to dtuple_t conversion. */
468};
469
470/** Merge the sorted files. */
471struct Merge_cursor : public Load_cursor {
472 /** File cursors to use for the scan. */
473 using File_readers = std::vector<File_reader *, ut::allocator<File_reader *>>;
474
475 /** Constructor.
476 @param[in,out] builder Index builder.
477 @param[in,out] dup If not nullptr, then report duplicates.
478 @param[in,out] stage PFS stage monitoring. */
479 explicit Merge_cursor(Builder *builder, Dup *dup,
480 Alter_stage *stage) noexcept;
481
482 /** Destructor. */
483 ~Merge_cursor() noexcept override;
484
485 /** Add the cursor to use for merge load.
486 @param[in] file File to merge from.
487 @param[in] buffer_size IO buffer size to use for reading.
488 @return DB_SUCCESS or error code. */
489 [[nodiscard]] dberr_t add_file(const ddl::file_t &file,
490 size_t buffer_size) noexcept;
491
492 /** Add the cursor to use for merge load.
493 @param[in] file File file to read.
494 @param[in] buffer_size IO buffer size to use for reading.
495 @param[in] offset Page to read from.
496 @return DB_SUCCESS or error code. */
497 [[nodiscard]] dberr_t add_file(const ddl::file_t &file, size_t buffer_size,
498 os_offset_t offset) noexcept;
499
500 /** Open the cursor.
501 @return DB_SUCCESS or error code. */
502 [[nodiscard]] dberr_t open() noexcept;
503
504 /** Fetch the current row as a tuple. Note: Tuple columns are shallow copies.
505 @param[out] dtuple Row represented as a tuple.
506 @return DB_SUCCESS, DB_END_OF_INDEX or error code. */
507 [[nodiscard]] dberr_t fetch(dtuple_t *&dtuple) noexcept override;
508
509 /** Fetch the current row.
510 @param[out] mrec Current merge record.
511 @param[out] offsets Columns offsets inside mrec.
512 @return DB_SUCCESS, DB_END_OF_INDEX or error code. */
513 [[nodiscard]] dberr_t fetch(const mrec_t *&mrec, ulint *&offsets) noexcept;
514
515 /** Move to the next record.
516 @return DB_SUCCESS, DB_END_OF_INDEX or error code. */
517 [[nodiscard]] dberr_t next() noexcept override;
518
519 /** @return the file reader instances. */
520 [[nodiscard]] File_readers file_readers() noexcept;
521
522 /** Add the active cursors to the priority queue. */
523 void clear_eof() noexcept;
524
525 /** @return the number of active readers. */
526 [[nodiscard]] size_t size() const noexcept { return m_pq.size(); }
527
528 /** @return the number of rows read from the files. */
529 [[nodiscard]] uint64_t get_n_rows() const noexcept;
530
531 /** @return the number of cursors being merged. */
532 [[nodiscard]] size_t number_of_cursors() const noexcept {
533 return m_cursors.size();
534 }
535
536 private:
537 /** @return the current cursor at the head of the queue. */
538 [[nodiscard]] File_cursor *pop() noexcept;
539
540 private:
541 /** Comparator. */
542 struct Compare {
543 /** Constructor.
544 @param[in] index Index that the rows belong to.
545 @param[in,out] dup For reporting duplicates, can be nullptr. */
546 explicit Compare(const dict_index_t *index, Dup *dup)
547 : m_dup(dup), m_index(index) {}
548
549 /** Destructor. */
550 Compare() = default;
551
552 /** Compare the keys of two cursors.
553 @param[in] lhs Left hand side.
554 @param[in] rhs Right hand side.
555 @return true if lhs strictly less than rhs. */
556 bool operator()(const File_cursor *lhs,
557 const File_cursor *rhs) const noexcept;
558
559 /** For reporting duplicates. */
561
562 /** Index being built. */
563 const dict_index_t *m_index{};
564 };
565
566 /** File cursors to use for the scan. */
567 using File_cursors = std::vector<File_cursor *, ut::allocator<File_cursor *>>;
568
569 /** Priority queue for ordering the rows. */
570 using Queue = std::priority_queue<File_cursor *, File_cursors, Compare>;
571
572 /** Priority queue for merging the file cursors. */
574
575 /** Cursors to use for parallel loading of the index. */
577
578 /** Current cursor. */
580
581 /** PFS stage monitoring. */
583};
584
585} // namespace ddl
586
587#endif /* !ddl0impl_builder_h */
The B-tree bulk load.
Class used to report ALTER TABLE progress via performance_schema.
Definition: ut0stage.h:80
Definition: btr0load.h:50
Implements a persistent FIFO using server List method names.
Definition: sql_profile.h:75
Build indexes on a table by reading a clustered index, creating a temporary file containing index ent...
Definition: ddl0impl-loader.h:43
Class that caches RTree index tuples made from a single cluster index page scan, and then insert into...
Definition: ddl0impl-rtree.h:43
Page size descriptor.
Definition: page0size.h:49
Allocator that allows std::* containers to manage their memory through ut::malloc* and ut::free libra...
Definition: ut0new.h:2179
dberr_t
Definition: db0err.h:38
@ DB_SUCCESS
Definition: db0err.h:42
DDL buffer infrastrucure.
For scanning the temporary file.
static ulint dict_index_is_unique(const dict_index_t *index)
Check whether the index is unique.
static ulint dict_index_is_spatial(const dict_index_t *index)
Check whether the index is a Spatial Index.
constexpr uint32_t DICT_FTS
FTS index; can't be combined with the other flags.
Definition: dict0mem.h:103
uint64_t doc_id_t
Document id type.
Definition: fts0fts.h:76
static my_thread_id thread_id
Definition: my_thr_init.cc:62
static struct st_file_buffer file_buffer
Definition: myisampack.cc:205
The general architecture is that the work is done in two phases, roughly the read and write phase.
Definition: btr0load.cc:41
std::deque< os_offset_t, ut::allocator< os_offset_t > > Merge_offsets
Start offsets in the file, from where to merge records.
Definition: ddl0impl.h:63
byte mrec_t
Merge record in Aligned_buffer.
Definition: ddl0ddl.h:77
std::function< dberr_t()> Latch_release
Called when a log free check is required.
Definition: ddl0impl.h:49
std::pair< byte *, os_offset_t > IO_buffer
Block size for DDL I/O operations.
Definition: ddl0impl.h:46
Definition: os0file.h:85
static Value err()
Create a Value object that represents an error condition.
Definition: json_binary.cc:909
size_t buffer_size(const ConstBufferSequence &buffers) noexcept
Definition: buffer.h:313
uint64_t os_offset_t
File offset in bytes.
Definition: os0file.h:83
Parallel read interface.
Interface to consume from.
Definition: btr0load.h:53
Heap wrapper that destroys the heap instance when it goes out of scope.
Definition: mem0mem.h:438
void clear() noexcept
Empty the heap.
Definition: mem0mem.h:474
Buffer used for reading and writing to the temporary files.
Definition: ddl0impl-buffer.h:82
State of a cluster index reader thread.
Definition: ddl0impl-builder.h:227
For loading indexes.
Definition: ddl0impl-builder.h:47
dberr_t finalize() noexcept
Flush all dirty pages, apply the row log and write the redo log record.
Definition: ddl0builder.cc:1902
dberr_t insert_direct(Cursor &cursor, size_t thread_id) noexcept
Sort the buffer in memory and insert directly in the BTree loader, don't write to a temporary file.
Definition: ddl0builder.cc:1208
dberr_t btree_build() noexcept
Load the sorted data into the B+Tree.
Definition: ddl0builder.cc:1703
void set_state(State state) noexcept
Set the next state.
Definition: ddl0impl-builder.h:131
std::atomic< size_t > m_n_sort_tasks
Number of active sort tasks.
Definition: ddl0impl-builder.h:433
void fts_wait_for_completion(const dict_index_t *index) noexcept
Wait for FTS completion.
dberr_t add_row(Cursor &cursor, Row &row, size_t thread_id, Latch_release &&latch_release) noexcept
Add a row to the merge buffer.
Definition: ddl0builder.cc:1531
void clear_virtual_heap() noexcept
Clear the heap used for virtual columns.
Definition: ddl0impl-builder.h:317
bool is_skip_file_sort() const noexcept
Definition: ddl0impl-builder.h:157
void set_next_state() noexcept
Set the next state.
Definition: ddl0builder.cc:2038
dberr_t fts_sort_and_build() noexcept
FTS: Sort and insert the rows read.
Definition: ddl0builder.cc:1877
static void write_redo(const dict_index_t *index) noexcept
Write an MLOG_INDEX_LOAD record to indicate in the redo-log that redo-logging of individual index pag...
Definition: ddl0builder.cc:1858
static void copy_blobs(const dict_index_t *index, const mrec_t *mrec, const ulint *offsets, const page_size_t &page_size, dtuple_t *tuple, bool is_sdi, mem_heap_t *heap) noexcept
Copy externally stored columns to the data tuple.
Definition: ddl0builder.cc:1549
bool is_spatial_index() const noexcept
Definition: ddl0impl-builder.h:109
bool create_file(ddl::file_t &file) noexcept
Create the merge file, if needed.
Definition: ddl0builder.cc:1147
std::atomic< State > m_state
Initial phase.
Definition: ddl0impl-builder.h:400
dberr_t key_buffer_sort(size_t thread_id) noexcept
Sort the data in the key buffer.
Definition: ddl0builder.cc:1188
static void convert(const dict_index_t *clust_index, const dfield_t *row_field, dfield_t *field, ulint len, const page_size_t &page_size, bool is_sdi, mem_heap_t *heap) noexcept
Convert the field data from compact to redundant format.
Definition: ddl0builder.cc:477
dberr_t create_merge_sort_tasks() noexcept
Create the tasks to merge Sort the file before we load the file into the Btree index.
Definition: ddl0builder.cc:1777
Context & ctx() noexcept
Definition: ddl0impl-builder.h:103
Alter_stage * m_local_stage
Stage per builder.
Definition: ddl0impl-builder.h:440
dberr_t check_state_of_online_build_log() noexcept
Check the state of the online build log for the index.
Definition: ddl0builder.cc:618
dberr_t dtuple_copy_blobs(dtuple_t *dtuple, ulint *offsets, const mrec_t *mrec, mem_heap_t *heap) noexcept
Copy blobs to the tuple.
Definition: ddl0builder.cc:1598
dberr_t init(Cursor &cursor, size_t n_threads) noexcept
Initialize the cursor.
Definition: ddl0builder.cc:628
Scoped_heap m_conv_heap
Scoped conversion heap.
Definition: ddl0impl-builder.h:427
void fallback_to_single_thread() noexcept
Parallel scan thread spawn failed, release the extra thread states.
Definition: ddl0builder.cc:2028
dict_index_t * m_sort_index
The index to be built, FTS or non-FTS.
Definition: ddl0impl-builder.h:430
Loader & m_loader
Loader that owns the instance.
Definition: ddl0impl-builder.h:406
const char * tmpdir() const noexcept
Definition: ddl0impl-builder.h:201
Builder(ddl::Context &ctx, Loader &loader, size_t i) noexcept
Constructor.
Definition: ddl0builder.cc:584
void set_error(dberr_t err) noexcept
Set the error code.
Definition: ddl0impl-builder.h:94
bool is_unique_index() const noexcept
Definition: ddl0impl-builder.h:119
std::vector< Thread_ctx *, Allocator > Thread_ctxs
Definition: ddl0impl-builder.h:261
dberr_t copy_fts_column(Copy_ctx &ctx, dfield_t *field) noexcept
Copy the FTS columns.
Definition: ddl0builder.cc:838
void fts_add_doc_id(dfield_t *dst, const dict_field_t *src, doc_id_t &write_doc_id) noexcept
Add the FTS document ID to the destination field.
Definition: ddl0builder.cc:757
Dup m_clust_dup
For collecting duplicate entries (error reporting).
Definition: ddl0impl-builder.h:421
dberr_t bulk_add_row(Cursor &cursor, Row &row, size_t thread_id, Latch_release &&latch_release) noexcept
Add a row to the merge buffer.
Definition: ddl0builder.cc:1377
dict_index_t * index() noexcept
Definition: ddl0impl-builder.h:100
ddl::Context & m_ctx
DDL Context.
Definition: ddl0impl-builder.h:403
State
Build phase/states.
Definition: ddl0impl-builder.h:49
@ STOP
Stop on success.
@ ADD
Collect the rows for the index to build.
@ SORT
Sort the collected rows, if required.
@ ERROR
Stop on error.
@ BTREE_BUILD
Build the btree.
@ SETUP_SORT
Setup the merge sort and add the tasks to the task queue.
@ FINISH
Finish the loading of the index.
@ FTS_SORT_AND_BUILD
FTS sort and build, this is done in one "step".
@ INIT
Initial phase.
void batch_insert_deep_copy_tuples(size_t thread_id) noexcept
Note that the latches are going to be released.
Definition: ddl0builder.cc:1182
dict_index_t * m_index
Index to create (if not FTS index).
Definition: ddl0impl-builder.h:409
dberr_t check_duplicates(Thread_ctxs &dupcheck, Dup *dup) noexcept
Check for duplicates in the first block.
Definition: ddl0builder.cc:1635
dberr_t merge_sort(size_t thread_id) noexcept
Non-FTS: Sort the rows read.
Definition: ddl0builder.cc:1938
~Builder() noexcept
Destructor/.
Definition: ddl0builder.cc:600
dberr_t append(ddl::file_t &file, IO_buffer file_buffer) noexcept
Write data to disk - in append mode.
Definition: ddl0builder.cc:1159
dberr_t finish() noexcept
Close temporary files, Flush all dirty pages, apply the row log and write the redo log record.
Definition: ddl0builder.cc:1991
dberr_t get_virtual_column(Copy_ctx &ctx, const dict_field_t *ifield, dict_col_t *col, dfield_t *&src_field, size_t &mv_rows_added) noexcept
Setup the virtual column src column.
Definition: ddl0builder.cc:774
dberr_t copy_columns(Copy_ctx &ctx, size_t &mv_rows_added, doc_id_t &write_doc_id) noexcept
Copy the columns to the temporary file buffer.
Definition: ddl0builder.cc:880
dberr_t add_to_key_buffer(Copy_ctx &ctx, size_t &mv_rows_added) noexcept
Add row to the key buffer.
Definition: ddl0builder.cc:1308
State get_state() const noexcept
Definition: ddl0impl-builder.h:125
dberr_t batch_add_row(Row &row, size_t thread_id) noexcept
Cache a row for batch inserts.
Definition: ddl0builder.cc:1281
dberr_t batch_insert(size_t thread_id, Latch_release &&latch_release) noexcept
Insert cached rows.
Definition: ddl0builder.cc:1172
size_t m_id
Buffer ID.
Definition: ddl0impl-builder.h:397
dberr_t copy_row(Copy_ctx &ctx, size_t &mv_rows_added) noexcept
Add a row to the write buffer.
Definition: ddl0builder.cc:1018
dberr_t setup_sort() noexcept
Non-FTS: Sort the rows read.
Definition: ddl0builder.cc:1975
dberr_t get_error() const noexcept
Definition: ddl0impl-builder.h:90
const char * m_tmpdir
Temporary file path.
Definition: ddl0impl-builder.h:412
Btree_load * m_btr_load
Cluster index bulk load instance to use, direct insert without a file sort.
Definition: ddl0impl-builder.h:437
Scoped_heap m_v_heap
Scoped virtual column heap.
Definition: ddl0impl-builder.h:424
dfield_t * m_prev_fields
For tracking duplicates.
Definition: ddl0impl-builder.h:418
size_t id() const noexcept
Definition: ddl0impl-builder.h:97
bool is_fts_index() const noexcept
Definition: ddl0impl-builder.h:114
Alter_stage * stage() noexcept
Definition: ddl0impl-builder.h:136
Thread_ctxs m_thread_ctxs
Per thread context.
Definition: ddl0impl-builder.h:415
DDL context/configuration.
Definition: ddl0ddl.h:320
void set_error(dberr_t err) noexcept
Set the error code, when it's not specific to an index.
Definition: ddl0ddl.h:426
bool m_skip_pk_sort
Skip the sorting phase if true.
Definition: ddl0ddl.h:630
dberr_t get_error() const noexcept
Definition: ddl0ddl.h:422
Context for copying cluster index row for the index to being created.
Definition: ddl0builder.cc:49
Cursor for reading the data.
Definition: ddl0impl-cursor.h:40
Structure for reporting duplicate records.
Definition: ddl0ddl.h:131
For loading a Btree index from a file.
Definition: ddl0builder.cc:176
Buffer for sorting in main memory.
Definition: ddl0impl-buffer.h:122
Definition: ddl0impl-builder.h:443
virtual ~Load_cursor() override=default
Default destructor.
Scoped_heap m_tuple_heap
Heap for the raw row to dtuple_t conversion.
Definition: ddl0impl-builder.h:467
Builder * m_builder
Index meta data.
Definition: ddl0impl-builder.h:464
dberr_t get_err() const noexcept
Definition: ddl0impl-builder.h:452
Load_cursor(Builder *builder, Dup *dup) noexcept
Default constructor.
Definition: ddl0impl-builder.h:445
bool duplicates_detected() const noexcept override
Definition: ddl0builder.cc:230
dberr_t m_err
Operation error code.
Definition: ddl0impl-builder.h:461
Dup * m_dup
Duplicate checking and reporting.
Definition: ddl0impl-builder.h:458
Comparator.
Definition: ddl0impl-builder.h:542
Compare(const dict_index_t *index, Dup *dup)
Constructor.
Definition: ddl0impl-builder.h:546
Compare()=default
Destructor.
Merge the sorted files.
Definition: ddl0impl-builder.h:471
std::vector< File_cursor *, ut::allocator< File_cursor * > > File_cursors
File cursors to use for the scan.
Definition: ddl0impl-builder.h:567
void clear_eof() noexcept
Add the active cursors to the priority queue.
Definition: ddl0builder.cc:380
dberr_t add_file(const ddl::file_t &file, size_t buffer_size) noexcept
Add the cursor to use for merge load.
Definition: ddl0builder.cc:350
std::vector< File_reader *, ut::allocator< File_reader * > > File_readers
File cursors to use for the scan.
Definition: ddl0impl-builder.h:473
size_t number_of_cursors() const noexcept
Definition: ddl0impl-builder.h:532
Merge_cursor(Builder *builder, Dup *dup, Alter_stage *stage) noexcept
Constructor.
Definition: ddl0builder.cc:319
dberr_t fetch(dtuple_t *&dtuple) noexcept override
Fetch the current row as a tuple.
Definition: ddl0builder.cc:428
~Merge_cursor() noexcept override
Destructor.
Definition: ddl0builder.cc:325
File_readers file_readers() noexcept
Definition: ddl0builder.cc:309
Queue m_pq
Priority queue for merging the file cursors.
Definition: ddl0impl-builder.h:573
File_cursor * pop() noexcept
Definition: ddl0builder.cc:416
dberr_t next() noexcept override
Move to the next record.
Definition: ddl0builder.cc:450
size_t size() const noexcept
Definition: ddl0impl-builder.h:526
Alter_stage * m_stage
PFS stage monitoring.
Definition: ddl0impl-builder.h:582
dberr_t open() noexcept
Open the cursor.
Definition: ddl0builder.cc:396
uint64_t get_n_rows() const noexcept
Definition: ddl0builder.cc:467
File_cursors m_cursors
Cursors to use for parallel loading of the index.
Definition: ddl0impl-builder.h:576
File_cursor * m_cursor
Current cursor.
Definition: ddl0impl-builder.h:579
Physical row context.
Definition: ddl0impl.h:120
Information about temporary files used in merge sort.
Definition: ddl0impl.h:66
Structure for an SQL data field.
Definition: data0data.h:604
Data structure for a column in a table.
Definition: dict0mem.h:488
Data structure for a field in an index.
Definition: dict0mem.h:894
Data structure for an index.
Definition: dict0mem.h:1045
unsigned type
index type (DICT_CLUSTERED, DICT_UNIQUE, DICT_IBUF, DICT_CORRUPT)
Definition: dict0mem.h:1072
bool is_clustered() const
Definition: dict0mem.h:1310
Structure for an SQL data tuple of fields (logical record)
Definition: data0data.h:681
The info structure stored at the beginning of a heap block.
Definition: mem0mem.h:301
#define IF_DEBUG(...)
Definition: univ.i:675
unsigned long int ulint
Definition: univ.i:407
#define ut_a(EXPR)
Abort execution if EXPR does not evaluate to nonzero.
Definition: ut0dbg.h:56