32#ifndef ddl0impl_builder_h
33#define ddl0impl_builder_h
98 [[nodiscard]]
size_t id() const noexcept {
return m_id; }
127 return m_state.load(std::memory_order_seq_cst);
133 m_state.store(state, std::memory_order_seq_cst);
347 size_t &mv_rows_added)
noexcept;
369 size_t &mv_rows_added)
noexcept;
487 using File_readers = std::vector<File_reader *, ut::allocator<File_reader *>>;
540 [[nodiscard]]
size_t size() const noexcept {
return m_pq.size(); }
543 [[nodiscard]] uint64_t
get_n_rows() const noexcept;
561 :
m_dup(dup), m_index(index) {}
581 using File_cursors = std::vector<File_cursor *, ut::allocator<File_cursor *>>;
584 using Queue = std::priority_queue<File_cursor *, File_cursors, Compare>;
static Mysys_charset_loader * loader
Definition: charset.cc:185
Class used to report ALTER TABLE progress via performance_schema.
Definition: ut0stage.h:81
Definition: btr0load.h:51
Implements a persistent FIFO using server List method names.
Definition: sql_profile.h:76
Build indexes on a table by reading a clustered index, creating a temporary file containing index ent...
Definition: ddl0impl-loader.h:44
Class that caches RTree index tuples made from a single cluster index page scan, and then insert into...
Definition: ddl0impl-rtree.h:44
Page size descriptor.
Definition: page0size.h:50
Allocator that allows std::* containers to manage their memory through ut::malloc* and ut::free libra...
Definition: ut0new.h:2182
dberr_t
Definition: db0err.h:39
@ DB_SUCCESS
Definition: db0err.h:43
DDL buffer infrastructure.
For scanning the temporary file.
static ulint dict_index_is_unique(const dict_index_t *index)
Check whether the index is unique.
static ulint dict_index_is_spatial(const dict_index_t *index)
Check whether the index is a Spatial Index.
constexpr uint32_t DICT_FTS
FTS index; can't be combined with the other flags.
Definition: dict0mem.h:104
uint64_t doc_id_t
Document id type.
Definition: fts0fts.h:79
static my_thread_id thread_id
Definition: my_thr_init.cc:63
static struct st_file_buffer file_buffer
Definition: myisampack.cc:209
The general architecture is that the work is done in two phases, roughly the read and write phase.
Definition: btr0load.cc:42
std::deque< os_offset_t, ut::allocator< os_offset_t > > Merge_offsets
Start offsets in the file, from where to merge records.
Definition: ddl0impl.h:64
byte mrec_t
Merge record in Aligned_buffer.
Definition: ddl0ddl.h:78
std::function< dberr_t()> Latch_release
Called when a log free check is required.
Definition: ddl0impl.h:50
std::pair< byte *, os_offset_t > IO_buffer
Block size for DDL I/O operations.
Definition: ddl0impl.h:47
static Value err()
Create a Value object that represents an error condition.
Definition: json_binary.cc:927
size_t buffer_size(const ConstBufferSequence &buffers) noexcept
Definition: buffer.h:313
std::conditional_t< !std::is_array< T >::value, std::unique_ptr< T, detail::Aligned_deleter< T > >, std::conditional_t< detail::is_unbounded_array_v< T >, std::unique_ptr< T, detail::Aligned_array_deleter< std::remove_extent_t< T > > >, void > > unique_ptr_aligned
The following is a common type that is returned by all the ut::make_unique_aligned (non-aligned) spec...
Definition: ut0new.h:2573
uint64_t os_offset_t
File offset in bytes.
Definition: os0file.h:87
Interface to consume from.
Definition: btr0load.h:54
Heap wrapper that destroys the heap instance when it goes out of scope.
Definition: mem0mem.h:439
void clear() noexcept
Empty the heap.
Definition: mem0mem.h:479
State of a cluster index reader thread.
Definition: ddl0impl-builder.h:228
IO_buffer m_io_buffer
Buffer to use for file writes.
Definition: ddl0impl-builder.h:255
For loading indexes.
Definition: ddl0impl-builder.h:48
dberr_t finalize() noexcept
Flush all dirty pages, apply the row log and write the redo log record.
Definition: ddl0builder.cc:1948
dberr_t insert_direct(Cursor &cursor, size_t thread_id) noexcept
Sort the buffer in memory and insert directly in the BTree loader, don't write to a temporary file.
Definition: ddl0builder.cc:1238
dberr_t btree_build() noexcept
Load the sorted data into the B+Tree.
Definition: ddl0builder.cc:1751
void set_state(State state) noexcept
Set the next state.
Definition: ddl0impl-builder.h:132
std::atomic< size_t > m_n_sort_tasks
Number of active sort tasks.
Definition: ddl0impl-builder.h:447
void fts_wait_for_completion(const dict_index_t *index) noexcept
Wait for FTS completion.
dberr_t add_row(Cursor &cursor, Row &row, size_t thread_id, Latch_release &&latch_release) noexcept
Add a row to the merge buffer.
Definition: ddl0builder.cc:1574
void clear_virtual_heap() noexcept
Clear the heap used for virtual columns.
Definition: ddl0impl-builder.h:321
bool is_skip_file_sort() const noexcept
Definition: ddl0impl-builder.h:158
void set_next_state() noexcept
Set the next state.
Definition: ddl0builder.cc:2084
dberr_t fts_sort_and_build() noexcept
FTS: Sort and insert the rows read.
Definition: ddl0builder.cc:1923
static void write_redo(const dict_index_t *index) noexcept
Write an MLOG_INDEX_LOAD record to indicate in the redo-log that redo-logging of individual index pag...
Definition: ddl0builder.cc:1904
static void copy_blobs(const dict_index_t *index, const mrec_t *mrec, const ulint *offsets, const page_size_t &page_size, dtuple_t *tuple, bool is_sdi, mem_heap_t *heap) noexcept
Copy externally stored columns to the data tuple.
Definition: ddl0builder.cc:1597
bool is_spatial_index() const noexcept
Definition: ddl0impl-builder.h:110
bool create_file(ddl::file_t &file) noexcept
Create the merge file, if needed.
Definition: ddl0builder.cc:1160
dberr_t online_build_handle_error(dberr_t err) noexcept
Cleanup DDL after error in online build Note: To be called if DDL must cleanup due to error in online...
Definition: ddl0builder.cc:1221
std::atomic< State > m_state
Initial phase.
Definition: ddl0impl-builder.h:414
dberr_t key_buffer_sort(size_t thread_id) noexcept
Sort the data in the key buffer.
Definition: ddl0builder.cc:1201
static void convert(const dict_index_t *clust_index, const dfield_t *row_field, dfield_t *field, ulint len, const page_size_t &page_size, bool is_sdi, mem_heap_t *heap) noexcept
Convert the field data from compact to redundant format.
Definition: ddl0builder.cc:478
dberr_t create_merge_sort_tasks() noexcept
Create the tasks to merge Sort the file before we load the file into the Btree index.
Definition: ddl0builder.cc:1823
Context & ctx() noexcept
Definition: ddl0impl-builder.h:104
Alter_stage * m_local_stage
Stage per builder.
Definition: ddl0impl-builder.h:454
dberr_t check_state_of_online_build_log() noexcept
Check the state of the online build log for the index.
Definition: ddl0builder.cc:620
dberr_t dtuple_copy_blobs(dtuple_t *dtuple, ulint *offsets, const mrec_t *mrec, mem_heap_t *heap) noexcept
Copy blobs to the tuple.
Definition: ddl0builder.cc:1646
dberr_t init(Cursor &cursor, size_t n_threads) noexcept
Initialize the cursor.
Definition: ddl0builder.cc:630
Scoped_heap m_conv_heap
Scoped conversion heap.
Definition: ddl0impl-builder.h:441
void fallback_to_single_thread() noexcept
Parallel scan thread spawn failed, release the extra thread states.
Definition: ddl0builder.cc:2074
dict_index_t * m_sort_index
The index to be built, FTS or non-FTS.
Definition: ddl0impl-builder.h:444
Loader & m_loader
Loader that owns the instance.
Definition: ddl0impl-builder.h:420
const char * tmpdir() const noexcept
Definition: ddl0impl-builder.h:202
Builder(ddl::Context &ctx, Loader &loader, size_t i) noexcept
Constructor.
Definition: ddl0builder.cc:585
void set_error(dberr_t err) noexcept
Set the error code.
Definition: ddl0impl-builder.h:95
bool is_unique_index() const noexcept
Definition: ddl0impl-builder.h:120
std::vector< Thread_ctx *, Allocator > Thread_ctxs
Definition: ddl0impl-builder.h:265
dberr_t copy_fts_column(Copy_ctx &ctx, dfield_t *field) noexcept
Copy the FTS columns.
Definition: ddl0builder.cc:847
void fts_add_doc_id(dfield_t *dst, const dict_field_t *src, doc_id_t &write_doc_id) noexcept
Add the FTS document ID to the destination field.
Definition: ddl0builder.cc:766
Dup m_clust_dup
For collecting duplicate entries (error reporting).
Definition: ddl0impl-builder.h:435
dberr_t bulk_add_row(Cursor &cursor, Row &row, size_t thread_id, Latch_release &&latch_release) noexcept
Add a row to the merge buffer.
Definition: ddl0builder.cc:1424
dict_index_t * index() noexcept
Definition: ddl0impl-builder.h:101
ddl::Context & m_ctx
DDL Context.
Definition: ddl0impl-builder.h:417
State
Build phase/states.
Definition: ddl0impl-builder.h:50
@ ADD
Collect the rows for the index to build.
@ SORT
Sort the collected rows, if required.
@ BTREE_BUILD
Build the btree.
@ SETUP_SORT
Setup the merge sort and add the tasks to the task queue.
@ FINISH
Finish the loading of the index.
@ FTS_SORT_AND_BUILD
FTS sort and build, this is done in one "step".
void batch_insert_deep_copy_tuples(size_t thread_id) noexcept
Note that the latches are going to be released.
Definition: ddl0builder.cc:1195
dict_index_t * m_index
Index to create (if not FTS index).
Definition: ddl0impl-builder.h:423
dberr_t check_duplicates(Thread_ctxs &dupcheck, Dup *dup) noexcept
Check for duplicates in the first block.
Definition: ddl0builder.cc:1683
dberr_t merge_sort(size_t thread_id) noexcept
Non-FTS: Sort the rows read.
Definition: ddl0builder.cc:1984
~Builder() noexcept
Destructor/.
Definition: ddl0builder.cc:602
dberr_t append(ddl::file_t &file, IO_buffer file_buffer) noexcept
Write data to disk - in append mode.
Definition: ddl0builder.cc:1172
dberr_t finish() noexcept
Close temporary files, Flush all dirty pages, apply the row log and write the redo log record.
Definition: ddl0builder.cc:2037
dberr_t get_virtual_column(Copy_ctx &ctx, const dict_field_t *ifield, dict_col_t *col, dfield_t *&src_field, size_t &mv_rows_added) noexcept
Setup the virtual column src column.
Definition: ddl0builder.cc:783
dberr_t copy_columns(Copy_ctx &ctx, size_t &mv_rows_added, doc_id_t &write_doc_id) noexcept
Copy the columns to the temporary file buffer.
Definition: ddl0builder.cc:889
dberr_t add_to_key_buffer(Copy_ctx &ctx, size_t &mv_rows_added) noexcept
Add row to the key buffer.
Definition: ddl0builder.cc:1355
State get_state() const noexcept
Definition: ddl0impl-builder.h:126
dberr_t batch_add_row(Row &row, size_t thread_id) noexcept
Cache a row for batch inserts.
Definition: ddl0builder.cc:1328
dberr_t batch_insert(size_t thread_id, Latch_release &&latch_release) noexcept
Insert cached rows.
Definition: ddl0builder.cc:1185
size_t m_id
Buffer ID.
Definition: ddl0impl-builder.h:411
dberr_t copy_row(Copy_ctx &ctx, size_t &mv_rows_added) noexcept
Add a row to the write buffer.
Definition: ddl0builder.cc:1031
dberr_t setup_sort() noexcept
Non-FTS: Sort the rows read.
Definition: ddl0builder.cc:2021
dberr_t get_error() const noexcept
Definition: ddl0impl-builder.h:91
const char * m_tmpdir
Temporary file path.
Definition: ddl0impl-builder.h:426
Btree_load * m_btr_load
Cluster index bulk load instance to use, direct insert without a file sort.
Definition: ddl0impl-builder.h:451
Scoped_heap m_v_heap
Scoped virtual column heap.
Definition: ddl0impl-builder.h:438
dfield_t * m_prev_fields
For tracking duplicates.
Definition: ddl0impl-builder.h:432
size_t id() const noexcept
Definition: ddl0impl-builder.h:98
bool is_fts_index() const noexcept
Definition: ddl0impl-builder.h:115
Alter_stage * stage() noexcept
Definition: ddl0impl-builder.h:137
Thread_ctxs m_thread_ctxs
Per thread context.
Definition: ddl0impl-builder.h:429
DDL context/configuration.
Definition: ddl0ddl.h:321
void set_error(dberr_t err) noexcept
Set the error code, when it's not specific to an index.
Definition: ddl0ddl.h:427
bool m_skip_pk_sort
Skip the sorting phase if true.
Definition: ddl0ddl.h:631
dberr_t get_error() const noexcept
Definition: ddl0ddl.h:423
Context for copying cluster index row for the index to being created.
Definition: ddl0builder.cc:50
Cursor for reading the data.
Definition: ddl0impl-cursor.h:41
Structure for reporting duplicate records.
Definition: ddl0ddl.h:132
For loading a Btree index from a file.
Definition: ddl0builder.cc:177
Buffer for sorting in main memory.
Definition: ddl0impl-buffer.h:41
Definition: ddl0impl-builder.h:457
virtual ~Load_cursor() override=default
Default destructor.
Scoped_heap m_tuple_heap
Heap for the raw row to dtuple_t conversion.
Definition: ddl0impl-builder.h:481
Builder * m_builder
Index meta data.
Definition: ddl0impl-builder.h:478
dberr_t get_err() const noexcept
Definition: ddl0impl-builder.h:466
Load_cursor(Builder *builder, Dup *dup) noexcept
Default constructor.
Definition: ddl0impl-builder.h:459
bool duplicates_detected() const noexcept override
Definition: ddl0builder.cc:231
dberr_t m_err
Operation error code.
Definition: ddl0impl-builder.h:475
Dup * m_dup
Duplicate checking and reporting.
Definition: ddl0impl-builder.h:472
Comparator.
Definition: ddl0impl-builder.h:556
Compare(const dict_index_t *index, Dup *dup)
Constructor.
Definition: ddl0impl-builder.h:560
Compare()=default
Destructor.
Merge the sorted files.
Definition: ddl0impl-builder.h:485
std::vector< File_cursor *, ut::allocator< File_cursor * > > File_cursors
File cursors to use for the scan.
Definition: ddl0impl-builder.h:581
void clear_eof() noexcept
Add the active cursors to the priority queue.
Definition: ddl0builder.cc:381
dberr_t add_file(const ddl::file_t &file, size_t buffer_size) noexcept
Add the cursor to use for merge load.
Definition: ddl0builder.cc:351
std::vector< File_reader *, ut::allocator< File_reader * > > File_readers
File cursors to use for the scan.
Definition: ddl0impl-builder.h:487
size_t number_of_cursors() const noexcept
Definition: ddl0impl-builder.h:546
Merge_cursor(Builder *builder, Dup *dup, Alter_stage *stage) noexcept
Constructor.
Definition: ddl0builder.cc:320
dberr_t fetch(dtuple_t *&dtuple) noexcept override
Fetch the current row as a tuple.
Definition: ddl0builder.cc:429
~Merge_cursor() noexcept override
Destructor.
Definition: ddl0builder.cc:326
File_readers file_readers() noexcept
Definition: ddl0builder.cc:310
Queue m_pq
Priority queue for merging the file cursors.
Definition: ddl0impl-builder.h:587
File_cursor * pop() noexcept
Definition: ddl0builder.cc:417
dberr_t next() noexcept override
Move to the next record.
Definition: ddl0builder.cc:451
size_t size() const noexcept
Definition: ddl0impl-builder.h:540
Alter_stage * m_stage
PFS stage monitoring.
Definition: ddl0impl-builder.h:596
dberr_t open() noexcept
Open the cursor.
Definition: ddl0builder.cc:397
uint64_t get_n_rows() const noexcept
Definition: ddl0builder.cc:468
File_cursors m_cursors
Cursors to use for parallel loading of the index.
Definition: ddl0impl-builder.h:590
File_cursor * m_cursor
Current cursor.
Definition: ddl0impl-builder.h:593
Physical row context.
Definition: ddl0impl.h:121
Information about temporary files used in merge sort.
Definition: ddl0impl.h:67
Structure for an SQL data field.
Definition: data0data.h:605
Data structure for a column in a table.
Definition: dict0mem.h:489
Data structure for a field in an index.
Definition: dict0mem.h:895
Data structure for an index.
Definition: dict0mem.h:1046
unsigned type
index type (DICT_CLUSTERED, DICT_UNIQUE, DICT_IBUF, DICT_CORRUPT)
Definition: dict0mem.h:1073
bool is_clustered() const
Definition: dict0mem.h:1311
Structure for an SQL data tuple of fields (logical record)
Definition: data0data.h:684
The info structure stored at the beginning of a heap block.
Definition: mem0mem.h:302
#define IF_DEBUG(...)
Definition: univ.i:674
unsigned long int ulint
Definition: univ.i:406
#define ut_a(EXPR)
Abort execution if EXPR does not evaluate to nonzero.
Definition: ut0dbg.h:93