31#ifndef ddl0impl_builder_h
32#define ddl0impl_builder_h
97 [[nodiscard]]
size_t id() const noexcept {
return m_id; }
126 return m_state.load(std::memory_order_seq_cst);
132 m_state.store(state, std::memory_order_seq_cst);
343 size_t &mv_rows_added)
noexcept;
365 size_t &mv_rows_added)
noexcept;
473 using File_readers = std::vector<File_reader *, ut::allocator<File_reader *>>;
526 [[nodiscard]]
size_t size() const noexcept {
return m_pq.size(); }
529 [[nodiscard]] uint64_t
get_n_rows() const noexcept;
547 :
m_dup(dup), m_index(index) {}
567 using File_cursors = std::vector<File_cursor *, ut::allocator<File_cursor *>>;
570 using Queue = std::priority_queue<File_cursor *, File_cursors, Compare>;
Class used to report ALTER TABLE progress via performance_schema.
Definition: ut0stage.h:80
Definition: btr0load.h:50
Implements a persistent FIFO using server List method names.
Definition: sql_profile.h:75
Build indexes on a table by reading a clustered index, creating a temporary file containing index ent...
Definition: ddl0impl-loader.h:43
Class that caches RTree index tuples made from a single cluster index page scan, and then insert into...
Definition: ddl0impl-rtree.h:43
Page size descriptor.
Definition: page0size.h:49
Allocator that allows std::* containers to manage their memory through ut::malloc* and ut::free libra...
Definition: ut0new.h:2179
dberr_t
Definition: db0err.h:38
@ DB_SUCCESS
Definition: db0err.h:42
DDL buffer infrastrucure.
For scanning the temporary file.
static ulint dict_index_is_unique(const dict_index_t *index)
Check whether the index is unique.
static ulint dict_index_is_spatial(const dict_index_t *index)
Check whether the index is a Spatial Index.
constexpr uint32_t DICT_FTS
FTS index; can't be combined with the other flags.
Definition: dict0mem.h:103
uint64_t doc_id_t
Document id type.
Definition: fts0fts.h:76
static my_thread_id thread_id
Definition: my_thr_init.cc:62
static struct st_file_buffer file_buffer
Definition: myisampack.cc:205
The general architecture is that the work is done in two phases, roughly the read and write phase.
Definition: btr0load.cc:41
std::deque< os_offset_t, ut::allocator< os_offset_t > > Merge_offsets
Start offsets in the file, from where to merge records.
Definition: ddl0impl.h:63
byte mrec_t
Merge record in Aligned_buffer.
Definition: ddl0ddl.h:77
std::function< dberr_t()> Latch_release
Called when a log free check is required.
Definition: ddl0impl.h:49
std::pair< byte *, os_offset_t > IO_buffer
Block size for DDL I/O operations.
Definition: ddl0impl.h:46
static Value err()
Create a Value object that represents an error condition.
Definition: json_binary.cc:909
size_t buffer_size(const ConstBufferSequence &buffers) noexcept
Definition: buffer.h:313
uint64_t os_offset_t
File offset in bytes.
Definition: os0file.h:83
Interface to consume from.
Definition: btr0load.h:53
Heap wrapper that destroys the heap instance when it goes out of scope.
Definition: mem0mem.h:438
void clear() noexcept
Empty the heap.
Definition: mem0mem.h:474
Buffer used for reading and writing to the temporary files.
Definition: ddl0impl-buffer.h:82
State of a cluster index reader thread.
Definition: ddl0impl-builder.h:227
For loading indexes.
Definition: ddl0impl-builder.h:47
dberr_t finalize() noexcept
Flush all dirty pages, apply the row log and write the redo log record.
Definition: ddl0builder.cc:1902
dberr_t insert_direct(Cursor &cursor, size_t thread_id) noexcept
Sort the buffer in memory and insert directly in the BTree loader, don't write to a temporary file.
Definition: ddl0builder.cc:1208
dberr_t btree_build() noexcept
Load the sorted data into the B+Tree.
Definition: ddl0builder.cc:1703
void set_state(State state) noexcept
Set the next state.
Definition: ddl0impl-builder.h:131
std::atomic< size_t > m_n_sort_tasks
Number of active sort tasks.
Definition: ddl0impl-builder.h:433
void fts_wait_for_completion(const dict_index_t *index) noexcept
Wait for FTS completion.
dberr_t add_row(Cursor &cursor, Row &row, size_t thread_id, Latch_release &&latch_release) noexcept
Add a row to the merge buffer.
Definition: ddl0builder.cc:1531
void clear_virtual_heap() noexcept
Clear the heap used for virtual columns.
Definition: ddl0impl-builder.h:317
bool is_skip_file_sort() const noexcept
Definition: ddl0impl-builder.h:157
void set_next_state() noexcept
Set the next state.
Definition: ddl0builder.cc:2038
dberr_t fts_sort_and_build() noexcept
FTS: Sort and insert the rows read.
Definition: ddl0builder.cc:1877
static void write_redo(const dict_index_t *index) noexcept
Write an MLOG_INDEX_LOAD record to indicate in the redo-log that redo-logging of individual index pag...
Definition: ddl0builder.cc:1858
static void copy_blobs(const dict_index_t *index, const mrec_t *mrec, const ulint *offsets, const page_size_t &page_size, dtuple_t *tuple, bool is_sdi, mem_heap_t *heap) noexcept
Copy externally stored columns to the data tuple.
Definition: ddl0builder.cc:1549
bool is_spatial_index() const noexcept
Definition: ddl0impl-builder.h:109
bool create_file(ddl::file_t &file) noexcept
Create the merge file, if needed.
Definition: ddl0builder.cc:1147
std::atomic< State > m_state
Initial phase.
Definition: ddl0impl-builder.h:400
dberr_t key_buffer_sort(size_t thread_id) noexcept
Sort the data in the key buffer.
Definition: ddl0builder.cc:1188
static void convert(const dict_index_t *clust_index, const dfield_t *row_field, dfield_t *field, ulint len, const page_size_t &page_size, bool is_sdi, mem_heap_t *heap) noexcept
Convert the field data from compact to redundant format.
Definition: ddl0builder.cc:477
dberr_t create_merge_sort_tasks() noexcept
Create the tasks to merge Sort the file before we load the file into the Btree index.
Definition: ddl0builder.cc:1777
Context & ctx() noexcept
Definition: ddl0impl-builder.h:103
Alter_stage * m_local_stage
Stage per builder.
Definition: ddl0impl-builder.h:440
dberr_t check_state_of_online_build_log() noexcept
Check the state of the online build log for the index.
Definition: ddl0builder.cc:618
dberr_t dtuple_copy_blobs(dtuple_t *dtuple, ulint *offsets, const mrec_t *mrec, mem_heap_t *heap) noexcept
Copy blobs to the tuple.
Definition: ddl0builder.cc:1598
dberr_t init(Cursor &cursor, size_t n_threads) noexcept
Initialize the cursor.
Definition: ddl0builder.cc:628
Scoped_heap m_conv_heap
Scoped conversion heap.
Definition: ddl0impl-builder.h:427
void fallback_to_single_thread() noexcept
Parallel scan thread spawn failed, release the extra thread states.
Definition: ddl0builder.cc:2028
dict_index_t * m_sort_index
The index to be built, FTS or non-FTS.
Definition: ddl0impl-builder.h:430
Loader & m_loader
Loader that owns the instance.
Definition: ddl0impl-builder.h:406
const char * tmpdir() const noexcept
Definition: ddl0impl-builder.h:201
Builder(ddl::Context &ctx, Loader &loader, size_t i) noexcept
Constructor.
Definition: ddl0builder.cc:584
void set_error(dberr_t err) noexcept
Set the error code.
Definition: ddl0impl-builder.h:94
bool is_unique_index() const noexcept
Definition: ddl0impl-builder.h:119
std::vector< Thread_ctx *, Allocator > Thread_ctxs
Definition: ddl0impl-builder.h:261
dberr_t copy_fts_column(Copy_ctx &ctx, dfield_t *field) noexcept
Copy the FTS columns.
Definition: ddl0builder.cc:838
void fts_add_doc_id(dfield_t *dst, const dict_field_t *src, doc_id_t &write_doc_id) noexcept
Add the FTS document ID to the destination field.
Definition: ddl0builder.cc:757
Dup m_clust_dup
For collecting duplicate entries (error reporting).
Definition: ddl0impl-builder.h:421
dberr_t bulk_add_row(Cursor &cursor, Row &row, size_t thread_id, Latch_release &&latch_release) noexcept
Add a row to the merge buffer.
Definition: ddl0builder.cc:1377
dict_index_t * index() noexcept
Definition: ddl0impl-builder.h:100
ddl::Context & m_ctx
DDL Context.
Definition: ddl0impl-builder.h:403
State
Build phase/states.
Definition: ddl0impl-builder.h:49
@ ADD
Collect the rows for the index to build.
@ SORT
Sort the collected rows, if required.
@ BTREE_BUILD
Build the btree.
@ SETUP_SORT
Setup the merge sort and add the tasks to the task queue.
@ FINISH
Finish the loading of the index.
@ FTS_SORT_AND_BUILD
FTS sort and build, this is done in one "step".
void batch_insert_deep_copy_tuples(size_t thread_id) noexcept
Note that the latches are going to be released.
Definition: ddl0builder.cc:1182
dict_index_t * m_index
Index to create (if not FTS index).
Definition: ddl0impl-builder.h:409
dberr_t check_duplicates(Thread_ctxs &dupcheck, Dup *dup) noexcept
Check for duplicates in the first block.
Definition: ddl0builder.cc:1635
dberr_t merge_sort(size_t thread_id) noexcept
Non-FTS: Sort the rows read.
Definition: ddl0builder.cc:1938
~Builder() noexcept
Destructor/.
Definition: ddl0builder.cc:600
dberr_t append(ddl::file_t &file, IO_buffer file_buffer) noexcept
Write data to disk - in append mode.
Definition: ddl0builder.cc:1159
dberr_t finish() noexcept
Close temporary files, Flush all dirty pages, apply the row log and write the redo log record.
Definition: ddl0builder.cc:1991
dberr_t get_virtual_column(Copy_ctx &ctx, const dict_field_t *ifield, dict_col_t *col, dfield_t *&src_field, size_t &mv_rows_added) noexcept
Setup the virtual column src column.
Definition: ddl0builder.cc:774
dberr_t copy_columns(Copy_ctx &ctx, size_t &mv_rows_added, doc_id_t &write_doc_id) noexcept
Copy the columns to the temporary file buffer.
Definition: ddl0builder.cc:880
dberr_t add_to_key_buffer(Copy_ctx &ctx, size_t &mv_rows_added) noexcept
Add row to the key buffer.
Definition: ddl0builder.cc:1308
State get_state() const noexcept
Definition: ddl0impl-builder.h:125
dberr_t batch_add_row(Row &row, size_t thread_id) noexcept
Cache a row for batch inserts.
Definition: ddl0builder.cc:1281
dberr_t batch_insert(size_t thread_id, Latch_release &&latch_release) noexcept
Insert cached rows.
Definition: ddl0builder.cc:1172
size_t m_id
Buffer ID.
Definition: ddl0impl-builder.h:397
dberr_t copy_row(Copy_ctx &ctx, size_t &mv_rows_added) noexcept
Add a row to the write buffer.
Definition: ddl0builder.cc:1018
dberr_t setup_sort() noexcept
Non-FTS: Sort the rows read.
Definition: ddl0builder.cc:1975
dberr_t get_error() const noexcept
Definition: ddl0impl-builder.h:90
const char * m_tmpdir
Temporary file path.
Definition: ddl0impl-builder.h:412
Btree_load * m_btr_load
Cluster index bulk load instance to use, direct insert without a file sort.
Definition: ddl0impl-builder.h:437
Scoped_heap m_v_heap
Scoped virtual column heap.
Definition: ddl0impl-builder.h:424
dfield_t * m_prev_fields
For tracking duplicates.
Definition: ddl0impl-builder.h:418
size_t id() const noexcept
Definition: ddl0impl-builder.h:97
bool is_fts_index() const noexcept
Definition: ddl0impl-builder.h:114
Alter_stage * stage() noexcept
Definition: ddl0impl-builder.h:136
Thread_ctxs m_thread_ctxs
Per thread context.
Definition: ddl0impl-builder.h:415
DDL context/configuration.
Definition: ddl0ddl.h:320
void set_error(dberr_t err) noexcept
Set the error code, when it's not specific to an index.
Definition: ddl0ddl.h:426
bool m_skip_pk_sort
Skip the sorting phase if true.
Definition: ddl0ddl.h:630
dberr_t get_error() const noexcept
Definition: ddl0ddl.h:422
Context for copying cluster index row for the index to being created.
Definition: ddl0builder.cc:49
Cursor for reading the data.
Definition: ddl0impl-cursor.h:40
Structure for reporting duplicate records.
Definition: ddl0ddl.h:131
For loading a Btree index from a file.
Definition: ddl0builder.cc:176
Buffer for sorting in main memory.
Definition: ddl0impl-buffer.h:122
Definition: ddl0impl-builder.h:443
virtual ~Load_cursor() override=default
Default destructor.
Scoped_heap m_tuple_heap
Heap for the raw row to dtuple_t conversion.
Definition: ddl0impl-builder.h:467
Builder * m_builder
Index meta data.
Definition: ddl0impl-builder.h:464
dberr_t get_err() const noexcept
Definition: ddl0impl-builder.h:452
Load_cursor(Builder *builder, Dup *dup) noexcept
Default constructor.
Definition: ddl0impl-builder.h:445
bool duplicates_detected() const noexcept override
Definition: ddl0builder.cc:230
dberr_t m_err
Operation error code.
Definition: ddl0impl-builder.h:461
Dup * m_dup
Duplicate checking and reporting.
Definition: ddl0impl-builder.h:458
Comparator.
Definition: ddl0impl-builder.h:542
Compare(const dict_index_t *index, Dup *dup)
Constructor.
Definition: ddl0impl-builder.h:546
Compare()=default
Destructor.
Merge the sorted files.
Definition: ddl0impl-builder.h:471
std::vector< File_cursor *, ut::allocator< File_cursor * > > File_cursors
File cursors to use for the scan.
Definition: ddl0impl-builder.h:567
void clear_eof() noexcept
Add the active cursors to the priority queue.
Definition: ddl0builder.cc:380
dberr_t add_file(const ddl::file_t &file, size_t buffer_size) noexcept
Add the cursor to use for merge load.
Definition: ddl0builder.cc:350
std::vector< File_reader *, ut::allocator< File_reader * > > File_readers
File cursors to use for the scan.
Definition: ddl0impl-builder.h:473
size_t number_of_cursors() const noexcept
Definition: ddl0impl-builder.h:532
Merge_cursor(Builder *builder, Dup *dup, Alter_stage *stage) noexcept
Constructor.
Definition: ddl0builder.cc:319
dberr_t fetch(dtuple_t *&dtuple) noexcept override
Fetch the current row as a tuple.
Definition: ddl0builder.cc:428
~Merge_cursor() noexcept override
Destructor.
Definition: ddl0builder.cc:325
File_readers file_readers() noexcept
Definition: ddl0builder.cc:309
Queue m_pq
Priority queue for merging the file cursors.
Definition: ddl0impl-builder.h:573
File_cursor * pop() noexcept
Definition: ddl0builder.cc:416
dberr_t next() noexcept override
Move to the next record.
Definition: ddl0builder.cc:450
size_t size() const noexcept
Definition: ddl0impl-builder.h:526
Alter_stage * m_stage
PFS stage monitoring.
Definition: ddl0impl-builder.h:582
dberr_t open() noexcept
Open the cursor.
Definition: ddl0builder.cc:396
uint64_t get_n_rows() const noexcept
Definition: ddl0builder.cc:467
File_cursors m_cursors
Cursors to use for parallel loading of the index.
Definition: ddl0impl-builder.h:576
File_cursor * m_cursor
Current cursor.
Definition: ddl0impl-builder.h:579
Physical row context.
Definition: ddl0impl.h:120
Information about temporary files used in merge sort.
Definition: ddl0impl.h:66
Structure for an SQL data field.
Definition: data0data.h:604
Data structure for a column in a table.
Definition: dict0mem.h:488
Data structure for a field in an index.
Definition: dict0mem.h:894
Data structure for an index.
Definition: dict0mem.h:1045
unsigned type
index type (DICT_CLUSTERED, DICT_UNIQUE, DICT_IBUF, DICT_CORRUPT)
Definition: dict0mem.h:1072
bool is_clustered() const
Definition: dict0mem.h:1310
Structure for an SQL data tuple of fields (logical record)
Definition: data0data.h:681
The info structure stored at the beginning of a heap block.
Definition: mem0mem.h:301
#define IF_DEBUG(...)
Definition: univ.i:675
unsigned long int ulint
Definition: univ.i:407
#define ut_a(EXPR)
Abort execution if EXPR does not evaluate to nonzero.
Definition: ut0dbg.h:56