Implements the row and column memory management for parse and load operations. More...

#include <bulk_data_service.h>

Public Member Functions
	Row_bunch (size_t n_cols)
	Create a new row bunch. More...

size_t	get_num_rows () const

size_t	get_num_cols () const

template<typename F >
bool	process_columns (size_t row_index, F &&cbk)
	Process all columns, invoking callback for each. More...

template<typename F >
bool	process_columns_by_offset (size_t row_offset, F &&cbk)

void	reset ()

size_t	get_row_offset (size_t row_index) const
	Get current row offset to access columns. More...

size_t	get_next_row_offset (size_t &offset) const
	Get next row offset from current row offset. More...

Column_type &	get_column (size_t row_offset, size_t col_index)
	Get column using row offset and column index. More...

Column_type &	get_col (size_t row_index, size_t col_index)
	Get column using row index and column index. More...

Column_type &	get_col (size_t col_offset)
	Get column using the column offset. More...

const Column_type &	read_column (size_t row_offset, size_t col_index) const
	Get constant column for reading using row offset and column index. More...

bool	set_num_rows (size_t n_rows)
	Set the number of rows. More...

Static Public Attributes
static const size_t	S_MAX_TOTAL_COLS = 600 * 1024 * 1024
	Limit allocation up to 600M columns. More...

Private Attributes
std::vector< Column_type >	m_columns
	All the columns. More...

size_t	m_num_rows {}
	Number of rows. More...

size_t	m_num_columns {}
	Number of columns in each row. More...

Detailed Description

template<typename Column_type>
class Row_bunch< Column_type >

Implements the row and column memory management for parse and load operations.

We try to pre-allocate the memory contiguously as much as we can to maximize the performance.

Template Parameters

Column_type Column_text when used in the CSV context, Column_sql when used in the InnoDB context.

Constructor & Destructor Documentation

◆ Row_bunch()

template<typename Column_type >

Row_bunch< Column_type >::Row_bunch ( size_t n_cols )

inline

Create a new row bunch.

Parameters

[in] n_cols number of columns

Member Function Documentation

◆ get_col() [1/2]

template<typename Column_type >

Column_type & Row_bunch< Column_type >::get_col ( size_t col_offset )

inline

Get column using the column offset.

Parameters

[in] col_offset column offset

Returns: column data

◆ get_col() [2/2]

template<typename Column_type >

Column_type & Row_bunch< Column_type >::get_col	(	size_t	row_index,
		size_t	col_index
	)

inline

Get column using row index and column index.

Parameters

[in]	row_index	index of the row in the bunch
[in]	col_index	index of the column within row

Returns: column data

◆ get_column()

template<typename Column_type >

Column_type & Row_bunch< Column_type >::get_column	(	size_t	row_offset,
		size_t	col_index
	)

inline

Get column using row offset and column index.

Parameters

[in]	row_offset	row offset in column vector
[in]	col_index	index of the column within row

Returns: column data

◆ get_next_row_offset()

template<typename Column_type >

size_t Row_bunch< Column_type >::get_next_row_offset ( size_t & offset ) const

inline

Get next row offset from current row offset.

Parameters

[in,out] offset row offset

Returns: true if there is a next row.

◆ get_num_cols()

template<typename Column_type >

size_t Row_bunch< Column_type >::get_num_cols ( ) const

inline

Returns: return number of columns in each row.

◆ get_num_rows()

template<typename Column_type >

size_t Row_bunch< Column_type >::get_num_rows ( ) const

inline

Returns: return number of rows in the bunch.

◆ get_row_offset()

template<typename Column_type >

size_t Row_bunch< Column_type >::get_row_offset ( size_t row_index ) const

inline

Get current row offset to access columns.

Parameters

[in] row_index row index

Returns: row offset in column vector.

◆ process_columns()

template<typename Column_type >

template<typename F >

bool Row_bunch< Column_type >::process_columns	(	size_t	row_index,
		F &&	cbk
	)

inline

Process all columns, invoking callback for each.

Parameters

[in]	row_index	index of the row
[in]	cbk	callback function

Returns: true if successful

◆ process_columns_by_offset()

template<typename Column_type >

template<typename F >

bool Row_bunch< Column_type >::process_columns_by_offset	(	size_t	row_offset,
		F &&	cbk
	)

inline

◆ read_column()

template<typename Column_type >

const Column_type & Row_bunch< Column_type >::read_column	(	size_t	row_offset,
		size_t	col_index
	)		const

inline

Get constant column for reading using row offset and column index.

Parameters

[in]	row_offset	row offset in column vector
[in]	col_index	index of the column within row

Returns: column data

◆ reset()

template<typename Column_type >

void Row_bunch< Column_type >::reset ( )

inline

◆ set_num_rows()

template<typename Column_type >

bool Row_bunch< Column_type >::set_num_rows ( size_t n_rows )

inline

Set the number of rows.

Adjust number of rows base on maximum column storage limit.

Parameters

[in,out] n_rows number of rows

Returns: true if successful, false if too many rows or columns.

Member Data Documentation

◆ m_columns

template<typename Column_type >

std::vector<Column_type> Row_bunch< Column_type >::m_columns

private

All the columns.

◆ m_num_columns

template<typename Column_type >

size_t Row_bunch< Column_type >::m_num_columns {}

private

Number of columns in each row.

◆ m_num_rows

template<typename Column_type >

size_t Row_bunch< Column_type >::m_num_rows {}

private

Number of rows.

◆ S_MAX_TOTAL_COLS

template<typename Column_type >

const size_t Row_bunch< Column_type >::S_MAX_TOTAL_COLS = 600 * 1024 * 1024

static

Limit allocation up to 600M columns.

This number is rounded up from an estimate of the number of columns with the max chunk size (1024M). In the worst case we can have 2 bytes per column so a chunk can contain around 512M columns, and because of rows that spill over chunk boundaries we assume we can append a full additional row (which should have at most 4096 columns). Rounded up to 600M.

The documentation for this class was generated from the following file:

include/mysql/components/services/bulk_data_service.h

Public Member Functions

Static Public Attributes

Private Attributes

Detailed Description

Constructor & Destructor Documentation

◆ Row_bunch()

Member Function Documentation

◆ get_col() [1/2]

◆ get_col() [2/2]

◆ get_column()

◆ get_next_row_offset()

◆ get_num_cols()

◆ get_num_rows()

◆ get_row_offset()

◆ process_columns()

◆ process_columns_by_offset()

◆ read_column()

◆ reset()

◆ set_num_rows()

Member Data Documentation

◆ m_columns

◆ m_num_columns

◆ m_num_rows

◆ S_MAX_TOTAL_COLS