Full Text Search optimize thread. More...

#include <math.h>
#include <stdlib.h>
#include <sys/types.h>
#include <time.h>
#include <zlib.h>
#include "current_thd.h"
#include "dict0dd.h"
#include "fts0fts.h"
#include "fts0opt.h"
#include "fts0priv.h"
#include "fts0types.h"
#include "ha_prototypes.h"
#include "os0thread-create.h"
#include "que0types.h"
#include "row0sel.h"
#include "sql_thd_internal_api.h"
#include "srv0start.h"
#include "ut0list.h"
#include "ut0wqueue.h"

Classes
struct	fts_zip_t
	Compressed list of words that have been read from FTS INDEX that needs to be optimized. More...

struct	fts_optimize_graph_t
	Prepared statemets used during optimize. More...

struct	fts_optimize_t
	Used by fts_optimize() to store state. More...

struct	fts_encode_t
	Used by the optimize, to keep state during compacting nodes. More...

struct	fts_slot_t
	We use this information to determine when to start the optimize cycle for a table. More...

struct	fts_msg_id_t
	A table remove message for the FTS optimize thread. More...

struct	fts_msg_t
	The FTS optimize message work queue message type. More...

Macros
#define	FTS_DEBUG_PRINT

Enumerations
enum	fts_state_t { FTS_STATE_LOADED , FTS_STATE_RUNNING , FTS_STATE_SUSPENDED , FTS_STATE_DONE , FTS_STATE_EMPTY }
	State of a table within the optimization sub system. More...

enum	fts_msg_type_t { FTS_MSG_START , FTS_MSG_PAUSE , FTS_MSG_STOP , FTS_MSG_ADD_TABLE , FTS_MSG_OPTIMIZE_TABLE , FTS_MSG_DEL_TABLE , FTS_MSG_SYNC_TABLE }
	FTS optimize thread message types. More...

Functions
static void	fts_zip_clear_word (fts_zip_t &zip)
	Set fts_zip_t::word to empty string. More...

static void	fts_zip_initialize (fts_zip_t *zip)
	Initialize fts_zip_t. More...

static fts_zip_t *	fts_zip_create (mem_heap_t *heap, ulint block_sz, ulint max_words)
	Create an instance of fts_zip_t. More...

static void	fts_zip_init (fts_zip_t *zip)
	Initialize an instance of fts_zip_t. More...

static fts_word_t *	fts_word_init (fts_word_t word, byte utf8, ulint len)
	Create a fts_optimizer_word_t instance. More...

static fts_node_t *	fts_optimize_read_node (fts_word_t word, que_node_t exp)
	Read the FTS INDEX row. More...

bool	fts_optimize_index_fetch_node (void row, void user_arg)
	Callback function to fetch the rows in an FTS INDEX record. More...

dberr_t	fts_index_fetch_nodes (trx_t trx, que_t graph, fts_table_t fts_table, const fts_string_t word, fts_fetch_t fetch)
	Read the rows from the FTS inde. More...

static bool	fts_zip_extract_bytes (fts_zip_t zip, byte buf, unsigned int size)
	Extract a given number of bytes into a buffer. More...

static bool	fts_zip_read_word (fts_zip_t zip, fts_string_t word)
	Read a word. More...

static bool	fts_fetch_index_words (void row, void user_arg)
	Callback function to fetch and compress the word in an FTS INDEX record. More...

static void	fts_zip_deflate_end (fts_zip_t *zip)
	Finish Zip deflate. More...

static dberr_t	fts_index_fetch_words (fts_optimize_t optim, const fts_string_t word, ulint n_words)
	Read the words from the FTS INDEX. More...

static bool	fts_fetch_doc_ids (void row, void user_arg)
	Callback function to fetch the doc id from the record. More...

dberr_t	fts_table_fetch_doc_ids (trx_t trx, fts_table_t fts_table, fts_doc_ids_t *doc_ids)
	Read the rows from a FTS common auxiliary table. More...

int	fts_bsearch (fts_update_t *array, int lower, int upper, doc_id_t doc_id)
	Do a binary search for a doc id in the array. More...

static int	fts_optimize_lookup (ib_vector_t *doc_ids, ulint lower, doc_id_t first_doc_id, doc_id_t last_doc_id)
	Search in the to delete array whether any of the doc ids within the [first, last] range are to be deleted. More...

static dberr_t	fts_optimize_encode_node (fts_node_t node, doc_id_t doc_id, fts_encode_t enc)
	Encode the word pos list into the node. More...

static dberr_t	fts_optimize_node (ib_vector_t del_vec, int del_pos, fts_node_t dst_node, fts_node_t src_node, fts_encode_t *enc)
	Optimize the data contained in a node. More...

static int	fts_optimize_deleted_pos (fts_optimize_t optim, fts_word_t word)
	Determine the starting pos within the deleted doc id vector for a word. More...

static ib_vector_t *	fts_optimize_word (fts_optimize_t optim, fts_word_t word)
	Compact the nodes for a word, we also remove any doc ids during the compaction pass. More...

static dberr_t	fts_optimize_write_word (trx_t trx, fts_table_t fts_table, fts_string_t word, ib_vector_t nodes)
	Update the FTS index table. More...

void	fts_word_free (fts_word_t *word)
	Free fts_optimizer_word_t instanace. More...

static dberr_t	fts_optimize_compact (fts_optimize_t optim, dict_index_t index, std::chrono::steady_clock::time_point start_time)
	Optimize the word ilist and rewrite data to the FTS index. More...

static fts_optimize_t *	fts_optimize_create (dict_table_t *table)
	Create an instance of fts_optimize_t. More...

static void	fts_optimize_graph_free (fts_optimize_graph_t *graph)
	Free the optimize prepared statements. More...

static void	fts_optimize_free (fts_optimize_t *optim)
	Free all optimize resources. More...

static std::chrono::seconds	fts_optimize_get_time_limit (trx_t trx, fts_table_t fts_table)
	Get the max time optimize should run. More...

static void	fts_optimize_words (fts_optimize_t optim, dict_index_t index, fts_string_t *word)
	Run OPTIMIZE on the given table. More...

static dberr_t	fts_optimize_index_completed (fts_optimize_t optim, dict_index_t index)
	Optimize is complete. More...

static dberr_t	fts_optimize_index_read_words (fts_optimize_t optim, dict_index_t index, fts_string_t *word)
	Read the list of words from the FTS auxiliary index that will be optimized in this pass. More...

static dberr_t	fts_optimize_index (fts_optimize_t optim, dict_index_t index)
	Run OPTIMIZE on the given FTS index. More...

static dberr_t	fts_optimize_purge_deleted_doc_ids (fts_optimize_t *optim)
	Delete the document ids in the delete, and delete cache tables. More...

static dberr_t	fts_optimize_purge_deleted_doc_id_snapshot (fts_optimize_t *optim)
	Delete the document ids in the pending delete, and delete tables. More...

static ulint	fts_optimize_being_deleted_count (fts_optimize_t *optim)
	Copy the deleted doc ids that will be purged during this optimize run to the being deleted FTS auxiliary tables. More...

static dberr_t	fts_optimize_create_deleted_doc_id_snapshot (fts_optimize_t *optim)
	Copy the deleted doc ids that will be purged during this optimize run to the being deleted FTS auxiliary tables. More...

static dberr_t	fts_optimize_read_deleted_doc_id_snapshot (fts_optimize_t *optim)
	Read in the document ids that are to be purged during optimize. More...

static dberr_t	fts_optimize_indexes (fts_optimize_t *optim)
	Optimize all the FTS indexes, skipping those that have already been optimized, since the FTS auxiliary indexes are not guaranteed to be of the same cardinality. More...

static dberr_t	fts_optimize_purge_snapshot (fts_optimize_t *optim)
	Cleanup the snapshot tables and the master deleted table. More...

static dberr_t	fts_optimize_reset_start_time (fts_optimize_t *optim)
	Reset the start time to 0 so that a new optimize can be started. More...

static dberr_t	fts_optimize_table_bk (fts_slot_t *slot)
	Run OPTIMIZE on the given table by a background thread. More...

dberr_t	fts_optimize_table (dict_table_t *table)
	Run OPTIMIZE on the given table. More...

static fts_msg_t *	fts_optimize_create_msg (fts_msg_type_t type, void *ptr)
	Add the table to add to the OPTIMIZER's list. More...

void	fts_optimize_add_table (dict_table_t *table)
	Add the table to add to the OPTIMIZER's list. More...

void	fts_optimize_remove_table (dict_table_t *table)
	Remove the table from the OPTIMIZER's list. More...

void	fts_optimize_request_sync_table (dict_table_t *table)
	Send sync fts cache for the table. More...

static fts_slot_t *	fts_optimize_find_slot (ib_vector_t *tables, table_id_t table_id)
	Find the slot for a particular table. More...

static void	fts_optimize_start_table (ib_vector_t tables, dict_table_t table)
	Start optimizing table. More...

static bool	fts_optimize_new_table (ib_vector_t tables, fts_msg_id_t msg)
	Add the table to the vector if it doesn't already exist. More...

static bool	fts_optimize_del_table (ib_vector_t tables, fts_msg_id_t msg)
	Remove the table from the vector if it exists. More...

static ulint	fts_optimize_how_many (const ib_vector_t *tables)
	Calculate how many of the registered tables need to be optimized. More...

static bool	fts_is_sync_needed (const ib_vector_t *tables)
	Check if the total memory used by all FTS table exceeds the maximum limit. More...

void	fts_optimize_sync_table (table_id_t table_id)
	Sync fts cache of a table. More...

static void	fts_optimize_thread (ib_wqueue_t *wq)
	Optimize all FTS tables. More...

void	fts_optimize_init (void)
	Startup the optimize thread and create the work queue. More...

void	fts_optimize_shutdown ()
	Shutdown fts optimize thread. More...

Variables
static ib_wqueue_t *	fts_optimize_wq
	The FTS optimize thread's work queue. More...

constexpr std::chrono::seconds	FTS_QUEUE_WAIT {5}
	Time to wait for a message. More...

constexpr std::chrono::minutes	FTS_OPTIMIZE_INTERVAL {5}
	Default optimize interval. More...

static bool	fts_opt_start_shutdown = false
	Server is shutting down, so does we exiting the optimize thread. More...

static const ulint	FTS_WORD_NODES_INIT_SIZE = 64
	Initial size of nodes in fts_word_t. More...

static std::chrono::steady_clock::time_point	last_check_sync_time
	Last time we did check whether system need a sync. More...

ulong	fts_num_word_optimize
	The number of words to read and optimize in a single pass. More...

bool	fts_enable_diag_print
	Variable specifying whether we do additional FTS diagnostic printout in the log. More...

static ulint	FTS_ZIP_BLOCK_SIZE = 1024
	ZLib compressed block size. More...

static std::chrono::milliseconds	fts_optimize_time_limit {0}
	The amount of time optimizing in a single pass. More...

const char *	fts_common_tables []
	It's defined in fts0fts.cc More...

static const char *	fts_init_delete_sql
	SQL Statement for changing state of rows to be deleted from FTS Index. More...

static const char *	fts_delete_doc_ids_sql

static const char *	fts_end_delete_sql

Detailed Description

Full Text Search optimize thread.

Created 2007/03/27 Sunny Bains Completed 2011/7/10 Sunny and Jimmy Yang

Macro Definition Documentation

◆ FTS_DEBUG_PRINT

#define FTS_DEBUG_PRINT

Enumeration Type Documentation

◆ fts_msg_type_t

enum fts_msg_type_t

FTS optimize thread message types.

Enumerator
FTS_MSG_START	Start optimizing thread.
FTS_MSG_PAUSE	Pause optimizing thread.
FTS_MSG_STOP	Stop optimizing and exit thread.
FTS_MSG_ADD_TABLE	Add table to the optimize thread's work queue.
FTS_MSG_OPTIMIZE_TABLE	Optimize a table.
FTS_MSG_DEL_TABLE	Remove a table from the optimize threads work queue.
FTS_MSG_SYNC_TABLE	Sync fts cache of a table.

◆ fts_state_t

enum fts_state_t

State of a table within the optimization sub system.

Enumerator
FTS_STATE_LOADED
FTS_STATE_RUNNING
FTS_STATE_SUSPENDED
FTS_STATE_DONE
FTS_STATE_EMPTY

Function Documentation

◆ fts_bsearch()

int fts_bsearch	(	fts_update_t *	array,
		int	lower,
		int	upper,
		doc_id_t	doc_id
	)

Do a binary search for a doc id in the array.

Returns: +ve index if found -ve index where it should be inserted if not found

Parameters

array	in: array to sort
lower	in: the array lower bound
upper	in: the array upper bound
doc_id	in: the doc id to search for

◆ fts_fetch_doc_ids()

static bool fts_fetch_doc_ids	(	void *	row,
		void *	user_arg
	)

static

Callback function to fetch the doc id from the record.

Returns: always returns true

Parameters

row	in: sel_node_t*
user_arg	in: pointer to ib_vector_t

◆ fts_fetch_index_words()

static bool fts_fetch_index_words	(	void *	row,
		void *	user_arg
	)

static

Callback function to fetch and compress the word in an FTS INDEX record.

Returns: false on EOF

Parameters

row	in: sel_node_t*
user_arg	in: pointer to fts_optimize_t

◆ fts_index_fetch_nodes()

dberr_t fts_index_fetch_nodes	(	trx_t *	trx,
		que_t **	graph,
		fts_table_t *	fts_table,
		const fts_string_t *	word,
		fts_fetch_t *	fetch
	)

Read the rows from the FTS inde.

Returns: DB_SUCCESS or error code

Parameters

trx	in: transaction
graph	in: prepared statement
fts_table	in: table of the FTS INDEX
word	in: the word to fetch
fetch	in: fetch callback.

◆ fts_index_fetch_words()

static dberr_t fts_index_fetch_words	(	fts_optimize_t *	optim,
		const fts_string_t *	word,
		ulint	n_words
	)

static

Read the words from the FTS INDEX.

Returns: DB_SUCCESS if all OK, DB_TABLE_NOT_FOUND if no more indexes to search else error code

Parameters

optim	in: optimize scratch pad
word	in: get words greater than this word
n_words	in: max words to read

◆ fts_is_sync_needed()

static bool fts_is_sync_needed ( const ib_vector_t * tables )

static

Check if the total memory used by all FTS table exceeds the maximum limit.

Returns: true if a sync is needed, false otherwise

Parameters

tables in: registered tables vector

◆ fts_optimize_add_table()

void fts_optimize_add_table ( dict_table_t * table )

Add the table to add to the OPTIMIZER's list.

in: table to add

Parameters

table in: table to add

◆ fts_optimize_being_deleted_count()

static ulint fts_optimize_being_deleted_count ( fts_optimize_t * optim )

static

Copy the deleted doc ids that will be purged during this optimize run to the being deleted FTS auxiliary tables.

The transaction is committed upon successful copy and rolled back on DB_DUPLICATE_KEY error.

Returns: DB_SUCCESS if all OK

Parameters

optim in: optimize instance

◆ fts_optimize_compact()

static dberr_t fts_optimize_compact	(	fts_optimize_t *	optim,
		dict_index_t *	index,
		std::chrono::steady_clock::time_point	start_time
	)

static

Optimize the word ilist and rewrite data to the FTS index.

Returns: status one of RESTART, EXIT, ERROR

Parameters

optim	in: optimize state data
index	in: current FTS being optimized
start_time	in: optimize start time

◆ fts_optimize_create()

static fts_optimize_t * fts_optimize_create ( dict_table_t * table )

static

Create an instance of fts_optimize_t.

Also create a new background transaction.

Parameters

table in: table with FTS indexes

◆ fts_optimize_create_deleted_doc_id_snapshot()

static dberr_t fts_optimize_create_deleted_doc_id_snapshot ( fts_optimize_t * optim )

static

Copy the deleted doc ids that will be purged during this optimize run to the being deleted FTS auxiliary tables.

The transaction is committed upon successful copy and rolled back on DB_DUPLICATE_KEY error.

Returns: DB_SUCCESS if all OK

Parameters

optim in: optimize instance

◆ fts_optimize_create_msg()

static fts_msg_t * fts_optimize_create_msg	(	fts_msg_type_t	type,
		void *	ptr
	)

static

Add the table to add to the OPTIMIZER's list.

Returns: new message instance

Parameters

type	in: type of message
ptr	in: message payload

◆ fts_optimize_del_table()

static bool fts_optimize_del_table	(	ib_vector_t *	tables,
		fts_msg_id_t *	msg
	)

static

Remove the table from the vector if it exists.

Parameters

tables	in/out: vector of tables
msg	in: table to delete

◆ fts_optimize_deleted_pos()

static int fts_optimize_deleted_pos	(	fts_optimize_t *	optim,
		fts_word_t *	word
	)

static

Determine the starting pos within the deleted doc id vector for a word.

Returns: delete position

Parameters

optim	in: optimize state data
word	in: the word data to check

◆ fts_optimize_encode_node()

static dberr_t fts_optimize_encode_node	(	fts_node_t *	node,
		doc_id_t	doc_id,
		fts_encode_t *	enc
	)

static

Encode the word pos list into the node.

Returns: DB_SUCCESS or error code

Parameters

node	in: node to fill
doc_id	in: doc id to encode
enc	in: encoding state.

◆ fts_optimize_find_slot()

static fts_slot_t * fts_optimize_find_slot	(	ib_vector_t *	tables,
		table_id_t	table_id
	)

static

Find the slot for a particular table.

Returns: slot if found else NULL.

Parameters

tables	in: vector of tables
table_id	in: table id to find

◆ fts_optimize_free()

static void fts_optimize_free ( fts_optimize_t * optim )

static

Free all optimize resources.

Parameters

optim in: table with on FTS index

◆ fts_optimize_get_time_limit()

static std::chrono::seconds fts_optimize_get_time_limit	(	trx_t *	trx,
		fts_table_t *	fts_table
	)

static

Get the max time optimize should run.

Returns: max optimize time limit.

Parameters

trx	in: transaction
fts_table	in: aux table

◆ fts_optimize_graph_free()

static void fts_optimize_graph_free ( fts_optimize_graph_t * graph )

static

Free the optimize prepared statements.

Parameters

graph in/out: The graph instances to free

◆ fts_optimize_how_many()

static ulint fts_optimize_how_many ( const ib_vector_t * tables )

static

Calculate how many of the registered tables need to be optimized.

Returns: no. of tables to optimize

Parameters

tables in: registered tables vector

◆ fts_optimize_index()

static dberr_t fts_optimize_index	(	fts_optimize_t *	optim,
		dict_index_t *	index
	)

static

Run OPTIMIZE on the given FTS index.

Note: this can take a very long time (hours).

Returns: DB_SUCCESS if all OK

Parameters

optim	in: optimize instance
index	in: table with one FTS index

◆ fts_optimize_index_completed()

static dberr_t fts_optimize_index_completed	(	fts_optimize_t *	optim,
		dict_index_t *	index
	)

static

Optimize is complete.

Set the completion time, and reset the optimize start string for this FTS index to "".

Returns: DB_SUCCESS if all OK

Parameters

optim	in: optimize instance
index	in: table with one FTS index

◆ fts_optimize_index_fetch_node()

bool fts_optimize_index_fetch_node	(	void *	row,
		void *	user_arg
	)

Callback function to fetch the rows in an FTS INDEX record.

Returns: always returns non-NULL

Parameters

row	in: sel_node_t*
user_arg	in: pointer to ib_vector_t

◆ fts_optimize_index_read_words()

static dberr_t fts_optimize_index_read_words	(	fts_optimize_t *	optim,
		dict_index_t *	index,
		fts_string_t *	word
	)

static

Read the list of words from the FTS auxiliary index that will be optimized in this pass.

Returns: DB_SUCCESS if all OK

Parameters

optim	in: optimize instance
index	in: table with one FTS index
word	in: buffer to use

◆ fts_optimize_indexes()

static dberr_t fts_optimize_indexes ( fts_optimize_t * optim )

static

Optimize all the FTS indexes, skipping those that have already been optimized, since the FTS auxiliary indexes are not guaranteed to be of the same cardinality.

Returns: DB_SUCCESS if all OK

Parameters

optim in: optimize instance

◆ fts_optimize_init()

void fts_optimize_init ( void )

Startup the optimize thread and create the work queue.

◆ fts_optimize_lookup()

static int fts_optimize_lookup	(	ib_vector_t *	doc_ids,
		ulint	lower,
		doc_id_t	first_doc_id,
		doc_id_t	last_doc_id
	)

static

Search in the to delete array whether any of the doc ids within the [first, last] range are to be deleted.

Returns: +ve index if found -ve index where it should be inserted if not found

Parameters

doc_ids	in: array to search
lower	in: lower limit of array
first_doc_id	in: doc id to lookup
last_doc_id	in: doc id to lookup

◆ fts_optimize_new_table()

static bool fts_optimize_new_table	(	ib_vector_t *	tables,
		fts_msg_id_t *	msg
	)

static

Add the table to the vector if it doesn't already exist.

Parameters

tables	in/out: vector of tables
msg	in: table to delete

◆ fts_optimize_node()

static dberr_t fts_optimize_node	(	ib_vector_t *	del_vec,
		int *	del_pos,
		fts_node_t *	dst_node,
		fts_node_t *	src_node,
		fts_encode_t *	enc
	)

static

Optimize the data contained in a node.

Returns: DB_SUCCESS or error code

Parameters

del_vec	in: vector of doc ids to delete
del_pos	in: offset into above vector
dst_node	in: node to fill
src_node	in: source node for data
enc	in: encoding state

◆ fts_optimize_purge_deleted_doc_id_snapshot()

static dberr_t fts_optimize_purge_deleted_doc_id_snapshot ( fts_optimize_t * optim )

static

Delete the document ids in the pending delete, and delete tables.

Returns: DB_SUCCESS if all OK

Parameters

optim in: optimize instance

◆ fts_optimize_purge_deleted_doc_ids()

static dberr_t fts_optimize_purge_deleted_doc_ids ( fts_optimize_t * optim )

static

Delete the document ids in the delete, and delete cache tables.

Returns: DB_SUCCESS if all OK

Parameters

optim in: optimize instance

◆ fts_optimize_purge_snapshot()

static dberr_t fts_optimize_purge_snapshot ( fts_optimize_t * optim )

static

Cleanup the snapshot tables and the master deleted table.

Returns: DB_SUCCESS if all OK

Parameters

optim in: optimize instance

◆ fts_optimize_read_deleted_doc_id_snapshot()

static dberr_t fts_optimize_read_deleted_doc_id_snapshot ( fts_optimize_t * optim )

static

Read in the document ids that are to be purged during optimize.

The transaction is committed upon successfully read.

Returns: DB_SUCCESS if all OK

Parameters

optim in: optimize instance

◆ fts_optimize_read_node()

static fts_node_t * fts_optimize_read_node	(	fts_word_t *	word,
		que_node_t *	exp
	)

static

Read the FTS INDEX row.

Returns: fts_node_t instance

Parameters

word	in:
exp	in:

◆ fts_optimize_remove_table()

void fts_optimize_remove_table ( dict_table_t * table )

Remove the table from the OPTIMIZER's list.

We do wait for acknowledgement from the consumer of the message.

Parameters

table in: table to remove

◆ fts_optimize_request_sync_table()

void fts_optimize_request_sync_table ( dict_table_t * table )

Send sync fts cache for the table.

Parameters

[in] table table to sync

◆ fts_optimize_reset_start_time()

static dberr_t fts_optimize_reset_start_time ( fts_optimize_t * optim )

static

Reset the start time to 0 so that a new optimize can be started.

Returns: DB_SUCCESS if all OK

Parameters

optim in: optimize instance

◆ fts_optimize_shutdown()

void fts_optimize_shutdown ( )

Shutdown fts optimize thread.

◆ fts_optimize_start_table()

static void fts_optimize_start_table	(	ib_vector_t *	tables,
		dict_table_t *	table
	)

static

Start optimizing table.

Parameters

tables	in/out: vector of tables
table	in: table to optimize

◆ fts_optimize_sync_table()

void fts_optimize_sync_table ( table_id_t table_id )

Sync fts cache of a table.

Parameters

[in] table_id table id

◆ fts_optimize_table()

dberr_t fts_optimize_table ( dict_table_t * table )

Run OPTIMIZE on the given table.

Returns: DB_SUCCESS if all OK

Parameters

table in: table to optimize

◆ fts_optimize_table_bk()

static dberr_t fts_optimize_table_bk ( fts_slot_t * slot )

static

Run OPTIMIZE on the given table by a background thread.

Returns: DB_SUCCESS if all OK

Parameters

slot	in: table to optimize

◆ fts_optimize_thread()

static void fts_optimize_thread ( ib_wqueue_t * wq )

static

Optimize all FTS tables.

◆ fts_optimize_word()

static ib_vector_t * fts_optimize_word	(	fts_optimize_t *	optim,
		fts_word_t *	word
	)

static

Compact the nodes for a word, we also remove any doc ids during the compaction pass.

Returns: DB_SUCCESS or error code.

Parameters

optim	in: optimize state data
word	in: the word to optimize

◆ fts_optimize_words()

static void fts_optimize_words	(	fts_optimize_t *	optim,
		dict_index_t *	index,
		fts_string_t *	word
	)

static

Run OPTIMIZE on the given table.

Note: this can take a very long time (hours).

Parameters

optim	in: optimize instance
index	in: current FTS being optimized
word	in: the starting word to optimize

◆ fts_optimize_write_word()

static dberr_t fts_optimize_write_word	(	trx_t *	trx,
		fts_table_t *	fts_table,
		fts_string_t *	word,
		ib_vector_t *	nodes
	)

static

Update the FTS index table.

This is a delete followed by an insert.

Returns: DB_SUCCESS or error code

Parameters

trx	in: transaction
fts_table	in: table of FTS index
word	in: word data to write
nodes	in: the nodes to write

◆ fts_table_fetch_doc_ids()

dberr_t fts_table_fetch_doc_ids	(	trx_t *	trx,
		fts_table_t *	fts_table,
		fts_doc_ids_t *	doc_ids
	)

Read the rows from a FTS common auxiliary table.

Read the rows from the FTS index.

Returns: DB_SUCCESS or error code

Parameters

trx	in: transaction
fts_table	in: table
doc_ids	in: For collecting doc ids

◆ fts_word_free()

void fts_word_free ( fts_word_t * word )

Free fts_optimizer_word_t instanace.

in: instance to free.

Parameters

word	in: instance to free.

◆ fts_word_init()

static fts_word_t * fts_word_init	(	fts_word_t *	word,
		byte *	utf8,
		ulint	len
	)

static

Create a fts_optimizer_word_t instance.

Returns: new instance

Parameters

word	in: word to initialize
utf8	in: UTF-8 string
len	in: length of string in bytes

◆ fts_zip_clear_word()

static void fts_zip_clear_word ( fts_zip_t & zip )

static

Set fts_zip_t::word to empty string.

◆ fts_zip_create()

static fts_zip_t * fts_zip_create	(	mem_heap_t *	heap,
		ulint	block_sz,
		ulint	max_words
	)

static

Create an instance of fts_zip_t.

Returns: a new instance of fts_zip_t

Parameters

heap	in: heap
block_sz	in: size of a zip block.
max_words	in: max words to read

◆ fts_zip_deflate_end()

static void fts_zip_deflate_end ( fts_zip_t * zip )

static

Finish Zip deflate.

Parameters

zip	in: instance that should be closed

◆ fts_zip_extract_bytes()

static bool fts_zip_extract_bytes	(	fts_zip_t *	zip,
		byte *	buf,
		unsigned int	size
	)

static

Extract a given number of bytes into a buffer.

Parameters

zip	in: Zip state + data
buf	out: buffer to be filled
size	in: size of buffer

◆ fts_zip_init()

static void fts_zip_init ( fts_zip_t * zip )

static

Initialize an instance of fts_zip_t.

Parameters

zip	in: zip instance to init

◆ fts_zip_initialize()

static void fts_zip_initialize ( fts_zip_t * zip )

static

Initialize fts_zip_t.

Parameters

zip	out: zip instance to initialize

◆ fts_zip_read_word()

static bool fts_zip_read_word	(	fts_zip_t *	zip,
		fts_string_t *	word
	)

static

Read a word.

Parameters

zip	in: Zip state + data
word	out: uncompressed word

Variable Documentation

◆ fts_common_tables

const char* fts_common_tables[]

extern

It's defined in fts0fts.cc

◆ fts_delete_doc_ids_sql

const char* fts_delete_doc_ids_sql

static

Initial value:

=
    "BEGIN\n"
    "\n"
    "DELETE FROM $deleted WHERE doc_id = :doc_id1;\n"
    "DELETE FROM $deleted_cache WHERE doc_id = :doc_id2;\n"

◆ fts_enable_diag_print

bool fts_enable_diag_print

Variable specifying whether we do additional FTS diagnostic printout in the log.

◆ fts_end_delete_sql

const char* fts_end_delete_sql

static

Initial value:

=
    "BEGIN\n"
    "\n"
    "DELETE FROM $being_deleted;\n"
    "DELETE FROM $being_deleted_cache;\n"

◆ fts_init_delete_sql

const char* fts_init_delete_sql

static

Initial value:

=
    "BEGIN\n"
    "\n"
    "INSERT INTO $being_deleted\n"
    "SELECT doc_id FROM $deleted;\n"
    "\n"
    "INSERT INTO $being_deleted_cache\n"
    "SELECT doc_id FROM $deleted_cache;\n"

SQL Statement for changing state of rows to be deleted from FTS Index.

◆ fts_num_word_optimize

ulong fts_num_word_optimize

The number of words to read and optimize in a single pass.

Variable specifying the number of word to optimize for each optimize table call.

◆ fts_opt_start_shutdown

bool fts_opt_start_shutdown = false

static

Server is shutting down, so does we exiting the optimize thread.

◆ FTS_OPTIMIZE_INTERVAL

constexpr std::chrono::minutes FTS_OPTIMIZE_INTERVAL {5}

constexpr

Default optimize interval.

◆ fts_optimize_time_limit

std::chrono::milliseconds fts_optimize_time_limit {0}

static

The amount of time optimizing in a single pass.

◆ fts_optimize_wq

ib_wqueue_t* fts_optimize_wq

static

The FTS optimize thread's work queue.

◆ FTS_QUEUE_WAIT

constexpr std::chrono::seconds FTS_QUEUE_WAIT {5}

constexpr

Time to wait for a message.

◆ FTS_WORD_NODES_INIT_SIZE

const ulint FTS_WORD_NODES_INIT_SIZE = 64

static

Initial size of nodes in fts_word_t.

◆ FTS_ZIP_BLOCK_SIZE

ulint FTS_ZIP_BLOCK_SIZE = 1024

static

ZLib compressed block size.

◆ last_check_sync_time

std::chrono::steady_clock::time_point last_check_sync_time

static

Last time we did check whether system need a sync.

Classes

Macros

Enumerations

Functions

Variables

Detailed Description

Macro Definition Documentation

◆ FTS_DEBUG_PRINT

Enumeration Type Documentation

◆ fts_msg_type_t

◆ fts_state_t

Function Documentation

◆ fts_bsearch()

◆ fts_fetch_doc_ids()

◆ fts_fetch_index_words()

◆ fts_index_fetch_nodes()

◆ fts_index_fetch_words()

◆ fts_is_sync_needed()

◆ fts_optimize_add_table()

◆ fts_optimize_being_deleted_count()

◆ fts_optimize_compact()

◆ fts_optimize_create()

◆ fts_optimize_create_deleted_doc_id_snapshot()

◆ fts_optimize_create_msg()

◆ fts_optimize_del_table()

◆ fts_optimize_deleted_pos()

◆ fts_optimize_encode_node()

◆ fts_optimize_find_slot()

◆ fts_optimize_free()

◆ fts_optimize_get_time_limit()

◆ fts_optimize_graph_free()

◆ fts_optimize_how_many()

◆ fts_optimize_index()

◆ fts_optimize_index_completed()

◆ fts_optimize_index_fetch_node()

◆ fts_optimize_index_read_words()

◆ fts_optimize_indexes()

◆ fts_optimize_init()

◆ fts_optimize_lookup()

◆ fts_optimize_new_table()

◆ fts_optimize_node()

◆ fts_optimize_purge_deleted_doc_id_snapshot()

◆ fts_optimize_purge_deleted_doc_ids()

◆ fts_optimize_purge_snapshot()

◆ fts_optimize_read_deleted_doc_id_snapshot()

◆ fts_optimize_read_node()

◆ fts_optimize_remove_table()

◆ fts_optimize_request_sync_table()

◆ fts_optimize_reset_start_time()

◆ fts_optimize_shutdown()

◆ fts_optimize_start_table()

◆ fts_optimize_sync_table()

◆ fts_optimize_table()

◆ fts_optimize_table_bk()

◆ fts_optimize_thread()

◆ fts_optimize_word()

◆ fts_optimize_words()

◆ fts_optimize_write_word()

◆ fts_table_fetch_doc_ids()

◆ fts_word_free()

◆ fts_word_init()

◆ fts_zip_clear_word()

◆ fts_zip_create()

◆ fts_zip_deflate_end()

◆ fts_zip_extract_bytes()

◆ fts_zip_init()

◆ fts_zip_initialize()

◆ fts_zip_read_word()

Variable Documentation

◆ fts_common_tables

◆ fts_delete_doc_ids_sql

◆ fts_enable_diag_print

◆ fts_end_delete_sql

◆ fts_init_delete_sql

◆ fts_num_word_optimize

◆ fts_opt_start_shutdown

◆ FTS_OPTIMIZE_INTERVAL

◆ fts_optimize_time_limit

◆ fts_optimize_wq

◆ FTS_QUEUE_WAIT