Disk Collector. More...

#include <health_monitor_disk.h>

Inheritance diagram for hm::Disk_Collector:

Public Types
enum class	Input_failure { filesystem , datadir_lookup }

Public Member Functions
	Disk_Collector ()
	Disk Collector constructor. More...

void	initialize ()
	Initialize the Disk Collector. More...

void	set_buffer_size () final
	Resize the stat buffer if necessary. More...

bool	settings_changed () override
	Indicate if the Disk Collector settings have changed. More...

void	set_params () final
	Set collection parameters from the system variables. More...

bool	running () const override
	Return health_monitor.disk_running. More...

std::shared_ptr< Stat_Buffer< Disk_Info > >	buffer ()

int	get_disk_info (const std::string &target_path, Disk_Info &disk_info, Filesystem_sample_state &sample_state, Filesystem_change &change)
	Get available free space for a monitored filesystem. More...

bool	resolve_monitored_path ()
	Resolve the automatic monitored path before filesystem collection. More...

void	update_disk_info ()
	Update disk info for display only. More...

void	update_per_sample_totals (const Disk_Info &disk_info)
	Update totals that change for each sample, convert to MiB. More...

void	set_storage_profile (const Disk_Info &disk_info)
	Set the storage threshold parameters for the monitored filesystem. More...

Fallback_State &	fallback ()

Mode_Timer &	fallback_timer ()

void	start_fallback_timer ()
	Start the fallback timer. More...

void	stop_fallback_timer ()
	Stop the fallback timer. More...

void	cancel_fallback_timer ()
	Cancel the fallback timer: More...

bool	fallback_eligible (std::string &reason)
	Fallback eligible: Server is eligible to be placed in fallback mode if: More...

bool	fallback_eligible (bool print_reason=false)
	Fallback eligible: Print reason if ineligible. More...

bool	forced_fallback_eligible ()
	Return true if forced fallback is allowed. More...

void	set_fallback_mode (bool force)
	Set the server fallback mode, OFFLINE_MODE and SUPER_READ_ONLY. More...

void	clear_fallback_mode ()
	Clear OFFLINE_MODE and SUPER_READ_ONLY, reset all fallback state variables. More...

bool	check_fenced_server_state ()
	Check if the state of a fenced server changed: More...

void	process_expired_fallback_timer ()
	Process expired fallback timer. More...

void	reset_fallback_and_recovery ()
	Reset fallback and recovery modes, cancel all timers. More...

void	cancel_fallback_process ()
	Cancel the ongoing fallback process. More...

bool	set_offline_mode (int max_retries, bool &already_set)
	Attempt to set OFFLINE_MODE. More...

bool	clear_offline_mode ()
	Clear OFFLINE_MODE. More...

bool	set_super_read_only (int max_retries, bool &already_set)
	Set SUPER_READ_ONLY. More...

bool	clear_super_read_only (int max_retries)
	Clear READ_ONLY and SUPER_READ_ONLY. More...

bool	terminate_and_retry (const Processlist_Ids &plist_ids, int group_size, bool query_only)
	Terminate queries or connections until SUPER_READ_ONLY can be set. More...

bool	terminate_queries (const Processlist_Ids &plist_ids, int group_size)

bool	terminate_connections (const Processlist_Ids &plist_ids, int group_size)

Recovery_State &	recovery ()

bool	recovery_eligible (std::string &reason)
	Recovery eligible. More...

void	check_recovery_status ()
	Check the recovery status after the server has been fenced. More...

bool	above_recovery_level () const

bool	below_recovery_level () const

void	complete_recovery_process ()
	Complete the recovery process: More...

void	cancel_recovery_process ()
	Cancel the recovery process: More...

bool	max_recovery_cycles ()
	Check if the max recovery cycle count has been reached. More...

void	set_recovery_window (bool print_status)
	Set the end of the recovery window. More...

void	check_recovery_window ()
	Check the recovery window. More...

Mode_Timer &	recovery_timer_1 ()

void	set_recovery_timeout_1 ()
	Set the timeout for recovery timer 1. More...

void	start_recovery_timer_1 ()
	Start the recovery_1 timer: More...

void	stop_recovery_timer_1 ()
	Stop the recovery_1 timer: More...

void	cancel_recovery_timer_1 ()
	Cancel the recovery_1 timer: More...

Mode_Timer &	recovery_timer_2 ()

void	set_recovery_timeout_2 ()
	Set the timeout for recovery timer 2. More...

void	start_recovery_timer_2 ()
	Start the recovery_2 timer: More...

void	stop_recovery_timer_2 ()
	Stop the recovery_2 timer: More...

void	cancel_recovery_timer_2 ()
	Cancel the recovery_2 timer: More...

bool *	super_read_only_set_ptr ()

bool *	offline_mode_set_ptr ()

bool	offline_mode_disk_full () const

bool	super_read_only_disk_full () const

void	update_super_read_only_disk_full ()
	Update the 'super_read_only_disk_full' status variable. More...

void	update_offline_mode_disk_full ()
	Update the 'offline_mode_disk_full' status variable. More...

void	update_fallback_status_vars ()
	Update fallback status variables, log changes. More...

std::string	msg_setting_super_read_only_disk_full () const
	Log messages for updates to status variables 'super_read_only_disk_full' and 'offline_mode_disk_full'. More...

std::string	msg_setting_offline_mode_disk_full () const

int *	recovery_count_ptr ()

int	recovery_count () const

void	set_state (ds::Storage_State &new_state)
	Set the next state based on the available space. More...

void	set_start_state ()

void	set_stop_state ()

void	set_normal_state ()

void	set_warning_state ()

void	set_low_limit_state ()

void	set_critical_state ()

void	set_error_state ()

void	check_storage_levels ()
	Check the available space against the storage thresholds: More...

void	set_disk_metric_buffer (Disk_Metric_Buffer buffer, Disk_Forecaster *forecaster)
	Set the time series buffer. More...

std::string	format_settings () override
	Disk Collector settings formatted string. More...

std::string	format_status_info () override
	Storage info formatted string. More...

std::string	tag_collector_state () override
	Collector state tag: Start, Stop, etc. More...

std::string	tag_resource_state () override
	Resource state tag: Normal Level, Warning Level, etc. More...

std::string	tag_mode () override
	Mode string: Fallback, Timer, etc. More...

std::string	tag_fallback ()
	Fallback mode string: "Fallback". More...

std::string	tag_recovery ()
	Recovery mode string: "Recovery". More...

std::string	storage_status () const
	Build storage state string. More...

std::string	timer_status (Mode_Timer &timer, const std::string &msg) const
	Timer status with optional message: Started, Running, Expired, ... More...

std::string	warning_status () const
	Warning level string: Warning Level 1, Warning Level 2, ... More...

void	log_info_message (const std::string &msg, bool levels)
	Log informational message with storage state, storage levels and message. More...

void	log_info_message (const std::string &msg)
	Log informational message with storage state, no levels and message. More...

void	log_warning_status (const std::string &msg, bool levels)
	Log warning with storage state, fallback state, storage levels and message. More...

void	log_warning_status (const std::string &msg)
	Log warning with storage state and message. More...

void	log_warning_level (int level)
	Low space warning level 1/2/3. More...

void	log_warning_level ()

void	log_warning_low_limit ()
	Low space warning with timer state. More...

void	log_warning_error_condition ()
	Error condition warning. More...

void	log_filesystem_recovery (const std::string &target_path)
	Log filesystem recovery after an unavailable transition. More...

void	log_timer_status (Mode_Timer &timer)
	Timer status, no levels. More...

void	log_timer_status (Mode_Timer &timer, bool levels)
	Timer status with optional storage levels. More...

void	log_timer_status (Mode_Timer &timer, const std::string &msg, bool levels)
	Timer status with optional message and storage levels. More...

void	log_volume_change ()
	The monitored disk has changed. More...

void	log_volume_size_change ()
	Size of the monitored disk has changed. More...

void	log_resource_status () override
	Collector or storage status with levels, depending on state. More...

void	log_resource_levels () override
	Log the current storage levels, no message. More...

Public Member Functions inherited from hm::Collector
	Collector ()=default

	Collector (const char name, const char resource_name, const char instr_name, const char os_name, const char *description)

	~Collector () override=default

	Collector (const Collector &)=delete

Collector &	operator= (const Collector &)=delete

	Collector (Collector &&)=delete

Collector &	operator= (Collector &&)=delete

const char *	resource_name () const

void	run_process () override
	Sample collection loop. More...

virtual bool	dynamic_settings_changed ()

virtual bool	static_settings_changed ()

void	mark_settings_pending ()
	Mark a settings update for evaluation by the Supervisor. More...

bool	settings_pending () const

void	set_settings_pending (bool pending)

virtual bool	update_settings ()
	Update system variable settings for the Collector. More...

int	base_interval () const

int	sample_interval () const

void	set_sample_interval (int sample_interval)
	Set the sample interval, adjust to the nearest multiple of the base interval. More...

void	set_collector_intervals (int base_interval, int sample_interval)
	Set the base and sample intervals together to ensure that the sample interval is a multiple of the base interval. More...

virtual void	log_collector_status (bool settings)
	Collector status with optional settings. More...

virtual void	log_collector_status (const std::string &state_msg, bool settings)
	Collector status with optional message and optional settings. More...

virtual void	log_status ()
	Issue a log status message for the collector. More...

virtual std::string	build_collector_message (const std::string &status, const std::string &mode, const std::string &message, bool settings)
	Formatted collector message with optional settings. More...

virtual std::string	build_resource_message (const std::string &status, const std::string &mode, const std::string &message, bool levels)
	Formatted resource message, no levels or settings. More...

Public Member Functions inherited from hm::Thread
	Thread ()=default

	Thread (const char name, const char instr_name, const char os_name, const char desc)

virtual	~Thread ()=0

const char *	name () const

virtual int	start ()

virtual void	stop ()

void	signal ()
	Signal the processing loop to end the current cycle. More...

Static Public Member Functions
static void	set_state_params ()
	Set threshold params for all storage states. More...

Static Public Member Functions inherited from hm::Collector
static std::string	format_level (uint64_t mbytes)
	Format storage amount in MiB, no units. More...

static std::string	format_mb (uint64_t mbytes, int prec, bool show_units)
	Format storage amount. More...

Static Public Member Functions inherited from hm::Thread
static void	cleanup ()
	Cleanup thread instrumentation. More...

static int	join (my_thread_handle thread, void *value_ptr)

static bool	joinable (const my_thread_handle &thread)

Public Attributes
ds::Storage_State *	m_current_state {nullptr}

ds::Start_State	m_start_state

ds::Stop_State	m_stop_state

ds::Normal_State	m_normal_state

ds::Warning_State	m_warning_state

ds::Low_Limit_State	m_low_limit_state

ds::Critical_State	m_critical_state

ds::Error_State	m_error_state

std::string	m_configured_monitored

std::string	m_monitored

uint64_t	m_monitored_generation {0}

int	m_low_space_duration {0}

uint64_t	m_low_space_level {0}

int	m_low_space_pct {0}

uint64_t	m_low_space_critical_level {0}

int	m_warn_level_1_pct {0}

int	m_warn_level_2_pct {0}

int	m_warn_level_3_pct {0}

std::atomic< bool >	m_settings_changed {false}

uint64_t	m_total_space_mb {0}

uint64_t	m_operative_space_mb {0}

uint64_t	m_warn_level_1_mb {0}

uint64_t	m_warn_level_2_mb {0}

uint64_t	m_warn_level_3_mb {0}

uint64_t	m_reserve_mb {0}

uint64_t	m_critical_level_mb {0}

bool	m_refresh_storage_profile {false}

uint64_t	m_avail_mb {0}

uint64_t	m_last_avail_mb {0}

float	m_used_pct {0}

int	m_delta_pct {k_delta_pct}

uint64_t	m_max_warn_level_mb {no_warn_level}

uint64_t	m_last_warn_level_mb {no_warn_level}

Disk_Info	m_info

Fallback_State	m_fallback

Mode_Timer	m_fallback_timer

Recovery_State	m_recovery

Mode_Timer	m_recovery_timer_1

Mode_Timer	m_recovery_timer_2

bool	m_status_offline_mode_disk_full {false}

bool	m_status_super_read_only_disk_full {false}

const std::string	k_sample_timer {"Disk Sample Timer"}

const std::string	k_fallback_mode {k_super_read_only}

const std::string	k_fallback {"Fallback"}

const std::string	k_fallback_timer {"Fallback Timer"}

const std::string	k_fallback_error {"Fallback Error"}

const std::string	k_recovery {"Recovery"}

const std::string	k_recovery_error {"Recovery Error"}

const std::string	k_recovery_timer_1 {"Timer 1"}

const std::string	k_recovery_timer_2 {"Timer 2"}

const std::string	k_recovery_1_mode {k_offline_mode}

const std::string	k_recovery_2_mode {k_super_read_only}

const std::string	k_normal_level {"Normal Level"}

const std::string	k_low_level {"Low Level"}

const std::string	k_warning_level {"Warning Level"}

const std::string	k_critical_level {"Critical Level"}

const std::string	k_running {"Running"}

const std::string	k_expired {"Expired"}

const std::string	k_canceled {"Canceled"}

Filesystem_sample_state	m_filesystem_sample_state

Filesystem_change	m_filesystem_change {Filesystem_change::initial}

bool	m_new_volume {false}

bool	m_new_size {false}

bool	m_disk_error {false}

std::optional< std::string >	m_unavailable_path

Input_failure	m_input_failure {Input_failure::filesystem}

Public Attributes inherited from hm::Collector
const std::string	k_started {"Started"}

const std::string	k_stopped {"Stopped"}

const std::string	k_offline_mode {"OFFLINE_MODE"}

const std::string	k_super_read_only {"SUPER_READ_ONLY"}

const std::string	k_invalid_path {"Invalid Path"}

const std::string	k_error {"ERROR"}

const std::string	k_system_error {"SYSTEM ERROR"}

const std::string	k_failed {"failed"}

const std::string	k_remaining {"attempts remaining"}

int	m_retention {0}

std::atomic< bool >	m_settings_pending {false}

int	m_status_interval {0}

int	m_status_counter {0}

bool	m_verbose_log {false}

bool	m_force_status {false}

Cycle_Timer	m_sample_timer

Cycle_Timer	m_status_timer

Static Public Attributes
static constexpr int	k_disk_base_interval {10}

static constexpr int	k_fallback_report_interval {k_disk_base_interval}

static constexpr int	k_recovery_report_interval {k_disk_base_interval}

static constexpr int	k_fallback_retries {5}

static constexpr int	k_delta_pct {10}

Static Public Attributes inherited from hm::Collector
static constexpr uint64_t	no_warn_level {std::numeric_limits<uint64_t>::max()}

static constexpr int	k_min_interval {1}

static constexpr int	k_max_interval {86400}

static constexpr int	k_default_base_interval {10}

static constexpr int	k_default_sample_interval {k_default_base_interval * 6}

Protected Member Functions
void	pre_process () override
	Put the collector into the START state. More...

void	process () override
	Query and evaluate filesystem usage at the base interval. More...

void	post_process () override
	Put the collector into the STOP state, which will cancel active timers, issue status messages, etc. More...

Protected Member Functions inherited from hm::Collector
void	consume_settings_pending ()
	Consume one pending notification without clearing a concurrent update. More...

void	apply_pending_settings ()
	Apply settings while the collector command lock is held. More...

Cycle_Timer &	sample_timer ()

Cycle_Timer &	status_message_timer ()

void	initialize_sample_timers ()
	Initialize collection timers. More...

void	advance_sample_timers ()
	Process collector sample and status timers. More...

Protected Member Functions inherited from hm::Thread
int	start_thread ()

void	stop_thread ()

void	request_thread_stop ()
	Request that the worker stop without waiting for it to exit. More...

void	join_thread ()
	Wait for a stopped worker to exit. More...

bool	stop_requested ()
	Return true when worker shutdown has been requested. More...

Private Member Functions
void	initialize_timers ()
	Initialize the fallback and recovery timers. More...

void	advance_timers ()
	Advance the timers driven by the collection interval. More...

Private Attributes
std::shared_ptr< Stat_Buffer< Disk_Info > >	m_buffer

Disk_Metric_Buffer	m_disk_metric_buffer

Disk_Forecaster *	m_disk_forecaster {nullptr}

PSI_rwlock_key	m_rwlock_key {0}

PSI_rwlock_info	m_rwlock_info

PSI_memory_key	m_memory_key {0}

PSI_memory_info	m_memory_info

Server	m_server

Additional Inherited Members
Protected Attributes inherited from hm::Collector
const std::string	m_resource_name

bool	m_running {false}

Protected Attributes inherited from hm::Thread
std::mutex	m_command_lock

const std::string	m_name

const std::string	m_instr_name

const std::string	m_os_name

const std::string	m_description

std::atomic< bool >	m_thread_running {false}

bool	m_stop {false}

my_thread_handle	m_thread

PSI_thread_key	m_thread_key {0}

PSI_thread_info	m_thread_info

mysql_mutex_t	m_wait_lock

PSI_mutex_key	m_mutex_key {0}

PSI_mutex_info	m_mutex_info

mysql_cond_t	m_wait

PSI_cond_key	m_cond_key {0}

PSI_cond_info	m_cond_info

Detailed Description

Disk Collector.

The Disk Collector monitors space on the filesystem path specified by health_monitor.disk_monitored. It collects filesystem usage, detects filesystem and capacity changes, publishes samples and supplies disk metrics to the Disk Forecaster.

The collector classifies available space as normal, warning, low or critical. When fencing is enabled, low and critical states can cause the collector to set OFFLINE_MODE and SUPER_READ_ONLY. The collector records which modes it set so that when disk space recovers, the Health Monitor clears only changes that it made.

If the monitored path or filesystem is unavailable, then the Disk Collector suspends sample collection and retries on subsequent collection cycles.

Member Enumeration Documentation

◆ Input_failure

enum class hm::Disk_Collector::Input_failure

strong

Enumerator
filesystem
datadir_lookup

Constructor & Destructor Documentation

◆ Disk_Collector()

hm::Disk_Collector::Disk_Collector ( )

Disk Collector constructor.

Initialize the stat buffer and instrumentation.

Member Function Documentation

◆ above_recovery_level()

bool hm::Disk_Collector::above_recovery_level ( ) const

inline

◆ advance_timers()

void hm::Disk_Collector::advance_timers ( )

private

Advance the timers driven by the collection interval.

◆ below_recovery_level()

bool hm::Disk_Collector::below_recovery_level ( ) const

inline

◆ buffer()

std::shared_ptr< Stat_Buffer< Disk_Info > > hm::Disk_Collector::buffer ( )

inline

◆ cancel_fallback_process()

void hm::Disk_Collector::cancel_fallback_process ( )

Cancel the ongoing fallback process.

Leave fallback state unchanged.

◆ cancel_fallback_timer()

void hm::Disk_Collector::cancel_fallback_timer ( )

Cancel the fallback timer:

stop the fallback timer
set the timer state to canceled
issue a warning that timer was canceled
restore the timer state

◆ cancel_recovery_process()

void hm::Disk_Collector::cancel_recovery_process ( )

Cancel the recovery process:

Reset the recovery timers
Issue log messages indicating that recovery has been canceled

◆ cancel_recovery_timer_1()

void hm::Disk_Collector::cancel_recovery_timer_1 ( )

Cancel the recovery_1 timer:

stop the recovery_1 timer
set the timer state to canceled
issue a warning that timer was canceled

◆ cancel_recovery_timer_2()

void hm::Disk_Collector::cancel_recovery_timer_2 ( )

Cancel the recovery_2 timer:

stop the recovery_2 timer
set the timer state to canceled
issue a warning that timer was canceled

◆ check_fenced_server_state()

bool hm::Disk_Collector::check_fenced_server_state ( )

Check if the state of a fenced server changed:

HA designation changes from Primary to Secondary
SUPER_READ_ONLY or OFFLINE_MODE were cleared externally

If the server state changed, then sync the fallback state accordingly.

This is independent of the recovery process, changes in storage levels or updates to the Health Monitor settings.

Returns: true if the server state changed

◆ check_recovery_status()

void hm::Disk_Collector::check_recovery_status ( )

Check the recovery status after the server has been fenced.

This is only called if the server was fenced by the Health Monitor.

Recovery from fallback mode is done in two phases. If storage rises above the recovery level, then:

after a delay of 'recovery_timer_1' set OFFLINE_MODE = OFF, then
after an additional delay of 'recovery_timer_2', set SUPER_READ_ONLY = OFF.

Increase the recovery delays by 'disk_recovery_increment' for each recovery cycle.

◆ check_recovery_window()

void hm::Disk_Collector::check_recovery_window ( )

Check the recovery window.

If sufficient time has elapsed since the last recovery, then reset the recovery cycle count.

◆ check_storage_levels()

void hm::Disk_Collector::check_storage_levels ( )

Check the available space against the storage thresholds:

available space < critical low space level, trigger fallback mode
available space < reserve level, start fallback timer
fallback timer >= low space duration, trigger fallback mode
available space > reserve level, cancel fallback timer if running
available space < a warning level, issue warning message Otherwise, if status duration has expired, issue a status message.

◆ clear_fallback_mode()

void hm::Disk_Collector::clear_fallback_mode ( )

Clear OFFLINE_MODE and SUPER_READ_ONLY, reset all fallback state variables.

This is called when the Health Monitor settings change or a new or resized volume is detected.

◆ clear_offline_mode()

bool hm::Disk_Collector::clear_offline_mode ( )

Clear OFFLINE_MODE.

Returns: True if OFFLINE_MODE = OFF

◆ clear_super_read_only()

bool hm::Disk_Collector::clear_super_read_only ( int max_retries )

Clear READ_ONLY and SUPER_READ_ONLY.

Retry if necessary.

Parameters

max_retries Number of attempts to make

Returns: True if READ_ONLY and SUPER_READ_ONLY = OFF

◆ complete_recovery_process()

void hm::Disk_Collector::complete_recovery_process ( )

Complete the recovery process:

Reset the fallback and recovery states
Reset the recovery timers
Clear the status variables
Issue messages with the current values of SUPER_READ_ONLY and OFFLINE_MODE

◆ fallback()

Fallback_State & hm::Disk_Collector::fallback ( )

inline

◆ fallback_eligible() [1/2]

bool hm::Disk_Collector::fallback_eligible ( bool print_reason = false )

Fallback eligible: Print reason if ineligible.

Parameters

print_reason Print the reason why the server cannot be fenced

Returns: true if eligible

◆ fallback_eligible() [2/2]

bool hm::Disk_Collector::fallback_eligible ( std::string & reason )

Fallback eligible: Server is eligible to be placed in fallback mode if:

fallback is enabled
Health Monitor is not already in fallback mode
server is not a group replication secondary
SUPER_READ_ONLY is not already set (i.e. externally)
Parameters

reason Reason for ineligibility

Returns
true if eligible

◆ fallback_timer()

Mode_Timer & hm::Disk_Collector::fallback_timer ( )

inline

◆ forced_fallback_eligible()

bool hm::Disk_Collector::forced_fallback_eligible ( )

Return true if forced fallback is allowed.

Criteria:

health_monitor.disk_fallback_force = true
forced fallback has not been attempted
SUPER_READ_ONLY has not been set by the Health Monitor Assumes:
health_monitor.disk_fallback_enable = true
available storage <= health_monitor.disk_low_space_critical_level

◆ format_settings()

std::string hm::Disk_Collector::format_settings ( )

overridevirtual

Disk Collector settings formatted string.

Reimplemented from hm::Collector.

◆ format_status_info()

std::string hm::Disk_Collector::format_status_info ( )

overridevirtual

Storage info formatted string.

Reimplemented from hm::Collector.

◆ get_disk_info()

int hm::Disk_Collector::get_disk_info	(	const std::string &	target_path,
		Disk_Info &	disk_info,
		Filesystem_sample_state &	sample_state,
		Filesystem_change &	change
	)

Get available free space for a monitored filesystem.

Parameters

[in]	target_path	A valid path
[out]	disk_info	Storage info for the filesystem
[in,out]	sample_state	Last successful filesystem sample state
[out]	change	Current sample classification

Returns: 0 success, 1 failure

◆ initialize()

void hm::Disk_Collector::initialize ( )

Initialize the Disk Collector.

Performs initialization after construction. Call this immediately after creating the collector.

◆ initialize_timers()

void hm::Disk_Collector::initialize_timers ( )

private

Initialize the fallback and recovery timers.

◆ log_filesystem_recovery()

void hm::Disk_Collector::log_filesystem_recovery ( const std::string & target_path )

Log filesystem recovery after an unavailable transition.

◆ log_info_message() [1/2]

void hm::Disk_Collector::log_info_message ( const std::string & msg )

Log informational message with storage state, no levels and message.

◆ log_info_message() [2/2]

void hm::Disk_Collector::log_info_message	(	const std::string &	msg,
		bool	levels
	)

Log informational message with storage state, storage levels and message.

◆ log_resource_levels()

void hm::Disk_Collector::log_resource_levels ( )

overridevirtual

Log the current storage levels, no message.

Reimplemented from hm::Collector.

◆ log_resource_status()

void hm::Disk_Collector::log_resource_status ( )

overridevirtual

Collector or storage status with levels, depending on state.

Implements hm::Collector.

◆ log_timer_status() [1/3]

void hm::Disk_Collector::log_timer_status ( Mode_Timer & timer )

Timer status, no levels.

◆ log_timer_status() [2/3]

void hm::Disk_Collector::log_timer_status	(	Mode_Timer &	timer,
		bool	levels
	)

Timer status with optional storage levels.

◆ log_timer_status() [3/3]

void hm::Disk_Collector::log_timer_status	(	Mode_Timer &	timer,
		const std::string &	msg,
		bool	levels
	)

Timer status with optional message and storage levels.

◆ log_volume_change()

void hm::Disk_Collector::log_volume_change ( )

The monitored disk has changed.

◆ log_volume_size_change()

void hm::Disk_Collector::log_volume_size_change ( )

Size of the monitored disk has changed.

◆ log_warning_error_condition()

void hm::Disk_Collector::log_warning_error_condition ( )

Error condition warning.

◆ log_warning_level() [1/2]

void hm::Disk_Collector::log_warning_level ( )

inline

◆ log_warning_level() [2/2]

void hm::Disk_Collector::log_warning_level ( int level )

Low space warning level 1/2/3.

◆ log_warning_low_limit()

void hm::Disk_Collector::log_warning_low_limit ( )

Low space warning with timer state.

Only use the "Fallback" tag if the server is fenced or in the process of being fenced.

◆ log_warning_status() [1/2]

void hm::Disk_Collector::log_warning_status ( const std::string & msg )

Log warning with storage state and message.

◆ log_warning_status() [2/2]

void hm::Disk_Collector::log_warning_status	(	const std::string &	msg,
		bool	levels
	)

Log warning with storage state, fallback state, storage levels and message.

◆ max_recovery_cycles()

bool hm::Disk_Collector::max_recovery_cycles ( )

Check if the max recovery cycle count has been reached.

Returns: True if recovery cycles have reached the maximum within the recovery expiration window

◆ msg_setting_offline_mode_disk_full()

std::string hm::Disk_Collector::msg_setting_offline_mode_disk_full ( ) const

◆ msg_setting_super_read_only_disk_full()

std::string hm::Disk_Collector::msg_setting_super_read_only_disk_full ( ) const

Log messages for updates to status variables 'super_read_only_disk_full' and 'offline_mode_disk_full'.

Returns: "Setting status" message

◆ offline_mode_disk_full()

bool hm::Disk_Collector::offline_mode_disk_full ( ) const

inline

◆ offline_mode_set_ptr()

bool * hm::Disk_Collector::offline_mode_set_ptr ( )

inline

◆ post_process()

void hm::Disk_Collector::post_process ( )

overrideprotectedvirtual

Put the collector into the STOP state, which will cancel active timers, issue status messages, etc.

Reimplemented from hm::Thread.

◆ pre_process()

void hm::Disk_Collector::pre_process ( )

overrideprotectedvirtual

Put the collector into the START state.

Reimplemented from hm::Thread.

◆ process()

void hm::Disk_Collector::process ( )

overrideprotectedvirtual

Query and evaluate filesystem usage at the base interval.

Publish a Performance Schema record only when the sample interval expires and publish a disk metric record when its metric buffer is writable. Check the available space against the storage limits and take appropriate action if necessary.

Reimplemented from hm::Collector.

◆ process_expired_fallback_timer()

void hm::Disk_Collector::process_expired_fallback_timer ( )

Process expired fallback timer.

Issue warning that the server will be fenced
Stop the timer
Fence the server

◆ recovery()

Recovery_State & hm::Disk_Collector::recovery ( )

inline

◆ recovery_count()

int hm::Disk_Collector::recovery_count ( ) const

inline

◆ recovery_count_ptr()

int * hm::Disk_Collector::recovery_count_ptr ( )

inline

◆ recovery_eligible()

bool hm::Disk_Collector::recovery_eligible ( std::string & reason )

Recovery eligible.

Server is eligible to recover from fallback mode if:

Recovery is enabled
HA role is Primary or N/A
Parameters

reason Message string giving reason for ineligibility

Returns
true if eligible to recover from fallback mode

◆ recovery_timer_1()

Mode_Timer & hm::Disk_Collector::recovery_timer_1 ( )

inline

◆ recovery_timer_2()

Mode_Timer & hm::Disk_Collector::recovery_timer_2 ( )

inline

◆ reset_fallback_and_recovery()

void hm::Disk_Collector::reset_fallback_and_recovery ( )

Reset fallback and recovery modes, cancel all timers.

Invoked after the monitored filesystem capacity changes.

◆ resolve_monitored_path()

bool hm::Disk_Collector::resolve_monitored_path ( )

Resolve the automatic monitored path before filesystem collection.

Returns: true if a path is available, false otherwise

◆ running()

bool hm::Disk_Collector::running ( ) const

overridevirtual

Return health_monitor.disk_running.

Reimplemented from hm::Thread.

◆ set_buffer_size()

void hm::Disk_Collector::set_buffer_size ( )

finalvirtual

Resize the stat buffer if necessary.

Must be inside collector lock.

Reimplemented from hm::Collector.

◆ set_critical_state()

void hm::Disk_Collector::set_critical_state ( )

inline

◆ set_disk_metric_buffer()

void hm::Disk_Collector::set_disk_metric_buffer	(	Disk_Metric_Buffer	buffer,
		Disk_Forecaster *	forecaster
	)

Set the time series buffer.

Parameters

buffer	Disk time series buffer
forecaster	Disk Forecaster associated with the buffer

◆ set_error_state()

void hm::Disk_Collector::set_error_state ( )

inline

◆ set_fallback_mode()

void hm::Disk_Collector::set_fallback_mode ( bool force )

Set the server fallback mode, OFFLINE_MODE and SUPER_READ_ONLY.

Parameters

force If true then terminate queries blocking SUPER_READ_ONLY

◆ set_low_limit_state()

void hm::Disk_Collector::set_low_limit_state ( )

inline

◆ set_normal_state()

void hm::Disk_Collector::set_normal_state ( )

inline

◆ set_offline_mode()

bool hm::Disk_Collector::set_offline_mode	(	int	max_retries,
		bool &	already_set
	)

Attempt to set OFFLINE_MODE.

Retry if necessary.

Parameters

max_retries	Number of attempts to make
already_set	True if OFFLINE_MODE was already set

Returns: True if OFFLINE_MODE was set

◆ set_params()

void hm::Disk_Collector::set_params ( )

finalvirtual

Set collection parameters from the system variables.

Reimplemented from hm::Collector.

◆ set_recovery_timeout_1()

void hm::Disk_Collector::set_recovery_timeout_1 ( )

Set the timeout for recovery timer 1.

◆ set_recovery_timeout_2()

void hm::Disk_Collector::set_recovery_timeout_2 ( )

Set the timeout for recovery timer 2.

◆ set_recovery_window()

void hm::Disk_Collector::set_recovery_window ( bool print_status )

Set the end of the recovery window.

param[in] print_status If true, issue log message

◆ set_start_state()

void hm::Disk_Collector::set_start_state ( )

inline

◆ set_state()

void hm::Disk_Collector::set_state ( ds::Storage_State & new_state )

Set the next state based on the available space.

◆ set_state_params()

void hm::Disk_Collector::set_state_params ( )

static

Set threshold params for all storage states.

◆ set_stop_state()

void hm::Disk_Collector::set_stop_state ( )

inline

◆ set_storage_profile()

void hm::Disk_Collector::set_storage_profile ( const Disk_Info & disk_info )

Set the storage threshold parameters for the monitored filesystem.

This requires that a disk sample has been collected.

Convert units to MiB.
Adjust levels to be relative to the total space.
Parameters

[in] disk_info Current block device info

◆ set_super_read_only()

bool hm::Disk_Collector::set_super_read_only	(	int	max_retries,
		bool &	already_set
	)

Set SUPER_READ_ONLY.

Retry if necessary. Setting SUPER_READ_ONLY also sets READ_ONLY.

Parameters

max_retries	Number of attempts to make
already_set	True if SUPER_READ_ONLY already set

Returns: True if SUPER_READ_ONLY is set

◆ set_warning_state()

void hm::Disk_Collector::set_warning_state ( )

inline

◆ settings_changed()

bool hm::Disk_Collector::settings_changed ( )

overridevirtual

Indicate if the Disk Collector settings have changed.

Returns: true if settings have changed

Reimplemented from hm::Collector.

◆ start_fallback_timer()

void hm::Disk_Collector::start_fallback_timer ( )

Start the fallback timer.

◆ start_recovery_timer_1()

void hm::Disk_Collector::start_recovery_timer_1 ( )

Start the recovery_1 timer:

reset status update message counter
assumes timeout has been set

◆ start_recovery_timer_2()

void hm::Disk_Collector::start_recovery_timer_2 ( )

Start the recovery_2 timer:

reset status update message counter
assumes timeout has been set

◆ stop_fallback_timer()

void hm::Disk_Collector::stop_fallback_timer ( )

Stop the fallback timer.

◆ stop_recovery_timer_1()

void hm::Disk_Collector::stop_recovery_timer_1 ( )

Stop the recovery_1 timer:

set the recovery_1 timer to OFF
renable status update messages

◆ stop_recovery_timer_2()

void hm::Disk_Collector::stop_recovery_timer_2 ( )

Stop the recovery_2 timer:

set the recovery_2 timer to OFF
renable status update messages

◆ storage_status()

std::string hm::Disk_Collector::storage_status ( ) const

Build storage state string.

◆ super_read_only_disk_full()

bool hm::Disk_Collector::super_read_only_disk_full ( ) const

inline

◆ super_read_only_set_ptr()

bool * hm::Disk_Collector::super_read_only_set_ptr ( )

inline

◆ tag_collector_state()

std::string hm::Disk_Collector::tag_collector_state ( )

overridevirtual

Collector state tag: Start, Stop, etc.

Reimplemented from hm::Collector.

◆ tag_fallback()

std::string hm::Disk_Collector::tag_fallback ( )

Fallback mode string: "Fallback".

◆ tag_mode()

std::string hm::Disk_Collector::tag_mode ( )

overridevirtual

Mode string: Fallback, Timer, etc.

Reimplemented from hm::Collector.

◆ tag_recovery()

std::string hm::Disk_Collector::tag_recovery ( )

Recovery mode string: "Recovery".

◆ tag_resource_state()

std::string hm::Disk_Collector::tag_resource_state ( )

overridevirtual

Resource state tag: Normal Level, Warning Level, etc.

Reimplemented from hm::Collector.

◆ terminate_and_retry()

bool hm::Disk_Collector::terminate_and_retry	(	const Processlist_Ids &	plist_ids,
		int	group_size,
		bool	query_only
	)

Terminate queries or connections until SUPER_READ_ONLY can be set.

Parameters

plist_ids	Processlist ids of connections to terminate
group_size	Terminate connections/queries in groups of this size
query_only	Terminate the queries, not the connections

Returns: true if SUPER_READ_ONLY was set, false otherwise

◆ terminate_connections()

bool hm::Disk_Collector::terminate_connections	(	const Processlist_Ids &	plist_ids,
		int	group_size
	)

inline

◆ terminate_queries()

bool hm::Disk_Collector::terminate_queries	(	const Processlist_Ids &	plist_ids,
		int	group_size
	)

inline

◆ timer_status()

std::string hm::Disk_Collector::timer_status	(	Mode_Timer &	timer,
		const std::string &	msg
	)		const

Timer status with optional message: Started, Running, Expired, ...

◆ update_disk_info()

void hm::Disk_Collector::update_disk_info ( )

Update disk info for display only.

Do not write to the stat buffer, do not check storage levels, do not change collector state.

◆ update_fallback_status_vars()

void hm::Disk_Collector::update_fallback_status_vars ( )

Update fallback status variables, log changes.

◆ update_offline_mode_disk_full()

void hm::Disk_Collector::update_offline_mode_disk_full ( )

Update the 'offline_mode_disk_full' status variable.

◆ update_per_sample_totals()

void hm::Disk_Collector::update_per_sample_totals ( const Disk_Info & disk_info )

Update totals that change for each sample, convert to MiB.

◆ update_super_read_only_disk_full()

void hm::Disk_Collector::update_super_read_only_disk_full ( )

Update the 'super_read_only_disk_full' status variable.

◆ warning_status()

std::string hm::Disk_Collector::warning_status ( ) const

Warning level string: Warning Level 1, Warning Level 2, ...

Member Data Documentation

◆ k_canceled

const std::string hm::Disk_Collector::k_canceled {"Canceled"}

◆ k_critical_level

const std::string hm::Disk_Collector::k_critical_level {"Critical Level"}

◆ k_delta_pct

constexpr int hm::Disk_Collector::k_delta_pct {10}

staticconstexpr

◆ k_disk_base_interval

constexpr int hm::Disk_Collector::k_disk_base_interval {10}

staticconstexpr

◆ k_expired

const std::string hm::Disk_Collector::k_expired {"Expired"}

◆ k_fallback

const std::string hm::Disk_Collector::k_fallback {"Fallback"}

◆ k_fallback_error

const std::string hm::Disk_Collector::k_fallback_error {"Fallback Error"}

◆ k_fallback_mode

const std::string hm::Disk_Collector::k_fallback_mode {k_super_read_only}

◆ k_fallback_report_interval

constexpr int hm::Disk_Collector::k_fallback_report_interval {k_disk_base_interval}

staticconstexpr

◆ k_fallback_retries

constexpr int hm::Disk_Collector::k_fallback_retries {5}

staticconstexpr

◆ k_fallback_timer

const std::string hm::Disk_Collector::k_fallback_timer {"Fallback Timer"}

◆ k_low_level

const std::string hm::Disk_Collector::k_low_level {"Low Level"}

◆ k_normal_level

const std::string hm::Disk_Collector::k_normal_level {"Normal Level"}

◆ k_recovery

const std::string hm::Disk_Collector::k_recovery {"Recovery"}

◆ k_recovery_1_mode

const std::string hm::Disk_Collector::k_recovery_1_mode {k_offline_mode}

◆ k_recovery_2_mode

const std::string hm::Disk_Collector::k_recovery_2_mode {k_super_read_only}

◆ k_recovery_error

const std::string hm::Disk_Collector::k_recovery_error {"Recovery Error"}

◆ k_recovery_report_interval

constexpr int hm::Disk_Collector::k_recovery_report_interval {k_disk_base_interval}

staticconstexpr

◆ k_recovery_timer_1

const std::string hm::Disk_Collector::k_recovery_timer_1 {"Timer 1"}

◆ k_recovery_timer_2

const std::string hm::Disk_Collector::k_recovery_timer_2 {"Timer 2"}

◆ k_running

const std::string hm::Disk_Collector::k_running {"Running"}

◆ k_sample_timer

const std::string hm::Disk_Collector::k_sample_timer {"Disk Sample Timer"}

◆ k_warning_level

const std::string hm::Disk_Collector::k_warning_level {"Warning Level"}

◆ m_avail_mb

uint64_t hm::Disk_Collector::m_avail_mb {0}

◆ m_buffer

std::shared_ptr<Stat_Buffer<Disk_Info> > hm::Disk_Collector::m_buffer

private

◆ m_configured_monitored

std::string hm::Disk_Collector::m_configured_monitored

◆ m_critical_level_mb

uint64_t hm::Disk_Collector::m_critical_level_mb {0}

◆ m_critical_state

ds::Critical_State hm::Disk_Collector::m_critical_state

◆ m_current_state

ds::Storage_State* hm::Disk_Collector::m_current_state {nullptr}

◆ m_delta_pct

int hm::Disk_Collector::m_delta_pct {k_delta_pct}

◆ m_disk_error

bool hm::Disk_Collector::m_disk_error {false}

◆ m_disk_forecaster

Disk_Forecaster* hm::Disk_Collector::m_disk_forecaster {nullptr}

private

◆ m_disk_metric_buffer

Disk_Metric_Buffer hm::Disk_Collector::m_disk_metric_buffer

private

◆ m_error_state

ds::Error_State hm::Disk_Collector::m_error_state

◆ m_fallback

Fallback_State hm::Disk_Collector::m_fallback

◆ m_fallback_timer

Mode_Timer hm::Disk_Collector::m_fallback_timer

◆ m_filesystem_change

Filesystem_change hm::Disk_Collector::m_filesystem_change {Filesystem_change::initial}

◆ m_filesystem_sample_state

Filesystem_sample_state hm::Disk_Collector::m_filesystem_sample_state

◆ m_info

Disk_Info hm::Disk_Collector::m_info

◆ m_input_failure

Input_failure hm::Disk_Collector::m_input_failure {Input_failure::filesystem}

◆ m_last_avail_mb

uint64_t hm::Disk_Collector::m_last_avail_mb {0}

◆ m_last_warn_level_mb

uint64_t hm::Disk_Collector::m_last_warn_level_mb {no_warn_level}

◆ m_low_limit_state

ds::Low_Limit_State hm::Disk_Collector::m_low_limit_state

◆ m_low_space_critical_level

uint64_t hm::Disk_Collector::m_low_space_critical_level {0}

◆ m_low_space_duration

int hm::Disk_Collector::m_low_space_duration {0}

◆ m_low_space_level

uint64_t hm::Disk_Collector::m_low_space_level {0}

◆ m_low_space_pct

int hm::Disk_Collector::m_low_space_pct {0}

◆ m_max_warn_level_mb

uint64_t hm::Disk_Collector::m_max_warn_level_mb {no_warn_level}

◆ m_memory_info

PSI_memory_info hm::Disk_Collector::m_memory_info

private

Initial value:

{
      &m_memory_key, "disk_collector", PSI_FLAG_ONLY_GLOBAL_STAT,
      PSI_VOLATILITY_PROVISIONING, "Health Monitor disk metric buffer"}

◆ m_memory_key

PSI_memory_key hm::Disk_Collector::m_memory_key {0}

private

◆ m_monitored

std::string hm::Disk_Collector::m_monitored

◆ m_monitored_generation

uint64_t hm::Disk_Collector::m_monitored_generation {0}

◆ m_new_size

bool hm::Disk_Collector::m_new_size {false}

◆ m_new_volume

bool hm::Disk_Collector::m_new_volume {false}

◆ m_normal_state

ds::Normal_State hm::Disk_Collector::m_normal_state

◆ m_operative_space_mb

uint64_t hm::Disk_Collector::m_operative_space_mb {0}

◆ m_recovery

Recovery_State hm::Disk_Collector::m_recovery

◆ m_recovery_timer_1

Mode_Timer hm::Disk_Collector::m_recovery_timer_1

◆ m_recovery_timer_2

Mode_Timer hm::Disk_Collector::m_recovery_timer_2

◆ m_refresh_storage_profile

bool hm::Disk_Collector::m_refresh_storage_profile {false}

◆ m_reserve_mb

uint64_t hm::Disk_Collector::m_reserve_mb {0}

◆ m_rwlock_info

PSI_rwlock_info hm::Disk_Collector::m_rwlock_info

private

Initial value:

{&m_rwlock_key, "disk_collector",
                                PSI_FLAG_SINGLETON, PSI_VOLATILITY_PROVISIONING,
                                PSI_DOCUMENT_ME}

◆ m_rwlock_key

PSI_rwlock_key hm::Disk_Collector::m_rwlock_key {0}

private

◆ m_server

Server hm::Disk_Collector::m_server

private

◆ m_settings_changed

std::atomic<bool> hm::Disk_Collector::m_settings_changed {false}

◆ m_start_state

ds::Start_State hm::Disk_Collector::m_start_state

◆ m_status_offline_mode_disk_full

bool hm::Disk_Collector::m_status_offline_mode_disk_full {false}

◆ m_status_super_read_only_disk_full

bool hm::Disk_Collector::m_status_super_read_only_disk_full {false}

◆ m_stop_state

ds::Stop_State hm::Disk_Collector::m_stop_state

◆ m_total_space_mb

uint64_t hm::Disk_Collector::m_total_space_mb {0}

◆ m_unavailable_path

std::optional<std::string> hm::Disk_Collector::m_unavailable_path

◆ m_used_pct

float hm::Disk_Collector::m_used_pct {0}

◆ m_warn_level_1_mb

uint64_t hm::Disk_Collector::m_warn_level_1_mb {0}

◆ m_warn_level_1_pct

int hm::Disk_Collector::m_warn_level_1_pct {0}

◆ m_warn_level_2_mb

uint64_t hm::Disk_Collector::m_warn_level_2_mb {0}

◆ m_warn_level_2_pct

int hm::Disk_Collector::m_warn_level_2_pct {0}

◆ m_warn_level_3_mb

uint64_t hm::Disk_Collector::m_warn_level_3_mb {0}

◆ m_warn_level_3_pct

int hm::Disk_Collector::m_warn_level_3_pct {0}

◆ m_warning_state

ds::Warning_State hm::Disk_Collector::m_warning_state

The documentation for this class was generated from the following files:

components/health_monitor/health_monitor_disk.h
components/health_monitor/health_monitor_disk.cc

Public Types

Public Member Functions

Static Public Member Functions

Public Attributes

Static Public Attributes

Protected Member Functions

Private Member Functions

Private Attributes

Additional Inherited Members

Detailed Description

Member Enumeration Documentation

◆ Input_failure

Constructor & Destructor Documentation

◆ Disk_Collector()

Member Function Documentation

◆ above_recovery_level()

◆ advance_timers()

◆ below_recovery_level()

◆ buffer()

◆ cancel_fallback_process()

◆ cancel_fallback_timer()

◆ cancel_recovery_process()

◆ cancel_recovery_timer_1()

◆ cancel_recovery_timer_2()

◆ check_fenced_server_state()

◆ check_recovery_status()

◆ check_recovery_window()

◆ check_storage_levels()

◆ clear_fallback_mode()

◆ clear_offline_mode()

◆ clear_super_read_only()

◆ complete_recovery_process()

◆ fallback()

◆ fallback_eligible() [1/2]

◆ fallback_eligible() [2/2]

◆ fallback_timer()

◆ forced_fallback_eligible()

◆ format_settings()

◆ format_status_info()

◆ get_disk_info()

◆ initialize()

◆ initialize_timers()

◆ log_filesystem_recovery()

◆ log_info_message() [1/2]

◆ log_info_message() [2/2]

◆ log_resource_levels()

◆ log_resource_status()

◆ log_timer_status() [1/3]

◆ log_timer_status() [2/3]

◆ log_timer_status() [3/3]

◆ log_volume_change()

◆ log_volume_size_change()

◆ log_warning_error_condition()

◆ log_warning_level() [1/2]

◆ log_warning_level() [2/2]

◆ log_warning_low_limit()

◆ log_warning_status() [1/2]

◆ log_warning_status() [2/2]

◆ max_recovery_cycles()

◆ msg_setting_offline_mode_disk_full()

◆ msg_setting_super_read_only_disk_full()

◆ offline_mode_disk_full()

◆ offline_mode_set_ptr()

◆ post_process()

◆ pre_process()

◆ process()

◆ process_expired_fallback_timer()

◆ recovery()

◆ recovery_count()

◆ recovery_count_ptr()

◆ recovery_eligible()

◆ recovery_timer_1()

◆ recovery_timer_2()

◆ reset_fallback_and_recovery()

◆ resolve_monitored_path()

◆ running()

◆ set_buffer_size()

◆ set_critical_state()

◆ set_disk_metric_buffer()

◆ set_error_state()