MySQL 8.1.0
Source Code Documentation
page_alloc.h
Go to the documentation of this file.
1/*****************************************************************************
2
3Copyright (c) 2021, 2023, Oracle and/or its affiliates.
4
5This program is free software; you can redistribute it and/or modify it under
6the terms of the GNU General Public License, version 2.0, as published by the
7Free Software Foundation.
8
9This program is also distributed with certain software (including but not
10limited to OpenSSL) that is licensed under separate terms, as designated in a
11particular file or component or in included license documentation. The authors
12of MySQL hereby grant you an additional permission to link the program and
13your derivative works with the separately licensed software that they have
14included with MySQL.
15
16This program is distributed in the hope that it will be useful, but WITHOUT
17ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
18FOR A PARTICULAR PURPOSE. See the GNU General Public License, version 2.0,
19for more details.
20
21You should have received a copy of the GNU General Public License along with
22this program; if not, write to the Free Software Foundation, Inc.,
2351 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
24
25*****************************************************************************/
26
27/** @file include/detail/ut/page_alloc.h
28 Implementation bits and pieces for page-aligned allocations. */
29
30#ifndef detail_ut_page_alloc_h
31#define detail_ut_page_alloc_h
32
33#ifdef _WIN32
34#include <windows.h>
35// _must_ go after windows.h, this comment makes clang-format to preserve
36// the include order
37#include <memoryapi.h>
38#else
39#include <sys/mman.h>
40#endif
41
42#include <cassert>
43#include <cstddef>
44#include <cstdint>
45#include <cstdlib>
46#include <memory>
47#include <utility>
48
49#include "my_compiler.h"
50#include "my_config.h"
51#include "mysqld_error.h"
57
58namespace ut {
59namespace detail {
60
61/** Allocates system page-aligned memory.
62
63 @param[in] n_bytes Size of storage (in bytes) requested to be allocated.
64 @return Pointer to the allocated storage. nullptr if allocation failed.
65*/
66inline void *page_aligned_alloc(size_t n_bytes) {
67#ifdef _WIN32
68 // With lpAddress set to nullptr, VirtualAlloc will internally round n_bytes
69 // to the multiple of system page size if it is not already
70 void *ptr =
71 VirtualAlloc(nullptr, n_bytes, MEM_COMMIT | MEM_RESERVE, PAGE_READWRITE);
72 if (unlikely(!ptr)) {
73 ib::log_warn(ER_IB_MSG_856) << "page_aligned_alloc VirtualAlloc(" << n_bytes
74 << " bytes) failed;"
75 " Windows error "
76 << GetLastError();
77 }
78 return ptr;
79#else
80 // With addr set to nullptr, mmap will internally round n_bytes to the
81 // multiple of system page size if it is not already
82 void *ptr = mmap(nullptr, n_bytes, PROT_READ | PROT_WRITE,
83 MAP_PRIVATE | MAP_ANON, -1, 0);
84 if (unlikely(ptr == (void *)-1)) {
85 ib::log_warn(ER_IB_MSG_856) << "page_aligned_alloc mmap(" << n_bytes
86 << " bytes) failed;"
87 " errno "
88 << errno;
89 }
90 return (ptr != (void *)-1) ? ptr : nullptr;
91#endif
92}
93
94/** Releases system page-aligned storage.
95
96 @param[in] ptr Pointer to system page-aligned storage.
97 @param[in] n_bytes Size of the storage.
98 @return True if releasing the page-aligned memory was successful.
99 */
100inline bool page_aligned_free(void *ptr, size_t n_bytes [[maybe_unused]]) {
101 if (unlikely(!ptr)) return false;
102#ifdef _WIN32
103 auto ret = VirtualFree(ptr, 0, MEM_RELEASE);
104 if (unlikely(ret == 0)) {
105 ib::log_error(ER_IB_MSG_858)
106 << "large_page_aligned_free VirtualFree(" << ptr
107 << ") failed;"
108 " Windows error "
109 << GetLastError();
110 }
111 return ret != 0;
112#else
113 // length aka n_bytes does not need to be aligned to page-size
114 auto ret = munmap(ptr, n_bytes);
115 if (unlikely(ret != 0)) {
116 ib::log_error(ER_IB_MSG_858)
117 << "page_aligned_free munmap(" << ptr << ", " << n_bytes
118 << ") failed;"
119 " errno "
120 << errno;
121 }
122 return ret == 0;
123#endif
124}
125
126/** Allocation routines which are purposed for allocating system page-aligned
127 memory.
128
129 page_aligned_alloc() and page_aligned_free() are taking care of
130 OS specific details and Page_alloc is a convenience wrapper which only
131 makes the use of system page-aligned memory more ergonomic so that it
132 serializes the actual size being allocated into the raw memory. This size
133 is then automagically deduced when system page-aligned memory is being
134 freed. Otherwise, client code would have been responsible to store and keep
135 that value somewhere until the memory segment is freed. Additionally,
136 information on type of page used to back up requested allocation is also
137 serialized into the memory allowing to build higher-kinded abstractions more
138 easily. See ut::malloc_large_page with option to fallback to regular pages
139 through ut::malloc_page.
140
141 Cost associated with this abstraction is the size of a single CPU page. In
142 terms of virtual memory, especially in 64-bit address space, this cost is
143 negligible. In practice this means that for each N pages allocation
144 request there will be N+1 pages allocated beneath.
145
146 Memory layout representation looks like the following:
147
148 ------------------------------------------
149 | PAGE-ALLOC-METADATA | ... DATA ... |
150 ------------------------------------------
151 ^ ^
152 | |
153 | |
154 | ptr (system-page) to be
155 | returned to call-site
156 |
157 --------------------------------
158 | DATALEN | PAGE-TYPE | VARLEN |
159 --------------------------------
160 ^
161 |
162 |
163 ptr returned by
164 page_aligned_alloc
165
166
167 For details on DATALEN, PAGE-TYPE and VARLEN fields see Page_alloc_metadata.
168
169 DATA is an actual page-aligned segment that will be returned to the
170 call-site and which the client code will be able to use for the application
171 data.
172 */
173struct Page_alloc : public allocator_traits<false> {
175
176 /** Allocates memory through large-page support.
177
178 @param[in] size Size of storage (in bytes) requested to be allocated.
179 @return Pointer to the allocated storage. nullptr if allocation failed.
180 */
181 static inline void *alloc(std::size_t size) {
182 auto total_len = round_to_next_multiple(
184 auto mem = page_aligned_alloc(total_len);
185 if (unlikely(!mem)) return nullptr;
188 return static_cast<uint8_t *>(mem) + page_allocation_metadata::len;
189 }
190
191 /** Releases storage allocated through Page_alloc::alloc().
192
193 @param[in] data Pointer to storage allocated through Page_alloc::alloc()
194 @return True if releasing the page-aligned memory was successful.
195 */
196 static inline bool free(void *data) noexcept {
197 if (unlikely(!data)) return false;
199 return page_aligned_free(deduce(data),
201 }
202
203 /** Returns the number of bytes that have been allocated.
204
205 @param[in] data Pointer to storage allocated through Page_alloc::alloc()
206 @return Number of bytes.
207 */
212 }
213
214 /** Returns the the type of the page.
215
216 @param[in] data Pointer to storage allocated through Page_alloc::alloc()
217 @return Page type.
218 */
219 static inline Page_type page_type(void *data) {
222 }
223
224 /** Retrieves the pointer and size of the allocation provided by the OS. It is
225 a low level information, and is needed only to call low level
226 memory-related OS functions.
227
228 @param[in] data Pointer to storage allocated through Page_alloc::alloc()
229 @return Low level allocation info.
230 */
231 static inline allocation_low_level_info low_level_info(void *data) {
233 return {deduce(data), page_allocation_metadata::datalen(data)};
234 }
235
236 private:
237 /** Helper function which deduces the original pointer returned by
238 Page_alloc from a pointer which is passed to us by the call-site.
239 */
240 static inline void *deduce(void *data) noexcept {
242 const auto res = reinterpret_cast<void *>(static_cast<uint8_t *>(data) -
244 ut_ad(reinterpret_cast<std::uintptr_t>(res) % CPU_PAGE_SIZE == 0);
245 return res;
246 }
247};
248
249/** Allocation routines which are purposed for allocating system page-aligned
250 memory. This is a PFS (performance-schema) variant of Page_alloc.
251 Implemented in terms of Page_alloc_metadata_pfs.
252
253 page_aligned_alloc() and page_aligned_free() are taking care of
254 OS specific details and Page_alloc_pfs is a convenience wrapper which
255 only makes the use of system page-aligned memory more ergonomic so that
256 it serializes all the relevant PFS details into the raw memory. Otherwise,
257 client code would have been responsible to store and keep those details
258 somewhere until the memory segment is freed. Additionally, information on
259 type of page used to back up requested allocation is also serialized into
260 the memory allowing to build higher-kinded abstractions more easily. See
261 ut::malloc_large_page with option to fallback to regular pages through
262 ut::malloc_page.
263
264 Cost associated with this abstraction is the size of a single CPU page. In
265 terms of virtual memory, especially in 64-bit address space, this cost is
266 negligible. In practice this means that for each N pages allocation
267 request there will be N+1 pages allocated beneath.
268
269 Memory layout representation looks like the following:
270
271 ----------------------------------------------
272 | PAGE-ALLOC-METADATA-PFS | ... DATA ... |
273 ----------------------------------------------
274 ^ ^
275 | |
276 | |
277 | ptr (system-page) to be
278 | returned to call-site
279 |
280 ---------------------------------------------------
281 | PFS-META | PAGE-TYPE | VARLEN | PFS-META-OFFSET |
282 ---------------------------------------------------
283 ^ ^
284 | |
285 | ---------------------------
286 | | OWNER | DATALEN | KEY |
287 | ---------------------------
288 |
289 ptr returned by
290 page_aligned_alloc
291
292 For details on PFS-META, PAGE-TYPE, VARLEN and PFS-META-OFFSET fields
293 see Page_alloc_metadata_pfs.
294
295 DATA is an actual page-aligned segment that will be returned to the
296 call-site and which the client code will be able to use for the application
297 data.
298 */
299struct Page_alloc_pfs : public allocator_traits<true> {
301
302 /** Allocates system page-aligned memory.
303
304 @param[in] size Size of storage (in bytes) requested to be allocated.
305 @param[in] key PSI memory key to be used for PFS memory instrumentation.
306 @return Pointer to the allocated storage. nullptr if allocation failed.
307 */
308 static inline void *alloc(
309 std::size_t size,
311 auto total_len = round_to_next_multiple(
313 auto mem = page_aligned_alloc(total_len);
314 if (unlikely(!mem)) return nullptr;
315
316#ifdef HAVE_PSI_MEMORY_INTERFACE
317 // The point of this allocator variant is to trace the memory allocations
318 // through PFS (PSI) so do it.
320 key = PSI_MEMORY_CALL(memory_alloc)(key, total_len, &owner);
321 // To be able to do the opposite action of tracing when we are releasing the
322 // memory, we need right about the same data we passed to the tracing
323 // memory_alloc function. Let's encode this it into our allocator so we
324 // don't have to carry and keep this data around.
331#endif
332
333 return static_cast<uint8_t *>(mem) + page_allocation_metadata::len;
334 }
335
336 /** Releases storage allocated through Page_alloc_pfs::alloc().
337
338 @param[in] data Pointer to storage allocated through
339 Page_alloc_pfs::alloc()
340 @return True if releasing the page-aligned memory was successful.
341 */
342 static inline bool free(PFS_metadata::data_segment_ptr data) noexcept {
343 if (unlikely(!data)) return false;
345
346 PFS_metadata::pfs_datalen_t total_len = {};
347#ifdef HAVE_PSI_MEMORY_INTERFACE
348 // Deduce the PFS data we encoded in Page_alloc_pfs::alloc()
350 auto owner =
353 // With the deduced PFS data, now trace the memory release action.
355 (key, total_len, owner);
356#endif
357
358 return page_aligned_free(deduce(data), total_len);
359 }
360
361 /** Returns the number of bytes that have been allocated.
362
363 @param[in] data Pointer to storage allocated through
364 Page_alloc_pfs::alloc()
365 @return Number of bytes.
366 */
367 static inline size_t datalen(PFS_metadata::data_segment_ptr data) {
371 }
372
373 /** Returns the Page_type.
374
375 @param[in] data Pointer to storage allocated through
376 Page_alloc_pfs::alloc()
377 @return Page type.
378 */
382 }
383
384 /** Retrieves the pointer and size of the allocation provided by the OS. It is
385 a low level information, and is needed only to call low level
386 memory-related OS functions.
387
388 @param[in] data Pointer to storage allocated through
389 Page_alloc_pfs::alloc()
390 @return Low level allocation info.
391 */
392 static inline allocation_low_level_info low_level_info(void *data) {
394 return {deduce(data),
396 }
397
398 private:
399 /** Helper function which deduces the original pointer returned by
400 Page_alloc_pfs from a pointer which is passed to us by the call-site.
401 */
402 static inline void *deduce(PFS_metadata::data_segment_ptr data) noexcept {
404 const auto res =
406 ut_ad(reinterpret_cast<std::uintptr_t>(res) % CPU_PAGE_SIZE == 0);
407 return res;
408 }
409};
410
411/** Simple utility meta-function which selects appropriate allocator variant
412 (implementation) depending on the input parameter(s).
413 */
414template <bool Pfs_memory_instrumentation_on>
416 using type = Page_alloc; // When PFS is OFF, pick ordinary, non-PFS, variant
417};
418
419template <>
421 using type = Page_alloc_pfs; // Otherwise, pick PFS variant
422};
423
424/** Just a small helper type which saves us some keystrokes. */
425template <bool Pfs_memory_instrumentation_on>
428
429/** Small wrapper which utilizes SFINAE to dispatch the call to appropriate
430 aligned allocator implementation.
431 */
432template <typename Impl>
434 template <typename T = Impl>
436 alloc(size_t size, PSI_memory_key key) {
437 return Impl::alloc(size, key);
438 }
439 template <typename T = Impl>
441 alloc(size_t size, PSI_memory_key /*key*/) {
442 return Impl::alloc(size);
443 }
444 static inline bool free(void *ptr) { return Impl::free(ptr); }
445 static inline size_t datalen(void *ptr) { return Impl::datalen(ptr); }
446 static inline Page_type page_type(void *ptr) { return Impl::page_type(ptr); }
447 static inline allocation_low_level_info low_level_info(void *ptr) {
448 return Impl::low_level_info(ptr);
449 }
450};
451
452} // namespace detail
453} // namespace ut
454
455#endif
Simple allocator traits.
#define PSI_MEMORY_CALL(M)
Definition: psi_memory.h:35
unsigned int PSI_memory_key
Instrumented memory key.
Definition: psi_memory_bits.h:48
Small helper functions.
#define CPU_PAGE_SIZE
Definition: config.h:313
#define free(A)
Definition: lexyy.cc:915
Header for compiler-dependent features.
constexpr bool unlikely(bool expr)
Definition: my_compiler.h:57
Definition: ut0tuple.h:56
static auto log_warn()
Definition: ut0log.h:451
static auto log_error()
Definition: ut0log.h:452
static PSI_memory_key memory_alloc(PSI_memory_key, size_t, struct PSI_thread **owner)
Definition: psi_memory_v2_empty.cc:34
static void memory_free(PSI_memory_key, size_t, struct PSI_thread *)
Definition: psi_memory_v2_empty.cc:52
Page_type
Types of pages currently supported by ut:: library functions.
Definition: page_metadata.h:42
constexpr size_t round_to_next_multiple(size_t n, size_t m)
Calculates the next multiple of m that is bigger or equal to n.
Definition: helper.h:67
typename select_page_alloc_impl< Pfs_memory_instrumentation_on >::type select_page_alloc_impl_t
Just a small helper type which saves us some keystrokes.
Definition: page_alloc.h:427
void * page_aligned_alloc(size_t n_bytes)
Allocates system page-aligned memory.
Definition: page_alloc.h:66
bool page_aligned_free(void *ptr, size_t n_bytes)
Releases system page-aligned storage.
Definition: page_alloc.h:100
This file contains a set of libraries providing overloads for regular dynamic allocation routines whi...
Definition: aligned_alloc.h:47
Implementation bits and pieces for metadata for normal and large (huge) page allocations.
required string key
Definition: replication_asynchronous_connection_failover.proto:59
required string type
Definition: replication_group_member_actions.proto:33
static MEM_ROOT mem
Definition: sql_servers.cc:99
Implementation bits and pieces for PFS metadata handling.
Can be used to extract pointer and size of the allocation provided by the OS.
Definition: ut0new.h:139
void * data_segment_ptr
Definition: pfs.h:93
std::size_t pfs_datalen_t
Definition: pfs.h:90
static void pfs_datalen(data_segment_ptr data, size_t datalen) noexcept
Helper function which stores the PFS datalen info into the DATALEN field.
Definition: pfs.h:108
static void pfs_key(data_segment_ptr data, pfs_memory_key_t key) noexcept
Helper function which stores the PFS key info into the KEY field.
Definition: pfs.h:114
PSI_thread * pfs_owning_thread_t
Convenience types that we will be using to serialize necessary details into the Aligned_alloc metadat...
Definition: pfs.h:89
static void pfs_owning_thread(data_segment_ptr data, pfs_owning_thread_t thread) noexcept
Helper function which stores the PFS thread info into the OWNER field.
Definition: pfs.h:102
static void pfs_metaoffset(data_segment_ptr data, std::size_t alignment) noexcept
Helper function which stores the offset to PFS metadata segment into the PFS-META-OFFSET field.
Definition: pfs.h:121
PSI_memory_key pfs_memory_key_t
Definition: pfs.h:91
static void * deduce_pfs_meta(data_segment_ptr data) noexcept
Helper function which deduces the pointer to the beginning of PFS metadata segment given the pointer ...
Definition: pfs.h:155
Small wrapper which utilizes SFINAE to dispatch the call to appropriate aligned allocator implementat...
Definition: page_alloc.h:433
static std::enable_if<!T::is_pfs_instrumented_v, void * >::type alloc(size_t size, PSI_memory_key)
Definition: page_alloc.h:441
static std::enable_if< T::is_pfs_instrumented_v, void * >::type alloc(size_t size, PSI_memory_key key)
Definition: page_alloc.h:436
static Page_type page_type(void *ptr)
Definition: page_alloc.h:446
static size_t datalen(void *ptr)
Definition: page_alloc.h:445
static bool free(void *ptr)
Definition: page_alloc.h:444
static allocation_low_level_info low_level_info(void *ptr)
Definition: page_alloc.h:447
Helper struct implementing the type which represents the metadata for all types of PFS-aware page-ali...
Definition: page_metadata.h:197
static constexpr auto len
This is how much this metadata segment will be big.
Definition: page_metadata.h:202
static Page_type page_type(void *data)
Sanity check so that we can be sure that the size of our metadata segment is such so that the pointer...
Definition: page_metadata.h:220
Helper struct implementing the type which represents the metadata for all types of page-aligned alloc...
Definition: page_metadata.h:74
static constexpr auto len
This is how much tise metadata segment will be big.
Definition: page_metadata.h:76
static Page_type page_type(void *data)
Accessor to the page_type_t field.
Definition: page_metadata.h:109
static datalen_t datalen(void *data)
Sanity check so that we can be sure that the size of our metadata segment is such so that the next se...
Definition: page_metadata.h:100
size_t datalen_t
These are the types representing our memory fields.
Definition: page_metadata.h:79
Allocation routines which are purposed for allocating system page-aligned memory.
Definition: page_alloc.h:299
static bool free(PFS_metadata::data_segment_ptr data) noexcept
Releases storage allocated through Page_alloc_pfs::alloc().
Definition: page_alloc.h:342
static allocation_low_level_info low_level_info(void *data)
Retrieves the pointer and size of the allocation provided by the OS.
Definition: page_alloc.h:392
static size_t datalen(PFS_metadata::data_segment_ptr data)
Returns the number of bytes that have been allocated.
Definition: page_alloc.h:367
static void * alloc(std::size_t size, page_allocation_metadata::pfs_metadata::pfs_memory_key_t key)
Allocates system page-aligned memory.
Definition: page_alloc.h:308
static Page_type page_type(PFS_metadata::data_segment_ptr data)
Returns the Page_type.
Definition: page_alloc.h:379
static void * deduce(PFS_metadata::data_segment_ptr data) noexcept
Helper function which deduces the original pointer returned by Page_alloc_pfs from a pointer which is...
Definition: page_alloc.h:402
Allocation routines which are purposed for allocating system page-aligned memory.
Definition: page_alloc.h:173
static void * alloc(std::size_t size)
Allocates memory through large-page support.
Definition: page_alloc.h:181
static allocation_low_level_info low_level_info(void *data)
Retrieves the pointer and size of the allocation provided by the OS.
Definition: page_alloc.h:231
static void * deduce(void *data) noexcept
Helper function which deduces the original pointer returned by Page_alloc from a pointer which is pas...
Definition: page_alloc.h:240
static bool free(void *data) noexcept
Releases storage allocated through Page_alloc::alloc().
Definition: page_alloc.h:196
static Page_type page_type(void *data)
Returns the the type of the page.
Definition: page_alloc.h:219
static page_allocation_metadata::datalen_t datalen(void *data)
Returns the number of bytes that have been allocated.
Definition: page_alloc.h:208
Simple allocator traits.
Definition: allocator_traits.h:81
Simple utility meta-function which selects appropriate allocator variant (implementation) depending o...
Definition: page_alloc.h:415
#define ut_ad(EXPR)
Debug assertion.
Definition: ut0dbg.h:68
Base of InnoDB utilities.