MySQL 8.3.0
Source Code Documentation
composite_iterators.h
Go to the documentation of this file.
1#ifndef SQL_ITERATORS_COMPOSITE_ITERATORS_H_
2#define SQL_ITERATORS_COMPOSITE_ITERATORS_H_
3
4/* Copyright (c) 2018, 2023, Oracle and/or its affiliates.
5
6 This program is free software; you can redistribute it and/or modify
7 it under the terms of the GNU General Public License, version 2.0,
8 as published by the Free Software Foundation.
9
10 This program is also distributed with certain software (including
11 but not limited to OpenSSL) that is licensed under separate terms,
12 as designated in a particular file or component or in included license
13 documentation. The authors of MySQL hereby grant you an additional
14 permission to link the program and your derivative works with the
15 separately licensed software that they have included with MySQL.
16
17 This program is distributed in the hope that it will be useful,
18 but WITHOUT ANY WARRANTY; without even the implied warranty of
19 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 GNU General Public License, version 2.0, for more details.
21
22 You should have received a copy of the GNU General Public License
23 along with this program; if not, write to the Free Software
24 Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
25
26/**
27 @file composite_iterators.h
28
29 A composite row iterator is one that takes in one or more existing iterators
30 and processes their rows in some interesting way. They are usually not bound
31 to a single table or similar, but are the inner (non-leaf) nodes of the
32 iterator execution tree. They consistently own their source iterator, although
33 not its memory (since we never allocate row iterators on the heap--usually on
34 a MEM_ROOT>). This means that in the end, you'll end up with a single root
35 iterator which then owns everything else recursively.
36
37 SortingIterator and the two window iterators are also composite iterators,
38 but are defined in their own files.
39 */
40
41#include <assert.h>
42#include <stdint.h>
43#include <sys/types.h>
44#include <limits>
45#include <memory>
46#include <new>
47#include <string>
48#include <utility>
49#include <vector>
50
51#include "my_alloc.h"
52#include "my_base.h"
53#include "my_inttypes.h"
54#include "my_sys.h"
55#include "my_table_map.h"
56#include "mysqld_error.h"
62#include "sql/join_type.h"
63#include "sql/mem_root_array.h"
64#include "sql/pack_rows.h"
65#include "sql/sql_array.h"
66#include "sql_string.h"
67
68#include "extra/robin-hood-hashing/robin_hood.h"
69class Cached_item;
71class Item;
72class JOIN;
73class KEY;
75class SJ_TMP_TABLE;
76class Table_ref;
77class THD;
78class Table_function;
80struct TABLE;
81
82/**
83 An iterator that takes in a stream of rows and passes through only those that
84 meet some criteria (i.e., a condition evaluates to true). This is typically
85 used for WHERE/HAVING.
86 */
87class FilterIterator final : public RowIterator {
88 public:
90 Item *condition)
91 : RowIterator(thd), m_source(std::move(source)), m_condition(condition) {}
92
93 bool Init() override { return m_source->Init(); }
94
95 int Read() override;
96
97 void SetNullRowFlag(bool is_null_row) override {
98 m_source->SetNullRowFlag(is_null_row);
99 }
100
101 void StartPSIBatchMode() override { m_source->StartPSIBatchMode(); }
102 void EndPSIBatchModeIfStarted() override {
103 m_source->EndPSIBatchModeIfStarted();
104 }
105 void UnlockRow() override { m_source->UnlockRow(); }
106
107 private:
110};
111
112/**
113 Handles LIMIT and/or OFFSET; Init() eats the first "offset" rows, and Read()
114 stops as soon as it's seen "limit" rows (including any skipped by offset).
115 */
116class LimitOffsetIterator final : public RowIterator {
117 public:
118 /**
119 @param thd Thread context
120 @param source Row source
121 @param limit Maximum number of rows to read, including the ones skipped by
122 offset. Can be HA_POS_ERROR for no limit.
123 @param offset Number of initial rows to skip. Can be 0 for no offset.
124 @param count_all_rows If true, the query will run to completion to get
125 more accurate numbers for skipped_rows, so you will not get any
126 performance benefits of early end.
127 @param reject_multiple_rows True if a derived table transformed from a
128 scalar subquery needs a run-time cardinality check
129 @param skipped_rows If not nullptr, is incremented for each row skipped by
130 offset or limit.
131 */
133 ha_rows limit, ha_rows offset, bool count_all_rows,
134 bool reject_multiple_rows, ha_rows *skipped_rows)
135 : RowIterator(thd),
136 m_source(std::move(source)),
137 m_limit(limit),
138 m_offset(offset),
139 m_count_all_rows(count_all_rows),
140 m_reject_multiple_rows(reject_multiple_rows),
141 m_skipped_rows(skipped_rows) {
142 if (count_all_rows) {
143 assert(m_skipped_rows != nullptr);
144 }
145 }
146
147 bool Init() override;
148
149 int Read() override;
150
151 void SetNullRowFlag(bool is_null_row) override {
152 m_source->SetNullRowFlag(is_null_row);
153 }
154
155 void StartPSIBatchMode() override { m_source->StartPSIBatchMode(); }
156 void EndPSIBatchModeIfStarted() override {
157 m_source->EndPSIBatchModeIfStarted();
158 }
159 void UnlockRow() override { m_source->UnlockRow(); }
160
161 private:
163
164 // Note: The number of seen rows starts off at m_limit if we have OFFSET,
165 // which means we don't need separate LIMIT and OFFSET tests on the
166 // fast path of Read().
168
169 /**
170 Whether we have OFFSET rows that we still need to skip.
171 */
173
178};
179
180/**
181 Handles aggregation (typically used for GROUP BY) for the case where the rows
182 are already properly grouped coming in, ie., all rows that are supposed to be
183 part of the same group are adjacent in the input stream. (This could be
184 because they were sorted earlier, because we are scanning an index that
185 already gives us the rows in a group-compatible order, or because there is no
186 grouping.)
187
188 AggregateIterator needs to be able to save and restore rows; it doesn't know
189 when a group ends until it's seen the first row that is part of the _next_
190 group. When that happens, it needs to tuck away that next row, and then
191 restore the previous row so that the output row gets the correct grouped
192 values. A simple example, doing SELECT a, SUM(b) FROM t1 GROUP BY a:
193
194 t1.a t1.b SUM(b)
195 1 1 <-- first row, save it 1
196 1 2 3
197 1 3 6
198 2 1 <-- group changed, save row
199 [1 1] <-- restore first row, output 6
200 reset aggregate --> 0
201 [2 1] <-- restore new row, process it 1
202 2 10 11
203 <-- EOF, output 11
204
205 To save and restore rows like this, it uses the infrastructure from
206 pack_rows.h to pack and unpack all relevant rows into record[0] of every input
207 table. (Currently, there can only be one input table, but this may very well
208 change in the future.) It would be nice to have a more abstract concept of
209 sending a row around and taking copies of it if needed, as opposed to it
210 implicitly staying in the table's buffer. (This would also solve some
211 issues in EQRefIterator and when synthesizing NULL rows for outer joins.)
212 However, that's a large refactoring.
213 */
214class AggregateIterator final : public RowIterator {
215 public:
217 JOIN *join, pack_rows::TableCollection tables, bool rollup);
218
219 bool Init() override;
220 int Read() override;
221 void SetNullRowFlag(bool is_null_row) override {
222 m_source->SetNullRowFlag(is_null_row);
223 }
224
225 void StartPSIBatchMode() override { m_source->StartPSIBatchMode(); }
226 void EndPSIBatchModeIfStarted() override {
227 m_source->EndPSIBatchModeIfStarted();
228 }
229 void UnlockRow() override {
230 // Most likely, HAVING failed. Ideally, we'd like to backtrack and
231 // unlock all rows that went into this aggregate, but we can't do that,
232 // and we also can't unlock the _current_ row, since that belongs to a
233 // different group. Thus, do nothing.
234 }
235
236 private:
237 enum {
243
245
246 /**
247 The join we are part of. It would be nicer not to rely on this,
248 but we need a large number of members from there, like which
249 aggregate functions we have, the THD, temporary table parameters
250 and so on.
251 */
252 JOIN *m_join = nullptr;
253
254 /// Whether we have seen the last input row.
256
257 /**
258 Used to save NULL information in the specific case where we have
259 zero input rows.
260 */
262
263 /// Whether this is a rollup query.
264 const bool m_rollup;
265
266 /**
267 For rollup: The index of the first group item that did _not_ change when we
268 last switched groups. E.g., if we have group fields A,B,C,D and then switch
269 to group A,B,E,D, this value will become 1 (which means that we need
270 to output rollup rows for 2 -- A,B,E,NULL -- and then 1 -- A,B,NULL,NULL).
271 m_current_rollup_position will count down from the end until it becomes
272 less than this value.
273
274 If we do not have rollup, this value is perennially zero.
275 */
277
278 /**
279 If we are in state OUTPUTTING_ROLLUP_ROWS, where we are in the iteration.
280 This value will start at the index of the last group expression and then
281 count backwards down to and including m_last_unchanged_group_item_idx.
282 It is used to communicate to the rollup group items whether to turn
283 themselves into NULLs, and the sum items which of their sums to output.
284 */
286
287 /**
288 The list of tables we are reading from; they are the ones for which we need
289 to save and restore rows.
290 */
292
293 /// Packed version of the first row in the group we are currently processing.
295
296 /**
297 If applicable, packed version of the first row in the _next_ group. This is
298 used only in the LAST_ROW_STARTED_NEW_GROUP state; we just saw a row that
299 didn't belong to the current group, so we saved it here and went to output
300 a group. On the next Read() call, we need to process this deferred row
301 first of all.
302
303 Even when not in use, this string contains a buffer that is large enough to
304 pack a full row into, sans blobs. (If blobs are present,
305 StoreFromTableBuffers() will automatically allocate more space if needed.)
306 */
308
309 /**
310 The slice we're setting when returning rows. See the comment in the
311 constructor.
312 */
314
315 void SetRollupLevel(int level);
316};
317
318/**
319 A simple nested loop join, taking in two iterators (left/outer and
320 right/inner) and joining them together. This may, of course, scan the inner
321 iterator many times. It is currently the only form of join we have.
322
323 The iterator works as a state machine, where the state records whether we need
324 to read a new outer row or not, and whether we've seen any rows from the inner
325 iterator at all (if not, an outer join need to synthesize a new NULL row).
326
327 The iterator takes care of activating performance schema batch mode on the
328 right iterator if needed; this is typically only used if it is the innermost
329 table in the entire join (where the gains from turning on batch mode is the
330 largest, and the accuracy loss from turning it off are the least critical).
331 */
332class NestedLoopIterator final : public RowIterator {
333 public:
337 JoinType join_type, bool pfs_batch_mode)
338 : RowIterator(thd),
339 m_source_outer(std::move(source_outer)),
340 m_source_inner(std::move(source_inner)),
342 m_pfs_batch_mode(pfs_batch_mode) {
343 assert(m_source_outer != nullptr);
344 assert(m_source_inner != nullptr);
345
346 // Batch mode makes no sense for anti- or semijoins, since they should only
347 // be reading one row.
349 assert(!pfs_batch_mode);
350 }
351 }
352
353 bool Init() override;
354
355 int Read() override;
356
357 void SetNullRowFlag(bool is_null_row) override {
358 // TODO: write something here about why we can't do this lazily.
359 m_source_outer->SetNullRowFlag(is_null_row);
360 m_source_inner->SetNullRowFlag(is_null_row);
361 }
362
363 void EndPSIBatchModeIfStarted() override {
364 m_source_outer->EndPSIBatchModeIfStarted();
365 m_source_inner->EndPSIBatchModeIfStarted();
366 }
367
368 void UnlockRow() override {
369 // Since we don't know which condition that caused the row to be rejected,
370 // we can't know whether we could also unlock the outer row
371 // (it may still be used as parts of other joined rows).
373 m_source_inner->UnlockRow();
374 }
375 }
376
377 private:
378 enum {
384
388
389 /** Whether to use batch mode when scanning the inner iterator. */
391};
392
393/**
394 An iterator that helps invalidating caches. Every time a row passes through it
395 or it changes state in any other way, it increments its “generation” counter.
396 This allows MaterializeIterator to see whether any of its dependencies has
397 changed, and then force a rematerialization -- this is typically used for
398 LATERAL tables, where we're joining in a derived table that depends on
399 something earlier in the join.
400 */
402 public:
405 const std::string &name)
406 : RowIterator(thd),
407 m_source_iterator(std::move(source_iterator)),
408 m_name(name) {}
409
410 bool Init() override {
411 ++m_generation;
412 return m_source_iterator->Init();
413 }
414
415 int Read() override {
416 ++m_generation;
417 return m_source_iterator->Read();
418 }
419
420 void SetNullRowFlag(bool is_null_row) override {
421 ++m_generation;
422 m_source_iterator->SetNullRowFlag(is_null_row);
423 }
424
425 void UnlockRow() override { m_source_iterator->UnlockRow(); }
426
427 int64_t generation() const { return m_generation; }
428 std::string name() const { return m_name; }
429
430 private:
432 int64_t m_generation = 0;
433 std::string m_name;
434};
435
437/**
438 An operand (query block) to be materialized by MaterializeIterator.
439 (@see MaterializeIterator for details.)
440*/
441struct Operand {
442 /// The iterator to read the actual rows from.
444
445 /// Used only for optimizer trace.
447
448 /// The JOIN that this query block represents. Used for performance
449 /// schema batch mode: When materializing a query block that consists of
450 /// a single table, MaterializeIterator needs to set up schema batch mode,
451 /// since there is no nested loop iterator to do it. (This is similar to
452 /// what ExecuteIteratorQuery() needs to do at the top level.)
454
455 /// If true, de-duplication checking via hash key is disabled
456 /// when materializing this query block (ie., we simply avoid calling
457 /// check_unique_fields() for each row). Used when materializing
458 /// UNION DISTINCT and UNION ALL parts into the same table.
459 /// We'd like to just use a unique constraint via unique index instead,
460 /// but there might be other indexes on the destination table
461 /// that we'd like to keep, and the implementation doesn't allow
462 /// disabling only one index.
463 ///
464 /// If you use this on a query block, doing_hash_deduplication()
465 /// must be true.
467
468 /// If set to false, the Field objects in the output row are
469 /// presumed already to be filled out. This is the case iff
470 /// there's a windowing iterator earlier in the chain.
472
473 /// The number of operands (i.e. blocks) involved in the set operation:
474 /// used for INTERSECT to determine if a value is present in all operands
476 /// The current operand (i.e. block) number, starting at zero. We use this
477 /// for INTERSECT and EXCEPT materialization operand.
479 /// Used for EXCEPT computation: the index of the first operand involved in
480 /// a N-ary except operation which has DISTINCT. This is significant for
481 /// calculating whether to set the counter to zero or just decrement it
482 /// when we see a right side operand.
484
485 /// If copy_items is true, used for copying the Field objects
486 /// into the temporary table row. Otherwise unused.
488
489 // Whether this query block is a recursive reference back to the
490 // output of the materialization.
492
493 // If is_recursive_reference is true, contains the FollowTailIterator
494 // in the query block (there can be at most one recursive reference
495 // in a join list, as per the SQL standard, so there should be exactly one).
496 // Used for informing the iterators about various shared state in the
497 // materialization (including coordinating rematerializations).
499
500 /// The estimated number of rows produced by this block
502};
503
505
506/**
507 Contains spill state for set operations' use of in-memory hash map.
508
509 If we encounter a situation in which the hash map for set operations
510 overflows allowed memory, we initiate a spill to disk procedure. This class
511 encapsulates state using during this procedure. Spill to disk starts
512 with a call to \c handle_hash_map_full.
513
514 We built a mechanism with an in-memory hash map which can spill
515 gracefully to disk if the volume of rows gets large and still perform
516 well. In the presence of wrong table cardinality information, we may not be
517 able to complete the spill to disk procedure (if we still run out of memory
518 when hashing chunks, see below). If so, we fall back on de-duplicating
519 using the non-unique key of the output (materialized) result table.
520
521 The spill code is partially based on code developed for hash join: e.g. we
522 reuse packing/unpacking functions like
523 \verbatim
524 StoreFromTableBuffersRaw (pack_rows.h)
525 LoadImmutableStringIntoTableBuffers (hash_join_buffer.h)
526 \endverbatim
527 and furthermore, the Robin Hood hashing library (robin_hood.h), and the chunk
528 file abstraction.
529 \verbatim
530 Definitions:
531 A' - set of rows from operand 1 of set operation that fits in
532 the in-memory hash map, deduplicated, with counters
533 A - set of rows from operand 1 before deduplication
534 B - non-deduplicated set of rows from operand 1 that didn't
535 fit
536 C = A + B
537 - total set of rows in operand one; not known a priori, but we use
538 the statistics for an estimate.
539
540 M - (aka. m_num_chunks) total number of chunk files the tertiary
541 hash distributes the rows to. Multiple of 2, as used for hash join.
542
543 N - (aka. HashJoinIterator::kMaxChunks) the max number of HF and IF
544 files that may be open at one time. May be smaller than M.
545
546 S = ceiling(M/N) (aka. m_no_of_chunk_file_sets)
547 - number of sets of open files we need
548
549 s - the set of chunk files opened (aka. m_chunk_files), sets are
550 enumerated from 0..S-1, cf. m_current_chunk_file_set.
551
552 n - number of operands in set operation
553
554 REMAININGINPUT (aka. m_remaining_input) - tmp file needed if S > 1.
555 MATERIALIZEDTABLE (aka. m_materialized_table) - output for
556 EXCEPT/INTERSECT algorithm
557
558 primary hash
559 - MySQL record hash, aka. calc_row_hash(m_materialized_table)
560 secondary hash
561 - the hash function used by Robin Hood for the in-memory hash map
562 based on primary hash
563 tertiary hash
564 - hash function for distributing rows to chunk files, cf.
565 MY_XXH64 based on primary hash
566
567 ============
568 !In-memory ! Two kinds of tmp chunk files, HF and IF
569 !hash map ! HF: already Hashed and de-duplicated rows File
570 ! A' rows ! IF: Input File (not yet de-duplicated rows)
571 !==========!
572 | !---------! !----------------!
573 | ! B ! ! REMAININGINPUT !
574 | !---------! !----------------!
575 | |
576 ↓ tertiary hash → 0:M-1 ↓
577 +--------+------------\ +--------+------------\
578 ↓ ↓ ↓ ↓ ↓ ↓
579 !----! !----! !------! !----! !----! !------!
580 !HF_0! !HF_1! .. !HF_M-1! !IF_0! !IF_1! .. !IF_M-1!
581 !----! !----! !------! !----! !----! !------!
582 ↑ ↑
583 N N
584
585 !-------------------! !----------! !----------!
586 ! MATERIALIZEDTABLE ! ! operand-2! .. ! operand-n!
587 !-------------------! !----------! !----------!
588
589 If M > N, we cannot have open all chunk files at the same time, so in each
590 chunk file we have this structure:
591
592 +-------+
593 | | rows from set 0
594 +-------+
595 :
596 +-------+
597 | | rows from set S-1
598 +-------+
599
600 If we need more M than N, M will be a multiple of N as well as a multiple of
601 2, since N is also chosen a multiple of two (currently 128). So, the physical
602 tmp file contains several logical chunk files. For the HF chunks, we in
603 addition have several generations of these: each round of processing appends
604 a new generation (more updated) version of the chunks. For a 2 operand set
605 operation, we have three generations:
606
607 1. the initial row sets from the in-memory hash map (A' spread over M chunks)
608 2. updated sets with the rest of the left operand (C deduplicated and spread
609 over M chunks)
610 3. updated sets after we have processed the right operand
611
612 We keep track of the read and write positions on the tmp files, cf. methods
613 HashJoinChunk::SetAppend and HashJoinChunk::ContinueRead. This enables
614 reading back rows from the generation last written, and the writing of a new
615 generation at the tail of the chunk file. More set operands than two adds
616 further generations, one for each extra operand.
617
618 * Algorithm
619
620
621 1. The in-memory hash map can hit its memory limit when we read the
622 left set operand (block) after having read A rows, resulting in A' rows in
623 in-memory hash map. If we do not hit the limit, we are done, no spill to
624 disk is required.
625
626 Note: Spill can never happen when we read operand 2..n since operand 1 of
627 INTERSECT and EXCEPT determines the maximum rows in the result set and
628 hence the maximal size of the in-memory hash map.
629
630 So, we will have established the spill-over storage *before* reading of
631 operands 2..n starts.
632
633 2. Before looking at operand 2..n, we need to finish processing the remaining
634 rows in the left operand, cf. the details below:
635
636 3. When we hit limit, we:
637
638 Determine number N of chunk files based on the estimated number of rows in
639 operand 1 (the left operand). As mentioned, if number of chunks needed (M)
640 > maxOpenFiles, we still allow this but will keep open only a subset s at
641 any one time, presuming worst case of no deduplication, i.e. A'==A. In
642 this case, M == N * S, but M can be as low as 2 (M << N). This is
643 performed in the method `compute_chunk_file_sets' and
644 `initialize_first_HF_chunk_files'.
645
646 3.a)
647 For all file sets s in 1..S:
648
649 - rehash with tertiary hash and write A' to files HF-{0..N-1} all
650 rows in in-mem hash map. Save the computed primary hash value in
651 the hash column, so we do not need to compute it over again when
652 we read HF-k into hash map again. This is done in method
653 `spread_hash_map_to_HF_chunk_files'. HF chunk file sets are now
654 in generation one.
655
656 - When s contains hash for offending row, write the offending row
657 |A|+1 that did't fit the in-memory hash map to IF-k in s.
658 (aka. m_offending_row)
659
660 Note these rows (A') have been de-duplicated down to A' and
661 counters set accordingly.
662
663
664 3.b)
665 For all file sets s in 1..S:
666
667 3.b.1) read the rest of the left input (or re-read them via
668 REMAININGINPUT if s>1), hash and write to destination file IF-k
669 the rows which, based on its tertiary hash value, have index k
670 in the current set. If s is the first file set AND S>1 and row
671 didn't go to a file in s, also save input row to file
672 REMAININGINPUT since we need it for another file set (since we
673 cannot replay the source). See method
674 `save_rest_of_operand_to_IF_chunk_files' and
675 `reset_for_spill_handling'.
676
677 At this point we have the rest of the input rows B (that that have not
678 been matched against HFs) in IF-{0..N-1}. HF rows already are unique and
679 have set operation counters already set based on first part of input rows
680 that did fit in memory (so we have no need to "remember" that part of
681 input except as initialized counters): only the remaining input rows (from
682 operand 1) are of concern to us now.
683
684 From here on, the logic is driven from the read_next_row. The set counter
685 logic is still handled by process_row_hash. Most of the machinery
686 for reading, writing and switching chunk files are driven by a state
687 machine from read_next_row, (almost) invisible to
688 process_row_hash, except for a simplified handling when we
689 re-enter HF rows into the hash map ready to process operand 2..n, cf. call
690 to `load_HF_row_into_hash_map': these rows have already been
691 de-duplicated and the hash table will not grow in size compared to
692 operand one (intersect and except can't increase result set size), so we
693 can use a shorter logic path.
694
695 3.c)
696 For each s in 1..S do
697 For each pair of {HF-k, IF-k} in s do
698 3.c.1) Read HF-k into hash map: optimization: use saved hash value
699 Cf. ReadingState::SS_READING_LEFT_HF
700
701 3.c.2) Read rows from IF-k, continuing hash processing of
702 operand one. Cf. ReadingState::SS_READING_LEFT_IF.
703
704 If hash map overflows here, we recover by changing to
705 de-duplicating via the tmp table (we re-initialize it with a
706 non-unique index on the hash field in the row in
707 handle_hash_map_full). This overflow means we cannot fit
708 even 1/M-th of set of unique rows in input set of operand 1
709 in memory). If row estimates are reasonably good, it should
710 not happen. For details on secondary overflow recovery, see
711 handle_hash_map_full and comments in materialize_hash_map,
712 and logic in read_next_row_secondary_overflow.
713
714 3.c.3) We are done with pair {HF-k, IF-k}, append hash map to HF-k
715 and empty in-memory hash map, cf. `append_hash_map_to_HF'.
716
717 We are done with operand 1, and we have min(M,N) HF files with unique rows
718 (incl counters) on disk in one or more sets, in generation two.
719
720 4.a) For each operand 2..n do
721 4.a.0) Empty all IFs and REMAININGINPUT.
722 For each s in S do
723
724 4.a.1) Read input operand (from block or REMAININGINPUT if s>1),
725 hash to IF-k, and write. If s==1 AND S>1 also save input row
726 to file REMAININGINPUT since we need them for the next file
727 set s, cf. save_operand_to_IF_chunk_files.
728 4.a.2) Similar to same as 3.c, except with right side counter logic
729 cf. states ReadingState::SS_READING_RIGHT_{HF,IF}.
730
731 5) We now have min(N,M) HF files with unique rows sets (incl set logic
732 counters) on disk (generation three), all operands have been
733 processed. For each HF-k read it and write to MATERIALIZEDTABLE.
734 \endverbatim
735*/
737 public:
739 : m_thd(thd),
741 m_row_counts(mem_root, HashJoinIterator::kMaxChunks) {}
742
743 /**
744 Inquire spill handling state
745
746 @returns true if we are in spill to disk processing mode
747 */
749
750#ifndef NDEBUG
752
753 private:
754 size_t m_simulated_set_idx{std::numeric_limits<size_t>::max()};
755 size_t m_simulated_chunk_idx{std::numeric_limits<size_t>::max()};
756 size_t m_simulated_row_no{std::numeric_limits<size_t>::max()};
757
758 public:
759#endif
760
762
763 using hash_map_type = robin_hood::unordered_flat_map<
766
767 static void reset_hash_map(hash_map_type *hash_map) {
768 hash_map->~hash_map_type();
769 auto *map = new (hash_map)
770 hash_map_type(/*bucket_count=*/10, hash_join_buffer::KeyHasher());
771 if (map == nullptr) {
772 my_error(ER_OUTOFMEMORY, MYF(ME_FATALERROR), sizeof(hash_map_type));
773 }
774 }
775
776 /// Getter, cf. comment for \c m_secondary_overflow
780 m_secondary_overflow = false;
781 // free up resources from chunk files and hashmap
784 m_chunk_files.clear();
785 m_row_counts.clear();
786 }
787
788 enum class ReadingState : uint8_t {
789 SS_NONE,
792 SS_COPY_OPERAND_N_TO_IF, // done de-duplicating one operand, ready for next
795 SS_FLUSH_REST_OF_LEFT_IFS // only used after secondary overflow
796 };
797
799
800 /**
801 Initialize the spill to disk processing state with some variables.
802
803 @param left_operand the left-most operand in a N-ary set operation
804 @param hash_map the in-memory hash map that overflowed, causing the
805 spill to disk
806 @param rows_in_hash_table
807 the number of rows in the hash map
808 @param read_rows_before_dedup
809 the number of rows read from the left operand
810 before de-duplicating into the hash map
811 @param hash_map_mem_root
812 the mem_root used for allocating space for the hash
813 map's keys and values
814 @param t the materialized table that receive the result set of
815 the set operation
816 */
817 bool init(const Operand &left_operand, hash_map_type *hash_map,
818 size_t rows_in_hash_table, size_t read_rows_before_dedup,
819 MEM_ROOT *hash_map_mem_root, TABLE *t);
820
821 /**
822 Given current state of spill processing, return the next row up for
823 inserting into or matching against the hash map.
824 @param current_operand the operand (query block) we are currently reading
825 from
826 @retval
827 0 OK
828 @retval
829 -1 End of records
830 @retval
831 1 Error
832 */
833 int read_next_row(const Operand *current_operand);
834
835 /**
836 Given current state of secondary overflow processing, return the next row
837 up for inserting into or matching against the index in the result table (we
838 no longer use hashing, having fallen back on de-duplicating via index in
839 resulting output table.
840
841 First, return the row which caused the overflow as row #1. Next, we read
842 the rest of the IF rows of the current chunk we were processing when the
843 secondary overflow occured. Finally, we read all remaining left side IF
844 chunks, if any, which haven't been matched with their corresponding HF
845 chunk, i.e. we do not need to read IF files that have already been matched
846 up with their corresponding HF chunk files prior to the secondary overflow,
847 if any.
848
849 Processing of right operand(s) will proceed as for non-hashed
850 de-duplication (similarly to what is done for UNION), and is not handled
851 here. Cf. secondary_overflow_handling_done which completes secondary
852 overflow handling and reverts to normal non hashed de-duplication for
853 operands 2..n.
854
855 @retval
856 0 OK
857 @retval
858 -1 End of records
859 @retval
860 1 Error
861 */
863
864 /**
865 Used to write a complete (or incomplete in the case of secondary overflow)
866 HF chunk to the materialized tmp table. Will handle spill to disk if
867 needed.
868 @param thd Session state
869 @param set The set for which to write a chunk
870 @param chunk_idx The chunk for which to write rows
871 @param operands The operands of the set operation
872 @param [out] stored_rows Incremented with the number of row written from
873 the specified chunk to the materialized tmp table
874 @returns true if error, else false
875 */
876 bool write_HF(THD *thd, size_t set, size_t chunk_idx,
877 const Operands &operands, ha_rows *stored_rows);
878 /**
879 Write the contents of the final generation of HD chunks to the materialized
880 table which will hold the result set of the set operation.
881 TODO: avoid materializing more rows than required if LIMIT is present
882 TODO: stream rows as soon as final generation of a HF chunk file is ready?
883
884 @param thd Current session state
885 @param operands The operands of the set operation
886 @param [out] stored_rows Will be incremenented with the number of produced
887 rows
888 @returns true on error, else false.
889 */
890 bool write_completed_HFs(THD *thd, const Operands &operands,
891 ha_rows *stored_rows); // 5.
892
893 /**
894 Write the contents of the HD chunks that were completed when a secondary
895 memory overflow has occurred. In the general case it is a mix of 1.
896 and 2. generation HF chunks.
897
898 @param thd Current session state
899 @param operands The operands of the set operation
900 @param [out] stored_rows Will be updated with the written rows
901 @returns true on error
902 */
903 bool write_partially_completed_HFs(THD *thd, const Operands &operands,
904 ha_rows *stored_rows);
905
906 private:
912 else
914 }
915
921 else
923 }
924
925 public:
926 // Save away the contents of the row that made the hash table run out of
927 // memory - for later processing
928 bool save_offending_row();
929 THD *thd() { return m_thd; }
930
931 private:
932 /**
933 Compute sizing of and set aside space for the on-disk chunks and their
934 associated in-memory structures, based on the row estimate taken from
935 Operand::m_estimated_output_rows. Also save away the offending row (the one
936 that we read, but we couldn't put into the hash map) so that we can write
937 it to an IF chunk later.
938 @returns true on error
939 */
940 bool compute_chunk_file_sets(const Operand *current_operand); // 3.
942
943 /**
944 The initial hash map that overflowed will be spread over the determined
945 number of chunk files, cf. initialize_next_HF_chunk_files
946 @returns true on error
947 */
950 const Operand *current_operand); // 4.a.1)
952 const Operand *current_operand) { // 3.b
953 // "rest of": what didn't fit of left operand in initial hash map
954 return save_operand_to_IF_chunk_files(current_operand);
955 }
956 bool reset_for_spill_handling(); // 3.b/3.c
957 /**
958 We are done processing a {HF, IF} chunk pair. The results are
959 in the in-memory hash map, which we now append to the current
960 HF chunk file, i.e. m_chunk_files[offset].build_chunk; clear the
961 in-memory hash map, and make the HF chunk file ready for reading
962 of what we now append.
963 @returns true on error
964 */
966
968
969 /// If not SS_NONE, we have detected an overflow in the in-memory hash map
970 /// while reading the left(-most) operand of an INTERSECT or EXCEPT operation
971 /// and are ready for reading next row from an operand (left or right).
973
974 /// If true, we have seen memory overflow also during spill handling. This is
975 /// because a HF chunk won't fit in memory, i.e. the computation we made to
976 /// ensure it would fit, was not sufficient to make it so. This can be because
977 /// table cardinality statistics is not up to date, or data density is very
978 /// skewed. In this case we fall back on using tmp table unique key for
979 /// de-duplicating.
981
982 /// The materialized table we are eventualy writing the result of the set
983 /// operation to
985
986 /// Cached value for {m_materialized_table}.
988
989 /// The in-memory hash map that overflowed. We will use it also during
990 /// spill phase, so we need a pointer to it.
992
993 static constexpr uint32_t m_magic_prime = 4391;
994 /// Modify for each operator in a N-ary set operation to avoid initial
995 /// chunks filling up right away due to row order in previous operation
996 size_t m_hash_seed{0};
997
998 /// At the time of overflow: how many rows from left operand are in hash map
999 /// after deduplication
1001
1002 /// At the time of overflow: how many rows have we read from left operand
1004
1005 /// The mem_root of m_hash_map. We need it for reusing its space.
1007
1008 /// The number of chunks needed after rounding up to nearest power of two.
1009 /// It may be larger thank HashJoinIterator::kMaxChunks in which case
1010 /// m_no_of_chunk_file_sets > 1.
1011 size_t m_num_chunks{0};
1012
1013 /// The number of chunk file sets needed to process all m_num_chunks
1014 /// chunks.
1016
1017 /// The current chunk under processing. 0-based.
1019
1020 public:
1022
1023 private:
1024 /// Keeps the row that was just read from the left operand when we discovered
1025 /// that we were out of space in the in-memory hash map. Save it for
1026 /// writing it to IF-k.
1027 struct {
1030 size_t m_set{0};
1031 bool m_unsaved{true};
1033
1034 static size_t chunk_index_to_set(size_t chunk_index) {
1035 return chunk_index / HashJoinIterator::kMaxChunks;
1036 }
1037
1038 inline size_t hash_to_chunk_index(uint64_t hash) const {
1039 // put all entropy into two bytes
1040 uint16 word1 = 0xffff & (hash);
1041 uint16 word2 = 0xffff & (hash >> 16);
1042 uint16 word3 = 0xffff & (hash >> 32);
1043 uint16 word4 = 0xffff & (hash >> 48);
1044 uint16 folded_hash = word1 + word2 + word3 + word4;
1045 assert(m_num_chunks <= 65535);
1046 /// hash modulo m_num_chunks optimized calculation
1047 const size_t result = folded_hash & (m_num_chunks - 1);
1048 return result;
1049 }
1050
1051 static size_t chunk_offset(size_t chunk_index) {
1052 return chunk_index & (HashJoinIterator::kMaxChunks - 1);
1053 }
1054
1055 /// Temporary space for (de)serializing a row. Cf also
1056 /// m_offending_row.m_buffer for a similar dedicated space.
1058
1059 /// Array to hold the list of chunk files on disk in case we degrade into
1060 /// on-disk set EXCEPT/INTERSECT. Maximally kMaxChunks can be open and used at
1061 /// one time.
1063
1064 /// The index of the chunk pair being read, incremented before use
1066
1067 public:
1068 size_t current_chunk_idx() const { return m_current_chunk_idx; }
1069
1070 private:
1071 /// The current row no (1-based) in a chunk being read, incremented before
1072 /// use.
1074
1075 /// Used if m_no_of_chunk_file_sets > 1 so we can replay input rows from
1076 /// operands over sets 1..S-1, i.e. not used for rows from set 0.
1077 /// Not used if we only have one chunk file set.
1079
1080 /// For a given chunk file pair {HF, IF}, the count of rows in each chunk
1081 /// respectively.
1082 struct CountPair {
1083 size_t HF_count; // left set operation operand
1084 size_t IF_count; // right set operation operand
1085 };
1086
1087 /// For a chunk file pair, an array of counts indexed by
1088 /// m_current_chunk_file_set
1090
1091 /// A matrix of counters keeping track of how many rows have been stashed
1092 /// away in the chunk files for each set in each chunk file of the current
1093 /// generation. Used to allow us to read back the correct set of rows from
1094 /// each chunk given the current m_current_chunk_file_set.
1095 /// It is indexed thus:
1096 /// m_row_counts[ chunk index ][ set index ]
1098};
1099
1100/**
1101 Create an iterator that materializes a set of row into a temporary table
1102 and sets up a (pre-existing) iterator to access that.
1103 @see MaterializeIterator.
1104
1105 @param thd Thread handler.
1106 @param operands List of operands (query blocks) to materialize.
1107 @param path_params MaterializePath settings.
1108 @param table_iterator Iterator used for accessing the temporary table
1109 after materialization.
1110 @param join
1111 When materializing within the same JOIN (e.g., into a temporary table
1112 before sorting), as opposed to a derived table or a CTE, we may need
1113 to change the slice on the join before returning rows from the result
1114 table. If so, join and ref_slice would need to be set, and
1115 query_blocks_to_materialize should contain only one member, with the same
1116 join.
1117 @return the iterator.
1118*/
1121 const MaterializePathParameters *path_params,
1123
1124} // namespace materialize_iterator
1125
1127/**
1128 Create an iterator that aggregates the output rows from another iterator
1129 into a temporary table and then sets up a (pre-existing) iterator to
1130 access the temporary table.
1131 @see TemptableAggregateIterator.
1132
1133 @param thd Thread handler.
1134 @param subquery_iterator input to aggregation.
1135 @param temp_table_param temporary table settings.
1136 @param table_iterator Iterator used for scanning the temporary table
1137 after materialization.
1138 @param table the temporary table.
1139 @param join the JOIN in which we aggregate.
1140 @param ref_slice the slice to set when accessing temporary table;
1141 used if anything upstream wants to evaluate values based on its contents.
1142 @return the iterator.
1143*/
1145 THD *thd, unique_ptr_destroy_only<RowIterator> subquery_iterator,
1146 Temp_table_param *temp_table_param, TABLE *table,
1148 int ref_slice);
1149
1150} // namespace temptable_aggregate_iterator
1151
1152/**
1153 StreamingIterator is a minimal version of MaterializeIterator that does not
1154 actually materialize; instead, every Read() just forwards the call to the
1155 subquery iterator and does the required copying from one set of fields to
1156 another.
1157
1158 It is used for when the optimizer would normally set up a materialization,
1159 but you don't actually need one, ie. you don't want to read the rows multiple
1160 times after writing them, and you don't want to access them by index (only
1161 a single table scan). It also takes care of setting the NULL row flag
1162 on the temporary table.
1163 */
1165 public:
1166 /**
1167 @param thd Thread handle.
1168 @param subquery_iterator The iterator to read rows from.
1169 @param temp_table_param Parameters for the temp table.
1170 @param table The table we are streaming through. Will never actually
1171 be written to, but its fields will be used.
1172 @param provide_rowid If true, generate a row ID for each row we stream.
1173 This is used if the parent needs row IDs for deduplication, in particular
1174 weedout.
1175 @param join See MaterializeIterator.
1176 @param ref_slice See MaterializeIterator.
1177 */
1179 unique_ptr_destroy_only<RowIterator> subquery_iterator,
1180 Temp_table_param *temp_table_param, TABLE *table,
1181 bool provide_rowid, JOIN *join, int ref_slice);
1182
1183 bool Init() override;
1184
1185 int Read() override;
1186
1187 void StartPSIBatchMode() override {
1188 m_subquery_iterator->StartPSIBatchMode();
1189 }
1191 m_subquery_iterator->EndPSIBatchModeIfStarted();
1192 }
1193 void UnlockRow() override { m_subquery_iterator->UnlockRow(); }
1194
1195 private:
1199 JOIN *const m_join;
1202
1203 // Whether the iterator should generate and provide a row ID. Only true if the
1204 // iterator is part of weedout, where the iterator will create a fake row ID
1205 // to uniquely identify the rows it produces.
1207};
1208
1209/**
1210 An iterator that wraps a Table_function (e.g. JSON_TABLE) and allows you to
1211 iterate over the materialized temporary table. The table is materialized anew
1212 for every Init().
1213
1214 TODO: Just wrapping it is probably not the optimal thing to do;
1215 Table_function is highly oriented around materialization, but never caches.
1216 Thus, perhaps we should rewrite Table_function to return a RowIterator
1217 instead of going through a temporary table.
1218 */
1220 public:
1222 THD *thd, Table_function *table_function, TABLE *table,
1224
1225 bool Init() override;
1226 int Read() override { return m_table_iterator->Read(); }
1227 void SetNullRowFlag(bool is_null_row) override {
1228 m_table_iterator->SetNullRowFlag(is_null_row);
1229 }
1230
1231 void StartPSIBatchMode() override { m_table_iterator->StartPSIBatchMode(); }
1233 m_table_iterator->EndPSIBatchModeIfStarted();
1234 }
1235
1236 // The temporary table is private to us, so there's no need to worry about
1237 // locks to other transactions.
1238 void UnlockRow() override {}
1239
1240 private:
1242
1244};
1245
1246/**
1247 Like semijoin materialization, weedout works on the basic idea that a semijoin
1248 is just like an inner join as we long as we can get rid of the duplicates
1249 somehow. (This is advantageous, because inner joins can be reordered, whereas
1250 semijoins generally can't.) However, unlike semijoin materialization, weedout
1251 removes duplicates after the join, not before it. Consider something like
1252
1253 SELECT * FROM t1 WHERE a IN ( SELECT b FROM t2 );
1254
1255 Semijoin materialization solves this by materializing t2, with deduplication,
1256 and then joining. Weedout joins t1 to t2 and then leaves only one output row
1257 per t1 row. The disadvantage is that this potentially needs to discard more
1258 rows; the (potential) advantage is that we deduplicate on t1 instead of t2.
1259
1260 Weedout, unlike materialization, works in a streaming fashion; rows are output
1261 (or discarded) as they come in, with a temporary table used for recording the
1262 row IDs we've seen before. (We need to deduplicate on t1's row IDs, not its
1263 contents.) See SJ_TMP_TABLE for details about the table format.
1264 */
1265class WeedoutIterator final : public RowIterator {
1266 public:
1268 SJ_TMP_TABLE *sj, table_map tables_to_get_rowid_for);
1269
1270 bool Init() override;
1271 int Read() override;
1272
1273 void SetNullRowFlag(bool is_null_row) override {
1274 m_source->SetNullRowFlag(is_null_row);
1275 }
1276
1278 m_source->EndPSIBatchModeIfStarted();
1279 }
1280 void UnlockRow() override { m_source->UnlockRow(); }
1281
1282 private:
1286};
1287
1288/**
1289 An iterator that removes consecutive rows that are the same according to
1290 a set of items (typically the join key), so-called “loose scan”
1291 (not to be confused with “loose index scan”, which is made by the
1292 range optimizer). This is similar in spirit to WeedoutIterator above
1293 (removing duplicates allows us to treat the semijoin as a normal join),
1294 but is much cheaper if the data is already ordered/grouped correctly,
1295 as the removal can happen before the join, and it does not need a
1296 temporary table.
1297 */
1299 public:
1302 JOIN *join, Item **group_items,
1303 int group_items_size);
1304
1305 bool Init() override;
1306 int Read() override;
1307
1308 void SetNullRowFlag(bool is_null_row) override {
1309 m_source->SetNullRowFlag(is_null_row);
1310 }
1311
1312 void StartPSIBatchMode() override { m_source->StartPSIBatchMode(); }
1314 m_source->EndPSIBatchModeIfStarted();
1315 }
1316 void UnlockRow() override { m_source->UnlockRow(); }
1317
1318 private:
1322};
1323
1324/**
1325 Much like RemoveDuplicatesIterator, but works on the basis of a given index
1326 (or more accurately, its keypart), not an arbitrary list of grouped fields.
1327 This is only used in the non-hypergraph optimizer; the hypergraph optimizer
1328 can deal with groupings that come from e.g. sorts.
1329 */
1331 public:
1334 const TABLE *table, KEY *key, size_t key_len);
1335
1336 bool Init() override;
1337 int Read() override;
1338
1339 void SetNullRowFlag(bool is_null_row) override {
1340 m_source->SetNullRowFlag(is_null_row);
1341 }
1342
1343 void StartPSIBatchMode() override { m_source->StartPSIBatchMode(); }
1345 m_source->EndPSIBatchModeIfStarted();
1346 }
1347 void UnlockRow() override { m_source->UnlockRow(); }
1348
1349 private:
1353 uchar *m_key_buf; // Owned by the THD's MEM_ROOT.
1354 const size_t m_key_len;
1356};
1357
1358/**
1359 An iterator that is semantically equivalent to a semijoin NestedLoopIterator
1360 immediately followed by a RemoveDuplicatesOnIndexIterator. It is used to
1361 implement the “loose scan” strategy in queries with multiple tables on the
1362 inside of a semijoin, like
1363
1364 ... FROM t1 WHERE ... IN ( SELECT ... FROM t2 JOIN t3 ... )
1365
1366 In this case, the query tree without this iterator would ostensibly look like
1367
1368 -> Nested loop join
1369 -> Table scan on t1
1370 -> Remove duplicates on t2_idx
1371 -> Nested loop semijoin
1372 -> Index scan on t2 using t2_idx
1373 -> Filter (e.g. t3.a = t2.a)
1374 -> Table scan on t3
1375
1376 (t3 will be marked as “first match” on t2 when implementing loose scan,
1377 thus the semijoin.)
1378
1379 First note that we can't put the duplicate removal directly on t2 in this
1380 case, as the first t2 row doesn't necessarily match anything in t3, so it
1381 needs to be above. However, this is wasteful, because once we find a matching
1382 t2/t3 pair, we should stop scanning t3 until we have a new t2.
1383
1384 NestedLoopSemiJoinWithDuplicateRemovalIterator solves the problem by doing
1385 exactly this; it gets a row from the outer side, gets exactly one row from the
1386 inner side, and then skips over rows from the outer side (_without_ scanning
1387 the inner side) until its keypart changes.
1388 */
1390 : public RowIterator {
1391 public:
1395 KEY *key, size_t key_len);
1396
1397 bool Init() override;
1398
1399 int Read() override;
1400
1401 void SetNullRowFlag(bool is_null_row) override {
1402 m_source_outer->SetNullRowFlag(is_null_row);
1403 m_source_inner->SetNullRowFlag(is_null_row);
1404 }
1405
1407 m_source_outer->EndPSIBatchModeIfStarted();
1408 m_source_inner->EndPSIBatchModeIfStarted();
1409 }
1410
1411 void UnlockRow() override {
1412 m_source_outer->UnlockRow();
1413 m_source_inner->UnlockRow();
1414 }
1415
1416 private:
1419
1422 uchar *m_key_buf; // Owned by the THD's MEM_ROOT.
1423 const size_t m_key_len;
1425};
1426
1427/**
1428 MaterializeInformationSchemaTableIterator makes sure a given I_S temporary
1429 table is materialized (filled out) before we try to scan it.
1430 */
1432 public:
1435 Table_ref *table_list, Item *condition);
1436
1437 bool Init() override;
1438 int Read() override { return m_table_iterator->Read(); }
1439
1440 void SetNullRowFlag(bool is_null_row) override {
1441 m_table_iterator->SetNullRowFlag(is_null_row);
1442 }
1443
1444 void StartPSIBatchMode() override { m_table_iterator->StartPSIBatchMode(); }
1446 m_table_iterator->EndPSIBatchModeIfStarted();
1447 }
1448
1449 // The temporary table is private to us, so there's no need to worry about
1450 // locks to other transactions.
1451 void UnlockRow() override {}
1452
1453 private:
1454 /// The iterator that reads from the materialized table.
1458};
1459
1460/**
1461 Takes in two or more iterators and output rows from them sequentially
1462 (first all rows from the first one, the all from the second one, etc.).
1463 Used for implementing UNION ALL, typically together with StreamingIterator.
1464 */
1465class AppendIterator final : public RowIterator {
1466 public:
1468 THD *thd,
1470
1471 bool Init() override;
1472 int Read() override;
1473
1474 void StartPSIBatchMode() override;
1475 void EndPSIBatchModeIfStarted() override;
1476
1477 void SetNullRowFlag(bool is_null_row) override;
1478 void UnlockRow() override;
1479
1480 private:
1481 std::vector<unique_ptr_destroy_only<RowIterator>> m_sub_iterators;
1484};
1485
1486#endif // SQL_ITERATORS_COMPOSITE_ITERATORS_H_
Handles aggregation (typically used for GROUP BY) for the case where the rows are already properly gr...
Definition: composite_iterators.h:214
void UnlockRow() override
Definition: composite_iterators.h:229
int m_current_rollup_position
If we are in state OUTPUTTING_ROLLUP_ROWS, where we are in the iteration.
Definition: composite_iterators.h:285
void StartPSIBatchMode() override
Start performance schema batch mode, if supported (otherwise ignored).
Definition: composite_iterators.h:225
int Read() override
Read a single row.
Definition: composite_iterators.cc:249
JOIN * m_join
The join we are part of.
Definition: composite_iterators.h:252
bool m_seen_eof
Whether we have seen the last input row.
Definition: composite_iterators.h:255
pack_rows::TableCollection m_tables
The list of tables we are reading from; they are the ones for which we need to save and restore rows.
Definition: composite_iterators.h:291
String m_first_row_this_group
Packed version of the first row in the group we are currently processing.
Definition: composite_iterators.h:294
String m_first_row_next_group
If applicable, packed version of the first row in the next group.
Definition: composite_iterators.h:307
table_map m_save_nullinfo
Used to save NULL information in the specific case where we have zero input rows.
Definition: composite_iterators.h:261
bool Init() override
Initialize or reinitialize the iterator.
Definition: composite_iterators.cc:201
unique_ptr_destroy_only< RowIterator > m_source
Definition: composite_iterators.h:244
enum AggregateIterator::@57 m_state
int m_output_slice
The slice we're setting when returning rows.
Definition: composite_iterators.h:313
AggregateIterator(THD *thd, unique_ptr_destroy_only< RowIterator > source, JOIN *join, pack_rows::TableCollection tables, bool rollup)
Definition: composite_iterators.cc:188
@ LAST_ROW_STARTED_NEW_GROUP
Definition: composite_iterators.h:239
@ READING_FIRST_ROW
Definition: composite_iterators.h:238
@ OUTPUTTING_ROLLUP_ROWS
Definition: composite_iterators.h:240
@ DONE_OUTPUTTING_ROWS
Definition: composite_iterators.h:241
void EndPSIBatchModeIfStarted() override
Ends performance schema batch mode, if started.
Definition: composite_iterators.h:226
const bool m_rollup
Whether this is a rollup query.
Definition: composite_iterators.h:264
void SetNullRowFlag(bool is_null_row) override
Mark the current row buffer as containing a NULL row or not, so that if you read from it and the flag...
Definition: composite_iterators.h:221
void SetRollupLevel(int level)
Definition: composite_iterators.cc:454
int m_last_unchanged_group_item_idx
For rollup: The index of the first group item that did not change when we last switched groups.
Definition: composite_iterators.h:276
Takes in two or more iterators and output rows from them sequentially (first all rows from the first ...
Definition: composite_iterators.h:1465
size_t m_current_iterator_index
Definition: composite_iterators.h:1482
AppendIterator(THD *thd, std::vector< unique_ptr_destroy_only< RowIterator > > &&sub_iterators)
Definition: composite_iterators.cc:3728
void SetNullRowFlag(bool is_null_row) override
Mark the current row buffer as containing a NULL row or not, so that if you read from it and the flag...
Definition: composite_iterators.cc:3765
bool Init() override
Initialize or reinitialize the iterator.
Definition: composite_iterators.cc:3734
void EndPSIBatchModeIfStarted() override
Ends performance schema batch mode, if started.
Definition: composite_iterators.cc:3775
void UnlockRow() override
Definition: composite_iterators.cc:3783
std::vector< unique_ptr_destroy_only< RowIterator > > m_sub_iterators
Definition: composite_iterators.h:1481
int Read() override
Read a single row.
Definition: composite_iterators.cc:3740
bool m_pfs_batch_mode_enabled
Definition: composite_iterators.h:1483
void StartPSIBatchMode() override
Start performance schema batch mode, if supported (otherwise ignored).
Definition: composite_iterators.cc:3770
A wrapper class which provides array bounds checking.
Definition: sql_array.h:46
An iterator that helps invalidating caches.
Definition: composite_iterators.h:401
bool Init() override
Initialize or reinitialize the iterator.
Definition: composite_iterators.h:410
CacheInvalidatorIterator(THD *thd, unique_ptr_destroy_only< RowIterator > source_iterator, const std::string &name)
Definition: composite_iterators.h:403
std::string m_name
Definition: composite_iterators.h:433
void SetNullRowFlag(bool is_null_row) override
Mark the current row buffer as containing a NULL row or not, so that if you read from it and the flag...
Definition: composite_iterators.h:420
unique_ptr_destroy_only< RowIterator > m_source_iterator
Definition: composite_iterators.h:431
void UnlockRow() override
Definition: composite_iterators.h:425
int64_t m_generation
Definition: composite_iterators.h:432
int Read() override
Read a single row.
Definition: composite_iterators.h:415
std::string name() const
Definition: composite_iterators.h:428
int64_t generation() const
Definition: composite_iterators.h:427
This is used for segregating rows in groups (e.g.
Definition: item.h:6516
An iterator that takes in a stream of rows and passes through only those that meet some criteria (i....
Definition: composite_iterators.h:87
void EndPSIBatchModeIfStarted() override
Ends performance schema batch mode, if started.
Definition: composite_iterators.h:102
int Read() override
Read a single row.
Definition: composite_iterators.cc:87
void UnlockRow() override
Definition: composite_iterators.h:105
bool Init() override
Initialize or reinitialize the iterator.
Definition: composite_iterators.h:93
unique_ptr_destroy_only< RowIterator > m_source
Definition: composite_iterators.h:108
FilterIterator(THD *thd, unique_ptr_destroy_only< RowIterator > source, Item *condition)
Definition: composite_iterators.h:89
void StartPSIBatchMode() override
Start performance schema batch mode, if supported (otherwise ignored).
Definition: composite_iterators.h:101
Item * m_condition
Definition: composite_iterators.h:109
void SetNullRowFlag(bool is_null_row) override
Mark the current row buffer as containing a NULL row or not, so that if you read from it and the flag...
Definition: composite_iterators.h:97
FollowTailIterator is a special version of TableScanIterator that is used as part of WITH RECURSIVE q...
Definition: basic_row_iterators.h:470
Definition: hash_join_chunk.h:66
Definition: hash_join_iterator.h:263
static constexpr size_t kMaxChunks
Definition: hash_join_iterator.h:579
The variant with length (ImmutableStringWithLength) stores the length as a Varint128 (similar to prot...
Definition: immutable_string.h:62
Base class that is used to represent any kind of expression in a relational query.
Definition: item.h:933
Definition: sql_optimizer.h:132
Definition: key.h:112
Handles LIMIT and/or OFFSET; Init() eats the first "offset" rows, and Read() stops as soon as it's se...
Definition: composite_iterators.h:116
void UnlockRow() override
Definition: composite_iterators.h:159
LimitOffsetIterator(THD *thd, unique_ptr_destroy_only< RowIterator > source, ha_rows limit, ha_rows offset, bool count_all_rows, bool reject_multiple_rows, ha_rows *skipped_rows)
Definition: composite_iterators.h:132
bool Init() override
Initialize or reinitialize the iterator.
Definition: composite_iterators.cc:112
ha_rows m_seen_rows
Definition: composite_iterators.h:167
const bool m_count_all_rows
Definition: composite_iterators.h:175
ha_rows * m_skipped_rows
Definition: composite_iterators.h:177
void EndPSIBatchModeIfStarted() override
Ends performance schema batch mode, if started.
Definition: composite_iterators.h:156
const ha_rows m_limit
Definition: composite_iterators.h:174
const ha_rows m_offset
Definition: composite_iterators.h:174
bool m_needs_offset
Whether we have OFFSET rows that we still need to skip.
Definition: composite_iterators.h:172
void SetNullRowFlag(bool is_null_row) override
Mark the current row buffer as containing a NULL row or not, so that if you read from it and the flag...
Definition: composite_iterators.h:151
unique_ptr_destroy_only< RowIterator > m_source
Definition: composite_iterators.h:162
int Read() override
Read a single row.
Definition: composite_iterators.cc:126
const bool m_reject_multiple_rows
Definition: composite_iterators.h:176
void StartPSIBatchMode() override
Start performance schema batch mode, if supported (otherwise ignored).
Definition: composite_iterators.h:155
LinkedImmutableString is designed for storing rows (values) in hash join.
Definition: immutable_string.h:172
MaterializeInformationSchemaTableIterator makes sure a given I_S temporary table is materialized (fil...
Definition: composite_iterators.h:1431
Item * m_condition
Definition: composite_iterators.h:1457
void UnlockRow() override
Definition: composite_iterators.h:1451
MaterializeInformationSchemaTableIterator(THD *thd, unique_ptr_destroy_only< RowIterator > table_iterator, Table_ref *table_list, Item *condition)
Definition: composite_iterators.cc:3703
Table_ref * m_table_list
Definition: composite_iterators.h:1456
void SetNullRowFlag(bool is_null_row) override
Mark the current row buffer as containing a NULL row or not, so that if you read from it and the flag...
Definition: composite_iterators.h:1440
void EndPSIBatchModeIfStarted() override
Ends performance schema batch mode, if started.
Definition: composite_iterators.h:1445
int Read() override
Read a single row.
Definition: composite_iterators.h:1438
void StartPSIBatchMode() override
Start performance schema batch mode, if supported (otherwise ignored).
Definition: composite_iterators.h:1444
unique_ptr_destroy_only< RowIterator > m_table_iterator
The iterator that reads from the materialized table.
Definition: composite_iterators.h:1455
bool Init() override
Initialize or reinitialize the iterator.
Definition: composite_iterators.cc:3711
An iterator that wraps a Table_function (e.g.
Definition: composite_iterators.h:1219
Table_function * m_table_function
Definition: composite_iterators.h:1243
unique_ptr_destroy_only< RowIterator > m_table_iterator
Definition: composite_iterators.h:1241
void StartPSIBatchMode() override
Start performance schema batch mode, if supported (otherwise ignored).
Definition: composite_iterators.h:1231
void UnlockRow() override
Definition: composite_iterators.h:1238
int Read() override
Read a single row.
Definition: composite_iterators.h:1226
void SetNullRowFlag(bool is_null_row) override
Mark the current row buffer as containing a NULL row or not, so that if you read from it and the flag...
Definition: composite_iterators.h:1227
void EndPSIBatchModeIfStarted() override
Ends performance schema batch mode, if started.
Definition: composite_iterators.h:1232
MaterializedTableFunctionIterator(THD *thd, Table_function *table_function, TABLE *table, unique_ptr_destroy_only< RowIterator > table_iterator)
Definition: composite_iterators.cc:3460
bool Init() override
Initialize or reinitialize the iterator.
Definition: composite_iterators.cc:3467
A simple nested loop join, taking in two iterators (left/outer and right/inner) and joining them toge...
Definition: composite_iterators.h:332
bool Init() override
Initialize or reinitialize the iterator.
Definition: composite_iterators.cc:466
enum NestedLoopIterator::@58 m_state
int Read() override
Read a single row.
Definition: composite_iterators.cc:477
void UnlockRow() override
Definition: composite_iterators.h:368
const bool m_pfs_batch_mode
Whether to use batch mode when scanning the inner iterator.
Definition: composite_iterators.h:390
NestedLoopIterator(THD *thd, unique_ptr_destroy_only< RowIterator > source_outer, unique_ptr_destroy_only< RowIterator > source_inner, JoinType join_type, bool pfs_batch_mode)
Definition: composite_iterators.h:334
unique_ptr_destroy_only< RowIterator > const m_source_inner
Definition: composite_iterators.h:386
@ END_OF_ROWS
Definition: composite_iterators.h:382
@ READING_INNER_ROWS
Definition: composite_iterators.h:381
@ NEEDS_OUTER_ROW
Definition: composite_iterators.h:379
@ READING_FIRST_INNER_ROW
Definition: composite_iterators.h:380
void EndPSIBatchModeIfStarted() override
Ends performance schema batch mode, if started.
Definition: composite_iterators.h:363
const JoinType m_join_type
Definition: composite_iterators.h:387
void SetNullRowFlag(bool is_null_row) override
Mark the current row buffer as containing a NULL row or not, so that if you read from it and the flag...
Definition: composite_iterators.h:357
unique_ptr_destroy_only< RowIterator > const m_source_outer
Definition: composite_iterators.h:385
An iterator that is semantically equivalent to a semijoin NestedLoopIterator immediately followed by ...
Definition: composite_iterators.h:1390
int Read() override
Read a single row.
Definition: composite_iterators.cc:3647
void UnlockRow() override
Definition: composite_iterators.h:1411
KEY * m_key
Definition: composite_iterators.h:1421
unique_ptr_destroy_only< RowIterator > const m_source_outer
Definition: composite_iterators.h:1417
void SetNullRowFlag(bool is_null_row) override
Mark the current row buffer as containing a NULL row or not, so that if you read from it and the flag...
Definition: composite_iterators.h:1401
const size_t m_key_len
Definition: composite_iterators.h:1423
bool m_deduplicate_against_previous_row
Definition: composite_iterators.h:1424
uchar * m_key_buf
Definition: composite_iterators.h:1422
unique_ptr_destroy_only< RowIterator > const m_source_inner
Definition: composite_iterators.h:1418
NestedLoopSemiJoinWithDuplicateRemovalIterator(THD *thd, unique_ptr_destroy_only< RowIterator > source_outer, unique_ptr_destroy_only< RowIterator > source_inner, const TABLE *table, KEY *key, size_t key_len)
Definition: composite_iterators.cc:3624
void EndPSIBatchModeIfStarted() override
Ends performance schema batch mode, if started.
Definition: composite_iterators.h:1406
const TABLE * m_table_outer
Definition: composite_iterators.h:1420
bool Init() override
Initialize or reinitialize the iterator.
Definition: composite_iterators.cc:3639
An iterator that removes consecutive rows that are the same according to a set of items (typically th...
Definition: composite_iterators.h:1298
void StartPSIBatchMode() override
Start performance schema batch mode, if supported (otherwise ignored).
Definition: composite_iterators.h:1312
void EndPSIBatchModeIfStarted() override
Ends performance schema batch mode, if started.
Definition: composite_iterators.h:1313
void UnlockRow() override
Definition: composite_iterators.h:1316
int Read() override
Read a single row.
Definition: composite_iterators.cc:3558
void SetNullRowFlag(bool is_null_row) override
Mark the current row buffer as containing a NULL row or not, so that if you read from it and the flag...
Definition: composite_iterators.h:1308
bool m_first_row
Definition: composite_iterators.h:1321
bool Init() override
Initialize or reinitialize the iterator.
Definition: composite_iterators.cc:3553
Bounds_checked_array< Cached_item * > m_caches
Definition: composite_iterators.h:1320
RemoveDuplicatesIterator(THD *thd, unique_ptr_destroy_only< RowIterator > source, JOIN *join, Item **group_items, int group_items_size)
Definition: composite_iterators.cc:3541
unique_ptr_destroy_only< RowIterator > m_source
Definition: composite_iterators.h:1319
Much like RemoveDuplicatesIterator, but works on the basis of a given index (or more accurately,...
Definition: composite_iterators.h:1330
bool Init() override
Initialize or reinitialize the iterator.
Definition: composite_iterators.cc:3595
void UnlockRow() override
Definition: composite_iterators.h:1347
uchar * m_key_buf
Definition: composite_iterators.h:1353
bool m_first_row
Definition: composite_iterators.h:1355
const TABLE * m_table
Definition: composite_iterators.h:1351
void StartPSIBatchMode() override
Start performance schema batch mode, if supported (otherwise ignored).
Definition: composite_iterators.h:1343
void SetNullRowFlag(bool is_null_row) override
Mark the current row buffer as containing a NULL row or not, so that if you read from it and the flag...
Definition: composite_iterators.h:1339
const size_t m_key_len
Definition: composite_iterators.h:1354
void EndPSIBatchModeIfStarted() override
Ends performance schema batch mode, if started.
Definition: composite_iterators.h:1344
RemoveDuplicatesOnIndexIterator(THD *thd, unique_ptr_destroy_only< RowIterator > source, const TABLE *table, KEY *key, size_t key_len)
Definition: composite_iterators.cc:3585
KEY * m_key
Definition: composite_iterators.h:1352
int Read() override
Read a single row.
Definition: composite_iterators.cc:3600
unique_ptr_destroy_only< RowIterator > m_source
Definition: composite_iterators.h:1350
A context for reading through a single table using a chosen access method: index read,...
Definition: row_iterator.h:81
THD * thd() const
Definition: row_iterator.h:227
Definition: sql_executor.h:94
StreamingIterator is a minimal version of MaterializeIterator that does not actually materialize; ins...
Definition: composite_iterators.h:1164
JOIN *const m_join
Definition: composite_iterators.h:1199
void UnlockRow() override
Definition: composite_iterators.h:1193
void StartPSIBatchMode() override
Start performance schema batch mode, if supported (otherwise ignored).
Definition: composite_iterators.h:1187
int Read() override
Read a single row.
Definition: composite_iterators.cc:3052
StreamingIterator(THD *thd, unique_ptr_destroy_only< RowIterator > subquery_iterator, Temp_table_param *temp_table_param, TABLE *table, bool provide_rowid, JOIN *join, int ref_slice)
Definition: composite_iterators.cc:3000
Temp_table_param * m_temp_table_param
Definition: composite_iterators.h:1197
const bool m_provide_rowid
Definition: composite_iterators.h:1206
const int m_output_slice
Definition: composite_iterators.h:1200
void EndPSIBatchModeIfStarted() override
Ends performance schema batch mode, if started.
Definition: composite_iterators.h:1190
bool Init() override
Initialize or reinitialize the iterator.
Definition: composite_iterators.cc:3029
int m_input_slice
Definition: composite_iterators.h:1201
unique_ptr_destroy_only< RowIterator > m_subquery_iterator
Definition: composite_iterators.h:1196
ha_rows m_row_number
Definition: composite_iterators.h:1198
Using this class is fraught with peril, and you need to be very careful when doing so.
Definition: sql_string.h:166
For each client connection we create a separate thread with THD serving as a thread/connection descri...
Definition: sql_lexer_thd.h:35
Definition: row_iterator.h:233
TABLE * table() const
Definition: row_iterator.h:245
Class representing a table function.
Definition: table_function.h:52
Definition: table.h:2853
Object containing parameters used when creating and using temporary tables.
Definition: temp_table_param.h:94
Like semijoin materialization, weedout works on the basic idea that a semijoin is just like an inner ...
Definition: composite_iterators.h:1265
unique_ptr_destroy_only< RowIterator > m_source
Definition: composite_iterators.h:1283
void EndPSIBatchModeIfStarted() override
Ends performance schema batch mode, if started.
Definition: composite_iterators.h:1277
WeedoutIterator(THD *thd, unique_ptr_destroy_only< RowIterator > source, SJ_TMP_TABLE *sj, table_map tables_to_get_rowid_for)
Definition: composite_iterators.cc:3480
int Read() override
Read a single row.
Definition: composite_iterators.cc:3510
const table_map m_tables_to_get_rowid_for
Definition: composite_iterators.h:1285
SJ_TMP_TABLE * m_sj
Definition: composite_iterators.h:1284
void UnlockRow() override
Definition: composite_iterators.h:1280
void SetNullRowFlag(bool is_null_row) override
Mark the current row buffer as containing a NULL row or not, so that if you read from it and the flag...
Definition: composite_iterators.h:1273
bool Init() override
Initialize or reinitialize the iterator.
Definition: composite_iterators.cc:3493
Definition: hash_join_buffer.h:104
Definition: hash_join_buffer.h:126
Contains spill state for set operations' use of in-memory hash map.
Definition: composite_iterators.h:736
bool write_completed_HFs(THD *thd, const Operands &operands, ha_rows *stored_rows)
Write the contents of the final generation of HD chunks to the materialized table which will hold the...
Definition: composite_iterators.cc:2703
Mem_root_array< ChunkPair > m_chunk_files
Array to hold the list of chunk files on disk in case we degrade into on-disk set EXCEPT/INTERSECT.
Definition: composite_iterators.h:1062
bool reset_for_spill_handling()
Definition: composite_iterators.cc:2566
size_t current_chunk_idx() const
Definition: composite_iterators.h:1068
static constexpr uint32_t m_magic_prime
Definition: composite_iterators.h:993
String m_row_buffer
Temporary space for (de)serializing a row.
Definition: composite_iterators.h:1057
bool compute_chunk_file_sets(const Operand *current_operand)
Compute sizing of and set aside space for the on-disk chunks and their associated in-memory structure...
Definition: composite_iterators.cc:2254
void switch_to_IF()
Definition: composite_iterators.h:916
size_t m_hash_seed
Modify for each operator in a N-ary set operation to avoid initial chunks filling up right away due t...
Definition: composite_iterators.h:996
static size_t chunk_offset(size_t chunk_index)
Definition: composite_iterators.h:1051
void switch_to_HF()
Definition: composite_iterators.h:907
static size_t chunk_index_to_set(size_t chunk_index)
Definition: composite_iterators.h:1034
size_t m_set
Definition: composite_iterators.h:1030
struct materialize_iterator::SpillState::@59 m_offending_row
Keeps the row that was just read from the left operand when we discovered that we were out of space i...
bool simulated_secondary_overflow(bool *spill)
Definition: composite_iterators.cc:2715
bool spill()
Inquire spill handling state.
Definition: composite_iterators.h:748
size_t m_current_row_in_chunk
The current row no (1-based) in a chunk being read, incremented before use.
Definition: composite_iterators.h:1073
bool write_HF(THD *thd, size_t set, size_t chunk_idx, const Operands &operands, ha_rows *stored_rows)
Used to write a complete (or incomplete in the case of secondary overflow) HF chunk to the materializ...
Definition: composite_iterators.cc:2578
size_t m_num_chunks
The number of chunks needed after rounding up to nearest power of two.
Definition: composite_iterators.h:1011
size_t m_read_rows_before_dedup
At the time of overflow: how many rows have we read from left operand.
Definition: composite_iterators.h:1003
bool append_hash_map_to_HF()
We are done processing a {HF, IF} chunk pair.
Definition: composite_iterators.cc:2421
bool secondary_overflow() const
Getter, cf. comment for m_secondary_overflow.
Definition: composite_iterators.h:777
THD * m_thd
Definition: composite_iterators.h:967
bool save_operand_to_IF_chunk_files(const Operand *current_operand)
Definition: composite_iterators.cc:2467
hash_map_type * m_hash_map
The in-memory hash map that overflowed.
Definition: composite_iterators.h:991
void set_secondary_overflow()
Definition: composite_iterators.h:761
size_t m_current_chunk_idx
The index of the chunk pair being read, incremented before use.
Definition: composite_iterators.h:1065
HashJoinChunk m_remaining_input
Used if m_no_of_chunk_file_sets > 1 so we can replay input rows from operands over sets 1....
Definition: composite_iterators.h:1078
bool init(const Operand &left_operand, hash_map_type *hash_map, size_t rows_in_hash_table, size_t read_rows_before_dedup, MEM_ROOT *hash_map_mem_root, TABLE *t)
Initialize the spill to disk processing state with some variables.
Definition: composite_iterators.cc:2767
size_t m_current_chunk_file_set
The current chunk under processing. 0-based.
Definition: composite_iterators.h:1018
ReadingState
Definition: composite_iterators.h:788
bool spread_hash_map_to_HF_chunk_files()
The initial hash map that overflowed will be spread over the determined number of chunk files,...
Definition: composite_iterators.cc:2358
bool m_unsaved
Definition: composite_iterators.h:1031
Mem_root_array< SetCounts > m_row_counts
A matrix of counters keeping track of how many rows have been stashed away in the chunk files for eac...
Definition: composite_iterators.h:1097
bool write_partially_completed_HFs(THD *thd, const Operands &operands, ha_rows *stored_rows)
Write the contents of the HD chunks that were completed when a secondary memory overflow has occurred...
Definition: composite_iterators.cc:2632
bool initialize_first_HF_chunk_files()
Definition: composite_iterators.cc:2331
String m_buffer
Definition: composite_iterators.h:1028
void secondary_overflow_handling_done()
Definition: composite_iterators.h:778
size_t m_no_of_chunk_file_sets
The number of chunk file sets needed to process all m_num_chunks chunks.
Definition: composite_iterators.h:1015
size_t m_simulated_row_no
Definition: composite_iterators.h:756
pack_rows::TableCollection m_table_collection
Cached value for {m_materialized_table}.
Definition: composite_iterators.h:987
robin_hood::unordered_flat_map< ImmutableStringWithLength, LinkedImmutableString, hash_join_buffer::KeyHasher, hash_join_buffer::KeyEquals > hash_map_type
Definition: composite_iterators.h:765
size_t m_chunk_offset
Definition: composite_iterators.h:1029
static void reset_hash_map(hash_map_type *hash_map)
Definition: composite_iterators.h:767
bool m_secondary_overflow
If true, we have seen memory overflow also during spill handling.
Definition: composite_iterators.h:980
size_t m_simulated_set_idx
Definition: composite_iterators.h:754
ReadingState read_state()
Definition: composite_iterators.h:798
bool save_offending_row()
Definition: composite_iterators.cc:2238
MEM_ROOT * m_hash_map_mem_root
The mem_root of m_hash_map. We need it for reusing its space.
Definition: composite_iterators.h:1006
TABLE * m_materialized_table
The materialized table we are eventualy writing the result of the set operation to.
Definition: composite_iterators.h:984
size_t m_rows_in_hash_map
At the time of overflow: how many rows from left operand are in hash map after deduplication.
Definition: composite_iterators.h:1000
size_t hash_to_chunk_index(uint64_t hash) const
Definition: composite_iterators.h:1038
bool save_rest_of_operand_to_IF_chunk_files(const Operand *current_operand)
Definition: composite_iterators.h:951
SpillState(THD *thd, MEM_ROOT *mem_root)
Definition: composite_iterators.h:738
THD * thd()
Definition: composite_iterators.h:929
int read_next_row_secondary_overflow()
Given current state of secondary overflow processing, return the next row up for inserting into or ma...
Definition: composite_iterators.cc:2930
ReadingState m_spill_read_state
If not SS_NONE, we have detected an overflow in the in-memory hash map while reading the left(-most) ...
Definition: composite_iterators.h:972
int read_next_row(const Operand *current_operand)
Given current state of spill processing, return the next row up for inserting into or matching agains...
Definition: composite_iterators.cc:2806
size_t current_chunk_file_set() const
Definition: composite_iterators.h:1021
size_t m_simulated_chunk_idx
Definition: composite_iterators.h:755
A structure that contains a list of input tables for a hash join operation, BKA join operation or a s...
Definition: pack_rows.h:92
static MEM_ROOT mem_root
Definition: client_plugin.cc:113
#define word1(x)
Definition: dtoa.cc:598
void my_error(int nr, myf MyFlags,...)
Fill in and print a previously registered error message.
Definition: my_error.cc:215
#define ME_FATALERROR
Definition: my_sys.h:156
This file contains the HashJoinRowBuffer class and related functions/classes.
An iterator for joining two inputs by using hashing to match rows from the inputs.
ImmutableString defines a storage format for strings that is designed to be as compact as possible,...
JoinType
Definition: join_type.h:27
@ ANTI
Left antijoin, i.e.
@ SEMI
Left semijoin, i.e.
This file follows Google coding style, except for the name MEM_ROOT (which is kept for historical rea...
std::unique_ptr< T, Destroy_only< T > > unique_ptr_destroy_only
std::unique_ptr, but only destroying.
Definition: my_alloc.h:476
This file includes constants used by all storage engines.
my_off_t ha_rows
Definition: my_base.h:1140
Some integer typedefs for easier portability.
unsigned long long int ulonglong
Definition: my_inttypes.h:55
unsigned char uchar
Definition: my_inttypes.h:51
#define MYF(v)
Definition: my_inttypes.h:96
uint16_t uint16
Definition: my_inttypes.h:64
Common header for many mysys elements.
uint64_t table_map
Definition: my_table_map.h:29
static PFS_engine_table_share_proxy table
Definition: pfs.cc:60
Definition: composite_iterators.h:436
RowIterator * CreateIterator(THD *thd, Mem_root_array< materialize_iterator::Operand > operands, const MaterializePathParameters *path_params, unique_ptr_destroy_only< RowIterator > table_iterator, JOIN *join)
Create an iterator that materializes a set of row into a temporary table and sets up a (pre-existing)...
Definition: composite_iterators.cc:2978
std::string join(Container cont, const std::string &delim)
join elements of an container into a string separated by a delimiter.
Definition: string.h:150
Definition: varlen_sort.h:174
Definition: composite_iterators.h:1126
RowIterator * CreateIterator(THD *thd, unique_ptr_destroy_only< RowIterator > subquery_iterator, Temp_table_param *temp_table_param, TABLE *table, unique_ptr_destroy_only< RowIterator > table_iterator, JOIN *join, int ref_slice)
Create an iterator that aggregates the output rows from another iterator into a temporary table and t...
Definition: composite_iterators.cc:3432
std::vector< T, ut::allocator< T > > vector
Specialization of vector which uses allocator.
Definition: ut0new.h:2873
std::set< Key, Compare, ut::allocator< Key > > set
Specialization of set which uses ut_allocator.
Definition: ut0new.h:2881
std::map< Key, Value, Compare, ut::allocator< std::pair< const Key, Value > > > map
Specialization of map which uses ut_allocator.
Definition: ut0new.h:2891
Generic routines for packing rows (possibly from multiple tables at the same time) into strings,...
struct result result
Definition: result.h:33
required string key
Definition: replication_asynchronous_connection_failover.proto:59
repeated Source source
Definition: replication_asynchronous_connection_failover.proto:41
join_type
Definition: sql_opt_exec_shared.h:185
Our own string classes, used pervasively throughout the executor.
The MEM_ROOT is a simple arena, where allocations are carved out of larger blocks.
Definition: my_alloc.h:82
void Clear()
Deallocate all the RAM used.
Definition: my_alloc.cc:171
Definition: materialize_path_parameters.h:39
Definition: table.h:1403
An operand (query block) to be materialized by MaterializeIterator.
Definition: composite_iterators.h:441
unique_ptr_destroy_only< RowIterator > subquery_iterator
The iterator to read the actual rows from.
Definition: composite_iterators.h:443
bool copy_items
If set to false, the Field objects in the output row are presumed already to be filled out.
Definition: composite_iterators.h:471
Temp_table_param * temp_table_param
If copy_items is true, used for copying the Field objects into the temporary table row.
Definition: composite_iterators.h:487
double m_estimated_output_rows
The estimated number of rows produced by this block.
Definition: composite_iterators.h:501
ulonglong m_operand_idx
The current operand (i.e.
Definition: composite_iterators.h:478
bool is_recursive_reference
Definition: composite_iterators.h:491
FollowTailIterator * recursive_reader
Definition: composite_iterators.h:498
int select_number
Used only for optimizer trace.
Definition: composite_iterators.h:446
ulonglong m_total_operands
The number of operands (i.e.
Definition: composite_iterators.h:475
uint m_first_distinct
Used for EXCEPT computation: the index of the first operand involved in a N-ary except operation whic...
Definition: composite_iterators.h:483
bool disable_deduplication_by_hash_field
If true, de-duplication checking via hash key is disabled when materializing this query block (ie....
Definition: composite_iterators.h:466
JOIN * join
The JOIN that this query block represents.
Definition: composite_iterators.h:453
For a given chunk file pair {HF, IF}, the count of rows in each chunk respectively.
Definition: composite_iterators.h:1082
size_t IF_count
Definition: composite_iterators.h:1084
size_t HF_count
Definition: composite_iterators.h:1083
Definition: result.h:29