1 // Copyright (c) Microsoft. All rights reserved.
2 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
5 // Copyright (c) Microsoft. All rights reserved.
6 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
13 #pragma optimize( "t", on )
18 #define inline __attribute__((always_inline)) inline
20 #define inline __forceinline
30 #pragma warning(disable:4293)
31 #pragma warning(disable:4477)
34 inline void FATAL_GC_ERROR()
36 #ifndef DACCESS_COMPILE
37 GCToOSInterface::DebugBreak();
38 #endif // DACCESS_COMPILE
39 _ASSERTE(!"Fatal Error in GC.");
40 GCToEEInterface::HandleFatalError(COR_E_EXECUTIONENGINE);
44 #pragma inline_depth(20)
47 /* the following section defines the optional features */
49 // FEATURE_STRUCTALIGN was added by Midori. In CLR we are not interested
50 // in supporting custom alignments on LOH. Currently FEATURE_LOH_COMPACTION
51 // and FEATURE_STRUCTALIGN are mutually exclusive. It shouldn't be much
52 // work to make FEATURE_STRUCTALIGN not apply to LOH so they can be both
54 #define FEATURE_LOH_COMPACTION
56 #ifdef FEATURE_64BIT_ALIGNMENT
57 // We need the following feature as part of keeping 64-bit types aligned in the GC heap.
58 #define RESPECT_LARGE_ALIGNMENT //used to keep "double" objects aligned during
60 #endif //FEATURE_64BIT_ALIGNMENT
62 #define SHORT_PLUGS //used to keep ephemeral plugs short so they fit better into the oldest generation free items
65 #define DESIRED_PLUG_LENGTH (1000)
68 #define FEATURE_PREMORTEM_FINALIZATION
71 #ifndef FEATURE_REDHAWK
73 #define COLLECTIBLE_CLASS
74 #endif // !FEATURE_REDHAWK
77 #define initial_internal_roots (1024*16)
78 #endif // HEAP_ANALYZE
80 #define MARK_LIST //used sorted list to speed up plan phase
82 #define BACKGROUND_GC //concurrent background GC (requires WRITE_WATCH)
85 #define MH_SC_MARK //scalable marking
86 //#define SNOOP_STATS //diagnostic
87 #define PARALLEL_MARK_LIST_SORT //do the sorting and merging of the multiple mark lists in server gc in parallel
90 //This is used to mark some type volatile only when the scalable marking is used.
91 #if defined (SERVER_GC) && defined (MH_SC_MARK)
92 #define SERVER_SC_MARK_VOLATILE(x) VOLATILE(x)
93 #else //SERVER_GC&&MH_SC_MARK
94 #define SERVER_SC_MARK_VOLATILE(x) x
95 #endif //SERVER_GC&&MH_SC_MARK
97 //#define MULTIPLE_HEAPS //Allow multiple heaps for servers
99 #define INTERIOR_POINTERS //Allow interior pointers in the code manager
101 #define CARD_BUNDLE //enable card bundle feature.(requires WRITE_WATCH)
103 // If this is defined we use a map for segments in order to find the heap for
104 // a segment fast. But it does use more memory as we have to cover the whole
105 // heap range and for each entry we allocate a struct of 5 ptr-size words
106 // (3 for WKS as there's only one heap).
107 #define SEG_MAPPING_TABLE
109 // If allocating the heap mapping table for the available VA consumes too
110 // much memory, you can enable this to allocate only the portion that
111 // corresponds to rw segments and grow it when needed in grow_brick_card_table.
112 // However in heap_of you will need to always compare the address with
113 // g_lowest/highest before you can look at the heap mapping table.
114 #define GROWABLE_SEG_MAPPING_TABLE
117 #define MARK_ARRAY //Mark bit in an array
118 #endif //BACKGROUND_GC
120 #if defined(BACKGROUND_GC) || defined (CARD_BUNDLE) || defined(FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP)
121 #define WRITE_WATCH //Write Watch feature
122 #endif //BACKGROUND_GC || CARD_BUNDLE
125 #define array_size 100
128 //#define SHORT_PLUGS //keep plug short
130 #define FFIND_OBJECT //faster find_object, slower allocation
131 #define FFIND_DECAY 7 //Number of GC for which fast find will be active
134 #define MAX_LONGPATH 1024
135 #endif // MAX_LONGPATH
137 //#define DEBUG_WRITE_WATCH //Additional debug for write watch
139 //#define STRESS_PINNING //Stress pinning by pinning randomly
141 //#define TRACE_GC //debug trace gc operation
142 //#define SIMPLE_DPRINTF
144 //#define TIME_GC //time allocation and garbage collection
145 //#define TIME_WRITE_WATCH //time GetWriteWatch and ResetWriteWatch calls
146 //#define COUNT_CYCLES //Use cycle counter for timing
147 //#define JOIN_STATS //amount of time spent in the join
148 //also, see TIME_SUSPEND in switches.h.
150 //#define SYNCHRONIZATION_STATS
151 //#define SEG_REUSE_STATS
153 #if defined (SYNCHRONIZATION_STATS) || defined (STAGE_STATS)
154 #define BEGIN_TIMING(x) \
156 x##_start = GCToOSInterface::QueryPerformanceCounter()
158 #define END_TIMING(x) \
160 x##_end = GCToOSInterface::QueryPerformanceCounter(); \
161 x += x##_end - x##_start
164 #define BEGIN_TIMING(x)
165 #define END_TIMING(x)
166 #define BEGIN_TIMING_CYCLES(x)
167 #define END_TIMING_CYCLES(x)
168 #endif //SYNCHRONIZATION_STATS || STAGE_STATS
170 /* End of optional features */
172 #ifdef GC_CONFIG_DRIVEN
173 void GCLogConfig (const char *fmt, ... );
174 #define cprintf(x) {GCLogConfig x;}
175 #endif //GC_CONFIG_DRIVEN
181 // For the bestfit algorithm when we relocate ephemeral generations into an
182 // existing gen2 segment.
183 // We recorded sizes from 2^6, 2^7, 2^8...up to 2^30 (1GB). So that's 25 sizes total.
184 #define MIN_INDEX_POWER2 6
189 #define MAX_INDEX_POWER2 30
191 #define MAX_INDEX_POWER2 26
197 #define MAX_INDEX_POWER2 28
199 #define MAX_INDEX_POWER2 24
204 #define MAX_NUM_BUCKETS (MAX_INDEX_POWER2 - MIN_INDEX_POWER2 + 1)
206 #define MAX_NUM_FREE_SPACES 200
207 #define MIN_NUM_FREE_SPACES 5
209 //Please leave these definitions intact.
215 #ifdef FEATURE_STRUCTALIGN
216 #define REQD_ALIGN_DCL ,int requiredAlignment
217 #define REQD_ALIGN_ARG ,requiredAlignment
218 #define REQD_ALIGN_AND_OFFSET_DCL ,int requiredAlignment,size_t alignmentOffset
219 #define REQD_ALIGN_AND_OFFSET_DEFAULT_DCL ,int requiredAlignment=DATA_ALIGNMENT,size_t alignmentOffset=0
220 #define REQD_ALIGN_AND_OFFSET_ARG ,requiredAlignment,alignmentOffset
221 #else // FEATURE_STRUCTALIGN
222 #define REQD_ALIGN_DCL
223 #define REQD_ALIGN_ARG
224 #define REQD_ALIGN_AND_OFFSET_DCL
225 #define REQD_ALIGN_AND_OFFSET_DEFAULT_DCL
226 #define REQD_ALIGN_AND_OFFSET_ARG
227 #endif // FEATURE_STRUCTALIGN
229 #ifdef MULTIPLE_HEAPS
230 #define THREAD_NUMBER_DCL ,int thread
231 #define THREAD_NUMBER_ARG ,thread
232 #define THREAD_NUMBER_FROM_CONTEXT int thread = sc->thread_number;
233 #define THREAD_FROM_HEAP int thread = heap_number;
234 #define HEAP_FROM_THREAD gc_heap* hpt = gc_heap::g_heaps[thread];
236 #define THREAD_NUMBER_DCL
237 #define THREAD_NUMBER_ARG
238 #define THREAD_NUMBER_FROM_CONTEXT
239 #define THREAD_FROM_HEAP
240 #define HEAP_FROM_THREAD gc_heap* hpt = 0;
241 #endif //MULTIPLE_HEAPS
243 //These constants are ordered
244 const int policy_sweep = 0;
245 const int policy_compact = 1;
246 const int policy_expand = 2;
249 #define SEG_REUSE_LOG_0 7
250 #define SEG_REUSE_LOG_1 (SEG_REUSE_LOG_0 + 1)
251 #define DT_LOG_0 (SEG_REUSE_LOG_1 + 1)
252 #define BGC_LOG (DT_LOG_0 + 1)
253 #define GTC_LOG (DT_LOG_0 + 2)
254 #define GC_TABLE_LOG (DT_LOG_0 + 3)
255 #define JOIN_LOG (DT_LOG_0 + 4)
256 #define SPINLOCK_LOG (DT_LOG_0 + 5)
257 #define SNOOP_LOG (DT_LOG_0 + 6)
259 #ifndef DACCESS_COMPILE
261 #ifdef SIMPLE_DPRINTF
263 //#define dprintf(l,x) {if (trace_gc && ((l<=print_level)||gc_heap::settings.concurrent)) {printf ("\n");printf x ; fflush(stdout);}}
264 void GCLog (const char *fmt, ... );
265 //#define dprintf(l,x) {if (trace_gc && (l<=print_level)) {GCLog x;}}
266 //#define dprintf(l,x) {if ((l==SEG_REUSE_LOG_0) || (l==SEG_REUSE_LOG_1) || (trace_gc && (l<=3))) {GCLog x;}}
267 //#define dprintf(l,x) {if (l == DT_LOG_0) {GCLog x;}}
268 //#define dprintf(l,x) {if (trace_gc && ((l <= 2) || (l == BGC_LOG) || (l==GTC_LOG))) {GCLog x;}}
269 //#define dprintf(l,x) {if ((l == 1) || (l == 2222)) {GCLog x;}}
270 #define dprintf(l,x) {if ((l <= 1) || (l == GTC_LOG)) {GCLog x;}}
271 //#define dprintf(l,x) {if ((l==GTC_LOG) || (l <= 1)) {GCLog x;}}
272 //#define dprintf(l,x) {if (trace_gc && ((l <= print_level) || (l==GTC_LOG))) {GCLog x;}}
273 //#define dprintf(l,x) {if (l==GTC_LOG) {printf ("\n");printf x ; fflush(stdout);}}
274 #else //SIMPLE_DPRINTF
275 // Nobody used the logging mechanism that used to be here. If we find ourselves
276 // wanting to inspect GC logs on unmodified builds, we can use this define here
278 #define dprintf(l, x)
280 #endif //SIMPLE_DPRINTF
282 #else //DACCESS_COMPILE
284 #endif //DACCESS_COMPILE
289 #if !defined(FEATURE_REDHAWK) && !defined(BUILD_AS_STANDALONE)
291 #define assert _ASSERTE
293 #define ASSERT _ASSERTE
294 #endif // FEATURE_REDHAWK
296 struct GCDebugSpinLock {
297 VOLATILE(int32_t) lock; // -1 if free, 0 if held
299 VOLATILE(Thread *) holding_thread; // -1 if no thread holds the lock.
300 VOLATILE(BOOL) released_by_gc_p; // a GC thread released the lock.
302 #if defined (SYNCHRONIZATION_STATS)
303 // number of times we went into SwitchToThread in enter_spin_lock.
304 unsigned int num_switch_thread;
305 // number of times we went into WaitLonger.
306 unsigned int num_wait_longer;
307 // number of times we went to calling SwitchToThread in WaitLonger.
308 unsigned int num_switch_thread_w;
309 // number of times we went to calling DisablePreemptiveGC in WaitLonger.
310 unsigned int num_disable_preemptive_w;
316 , holding_thread((Thread*) -1)
318 #if defined (SYNCHRONIZATION_STATS)
319 , num_switch_thread(0), num_wait_longer(0), num_switch_thread_w(0), num_disable_preemptive_w(0)
324 #if defined (SYNCHRONIZATION_STATS)
327 num_switch_thread = 0;
329 num_switch_thread_w = 0;
330 num_disable_preemptive_w = 0;
334 typedef GCDebugSpinLock GCSpinLock;
342 class seg_free_spaces;
346 class exclusive_sync;
347 class recursive_gc_sync;
348 #endif //BACKGROUND_GC
350 // The following 2 modes are of the same format as in clr\src\bcl\system\runtime\gcsettings.cs
351 // make sure you change that one if you change this one!
354 pause_batch = 0, //We are not concerned about pause length
355 pause_interactive = 1, //We are running an interactive app
356 pause_low_latency = 2, //short pauses are essential
357 //avoid long pauses from blocking full GCs unless running out of memory
358 pause_sustained_low_latency = 3,
362 enum gc_loh_compaction_mode
364 loh_compaction_default = 1, // the default mode, don't compact LOH.
365 loh_compaction_once = 2, // only compact once the next time a blocking full GC happens.
366 loh_compaction_auto = 4 // GC decides when to compact LOH, to be implemented.
369 enum set_pause_mode_status
371 set_pause_mode_success = 0,
372 set_pause_mode_no_gc = 1 // NoGCRegion is in progress, can't change pause mode.
376 Latency modes required user to have specific GC knowledge (eg, budget, full blocking GC).
377 We are trying to move away from them as it makes a lot more sense for users to tell
378 us what's the most important out of the perf aspects that make sense to them.
380 In general there are 3 such aspects:
384 + pause predictibility
386 Currently the following levels are supported. We may (and will likely) add more
389 +----------+--------------------+---------------------------------------+
390 | Level | Optimization Goals | Latency Charactaristics |
391 +==========+====================+=======================================+
392 | 0 | memory footprint | pauses can be long and more frequent |
393 +----------+--------------------+---------------------------------------+
394 | 1 | balanced | pauses are more predictable and more |
395 | | | frequent. the longest pauses are |
396 | | | shorter than 1. |
397 +----------+--------------------+---------------------------------------+
399 enum gc_latency_level
401 latency_level_first = 0,
402 latency_level_memory_footprint = latency_level_first,
403 latency_level_balanced = 1,
404 latency_level_last = latency_level_balanced,
405 latency_level_default = latency_level_balanced
410 tuning_deciding_condemned_gen = 0,
411 tuning_deciding_full_gc = 1,
412 tuning_deciding_compaction = 2,
413 tuning_deciding_expansion = 3,
414 tuning_deciding_promote_ephemeral = 4,
415 tuning_deciding_short_on_seg = 5
418 #if defined(TRACE_GC) && defined(BACKGROUND_GC)
419 static const char * const str_bgc_state[] =
433 #endif // defined(TRACE_GC) && defined(BACKGROUND_GC)
435 enum allocation_state
438 a_state_can_allocate,
439 a_state_cant_allocate,
440 // This could be due to having to wait till a GC is done,
441 // or having to try a different heap.
442 a_state_retry_allocate,
444 a_state_try_fit_new_seg,
445 a_state_try_fit_after_cg,
446 a_state_try_fit_after_bgc,
447 a_state_try_free_full_seg_in_bgc,
448 a_state_try_free_after_bgc,
451 a_state_acquire_seg_after_cg,
452 a_state_acquire_seg_after_bgc,
453 a_state_check_and_wait_for_bgc,
454 a_state_trigger_full_compact_gc,
455 a_state_trigger_ephemeral_gc,
456 a_state_trigger_2nd_ephemeral_gc,
457 a_state_check_retry_seg,
463 gc_type_compacting = 0,
464 gc_type_blocking = 1,
466 gc_type_background = 2,
467 #endif //BACKGROUND_GC
471 //encapsulates the mechanism for the current gc
475 VOLATILE(size_t) gc_index; // starts from 1 for the first GC, like dd_collection_count
476 int condemned_generation;
484 int gen0_reduction_count;
485 BOOL should_lock_elevation;
486 int elevation_locked_count;
487 BOOL elevation_reduced;
490 gc_pause_mode pause_mode;
491 BOOL found_finalizers;
496 BOOL allocations_allowed;
497 #endif //BACKGROUND_GC
501 #endif // STRESS_HEAP
503 // These are opportunistically set
504 uint32_t entry_memory_load;
505 uint32_t exit_memory_load;
507 void init_mechanisms(); //for each GC
508 void first_init(); // for the life of the EE
510 void record (gc_history_global* history);
513 // This is a compact version of gc_mechanism that we use to save in the history.
514 class gc_mechanisms_store
525 bool should_lock_elevation;
526 int condemned_generation : 8;
527 int gen0_reduction_count : 8;
528 int elevation_locked_count : 8;
529 gc_reason reason : 8;
530 gc_pause_mode pause_mode : 8;
532 bgc_state b_state : 8;
533 #endif //BACKGROUND_GC
534 bool found_finalizers;
538 #endif //BACKGROUND_GC
542 #endif // STRESS_HEAP
545 uint32_t entry_memory_load;
548 void store (gc_mechanisms* gm)
550 gc_index = gm->gc_index;
551 condemned_generation = gm->condemned_generation;
552 promotion = (gm->promotion != 0);
553 compaction = (gm->compaction != 0);
554 loh_compaction = (gm->loh_compaction != 0);
555 heap_expansion = (gm->heap_expansion != 0);
556 concurrent = (gm->concurrent != 0);
557 demotion = (gm->demotion != 0);
558 card_bundles = (gm->card_bundles != 0);
559 gen0_reduction_count = gm->gen0_reduction_count;
560 should_lock_elevation = (gm->should_lock_elevation != 0);
561 elevation_locked_count = gm->elevation_locked_count;
563 pause_mode = gm->pause_mode;
564 found_finalizers = (gm->found_finalizers != 0);
567 background_p = (gm->background_p != 0);
568 b_state = gm->b_state;
569 #endif //BACKGROUND_GC
572 stress_induced = (gm->stress_induced != 0);
573 #endif // STRESS_HEAP
576 entry_memory_load = gm->entry_memory_load;
583 // GC specific statistics, tracking counts and timings for GCs occuring in the system.
584 // This writes the statistics to a file every 60 seconds, if a file is specified in
588 : public StatisticsBase
590 // initialized to the contents of COMPlus_GcMixLog, or NULL, if not present
591 static char* logFileName;
592 static FILE* logFile;
594 // number of times we executed a background GC, a foreground GC, or a
596 int cntBGC, cntFGC, cntNGC;
598 // min, max, and total time spent performing BGCs, FGCs, NGCs
599 // (BGC time includes everything between the moment the BGC starts until
600 // it completes, i.e. the times of all FGCs occuring concurrently)
601 MinMaxTot bgc, fgc, ngc;
603 // number of times we executed a compacting GC (sweeping counts can be derived)
604 int cntCompactNGC, cntCompactFGC;
607 int cntReasons[reason_max];
609 // count of condemned generation, by NGC and FGC:
610 int cntNGCGen[max_generation+1];
611 int cntFGCGen[max_generation];
613 ///////////////////////////////////////////////////////////////////////////////////////////////
614 // Internal mechanism:
616 virtual void Initialize();
617 virtual void DisplayAndUpdate();
621 static BOOL Enabled()
622 { return logFileName != NULL; }
624 void AddGCStats(const gc_mechanisms& settings, size_t timeInMSec);
627 extern GCStatistics g_GCStatistics;
628 extern GCStatistics g_LastGCStatistics;
632 typedef DPTR(class heap_segment) PTR_heap_segment;
633 typedef DPTR(class gc_heap) PTR_gc_heap;
634 typedef DPTR(PTR_gc_heap) PTR_PTR_gc_heap;
635 #ifdef FEATURE_PREMORTEM_FINALIZATION
636 typedef DPTR(class CFinalize) PTR_CFinalize;
637 #endif // FEATURE_PREMORTEM_FINALIZATION
639 //-------------------------------------
640 //generation free list. It is an array of free lists bucketed by size, starting at sizes lower than first_bucket_size
641 //and doubling each time. The last bucket (index == num_buckets) is for largest sizes with no limit
643 #define MAX_BUCKET_COUNT (13)//Max number of buckets for the small generations.
651 #ifdef FL_VERIFICATION
653 #endif //FL_VERIFICATION
655 uint8_t*& alloc_list_head () { return head;}
656 uint8_t*& alloc_list_tail () { return tail;}
657 size_t& alloc_list_damage_count(){ return damage_count; }
670 size_t frst_bucket_size;
671 alloc_list first_bucket;
673 alloc_list& alloc_list_of (unsigned int bn);
674 size_t& alloc_list_damage_count_of (unsigned int bn);
677 allocator (unsigned int num_b, size_t fbs, alloc_list* b);
681 frst_bucket_size = SIZE_T_MAX;
683 unsigned int number_of_buckets() {return (unsigned int)num_buckets;}
685 size_t first_bucket_size() {return frst_bucket_size;}
686 uint8_t*& alloc_list_head_of (unsigned int bn)
688 return alloc_list_of (bn).alloc_list_head();
690 uint8_t*& alloc_list_tail_of (unsigned int bn)
692 return alloc_list_of (bn).alloc_list_tail();
695 BOOL discard_if_no_fit_p()
697 return (num_buckets == 1);
700 // This is when we know there's nothing to repair because this free
701 // list has never gone through plan phase. Right now it's only used
702 // by the background ephemeral sweep when we copy the local free list
703 // to gen0's free list.
705 // We copy head and tail manually (vs together like copy_to_alloc_list)
706 // since we need to copy tail first because when we get the free items off
707 // of each bucket we check head first. We also need to copy the
708 // smaller buckets first so when gen0 allocation needs to thread
709 // smaller items back that bucket is guaranteed to have been full
711 void copy_with_no_repair (allocator* allocator_to_copy)
713 assert (num_buckets == allocator_to_copy->number_of_buckets());
714 for (unsigned int i = 0; i < num_buckets; i++)
716 alloc_list* al = &(allocator_to_copy->alloc_list_of (i));
717 alloc_list_tail_of(i) = al->alloc_list_tail();
718 alloc_list_head_of(i) = al->alloc_list_head();
722 void unlink_item (unsigned int bucket_number, uint8_t* item, uint8_t* previous_item, BOOL use_undo_p);
723 void thread_item (uint8_t* item, size_t size);
724 void thread_item_front (uint8_t* itme, size_t size);
725 void thread_free_item (uint8_t* free_item, uint8_t*& head, uint8_t*& tail);
726 void copy_to_alloc_list (alloc_list* toalist);
727 void copy_from_alloc_list (alloc_list* fromalist);
728 void commit_alloc_list_changes();
731 #define NUM_GEN_POWER2 (20)
732 #define BASE_GEN_SIZE (1*512)
734 // group the frequently used ones together (need intrumentation on accessors)
738 // Don't move these first two fields without adjusting the references
739 // from the __asm in jitinterface.cpp.
740 alloc_context allocation_context;
741 PTR_heap_segment start_segment;
742 uint8_t* allocation_start;
743 heap_segment* allocation_segment;
744 uint8_t* allocation_context_start_region;
745 allocator free_list_allocator;
746 size_t free_list_allocated;
747 size_t end_seg_allocated;
748 BOOL allocate_end_seg_p;
749 size_t condemned_allocated;
750 size_t free_list_space;
751 size_t free_obj_space;
752 size_t allocation_size;
753 uint8_t* plan_allocation_start;
754 size_t plan_allocation_start_size;
756 // this is the pinned plugs that got allocated into this gen.
757 size_t pinned_allocated;
758 size_t pinned_allocation_compact_size;
759 size_t pinned_allocation_sweep_size;
762 #ifdef FREE_USAGE_STATS
763 size_t gen_free_spaces[NUM_GEN_POWER2];
764 // these are non pinned plugs only
765 size_t gen_plugs[NUM_GEN_POWER2];
766 size_t gen_current_pinned_free_spaces[NUM_GEN_POWER2];
767 size_t pinned_free_obj_space;
768 // this is what got allocated into the pinned free spaces.
769 size_t allocated_in_pinned_free;
770 size_t allocated_since_last_pin;
771 #endif //FREE_USAGE_STATS
774 static_assert(offsetof(dac_generation, allocation_context) == offsetof(generation, allocation_context), "DAC generation offset mismatch");
775 static_assert(offsetof(dac_generation, start_segment) == offsetof(generation, start_segment), "DAC generation offset mismatch");
776 static_assert(offsetof(dac_generation, allocation_start) == offsetof(generation, allocation_start), "DAC generation offset mismatch");
778 // static data remains the same after it's initialized.
779 // It's per generation.
780 // TODO: for gen_time_tuning, we should put the multipliers in static data.
785 size_t fragmentation_limit;
786 float fragmentation_burden_limit;
789 size_t time_clock; // time after which to collect generation, in performance counts (see QueryPerformanceCounter)
790 size_t gc_clock; // nubmer of gcs after which to collect generation
793 // The dynamic data fields are grouped into 3 categories:
795 // calculated logical data (like desired_allocation)
796 // physical data (like fragmentation)
797 // const data (sdata), initialized at the beginning
801 ptrdiff_t new_allocation;
802 ptrdiff_t gc_new_allocation; // new allocation at beginning of gc
804 size_t desired_allocation;
806 // # of bytes taken by objects (ie, not free space) at the beginning
808 size_t begin_data_size;
809 // # of bytes taken by survived objects after mark.
810 size_t survived_size;
811 // # of bytes taken by survived pinned plugs after mark.
812 size_t pinned_survived_size;
813 size_t artificial_pinned_survived_size;
814 size_t added_pinned_size;
819 #if defined (RESPECT_LARGE_ALIGNMENT) || defined (FEATURE_STRUCTALIGN)
820 // # of plugs that are not pinned plugs.
821 size_t num_npinned_plugs;
822 #endif //RESPECT_LARGE_ALIGNMENT || FEATURE_STRUCTALIGN
823 //total object size after a GC, ie, doesn't include fragmentation
825 size_t collection_count;
826 size_t promoted_size;
827 size_t freach_previous_promotion;
828 size_t fragmentation; //fragmentation when we don't compact
829 size_t gc_clock; //gc# when last GC happened
830 size_t time_clock; //time when last gc started
831 size_t gc_elapsed_time; // Time it took for the gc to complete
832 float gc_speed; // speed in bytes/msec for the gc to complete
839 #define ro_in_entry 0x1
841 #ifdef SEG_MAPPING_TABLE
842 // Note that I am storing both h0 and seg0, even though in Server GC you can get to
843 // the heap* from the segment info. This is because heap_of needs to be really fast
844 // and we would not want yet another indirection.
847 // if an address is > boundary it belongs to h1; else h0.
848 // since we init h0 and h1 to 0, if we get 0 it means that
849 // address doesn't exist on managed segments. And heap_of
850 // would just return heap0 which is what it does now.
852 #ifdef MULTIPLE_HEAPS
855 #endif //MULTIPLE_HEAPS
856 // You could have an address that's inbetween 2 segments and
857 // this would return a seg, the caller then will use
858 // in_range_for_segment to determine if it's on that seg.
859 heap_segment* seg0; // this is what the seg for h0 is.
860 heap_segment* seg1; // this is what the seg for h1 is.
861 // Note that when frozen objects are used we mask seg1
862 // with 0x1 to indicate that there is a ro segment for
865 #endif //SEG_MAPPING_TABLE
868 //Alignment constant for allocation
869 #define ALIGNCONST (DATA_ALIGNMENT-1)
872 size_t Align (size_t nbytes, int alignment=ALIGNCONST)
874 return (nbytes + alignment) & ~alignment;
877 //return alignment constant for small object heap vs large object heap
879 int get_alignment_constant (BOOL small_object_p)
881 #ifdef FEATURE_STRUCTALIGN
882 // If any objects on the large object heap require 8-byte alignment,
883 // the compiler will tell us so. Let's not guess an alignment here.
885 #else // FEATURE_STRUCTALIGN
886 return small_object_p ? ALIGNCONST : 7;
887 #endif // FEATURE_STRUCTALIGN
892 size_t desired_allocation;
893 size_t new_allocation;
897 // Note, I am not removing the ones that are no longer used
898 // because the older versions of the runtime still use them
899 // and ETW interprets them.
900 enum alloc_wait_reason
902 // When we don't care about firing an event for
906 // when we detect we are in low memory
909 // when we detect the ephemeral segment is too full
910 awr_low_ephemeral = 1,
912 // we've given out too much budget for gen0.
915 // we've given out too much budget for loh.
918 // this event is really obsolete - it's for pre-XP
919 // OSs where low mem notification is not supported.
920 awr_alloc_loh_low_mem = 4,
922 // we ran out of VM spaced to reserve on loh.
925 // ran out of space when allocating a small object
926 awr_gen0_oos_bgc = 6,
928 // ran out of space when allocating a large object
931 // waiting for BGC to let FGC happen
932 awr_fgc_wait_for_bgc = 8,
934 // wait for bgc to finish to get loh seg.
935 // no longer used with the introduction of loh msl.
938 // we don't allow loh allocation during bgc planning.
939 // no longer used with the introduction of loh msl.
940 awr_loh_alloc_during_plan = 10,
942 // we don't allow too much loh allocation during bgc.
943 awr_loh_alloc_during_bgc = 11
946 struct alloc_thread_wait_data
953 mt_get_large_seg = 0,
977 msl_enter_state enter_state;
978 msl_take_state take_state;
979 EEThreadId thread_id;
983 #define HS_CACHE_LINE_SIZE 128
986 struct snoop_stats_data
990 // total number of objects that we called
992 size_t objects_checked_count;
993 // total number of time we called gc_mark
995 size_t zero_ref_count;
996 // total objects actually marked.
997 size_t objects_marked_count;
998 // number of objects written to the mark stack because
1000 size_t stolen_stack_count;
1001 // number of objects pushed onto the mark stack because
1002 // of the partial mark code path.
1003 size_t partial_stack_count;
1004 // number of objects pushed onto the mark stack because
1005 // of the non partial mark code path.
1006 size_t normal_stack_count;
1007 // number of references marked without mark stack.
1008 size_t non_stack_count;
1010 // number of times we detect next heap's mark stack
1012 size_t stack_idle_count;
1014 // number of times we do switch to thread.
1015 size_t switch_to_thread_count;
1017 // number of times we are checking if the next heap's
1018 // mark stack is busy.
1019 size_t check_level_count;
1020 // number of times next stack is busy and level is
1023 // how many interlocked exchange operations we did
1024 size_t interlocked_count;
1025 // numer of times parent objects stolen
1026 size_t partial_mark_parent_count;
1027 // numer of times we look at a normal stolen entry,
1028 // or the beginning/ending PM pair.
1029 size_t stolen_or_pm_count;
1030 // number of times we see 2 for the entry.
1031 size_t stolen_entry_count;
1032 // number of times we see a PM entry that's not ready.
1033 size_t pm_not_ready_count;
1034 // number of stolen normal marked objects and partial mark children.
1035 size_t normal_count;
1036 // number of times the bottom of mark stack was cleared.
1037 size_t stack_bottom_clear_count;
1039 #endif //SNOOP_STATS
1041 struct no_gc_region_info
1043 size_t soh_allocation_size;
1044 size_t loh_allocation_size;
1047 size_t num_gcs_induced;
1048 start_no_gc_region_status start_status;
1049 gc_pause_mode saved_pause_mode;
1050 size_t saved_gen0_min_size;
1051 size_t saved_gen3_min_size;
1055 // if you change these, make sure you update them for sos (strike.cpp) as well.
1058 // Right now I am only recording data from blocking GCs. When recording from BGC,
1059 // it should have its own copy just like gc_data_per_heap.
1060 // for BGCs we will have a very different set of datapoints to record.
1061 enum interesting_data_point
1066 idp_converted_pin = 3,
1069 idp_pre_and_post_pin = 6,
1070 idp_pre_short_padded = 7,
1071 idp_post_short_padded = 8,
1075 //class definition of the internal class
1078 friend class GCHeap;
1079 #ifdef FEATURE_PREMORTEM_FINALIZATION
1080 friend class CFinalize;
1081 #endif // FEATURE_PREMORTEM_FINALIZATION
1082 friend struct ::alloc_context;
1083 friend void ProfScanRootsHelper(Object** object, ScanContext *pSC, uint32_t dwFlags);
1084 friend void GCProfileWalkHeapWorker(BOOL fProfilerPinned, BOOL fShouldWalkHeapRootsForEtw, BOOL fShouldWalkHeapObjectsForEtw);
1085 friend class t_join;
1086 friend class gc_mechanisms;
1087 friend class seg_free_spaces;
1089 #ifdef BACKGROUND_GC
1090 friend class exclusive_sync;
1091 friend class recursive_gc_sync;
1092 #endif //BACKGROUND_GC
1094 #if defined (WRITE_BARRIER_CHECK) && !defined (SERVER_GC)
1095 friend void checkGCWriteBarrier();
1096 friend void initGCShadow();
1097 #endif //defined (WRITE_BARRIER_CHECK) && !defined (SERVER_GC)
1099 friend void PopulateDacVars(GcDacVars *gcDacVars);
1101 #ifdef MULTIPLE_HEAPS
1102 typedef void (gc_heap::* card_fn) (uint8_t**, int);
1103 #define call_fn(fn) (this->*fn)
1106 typedef void (* card_fn) (uint8_t**);
1107 #define call_fn(fn) (*fn)
1108 #define __this (gc_heap*)0
1115 void print_free_list (int gen, heap_segment* seg);
1118 #ifdef SYNCHRONIZATION_STATS
1121 void init_sync_stats()
1123 #ifdef MULTIPLE_HEAPS
1124 for (int i = 0; i < gc_heap::n_heaps; i++)
1126 gc_heap::g_heaps[i]->init_heap_sync_stats();
1128 #else //MULTIPLE_HEAPS
1129 init_heap_sync_stats();
1130 #endif //MULTIPLE_HEAPS
1134 void print_sync_stats(unsigned int gc_count_during_log)
1136 // bad/good gl acquire is accumulative during the log interval (because the numbers are too small)
1137 // min/max msl_acquire is the min/max during the log interval, not each GC.
1138 // Threads is however many allocation threads for the last GC.
1139 // num of msl acquired, avg_msl, high and low are all for each GC.
1140 printf("%2s%2s%10s%10s%12s%6s%4s%8s( st, wl, stw, dpw)\n",
1141 "H", "T", "good_sus", "bad_sus", "avg_msl", "high", "low", "num_msl");
1143 #ifdef MULTIPLE_HEAPS
1144 for (int i = 0; i < gc_heap::n_heaps; i++)
1146 gc_heap::g_heaps[i]->print_heap_sync_stats(i, gc_count_during_log);
1148 #else //MULTIPLE_HEAPS
1149 print_heap_sync_stats(0, gc_count_during_log);
1150 #endif //MULTIPLE_HEAPS
1153 #endif //SYNCHRONIZATION_STATS
1156 void verify_soh_segment_list();
1158 void verify_mark_array_cleared (heap_segment* seg);
1160 void verify_mark_array_cleared();
1162 void verify_seg_end_mark_array_cleared();
1164 void verify_partial();
1168 void verify_free_lists();
1170 void verify_heap (BOOL begin_gc_p);
1171 #endif //VERIFY_HEAP
1174 void fire_per_heap_hist_event (gc_history_per_heap* current_gc_data_per_heap, int heap_num);
1177 void fire_pevents();
1179 #ifdef FEATURE_BASICFREEZE
1180 static void walk_read_only_segment(heap_segment *seg, void *pvContext, object_callback_func pfnMethodTable, object_callback_func pfnObjRef);
1184 heap_segment* make_heap_segment (uint8_t* new_pages,
1188 l_heap* make_large_heap (uint8_t* new_pages, size_t size, BOOL managed);
1191 gc_heap* make_gc_heap(
1192 #if defined (MULTIPLE_HEAPS)
1195 #endif //MULTIPLE_HEAPS
1199 void destroy_gc_heap(gc_heap* heap);
1202 HRESULT initialize_gc (size_t segment_size,
1204 #ifdef MULTIPLE_HEAPS
1205 , unsigned number_of_heaps
1206 #endif //MULTIPLE_HEAPS
1212 // If the hard limit is specified, take that into consideration
1213 // and this means it may modify the # of heaps.
1215 size_t get_segment_size_hard_limit (uint32_t* num_heaps, bool should_adjust_num_heaps);
1218 bool should_retry_other_heap (size_t size);
1221 CObjectHeader* allocate (size_t jsize,
1222 alloc_context* acontext);
1224 #ifdef MULTIPLE_HEAPS
1225 static void balance_heaps (alloc_context* acontext);
1227 gc_heap* balance_heaps_loh (alloc_context* acontext, size_t size);
1229 void gc_thread_stub (void* arg);
1230 #endif //MULTIPLE_HEAPS
1232 // For LOH allocations we only update the alloc_bytes_loh in allocation
1233 // context - we don't actually use the ptr/limit from it so I am
1234 // making this explicit by not passing in the alloc_context.
1236 CObjectHeader* allocate_large_object (size_t size, int64_t& alloc_bytes);
1238 #ifdef FEATURE_STRUCTALIGN
1240 uint8_t* pad_for_alignment_large (uint8_t* newAlloc, int requiredAlignment, size_t size);
1241 #endif // FEATURE_STRUCTALIGN
1250 BOOL expand_soh_with_minimal_gc();
1252 // EE is always suspended when this method is called.
1253 // returning FALSE means we actually didn't do a GC. This happens
1254 // when we figured that we needed to do a BGC.
1256 void garbage_collect (int n);
1258 // Since we don't want to waste a join just to do this, I am doing
1259 // doing this at the last join in gc1.
1261 void pm_full_gc_init_or_clear();
1263 // This does a GC when pm_trigger_full_gc is set
1265 void garbage_collect_pm_full_gc();
1268 bool is_pm_ratio_exceeded();
1271 void init_records();
1274 uint32_t* make_card_table (uint8_t* start, uint8_t* end);
1277 void set_fgm_result (failure_get_memory f, size_t s, BOOL loh_p);
1280 int grow_brick_card_tables (uint8_t* start,
1283 heap_segment* new_seg,
1288 BOOL is_mark_set (uint8_t* o);
1290 #ifdef FEATURE_BASICFREEZE
1292 bool frozen_object_p(Object* obj);
1293 #endif // FEATURE_BASICFREEZE
1298 void walk_heap (walk_fn fn, void* context, int gen_number, BOOL walk_large_object_heap_p);
1301 void walk_heap_per_heap (walk_fn fn, void* context, int gen_number, BOOL walk_large_object_heap_p);
1303 struct walk_relocate_args
1307 mark* pinned_plug_entry;
1308 void* profiling_context;
1313 void walk_survivors (record_surv_fn fn, void* context, walk_surv_type type);
1316 void walk_plug (uint8_t* plug, size_t size, BOOL check_last_object_p,
1317 walk_relocate_args* args);
1320 void walk_relocation (void* profiling_context, record_surv_fn fn);
1323 void walk_relocation_in_brick (uint8_t* tree, walk_relocate_args* args);
1326 void walk_finalize_queue (fq_walk_fn fn);
1328 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
1330 void walk_survivors_for_bgc (void* profiling_context, record_surv_fn fn);
1331 #endif // defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
1333 // used in blocking GCs after plan phase so this walks the plugs.
1335 void walk_survivors_relocation (void* profiling_context, record_surv_fn fn);
1337 void walk_survivors_for_loh (void* profiling_context, record_surv_fn fn);
1340 int generation_to_condemn (int n,
1341 BOOL* blocking_collection_p,
1342 BOOL* elevation_requested_p,
1346 int joined_generation_to_condemn (BOOL should_evaluate_elevation,
1349 BOOL* blocking_collection
1350 STRESS_HEAP_ARG(int n_original));
1353 size_t min_reclaim_fragmentation_threshold (uint32_t num_heaps);
1356 uint64_t min_high_fragmentation_threshold (uint64_t available_mem, uint32_t num_heaps);
1359 void concurrent_print_time_delta (const char* msg);
1361 void free_list_info (int gen_num, const char* msg);
1363 // in svr GC on entry and exit of this method, the GC threads are not
1369 void save_data_for_no_gc();
1372 void restore_data_for_no_gc();
1375 void update_collection_counts_for_no_gc();
1378 BOOL should_proceed_with_gc();
1381 void record_gcs_during_no_gc();
1384 BOOL find_loh_free_for_no_gc();
1387 BOOL find_loh_space_for_no_gc();
1390 BOOL commit_loh_for_no_gc (heap_segment* seg);
1393 start_no_gc_region_status prepare_for_no_gc_region (uint64_t total_size,
1394 BOOL loh_size_known,
1396 BOOL disallow_full_blocking);
1399 BOOL loh_allocated_for_no_gc();
1402 void release_no_gc_loh_segments();
1405 void thread_no_gc_loh_segments();
1408 void check_and_set_no_gc_oom();
1411 void allocate_for_no_gc_after_gc();
1414 void set_loh_allocations_for_no_gc();
1417 void set_soh_allocations_for_no_gc();
1420 void prepare_for_no_gc_after_gc();
1423 void set_allocations_for_no_gc();
1426 BOOL should_proceed_for_no_gc();
1429 start_no_gc_region_status get_start_no_gc_region_status();
1432 end_no_gc_region_status end_no_gc_region();
1435 void handle_failure_for_no_gc();
1438 void fire_etw_allocation_event (size_t allocation_amount, int gen_number, uint8_t* object_address);
1441 void fire_etw_pin_object_event (uint8_t* object, uint8_t** ppObject);
1444 size_t limit_from_size (size_t size, size_t room, int gen_number,
1447 allocation_state try_allocate_more_space (alloc_context* acontext, size_t jsize,
1448 int alloc_generation_number);
1450 BOOL allocate_more_space (alloc_context* acontext, size_t jsize,
1451 int alloc_generation_number);
1454 size_t get_full_compact_gc_count();
1457 BOOL short_on_end_of_seg (int gen_number,
1462 BOOL a_fit_free_list_p (int gen_number,
1464 alloc_context* acontext,
1467 #ifdef BACKGROUND_GC
1469 void wait_for_background (alloc_wait_reason awr, bool loh_p);
1472 void wait_for_bgc_high_memory (alloc_wait_reason awr, bool loh_p);
1475 void bgc_loh_alloc_clr (uint8_t* alloc_start,
1477 alloc_context* acontext,
1482 #endif //BACKGROUND_GC
1484 #ifdef BACKGROUND_GC
1486 void bgc_track_loh_alloc();
1489 void bgc_untrack_loh_alloc();
1492 BOOL bgc_loh_should_allocate();
1493 #endif //BACKGROUND_GC
1495 #define max_saved_spinlock_info 48
1497 #ifdef SPINLOCK_HISTORY
1499 int spinlock_info_index;
1502 spinlock_info last_spinlock_info[max_saved_spinlock_info + 8];
1503 #endif //SPINLOCK_HISTORY
1506 void add_saved_spinlock_info (
1508 msl_enter_state enter_state,
1509 msl_take_state take_state);
1512 void trigger_gc_for_alloc (int gen_number, gc_reason reason,
1513 GCSpinLock* spin_lock, bool loh_p,
1514 msl_take_state take_state);
1517 BOOL a_fit_free_list_large_p (size_t size,
1518 alloc_context* acontext,
1522 BOOL a_fit_segment_end_p (int gen_number,
1525 alloc_context* acontext,
1527 BOOL* commit_failed_p);
1529 BOOL loh_a_fit_segment_end_p (int gen_number,
1531 alloc_context* acontext,
1533 BOOL* commit_failed_p,
1536 BOOL loh_get_new_seg (generation* gen,
1539 BOOL* commit_failed_p,
1543 size_t get_large_seg_size (size_t size);
1546 BOOL retry_full_compact_gc (size_t size);
1549 BOOL check_and_wait_for_bgc (alloc_wait_reason awr,
1550 BOOL* did_full_compact_gc,
1554 BOOL trigger_full_compact_gc (gc_reason gr,
1559 BOOL trigger_ephemeral_gc (gc_reason gr);
1562 BOOL soh_try_fit (int gen_number,
1564 alloc_context* acontext,
1566 BOOL* commit_failed_p,
1567 BOOL* short_seg_end_p);
1569 BOOL loh_try_fit (int gen_number,
1571 alloc_context* acontext,
1573 BOOL* commit_failed_p,
1577 allocation_state allocate_small (int gen_number,
1579 alloc_context* acontext,
1582 #ifdef RECORD_LOH_STATE
1583 #define max_saved_loh_states 12
1585 int loh_state_index;
1587 struct loh_state_info
1589 allocation_state alloc_state;
1590 EEThreadId thread_id;
1594 loh_state_info last_loh_states[max_saved_loh_states];
1596 void add_saved_loh_state (allocation_state loh_state_to_save, EEThreadId thread_id);
1597 #endif //RECORD_LOH_STATE
1599 allocation_state allocate_large (int gen_number,
1601 alloc_context* acontext,
1605 int init_semi_shared();
1607 int init_gc_heap (int heap_number);
1609 void self_destroy();
1611 void destroy_semi_shared();
1613 void repair_allocation_contexts (BOOL repair_p);
1615 void fix_allocation_contexts (BOOL for_gc_p);
1617 void fix_youngest_allocation_area (BOOL for_gc_p);
1619 void fix_allocation_context (alloc_context* acontext, BOOL for_gc_p,
1622 void fix_large_allocation_area (BOOL for_gc_p);
1624 void fix_older_allocation_area (generation* older_gen);
1626 void set_allocation_heap_segment (generation* gen);
1628 void reset_allocation_pointers (generation* gen, uint8_t* start);
1630 int object_gennum (uint8_t* o);
1632 int object_gennum_plan (uint8_t* o);
1634 void init_heap_segment (heap_segment* seg);
1636 void delete_heap_segment (heap_segment* seg, BOOL consider_hoarding=FALSE);
1637 #ifdef FEATURE_BASICFREEZE
1639 BOOL insert_ro_segment (heap_segment* seg);
1641 void remove_ro_segment (heap_segment* seg);
1642 #endif //FEATURE_BASICFREEZE
1644 BOOL set_ro_segment_in_range (heap_segment* seg);
1646 BOOL unprotect_segment (heap_segment* seg);
1648 heap_segment* soh_get_segment_to_expand();
1650 heap_segment* get_segment (size_t size, BOOL loh_p);
1652 void seg_mapping_table_add_segment (heap_segment* seg, gc_heap* hp);
1654 void seg_mapping_table_remove_segment (heap_segment* seg);
1656 heap_segment* get_large_segment (size_t size, BOOL* did_full_compact_gc);
1658 void thread_loh_segment (heap_segment* new_seg);
1660 heap_segment* get_segment_for_loh (size_t size
1661 #ifdef MULTIPLE_HEAPS
1663 #endif //MULTIPLE_HEAPS
1666 void reset_heap_segment_pages (heap_segment* seg);
1668 void decommit_heap_segment_pages (heap_segment* seg, size_t extra_space);
1670 void decommit_heap_segment (heap_segment* seg);
1672 bool virtual_alloc_commit_for_heap (void* addr, size_t size, int h_number);
1674 bool virtual_commit (void* address, size_t size, int h_number=-1, bool* hard_limit_exceeded_p=NULL);
1676 bool virtual_decommit (void* address, size_t size, int h_number=-1);
1678 void clear_gen0_bricks();
1679 #ifdef BACKGROUND_GC
1681 void rearrange_small_heap_segments();
1682 #endif //BACKGROUND_GC
1684 void rearrange_large_heap_segments();
1686 void rearrange_heap_segments(BOOL compacting);
1689 void reset_write_watch_for_gc_heap(void* base_address, size_t region_size);
1691 void get_write_watch_for_gc_heap(bool reset, void *base_address, size_t region_size, void** dirty_pages, uintptr_t* dirty_page_count_ref, bool is_runtime_suspended);
1694 void switch_one_quantum();
1696 void reset_ww_by_chunk (uint8_t* start_address, size_t total_reset_size);
1698 void switch_on_reset (BOOL concurrent_p, size_t* current_total_reset_size, size_t last_reset_size);
1700 void reset_write_watch (BOOL concurrent_p);
1702 void adjust_ephemeral_limits();
1704 void make_generation (generation& gen, heap_segment* seg,
1705 uint8_t* start, uint8_t* pointer);
1708 #define USE_PADDING_FRONT 1
1709 #define USE_PADDING_TAIL 2
1712 BOOL size_fit_p (size_t size REQD_ALIGN_AND_OFFSET_DCL, uint8_t* alloc_pointer, uint8_t* alloc_limit,
1713 uint8_t* old_loc=0, int use_padding=USE_PADDING_TAIL);
1715 BOOL a_size_fit_p (size_t size, uint8_t* alloc_pointer, uint8_t* alloc_limit,
1719 void handle_oom (int heap_num, oom_reason reason, size_t alloc_size,
1720 uint8_t* allocated, uint8_t* reserved);
1723 size_t card_of ( uint8_t* object);
1725 uint8_t* brick_address (size_t brick);
1727 size_t brick_of (uint8_t* add);
1729 uint8_t* card_address (size_t card);
1731 size_t card_to_brick (size_t card);
1733 void clear_card (size_t card);
1735 void set_card (size_t card);
1737 BOOL card_set_p (size_t card);
1739 void card_table_set_bit (uint8_t* location);
1743 void update_card_table_bundle();
1745 void reset_card_table_write_watch();
1747 void card_bundle_clear(size_t cardb);
1749 void card_bundle_set (size_t cardb);
1751 void card_bundles_set (size_t start_cardb, size_t end_cardb);
1753 void verify_card_bundle_bits_set(size_t first_card_word, size_t last_card_word);
1755 void verify_card_bundles();
1757 BOOL card_bundle_set_p (size_t cardb);
1759 BOOL find_card_dword (size_t& cardw, size_t cardw_end);
1761 void enable_card_bundles();
1763 BOOL card_bundles_enabled();
1765 #endif //CARD_BUNDLE
1768 BOOL find_card (uint32_t* card_table, size_t& card,
1769 size_t card_word_end, size_t& end_card);
1771 BOOL grow_heap_segment (heap_segment* seg, uint8_t* high_address, bool* hard_limit_exceeded_p=NULL);
1773 int grow_heap_segment (heap_segment* seg, uint8_t* high_address, uint8_t* old_loc, size_t size, BOOL pad_front_p REQD_ALIGN_AND_OFFSET_DCL);
1775 void copy_brick_card_range (uint8_t* la, uint32_t* old_card_table,
1776 short* old_brick_table,
1778 uint8_t* start, uint8_t* end);
1780 void init_brick_card_range (heap_segment* seg);
1782 void copy_brick_card_table_l_heap ();
1784 void copy_brick_card_table();
1786 void clear_brick_table (uint8_t* from, uint8_t* end);
1788 void set_brick (size_t index, ptrdiff_t val);
1790 int get_brick_entry (size_t index);
1793 unsigned int mark_array_marked (uint8_t* add);
1795 void mark_array_set_marked (uint8_t* add);
1797 BOOL is_mark_bit_set (uint8_t* add);
1799 void gmark_array_set_marked (uint8_t* add);
1801 void set_mark_array_bit (size_t mark_bit);
1803 BOOL mark_array_bit_set (size_t mark_bit);
1805 void mark_array_clear_marked (uint8_t* add);
1807 void clear_mark_array (uint8_t* from, uint8_t* end, BOOL check_only=TRUE
1808 #ifdef FEATURE_BASICFREEZE
1809 , BOOL read_only=FALSE
1810 #endif // FEATURE_BASICFREEZE
1812 #ifdef BACKGROUND_GC
1814 void seg_clear_mark_array_bits_soh (heap_segment* seg);
1816 void clear_batch_mark_array_bits (uint8_t* start, uint8_t* end);
1818 void bgc_clear_batch_mark_array_bits (uint8_t* start, uint8_t* end);
1820 void clear_mark_array_by_objects (uint8_t* from, uint8_t* end, BOOL loh_p);
1823 void set_batch_mark_array_bits (uint8_t* start, uint8_t* end);
1825 void check_batch_mark_array_bits (uint8_t* start, uint8_t* end);
1826 #endif //VERIFY_HEAP
1827 #endif //BACKGROUND_GC
1831 BOOL large_object_marked (uint8_t* o, BOOL clearp);
1833 #ifdef BACKGROUND_GC
1835 BOOL background_allowed_p();
1836 #endif //BACKGROUND_GC
1839 void send_full_gc_notification (int gen_num, BOOL due_to_alloc_p);
1842 void check_for_full_gc (int gen_num, size_t size);
1845 void adjust_limit (uint8_t* start, size_t limit_size, generation* gen,
1848 void adjust_limit_clr (uint8_t* start, size_t limit_size,
1849 alloc_context* acontext, heap_segment* seg,
1850 int align_const, int gen_number);
1852 void leave_allocation_segment (generation* gen);
1855 void init_free_and_plug();
1858 void print_free_and_plug (const char* msg);
1861 void add_gen_plug (int gen_number, size_t plug_size);
1864 void add_gen_free (int gen_number, size_t free_size);
1867 void add_item_to_current_pinned_free (int gen_number, size_t free_size);
1870 void remove_gen_free (int gen_number, size_t free_size);
1873 uint8_t* allocate_in_older_generation (generation* gen, size_t size,
1874 int from_gen_number,
1876 REQD_ALIGN_AND_OFFSET_DEFAULT_DCL);
1878 generation* ensure_ephemeral_heap_segment (generation* consing_gen);
1880 uint8_t* allocate_in_condemned_generations (generation* gen,
1882 int from_gen_number,
1884 BOOL* convert_to_pinned_p=NULL,
1885 uint8_t* next_pinned_plug=0,
1886 heap_segment* current_seg=0,
1887 #endif //SHORT_PLUGS
1889 REQD_ALIGN_AND_OFFSET_DEFAULT_DCL);
1890 #ifdef INTERIOR_POINTERS
1891 // Verifies that interior is actually in the range of seg; otherwise
1894 heap_segment* find_segment (uint8_t* interior, BOOL small_segment_only_p);
1897 heap_segment* find_segment_per_heap (uint8_t* interior, BOOL small_segment_only_p);
1900 uint8_t* find_object_for_relocation (uint8_t* o, uint8_t* low, uint8_t* high);
1901 #endif //INTERIOR_POINTERS
1904 gc_heap* heap_of (uint8_t* object);
1907 gc_heap* heap_of_gc (uint8_t* object);
1910 size_t& promoted_bytes (int);
1913 uint8_t* find_object (uint8_t* o, uint8_t* low);
1916 dynamic_data* dynamic_data_of (int gen_number);
1918 ptrdiff_t get_desired_allocation (int gen_number);
1920 ptrdiff_t get_new_allocation (int gen_number);
1922 ptrdiff_t get_allocation (int gen_number);
1924 bool new_allocation_allowed (int gen_number);
1925 #ifdef BACKGROUND_GC
1927 void allow_new_allocation (int gen_number);
1929 void disallow_new_allocation (int gen_number);
1930 #endif //BACKGROUND_GC
1932 void reset_pinned_queue();
1934 void reset_pinned_queue_bos();
1936 void set_allocator_next_pin (generation* gen);
1938 void set_allocator_next_pin (uint8_t* alloc_pointer, uint8_t*& alloc_limit);
1940 void enque_pinned_plug (generation* gen, uint8_t* plug, size_t len);
1942 void enque_pinned_plug (uint8_t* plug,
1943 BOOL save_pre_plug_info_p,
1944 uint8_t* last_object_in_last_plug);
1946 void merge_with_last_pinned_plug (uint8_t* last_pinned_plug, size_t plug_size);
1948 void set_pinned_info (uint8_t* last_pinned_plug,
1950 uint8_t* alloc_pointer,
1951 uint8_t*& alloc_limit);
1953 void set_pinned_info (uint8_t* last_pinned_plug, size_t plug_len, generation* gen);
1955 void save_post_plug_info (uint8_t* last_pinned_plug, uint8_t* last_object_in_last_plug, uint8_t* post_plug);
1957 size_t deque_pinned_plug ();
1959 mark* pinned_plug_of (size_t bos);
1961 mark* oldest_pin ();
1963 mark* before_oldest_pin();
1965 BOOL pinned_plug_que_empty_p ();
1967 void make_mark_stack (mark* arr);
1970 int& mark_stack_busy();
1972 VOLATILE(uint8_t*)& ref_mark_stack (gc_heap* hp, int index);
1974 #ifdef BACKGROUND_GC
1976 size_t& bpromoted_bytes (int);
1978 void make_background_mark_stack (uint8_t** arr);
1980 void make_c_mark_list (uint8_t** arr);
1981 #endif //BACKGROUND_GC
1983 generation* generation_of (int n);
1985 BOOL gc_mark1 (uint8_t* o);
1987 BOOL gc_mark (uint8_t* o, uint8_t* low, uint8_t* high);
1989 uint8_t* mark_object(uint8_t* o THREAD_NUMBER_DCL);
1992 void ha_mark_object_simple (uint8_t** o THREAD_NUMBER_DCL);
1993 #endif //HEAP_ANALYZE
1995 void mark_object_simple (uint8_t** o THREAD_NUMBER_DCL);
1997 void mark_object_simple1 (uint8_t* o, uint8_t* start THREAD_NUMBER_DCL);
2004 #ifdef BACKGROUND_GC
2007 BOOL background_marked (uint8_t* o);
2009 BOOL background_mark1 (uint8_t* o);
2011 BOOL background_mark (uint8_t* o, uint8_t* low, uint8_t* high);
2013 uint8_t* background_mark_object (uint8_t* o THREAD_NUMBER_DCL);
2015 void background_mark_simple (uint8_t* o THREAD_NUMBER_DCL);
2017 void background_mark_simple1 (uint8_t* o THREAD_NUMBER_DCL);
2019 void background_promote (Object**, ScanContext* , uint32_t);
2021 BOOL background_object_marked (uint8_t* o, BOOL clearp);
2023 void init_background_gc();
2025 uint8_t* background_next_end (heap_segment*, BOOL);
2026 // while we are in LOH sweep we can't modify the segment list
2027 // there so we mark them as to be deleted and deleted them
2028 // at the next chance we get.
2030 void background_delay_delete_loh_segments();
2032 void generation_delete_heap_segment (generation*,
2033 heap_segment*, heap_segment*, heap_segment*);
2035 void set_mem_verify (uint8_t*, uint8_t*, uint8_t);
2037 void process_background_segment_end (heap_segment*, generation*, uint8_t*,
2038 heap_segment*, BOOL*);
2040 void process_n_background_segments (heap_segment*, heap_segment*, generation* gen);
2042 BOOL fgc_should_consider_object (uint8_t* o,
2044 BOOL consider_bgc_mark_p,
2045 BOOL check_current_sweep_p,
2046 BOOL check_saved_sweep_p);
2048 void should_check_bgc_mark (heap_segment* seg,
2049 BOOL* consider_bgc_mark_p,
2050 BOOL* check_current_sweep_p,
2051 BOOL* check_saved_sweep_p);
2053 void background_ephemeral_sweep();
2055 void background_sweep ();
2057 void background_mark_through_object (uint8_t* oo THREAD_NUMBER_DCL);
2059 uint8_t* background_seg_end (heap_segment* seg, BOOL concurrent_p);
2061 uint8_t* background_first_overflow (uint8_t* min_add,
2064 BOOL small_object_p);
2066 void background_process_mark_overflow_internal (int condemned_gen_number,
2067 uint8_t* min_add, uint8_t* max_add,
2070 BOOL background_process_mark_overflow (BOOL concurrent_p);
2072 // for foreground GC to get hold of background structures containing refs
2075 scan_background_roots (promote_func* fn, int hn, ScanContext *pSC);
2078 BOOL bgc_mark_array_range (heap_segment* seg,
2080 uint8_t** range_beg,
2081 uint8_t** range_end);
2083 void bgc_verify_mark_array_cleared (heap_segment* seg);
2085 void verify_mark_bits_cleared (uint8_t* obj, size_t s);
2087 void clear_all_mark_array();
2088 #endif //BACKGROUND_GC
2091 uint8_t* next_end (heap_segment* seg, uint8_t* f);
2093 void mark_through_object (uint8_t* oo, BOOL mark_class_object_p THREAD_NUMBER_DCL);
2095 BOOL process_mark_overflow (int condemned_gen_number);
2097 void process_mark_overflow_internal (int condemned_gen_number,
2098 uint8_t* min_address, uint8_t* max_address);
2102 void print_snoop_stat();
2103 #endif //SNOOP_STATS
2108 BOOL check_next_mark_stack (gc_heap* next_heap);
2113 void scan_dependent_handles (int condemned_gen_number, ScanContext *sc, BOOL initial_scan_p);
2116 void mark_phase (int condemned_gen_number, BOOL mark_only_p);
2119 void pin_object (uint8_t* o, uint8_t** ppObject, uint8_t* low, uint8_t* high);
2121 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
2123 size_t get_total_pinned_objects();
2124 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
2127 void reset_mark_stack ();
2129 uint8_t* insert_node (uint8_t* new_node, size_t sequence_number,
2130 uint8_t* tree, uint8_t* last_node);
2132 size_t update_brick_table (uint8_t* tree, size_t current_brick,
2133 uint8_t* x, uint8_t* plug_end);
2136 void plan_generation_start (generation* gen, generation* consing_gen, uint8_t* next_plug_to_allocate);
2139 void realloc_plan_generation_start (generation* gen, generation* consing_gen);
2142 void plan_generation_starts (generation*& consing_gen);
2145 void advance_pins_for_demotion (generation* gen);
2148 void process_ephemeral_boundaries(uint8_t* x, int& active_new_gen_number,
2149 int& active_old_gen_number,
2150 generation*& consing_gen,
2151 BOOL& allocate_in_condemned);
2153 void seg_clear_mark_bits (heap_segment* seg);
2155 void sweep_ro_segments (heap_segment* start_seg);
2157 void convert_to_pinned_plug (BOOL& last_npinned_plug_p,
2158 BOOL& last_pinned_plug_p,
2159 BOOL& pinned_plug_p,
2161 size_t& artificial_pinned_size);
2163 void store_plug_gap_info (uint8_t* plug_start,
2165 BOOL& last_npinned_plug_p,
2166 BOOL& last_pinned_plug_p,
2167 uint8_t*& last_pinned_plug,
2168 BOOL& pinned_plug_p,
2169 uint8_t* last_object_in_last_plug,
2170 BOOL& merge_with_last_pin_p,
2171 // this is only for verification purpose
2172 size_t last_plug_len);
2174 void plan_phase (int condemned_gen_number);
2177 void record_interesting_data_point (interesting_data_point idp);
2179 #ifdef GC_CONFIG_DRIVEN
2181 void record_interesting_info_per_heap();
2183 void record_global_mechanisms();
2185 BOOL should_do_sweeping_gc (BOOL compact_p);
2186 #endif //GC_CONFIG_DRIVEN
2188 #ifdef FEATURE_LOH_COMPACTION
2189 // plan_loh can allocate memory so it can fail. If it fails, we will
2190 // fall back to sweeping.
2198 void relocate_in_loh_compact();
2201 void walk_relocation_for_loh (void* profiling_context, record_surv_fn fn);
2204 BOOL loh_enque_pinned_plug (uint8_t* plug, size_t len);
2207 void loh_set_allocator_next_pin();
2210 BOOL loh_pinned_plug_que_empty_p();
2213 size_t loh_deque_pinned_plug();
2216 mark* loh_pinned_plug_of (size_t bos);
2219 mark* loh_oldest_pin();
2222 BOOL loh_size_fit_p (size_t size, uint8_t* alloc_pointer, uint8_t* alloc_limit);
2225 uint8_t* loh_allocate_in_condemned (uint8_t* old_loc, size_t size);
2228 BOOL loh_object_p (uint8_t* o);
2231 BOOL loh_compaction_requested();
2233 // If the LOH compaction mode is just to compact once,
2234 // we need to see if we should reset it back to not compact.
2235 // We would only reset if every heap's LOH was compacted.
2237 void check_loh_compact_mode (BOOL all_heaps_compacted_p);
2238 #endif //FEATURE_LOH_COMPACTION
2241 void decommit_ephemeral_segment_pages (int condemned_gen_number);
2243 void fix_generation_bounds (int condemned_gen_number,
2244 generation* consing_gen);
2246 uint8_t* generation_limit (int gen_number);
2248 struct make_free_args
2250 int free_list_gen_number;
2251 uint8_t* current_gen_limit;
2252 generation* free_list_gen;
2253 uint8_t* highest_plug;
2256 uint8_t* allocate_at_end (size_t size);
2258 BOOL ensure_gap_allocation (int condemned_gen_number);
2259 // make_free_lists is only called by blocking GCs.
2261 void make_free_lists (int condemned_gen_number);
2263 void make_free_list_in_brick (uint8_t* tree, make_free_args* args);
2265 void thread_gap (uint8_t* gap_start, size_t size, generation* gen);
2267 void loh_thread_gap_front (uint8_t* gap_start, size_t size, generation* gen);
2269 void make_unused_array (uint8_t* x, size_t size, BOOL clearp=FALSE, BOOL resetp=FALSE);
2271 void clear_unused_array (uint8_t* x, size_t size);
2273 void relocate_address (uint8_t** old_address THREAD_NUMBER_DCL);
2274 struct relocate_args
2280 mark* pinned_plug_entry;
2284 void reloc_survivor_helper (uint8_t** pval);
2286 void check_class_object_demotion (uint8_t* obj);
2288 void check_class_object_demotion_internal (uint8_t* obj);
2291 void check_demotion_helper (uint8_t** pval, uint8_t* parent_obj);
2294 void relocate_survivor_helper (uint8_t* plug, uint8_t* plug_end);
2297 void verify_pins_with_post_plug_info (const char* msg);
2299 #ifdef COLLECTIBLE_CLASS
2301 void unconditional_set_card_collectible (uint8_t* obj);
2302 #endif //COLLECTIBLE_CLASS
2305 void relocate_shortened_survivor_helper (uint8_t* plug, uint8_t* plug_end, mark* pinned_plug_entry);
2308 void relocate_obj_helper (uint8_t* x, size_t s);
2311 void reloc_ref_in_shortened_obj (uint8_t** address_to_set_card, uint8_t** address_to_reloc);
2314 void relocate_pre_plug_info (mark* pinned_plug_entry);
2317 void relocate_shortened_obj_helper (uint8_t* x, size_t s, uint8_t* end, mark* pinned_plug_entry, BOOL is_pinned);
2320 void relocate_survivors_in_plug (uint8_t* plug, uint8_t* plug_end,
2321 BOOL check_last_object_p,
2322 mark* pinned_plug_entry);
2324 void relocate_survivors_in_brick (uint8_t* tree, relocate_args* args);
2327 void update_oldest_pinned_plug();
2330 void relocate_survivors (int condemned_gen_number,
2331 uint8_t* first_condemned_address );
2333 void relocate_phase (int condemned_gen_number,
2334 uint8_t* first_condemned_address);
2340 ptrdiff_t last_plug_relocation;
2341 uint8_t* before_last_plug;
2342 size_t current_compacted_brick;
2344 mark* pinned_plug_entry;
2345 BOOL check_gennum_p;
2350 dprintf (3, ("last plug: %Ix, last plug reloc: %Ix, before last: %Ix, b: %Ix",
2351 last_plug, last_plug_relocation, before_last_plug, current_compacted_brick));
2356 void copy_cards_range (uint8_t* dest, uint8_t* src, size_t len, BOOL copy_cards_p);
2358 void gcmemcopy (uint8_t* dest, uint8_t* src, size_t len, BOOL copy_cards_p);
2360 void compact_plug (uint8_t* plug, size_t size, BOOL check_last_object_p, compact_args* args);
2362 void compact_in_brick (uint8_t* tree, compact_args* args);
2365 mark* get_next_pinned_entry (uint8_t* tree,
2366 BOOL* has_pre_plug_info_p,
2367 BOOL* has_post_plug_info_p,
2371 mark* get_oldest_pinned_entry (BOOL* has_pre_plug_info_p, BOOL* has_post_plug_info_p);
2374 void recover_saved_pinned_info();
2377 void compact_phase (int condemned_gen_number, uint8_t*
2378 first_condemned_address, BOOL clear_cards);
2380 void clear_cards (size_t start_card, size_t end_card);
2382 void clear_card_for_addresses (uint8_t* start_address, uint8_t* end_address);
2384 void copy_cards (size_t dst_card, size_t src_card,
2385 size_t end_card, BOOL nextp);
2387 void copy_cards_for_addresses (uint8_t* dest, uint8_t* src, size_t len);
2389 #ifdef BACKGROUND_GC
2391 void copy_mark_bits (size_t dst_mark_bit, size_t src_mark_bit, size_t end_mark_bit);
2393 void copy_mark_bits_for_addresses (uint8_t* dest, uint8_t* src, size_t len);
2394 #endif //BACKGROUND_GC
2398 BOOL ephemeral_pointer_p (uint8_t* o);
2400 void fix_brick_to_highest (uint8_t* o, uint8_t* next_o);
2402 uint8_t* find_first_object (uint8_t* start_address, uint8_t* first_object);
2404 uint8_t* compute_next_boundary (uint8_t* low, int gen_number, BOOL relocating);
2406 void keep_card_live (uint8_t* o, size_t& n_gen,
2407 size_t& cg_pointers_found);
2409 void mark_through_cards_helper (uint8_t** poo, size_t& ngen,
2410 size_t& cg_pointers_found,
2411 card_fn fn, uint8_t* nhigh,
2412 uint8_t* next_boundary);
2415 BOOL card_transition (uint8_t* po, uint8_t* end, size_t card_word_end,
2416 size_t& cg_pointers_found,
2417 size_t& n_eph, size_t& n_card_set,
2418 size_t& card, size_t& end_card,
2419 BOOL& foundp, uint8_t*& start_address,
2420 uint8_t*& limit, size_t& n_cards_cleared);
2422 void mark_through_cards_for_segments (card_fn fn, BOOL relocating);
2425 void repair_allocation_in_expanded_heap (generation* gen);
2427 BOOL can_fit_in_spaces_p (size_t* ordered_blocks, int small_index, size_t* ordered_spaces, int big_index);
2429 BOOL can_fit_blocks_p (size_t* ordered_blocks, int block_index, size_t* ordered_spaces, int* space_index);
2431 BOOL can_fit_all_blocks_p (size_t* ordered_blocks, size_t* ordered_spaces, int count);
2432 #ifdef SEG_REUSE_STATS
2434 size_t dump_buckets (size_t* ordered_indices, int count, size_t* total_size);
2435 #endif //SEG_REUSE_STATS
2437 void build_ordered_free_spaces (heap_segment* seg);
2439 void count_plug (size_t last_plug_size, uint8_t*& last_plug);
2441 void count_plugs_in_brick (uint8_t* tree, uint8_t*& last_plug);
2443 void build_ordered_plug_indices ();
2445 void init_ordered_free_space_indices ();
2447 void trim_free_spaces_indices ();
2449 BOOL try_best_fit (BOOL end_of_segment_p);
2451 BOOL best_fit (size_t free_space, size_t largest_free_space, size_t additional_space, BOOL* use_additional_space);
2453 BOOL process_free_space (heap_segment* seg,
2455 size_t min_free_size,
2456 size_t min_cont_size,
2457 size_t* total_free_space,
2458 size_t* largest_free_space);
2460 size_t compute_eph_gen_starts_size();
2462 void compute_new_ephemeral_size();
2464 BOOL expand_reused_seg_p();
2466 BOOL can_expand_into_p (heap_segment* seg, size_t min_free_size,
2467 size_t min_cont_size, allocator* al);
2469 uint8_t* allocate_in_expanded_heap (generation* gen, size_t size,
2470 BOOL& adjacentp, uint8_t* old_loc,
2472 BOOL set_padding_on_saved_p,
2473 mark* pinned_plug_entry,
2474 #endif //SHORT_PLUGS
2475 BOOL consider_bestfit, int active_new_gen_number
2476 REQD_ALIGN_AND_OFFSET_DEFAULT_DCL);
2478 void realloc_plug (size_t last_plug_size, uint8_t*& last_plug,
2479 generation* gen, uint8_t* start_address,
2480 unsigned int& active_new_gen_number,
2481 uint8_t*& last_pinned_gap, BOOL& leftp,
2484 , mark* pinned_plug_entry
2485 #endif //SHORT_PLUGS
2488 void realloc_in_brick (uint8_t* tree, uint8_t*& last_plug, uint8_t* start_address,
2490 unsigned int& active_new_gen_number,
2491 uint8_t*& last_pinned_gap, BOOL& leftp);
2493 void realloc_plugs (generation* consing_gen, heap_segment* seg,
2494 uint8_t* start_address, uint8_t* end_address,
2495 unsigned active_new_gen_number);
2498 void set_expand_in_full_gc (int condemned_gen_number);
2501 void verify_no_pins (uint8_t* start, uint8_t* end);
2504 generation* expand_heap (int condemned_generation,
2505 generation* consing_gen,
2506 heap_segment* new_heap_segment);
2509 void save_ephemeral_generation_starts();
2512 size_t get_gen0_min_size();
2515 void set_static_data();
2518 void init_static_data();
2521 bool init_dynamic_data ();
2523 float surv_to_growth (float cst, float limit, float max_limit);
2525 size_t desired_new_allocation (dynamic_data* dd, size_t out,
2526 int gen_number, int pass);
2529 void trim_youngest_desired_low_memory();
2532 void decommit_ephemeral_segment_pages();
2536 size_t trim_youngest_desired (uint32_t memory_load,
2537 size_t total_new_allocation,
2538 size_t total_min_allocation);
2540 size_t joined_youngest_desired (size_t new_allocation);
2543 size_t get_total_heap_size ();
2545 size_t get_total_committed_size();
2547 size_t get_total_fragmentation();
2549 size_t get_total_gen_fragmentation (int gen_number);
2551 size_t get_total_gen_estimated_reclaim (int gen_number);
2553 void get_memory_info (uint32_t* memory_load,
2554 uint64_t* available_physical=NULL,
2555 uint64_t* available_page_file=NULL);
2557 size_t generation_size (int gen_number);
2559 size_t get_total_survived_size();
2560 // this also resets allocated_since_last_gc
2562 size_t get_total_allocated_since_last_gc();
2564 size_t get_current_allocated();
2566 size_t get_total_allocated();
2568 size_t current_generation_size (int gen_number);
2570 size_t generation_plan_size (int gen_number);
2572 void compute_promoted_allocation (int gen_number);
2574 size_t compute_in (int gen_number);
2576 void compute_new_dynamic_data (int gen_number);
2578 gc_history_per_heap* get_gc_data_per_heap();
2580 size_t new_allocation_limit (size_t size, size_t free_size, int gen_number);
2582 size_t generation_fragmentation (generation* gen,
2583 generation* consing_gen,
2586 size_t generation_sizes (generation* gen);
2588 size_t committed_size();
2590 size_t committed_size (bool loh_p, size_t* allocated);
2592 size_t approximate_new_allocation();
2594 size_t end_space_after_gc();
2596 size_t estimated_reclaim (int gen_number);
2598 BOOL decide_on_compacting (int condemned_gen_number,
2599 size_t fragmentation,
2600 BOOL& should_expand);
2602 BOOL sufficient_space_end_seg (uint8_t* start, uint8_t* seg_end,
2603 size_t end_space_required,
2604 gc_tuning_point tp);
2606 BOOL ephemeral_gen_fit_p (gc_tuning_point tp);
2608 void reset_large_object (uint8_t* o);
2610 void sweep_large_objects ();
2612 void relocate_in_large_objects ();
2614 void mark_through_cards_for_large_objects (card_fn fn, BOOL relocating);
2616 void descr_segment (heap_segment* seg);
2618 void descr_generations (BOOL begin_gc_p);
2621 void descr_generations_to_profiler (gen_walk_fn fn, void *context);
2623 /*------------ Multiple non isolated heaps ----------------*/
2624 #ifdef MULTIPLE_HEAPS
2626 BOOL create_thread_support (unsigned number_of_heaps);
2628 void destroy_thread_support ();
2630 bool create_gc_thread();
2632 void gc_thread_function();
2634 #ifdef PARALLEL_MARK_LIST_SORT
2636 void sort_mark_list();
2638 void merge_mark_lists();
2640 void append_to_mark_list(uint8_t **start, uint8_t **end);
2641 #else //PARALLEL_MARK_LIST_SORT
2643 void combine_mark_lists();
2644 #endif //PARALLEL_MARK_LIST_SORT
2646 #endif //MULTIPLE_HEAPS
2648 /*------------ End of Multiple non isolated heaps ---------*/
2650 #ifndef SEG_MAPPING_TABLE
2652 heap_segment* segment_of (uint8_t* add, ptrdiff_t & delta,
2653 BOOL verify_p = FALSE);
2654 #endif //SEG_MAPPING_TABLE
2656 #ifdef BACKGROUND_GC
2658 //this is called by revisit....
2660 uint8_t* high_page (heap_segment* seg, BOOL concurrent_p);
2663 void revisit_written_page (uint8_t* page, uint8_t* end, BOOL concurrent_p,
2664 heap_segment* seg, uint8_t*& last_page,
2665 uint8_t*& last_object, BOOL large_objects_p,
2666 size_t& num_marked_objects);
2668 void revisit_written_pages (BOOL concurrent_p, BOOL reset_only_p=FALSE);
2671 void concurrent_scan_dependent_handles (ScanContext *sc);
2677 void bgc_suspend_EE ();
2683 void background_verify_mark (Object*& object, ScanContext* sc, uint32_t flags);
2686 void background_scan_dependent_handles (ScanContext *sc);
2691 // Restores BGC settings if necessary.
2693 void recover_bgc_settings();
2696 void save_bgc_data_per_heap();
2699 BOOL should_commit_mark_array();
2702 void clear_commit_flag();
2705 void clear_commit_flag_global();
2708 void verify_mark_array_cleared (heap_segment* seg, uint32_t* mark_array_addr);
2711 void verify_mark_array_cleared (uint8_t* begin, uint8_t* end, uint32_t* mark_array_addr);
2714 BOOL commit_mark_array_by_range (uint8_t* begin,
2716 uint32_t* mark_array_addr);
2719 BOOL commit_mark_array_new_seg (gc_heap* hp,
2721 uint32_t* new_card_table = 0,
2722 uint8_t* new_lowest_address = 0);
2725 BOOL commit_mark_array_with_check (heap_segment* seg, uint32_t* mark_array_addr);
2727 // commit the portion of the mark array that corresponds to
2728 // this segment (from beginning to reserved).
2729 // seg and heap_segment_reserved (seg) are guaranteed to be
2732 BOOL commit_mark_array_by_seg (heap_segment* seg, uint32_t* mark_array_addr);
2734 // During BGC init, we commit the mark array for all in range
2735 // segments whose mark array hasn't been committed or fully
2736 // committed. All rw segments are in range, only ro segments
2737 // can be partial in range.
2739 BOOL commit_mark_array_bgc_init (uint32_t* mark_array_addr);
2742 BOOL commit_new_mark_array (uint32_t* new_mark_array);
2744 // We need to commit all segments that intersect with the bgc
2745 // range. If a segment is only partially in range, we still
2746 // should commit the mark array for the whole segment as
2747 // we will set the mark array commit flag for this segment.
2749 BOOL commit_new_mark_array_global (uint32_t* new_mark_array);
2751 // We can't decommit the first and the last page in the mark array
2752 // if the beginning and ending don't happen to be page aligned.
2754 void decommit_mark_array_by_seg (heap_segment* seg);
2757 void background_mark_phase();
2760 void background_drain_mark_list (int thread);
2763 void background_grow_c_mark_list();
2766 void background_promote_callback(Object** object, ScanContext* sc, uint32_t flags);
2769 void mark_absorb_new_alloc();
2775 BOOL prepare_bgc_thread(gc_heap* gh);
2777 BOOL create_bgc_thread(gc_heap* gh);
2779 BOOL create_bgc_threads_support (int number_of_heaps);
2781 BOOL create_bgc_thread_support();
2783 int check_for_ephemeral_alloc();
2785 void wait_to_proceed();
2787 void fire_alloc_wait_event_begin (alloc_wait_reason awr);
2789 void fire_alloc_wait_event_end (alloc_wait_reason awr);
2791 void background_gc_wait_lh (alloc_wait_reason awr = awr_ignored);
2793 uint32_t background_gc_wait (alloc_wait_reason awr = awr_ignored, int time_out_ms = INFINITE);
2797 void kill_gc_thread();
2799 void bgc_thread_function();
2801 void do_background_gc();
2803 void bgc_thread_stub (void* arg);
2804 #endif //BACKGROUND_GC
2809 VOLATILE(bool) internal_gc_done;
2811 #ifdef BACKGROUND_GC
2813 uint32_t cm_in_progress;
2815 // normally this is FALSE; we set it to TRUE at the end of the gen1 GC
2816 // we do right before the bgc starts.
2818 BOOL dont_restart_ee_p;
2821 GCEvent bgc_start_event;
2822 #endif //BACKGROUND_GC
2824 // The variables in this block are known to the DAC and must come first
2825 // in the gc_heap class.
2827 // Keeps track of the highest address allocated by Alloc
2829 uint8_t* alloc_allocated;
2831 // The ephemeral heap segment
2833 heap_segment* ephemeral_heap_segment;
2835 // The finalize queue.
2837 CFinalize* finalize_queue;
2841 oom_history oom_info;
2843 // Interesting data, recorded per-heap.
2845 size_t interesting_data_per_heap[max_idp_count];
2848 size_t compact_reasons_per_heap[max_compact_reasons_count];
2851 size_t expand_mechanisms_per_heap[max_expand_mechanisms_count];
2854 size_t interesting_mechanism_bits_per_heap[max_gc_mechanism_bits_count];
2857 uint8_t** internal_root_array;
2860 size_t internal_root_array_index;
2863 BOOL heap_analyze_success;
2865 // The generation table. Must always be last.
2867 generation generation_table [NUMBERGENERATIONS + 1];
2872 BOOL expanded_in_fgc;
2875 uint32_t wait_for_gc_done(int32_t timeOut = INFINITE);
2877 // Returns TRUE if the current thread used to be in cooperative mode
2878 // before calling this function.
2880 bool enable_preemptive ();
2882 void disable_preemptive (bool restore_cooperative);
2884 /* ------------------- per heap members --------------------------*/
2887 #ifndef MULTIPLE_HEAPS
2888 GCEvent gc_done_event;
2889 #else // MULTIPLE_HEAPS
2890 GCEvent gc_done_event;
2891 #endif // MULTIPLE_HEAPS
2894 VOLATILE(int32_t) gc_done_event_lock;
2897 VOLATILE(bool) gc_done_event_set;
2903 void reset_gc_done();
2906 void enter_gc_done_event_lock();
2909 void exit_gc_done_event_lock();
2912 uint8_t* ephemeral_low; //lowest ephemeral address
2915 uint8_t* ephemeral_high; //highest ephemeral address
2918 uint32_t* card_table;
2925 uint32_t* mark_array;
2930 uint32_t* card_bundle_table;
2931 #endif //CARD_BUNDLE
2933 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
2935 sorted_table* seg_table;
2936 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
2939 VOLATILE(BOOL) gc_started;
2941 // The following 2 events are there to support the gen2
2942 // notification feature which is only enabled if concurrent
2945 GCEvent full_gc_approach_event;
2948 GCEvent full_gc_end_event;
2950 // Full GC Notification percentages.
2952 uint32_t fgn_maxgen_percent;
2955 uint32_t fgn_loh_percent;
2958 VOLATILE(bool) full_gc_approach_event_set;
2960 #ifdef BACKGROUND_GC
2962 BOOL fgn_last_gc_was_concurrent;
2963 #endif //BACKGROUND_GC
2966 size_t fgn_last_alloc;
2968 static uint32_t user_thread_wait (GCEvent *event, BOOL no_mode_change, int time_out_ms=INFINITE);
2970 static wait_full_gc_status full_gc_wait (GCEvent *event, int time_out_ms);
2973 uint8_t* demotion_low;
2976 uint8_t* demotion_high;
2982 uint8_t* last_gen1_pin_end;
2985 gen_to_condemn_tuning gen_to_condemn_reasons;
2988 size_t etw_allocation_running_amount[2];
2991 int gc_policy; //sweep, compact, expand
2993 #ifdef MULTIPLE_HEAPS
2995 bool gc_thread_no_affinitize_p;
2998 GCEvent gc_start_event;
3001 GCEvent ee_suspend_event;
3004 heap_segment* new_heap_segment;
3006 #define alloc_quantum_balance_units (16)
3009 size_t min_balance_threshold;
3010 #else //MULTIPLE_HEAPS
3013 size_t allocation_running_time;
3016 size_t allocation_running_amount;
3018 #endif //MULTIPLE_HEAPS
3021 gc_latency_level latency_level;
3024 gc_mechanisms settings;
3027 gc_history_global gc_data_global;
3030 size_t gc_last_ephemeral_decommit_time;
3033 size_t gc_gen0_desired_high;
3036 size_t gen0_big_free_spaces;
3040 double short_plugs_pad_ratio;
3041 #endif //SHORT_PLUGS
3045 size_t youngest_gen_desired_th;
3049 uint32_t last_gc_memory_load;
3052 size_t last_gc_heap_size;
3055 size_t last_gc_fragmentation;
3058 uint32_t high_memory_load_th;
3061 uint32_t m_high_memory_load_th;
3064 uint32_t v_high_memory_load_th;
3067 uint64_t mem_one_percent;
3070 uint64_t total_physical_mem;
3073 uint64_t entry_available_physical_mem;
3075 // Hard limit for the heap, only supported on 64-bit.
3077 // Users can specify a hard limit for the GC heap via GCHeapHardLimit or
3078 // a percentage of the physical memory this process is allowed to use via
3079 // GCHeapHardLimitPercent. This is the maximum commit size the GC heap
3082 // The way the hard limit is decided is:
3084 // If the GCHeapHardLimit config is specified that's the value we use;
3085 // else if the GCHeapHardLimitPercent config is specified we use that
3087 // else if the process is running inside a container with a memory limit,
3088 // the hard limit is
3089 // max (20mb, 75% of the memory limit on the container).
3091 // Due to the different perf charicteristics of containers we make the
3092 // following policy changes:
3094 // 1) No longer affinitize Server GC threads by default because we wouldn't
3095 // want all the containers on the machine to only affinitize to use the
3096 // first few CPUs (and we don't know which CPUs are already used). You
3097 // can however override this by specifying the GCHeapAffinitizeMask
3098 // config which will decide which CPUs the process will affinitize the
3099 // Server GC threads to.
3101 // 2) Segment size is determined by limit / number of heaps but has a
3102 // minimum value of 16mb. This can be changed by specifying the number
3103 // of heaps via the GCHeapCount config. The minimum size is to avoid
3104 // the scenario where the hard limit is small but the process can use
3105 // many procs and we end up with tiny segments which doesn't make sense.
3107 // 3) LOH compaction occurs automatically if needed.
3109 // Since we do allow both gen0 and gen3 allocations, and we don't know
3110 // the distinction (and it's unrealistic to request users to specify
3111 // this distribution) we reserve memory this way -
3113 // For SOH we reserve (limit / number of heaps) per heap.
3114 // For LOH we reserve (limit * 2 / number of heaps) per heap.
3116 // This means the following -
3118 // + we never need to acquire new segments. This simplies the perf
3119 // calculations by a lot.
3121 // + we now need a different definition of "end of seg" because we
3122 // need to make sure the total does not exceed the limit.
3124 // + if we detect that we exceed the commit limit in the allocator we
3125 // wouldn't want to treat that as a normal commit failure because that
3126 // would mean we always do full compacting GCs.
3128 // TODO: some of the logic here applies to the general case as well
3129 // such as LOH automatic compaction. However it will require more
3130 //testing to change the general case.
3132 size_t heap_hard_limit;
3135 CLRCriticalSection check_commit_cs;
3138 size_t current_total_committed;
3140 // This is what GC uses for its own bookkeeping.
3142 size_t current_total_committed_bookkeeping;
3144 // This is what GC's own book keeping consumes.
3146 size_t current_total_committed_gc_own;
3148 // This is if large pages should be used.
3150 size_t use_large_pages_p;
3153 size_t last_gc_index;
3155 #ifdef SEG_MAPPING_TABLE
3157 size_t min_segment_size;
3160 size_t min_segment_size_shr;
3161 #endif //SEG_MAPPING_TABLE
3163 // For SOH we always allocate segments of the same
3164 // size unless no_gc_region requires larger ones.
3166 size_t soh_segment_size;
3169 size_t min_loh_segment_size;
3172 size_t segment_info_size;
3175 uint8_t* lowest_address;
3178 uint8_t* highest_address;
3181 BOOL ephemeral_promotion;
3183 uint8_t* saved_ephemeral_plan_start[NUMBERGENERATIONS-1];
3185 size_t saved_ephemeral_plan_start_size[NUMBERGENERATIONS-1];
3188 #ifdef MULTIPLE_HEAPS
3194 VOLATILE(int) alloc_context_count;
3195 #else //MULTIPLE_HEAPS
3196 #define vm_heap ((GCHeap*) g_theGCHeap)
3197 #define heap_number (0)
3198 #endif //MULTIPLE_HEAPS
3201 size_t time_bgc_last;
3204 uint8_t* gc_low; // lowest address being condemned
3207 uint8_t* gc_high; //highest address being condemned
3210 size_t mark_stack_tos;
3213 size_t mark_stack_bos;
3216 size_t mark_stack_array_length;
3219 mark* mark_stack_array;
3221 #if defined (_DEBUG) && defined (VERIFY_HEAP)
3223 BOOL verify_pinned_queue_p;
3224 #endif // _DEBUG && VERIFY_HEAP
3227 uint8_t* oldest_pinned_plug;
3229 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
3231 size_t num_pinned_objects;
3232 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
3234 #ifdef FEATURE_LOH_COMPACTION
3236 size_t loh_pinned_queue_tos;
3239 size_t loh_pinned_queue_bos;
3242 size_t loh_pinned_queue_length;
3245 int loh_pinned_queue_decay;
3248 mark* loh_pinned_queue;
3250 // This is for forced LOH compaction via the complus env var
3252 BOOL loh_compaction_always_p;
3254 // This is set by the user.
3256 gc_loh_compaction_mode loh_compaction_mode;
3258 // We may not compact LOH on every heap if we can't
3259 // grow the pinned queue. This is to indicate whether
3260 // this heap's LOH is compacted or not. So even if
3261 // settings.loh_compaction is TRUE this may not be TRUE.
3263 BOOL loh_compacted_p;
3264 #endif //FEATURE_LOH_COMPACTION
3266 #ifdef BACKGROUND_GC
3269 EEThreadId bgc_thread_id;
3273 uint8_t* background_written_addresses [array_size+2];
3274 #endif //WRITE_WATCH
3277 VOLATILE(c_gc_state) current_c_gc_state; //tells the large object allocator to
3278 //mark the object as new since the start of gc.
3281 gc_mechanisms saved_bgc_settings;
3284 gc_history_per_heap bgc_data_per_heap;
3287 BOOL bgc_thread_running; // gc thread is its main loop
3290 BOOL keep_bgc_threads_p;
3292 // This event is used by BGC threads to do something on
3293 // one specific thread while other BGC threads have to
3294 // wait. This is different from a join 'cause you can't
3295 // specify which thread should be doing some task
3296 // while other threads have to wait.
3297 // For example, to make the BGC threads managed threads
3298 // we need to create them on the thread that called
3299 // SuspendEE which is heap 0.
3301 GCEvent bgc_threads_sync_event;
3307 CLRCriticalSection bgc_threads_timeout_cs;
3310 GCEvent background_gc_done_event;
3313 GCEvent ee_proceed_event;
3316 GCEvent gc_lh_block_event;
3319 bool gc_can_use_concurrent;
3322 bool temp_disable_concurrent_p;
3325 BOOL do_ephemeral_gc_p;
3328 BOOL do_concurrent_p;
3331 VOLATILE(bgc_state) current_bgc_state;
3336 bgc_state current_bgc_state;
3337 uint32_t gc_time_ms;
3338 // This is in bytes per ms; consider breaking it
3339 // into the efficiency per phase.
3340 size_t gc_efficiency;
3342 uint8_t* gen0_start;
3344 uint8_t* bgc_highest;
3345 uint8_t* bgc_lowest;
3346 uint8_t* fgc_highest;
3347 uint8_t* fgc_lowest;
3352 #define max_history_count 64
3355 int gchist_index_per_heap;
3358 gc_history gchist_per_heap[max_history_count];
3364 gc_mechanisms_store gchist[max_history_count];
3367 void add_to_history_per_heap();
3370 void add_to_history();
3373 size_t total_promoted_bytes;
3376 size_t bgc_overflow_count;
3379 size_t bgc_begin_loh_size;
3381 size_t end_loh_size;
3383 // We need to throttle the LOH allocations during BGC since we can't
3384 // collect LOH when BGC is in progress.
3385 // We allow the LOH heap size to double during a BGC. So for every
3386 // 10% increase we will have the LOH allocating thread sleep for one more
3387 // ms. So we are already 30% over the original heap size the thread will
3390 uint32_t bgc_alloc_spin_loh;
3392 // This includes what we allocate at the end of segment - allocating
3393 // in free list doesn't increase the heap size.
3395 size_t bgc_loh_size_increased;
3398 size_t bgc_loh_allocated_in_free;
3401 size_t background_soh_alloc_count;
3404 size_t background_loh_alloc_count;
3407 VOLATILE(int32_t) loh_alloc_thread_count;
3410 uint8_t** background_mark_stack_tos;
3413 uint8_t** background_mark_stack_array;
3416 size_t background_mark_stack_array_length;
3419 uint8_t* background_min_overflow_address;
3422 uint8_t* background_max_overflow_address;
3424 // We can't process the soh range concurrently so we
3425 // wait till final mark to process it.
3427 BOOL processed_soh_overflow_p;
3430 uint8_t* background_min_soh_overflow_address;
3433 uint8_t* background_max_soh_overflow_address;
3436 heap_segment* saved_overflow_ephemeral_seg;
3439 heap_segment* saved_sweep_ephemeral_seg;
3442 uint8_t* saved_sweep_ephemeral_start;
3445 uint8_t* background_saved_lowest_address;
3448 uint8_t* background_saved_highest_address;
3450 // This is used for synchronization between the bgc thread
3451 // for this heap and the user threads allocating on this
3454 exclusive_sync* bgc_alloc_lock;
3458 snoop_stats_data snoop_stat;
3459 #endif //SNOOP_STATS
3463 uint8_t** c_mark_list;
3466 size_t c_mark_list_length;
3469 size_t c_mark_list_index;
3470 #endif //BACKGROUND_GC
3474 uint8_t** mark_list;
3477 size_t mark_list_size;
3480 uint8_t** mark_list_end;
3483 uint8_t** mark_list_index;
3486 uint8_t** g_mark_list;
3487 #ifdef PARALLEL_MARK_LIST_SORT
3489 uint8_t** g_mark_list_copy;
3491 uint8_t*** mark_list_piece_start;
3492 uint8_t*** mark_list_piece_end;
3493 #endif //PARALLEL_MARK_LIST_SORT
3497 uint8_t* min_overflow_address;
3500 uint8_t* max_overflow_address;
3502 #ifndef MULTIPLE_HEAPS
3504 uint8_t* shigh; //keeps track of the highest marked object
3507 uint8_t* slow; //keeps track of the lowest marked object
3508 #endif //MULTIPLE_HEAPS
3511 size_t allocation_quantum;
3514 size_t alloc_contexts_used;
3517 no_gc_region_info current_no_gc_region_info;
3520 size_t soh_allocation_no_gc;
3523 size_t loh_allocation_no_gc;
3529 heap_segment* saved_loh_segment_no_gc;
3532 BOOL proceed_with_gc_p;
3534 #define youngest_generation (generation_of (0))
3535 #define large_object_generation (generation_of (max_generation+1))
3537 // The more_space_lock and gc_lock is used for 3 purposes:
3539 // 1) to coordinate threads that exceed their quantum (UP & MP) (more_space_lock_soh)
3540 // 2) to synchronize allocations of large objects (more_space_lock_loh)
3541 // 3) to synchronize the GC itself (gc_lock)
3544 GCSpinLock gc_lock; //lock while doing GC
3547 GCSpinLock more_space_lock_soh; //lock while allocating more space for soh
3550 GCSpinLock more_space_lock_loh;
3552 #ifdef SYNCHRONIZATION_STATS
3555 unsigned int good_suspension;
3558 unsigned int bad_suspension;
3560 // Number of times when msl_acquire is > 200 cycles.
3562 unsigned int num_high_msl_acquire;
3564 // Number of times when msl_acquire is < 200 cycles.
3566 unsigned int num_low_msl_acquire;
3568 // Number of times the more_space_lock is acquired.
3570 unsigned int num_msl_acquired;
3572 // Total cycles it takes to acquire the more_space_lock.
3574 uint64_t total_msl_acquire;
3577 void init_heap_sync_stats()
3579 good_suspension = 0;
3581 num_msl_acquired = 0;
3582 total_msl_acquire = 0;
3583 num_high_msl_acquire = 0;
3584 num_low_msl_acquire = 0;
3585 more_space_lock.init();
3590 void print_heap_sync_stats(unsigned int heap_num, unsigned int gc_count_during_log)
3592 printf("%2d%2d%10u%10u%12u%6u%4u%8u(%4u,%4u,%4u,%4u)\n",
3594 alloc_contexts_used,
3597 (unsigned int)(total_msl_acquire / gc_count_during_log),
3598 num_high_msl_acquire / gc_count_during_log,
3599 num_low_msl_acquire / gc_count_during_log,
3600 num_msl_acquired / gc_count_during_log,
3601 more_space_lock.num_switch_thread / gc_count_during_log,
3602 more_space_lock.num_wait_longer / gc_count_during_log,
3603 more_space_lock.num_switch_thread_w / gc_count_during_log,
3604 more_space_lock.num_disable_preemptive_w / gc_count_during_log);
3607 #endif //SYNCHRONIZATION_STATS
3609 #define NUM_LOH_ALIST (7)
3610 #define BASE_LOH_ALIST (64*1024)
3612 alloc_list loh_alloc_list[NUM_LOH_ALIST-1];
3614 #define NUM_GEN2_ALIST (12)
3616 #define BASE_GEN2_ALIST (1*256)
3618 #define BASE_GEN2_ALIST (1*128)
3621 alloc_list gen2_alloc_list[NUM_GEN2_ALIST-1];
3623 //------------------------------------------
3626 dynamic_data dynamic_data_table [NUMBERGENERATIONS+1];
3629 gc_history_per_heap gc_data_per_heap;
3632 size_t maxgen_pinned_compact_before_advance;
3636 BOOL dt_low_ephemeral_space_p (gc_tuning_point tp);
3637 // if elevate_p is FALSE, it means we are determining fragmentation for a generation
3638 // to see if we should condemn this gen; otherwise it means we are determining if
3639 // we should elevate to doing max_gen from an ephemeral gen.
3641 BOOL dt_high_frag_p (gc_tuning_point tp, int gen_number, BOOL elevate_p=FALSE);
3644 dt_estimate_reclaim_space_p (gc_tuning_point tp, int gen_number);
3646 BOOL dt_estimate_high_frag_p (gc_tuning_point tp, int gen_number, uint64_t available_mem);
3648 BOOL dt_low_card_table_efficiency_p (gc_tuning_point tp);
3651 int generation_skip_ratio;//in %
3654 BOOL gen0_bricks_cleared;
3657 int gen0_must_clear_bricks;
3658 #endif //FFIND_OBJECT
3661 bool maxgen_size_inc_p;
3664 size_t full_gc_counts[gc_type_max];
3666 // the # of bytes allocates since the last full compacting GC.
3668 uint64_t loh_alloc_since_cg;
3671 BOOL elevation_requested;
3673 // if this is TRUE, we should always guarantee that we do a
3674 // full compacting GC before we OOM.
3676 BOOL last_gc_before_oom;
3679 BOOL should_expand_in_full_gc;
3681 // When we decide if we should expand the heap or not, we are
3682 // fine NOT to expand if we find enough free space in gen0's free
3683 // list or end of seg and we check this in decide_on_compacting.
3684 // This is an expensive check so we just record the fact and not
3685 // need to check in the allocator again.
3687 BOOL sufficient_gen0_space_p;
3689 #ifdef MULTIPLE_HEAPS
3691 bool gen0_allocated_after_gc_p;
3692 #endif //MULTIPLE_HEAPS
3694 // A provisional mode means we could change our mind in the middle of a GC
3695 // and want to do a different GC instead.
3697 // Right now there's only one such case which is in the middle of a gen1
3698 // GC we want to do a blocking gen2 instead. If/When we have more we should
3699 // have an enum that tells us which case in this provisional mode
3702 // When this mode is triggered, our current (only) condition says
3703 // we have high fragmentation in gen2 even after we do a compacting
3704 // full GC which is an indication of heavy pinning in gen2. In this
3705 // case we never do BGCs, we just do either gen0 or gen1's till a
3706 // gen1 needs to increase the gen2 size, in which case we finish up
3707 // the current gen1 as a sweeping GC and immediately do a compacting
3708 // full GC instead (without restarting EE).
3710 bool provisional_mode_triggered;
3713 bool pm_trigger_full_gc;
3715 // For testing only BEG
3716 // pm_stress_on currently means (since we just have one mode) we
3717 // randomly turn the mode on; and after a random # of NGC2s we
3719 // NOTE that this means concurrent will be disabled so we can
3720 // simulate what this mode is supposed to be used.
3725 size_t provisional_triggered_gc_count;
3728 size_t provisional_off_gc_count;
3729 // For testing only END
3732 size_t num_provisional_triggered;
3735 size_t allocated_since_last_gc;
3737 #ifdef BACKGROUND_GC
3739 size_t ephemeral_fgc_counts[max_generation];
3742 BOOL alloc_wait_event_p;
3745 uint8_t* next_sweep_obj;
3748 uint8_t* current_sweep_pos;
3750 #endif //BACKGROUND_GC
3753 fgm_history fgm_result;
3756 size_t eph_gen_starts_size;
3758 #ifdef GC_CONFIG_DRIVEN
3763 size_t time_since_init;
3765 // 0 stores compacting GCs;
3766 // 1 stores sweeping GCs;
3768 size_t compact_or_sweep_gcs[2];
3771 size_t interesting_data_per_gc[max_idp_count];
3772 #endif //GC_CONFIG_DRIVEN
3775 BOOL ro_segments_in_range;
3777 #ifdef BACKGROUND_GC
3779 heap_segment* freeable_small_heap_segment;
3780 #endif //BACKGROUND_GC
3783 heap_segment* freeable_large_heap_segment;
3786 heap_segment* segment_standby_list;
3789 size_t ordered_free_space_indices[MAX_NUM_BUCKETS];
3792 size_t saved_ordered_free_space_indices[MAX_NUM_BUCKETS];
3795 size_t ordered_plug_indices[MAX_NUM_BUCKETS];
3798 size_t saved_ordered_plug_indices[MAX_NUM_BUCKETS];
3801 BOOL ordered_plug_indices_init;
3807 uint8_t* bestfit_first_pin;
3810 BOOL commit_end_of_seg;
3813 size_t max_free_space_items; // dynamically adjusted.
3816 size_t free_space_buckets;
3819 size_t free_space_items;
3821 // -1 means we are using all the free
3822 // spaces we have (not including
3823 // end of seg space).
3825 int trimmed_free_space_index;
3828 size_t total_ephemeral_plugs;
3831 seg_free_spaces* bestfit_seg;
3833 // Note: we know this from the plan phase.
3834 // total_ephemeral_plugs actually has the same value
3835 // but while we are calculating its value we also store
3836 // info on how big the plugs are for best fit which we
3837 // don't do in plan phase.
3838 // TODO: get rid of total_ephemeral_plugs.
3840 size_t total_ephemeral_size;
3847 BOOL heap_analyze_enabled;
3850 size_t internal_root_array_length;
3852 // next two fields are used to optimize the search for the object
3853 // enclosing the current reference handled by ha_mark_object_simple.
3855 uint8_t* current_obj;
3858 size_t current_obj_size;
3860 #endif //HEAP_ANALYZE
3862 /* ----------------------- global members ----------------------- */
3866 int condemned_generation_num;
3869 BOOL blocking_collection;
3871 #ifdef MULTIPLE_HEAPS
3880 #ifdef BACKGROUND_GC
3882 size_t* g_bpromoted;
3883 #endif //BACKGROUND_GC
3886 int* g_mark_stack_busy;
3891 #ifdef BACKGROUND_GC
3894 #endif //BACKGROUND_GC
3895 #endif //MULTIPLE_HEAPS
3898 size_t reserved_memory;
3900 size_t reserved_memory_limit;
3902 BOOL g_low_memory_status;
3906 void update_collection_counts ();
3909 #define ASSERT_OFFSETS_MATCH(field) \
3910 static_assert(offsetof(dac_gc_heap, field) == offsetof(gc_heap, field), #field " offset mismatch")
3912 #ifdef MULTIPLE_HEAPS
3913 ASSERT_OFFSETS_MATCH(alloc_allocated);
3914 ASSERT_OFFSETS_MATCH(ephemeral_heap_segment);
3915 ASSERT_OFFSETS_MATCH(finalize_queue);
3916 ASSERT_OFFSETS_MATCH(oom_info);
3917 ASSERT_OFFSETS_MATCH(interesting_data_per_heap);
3918 ASSERT_OFFSETS_MATCH(compact_reasons_per_heap);
3919 ASSERT_OFFSETS_MATCH(expand_mechanisms_per_heap);
3920 ASSERT_OFFSETS_MATCH(interesting_mechanism_bits_per_heap);
3921 ASSERT_OFFSETS_MATCH(internal_root_array);
3922 ASSERT_OFFSETS_MATCH(internal_root_array_index);
3923 ASSERT_OFFSETS_MATCH(heap_analyze_success);
3924 ASSERT_OFFSETS_MATCH(generation_table);
3925 #endif // MULTIPLE_HEAPS
3927 #ifdef FEATURE_PREMORTEM_FINALIZATION
3930 #ifdef DACCESS_COMPILE
3931 friend class ::ClrDataAccess;
3932 #endif // DACCESS_COMPILE
3934 friend class CFinalizeStaticAsserts;
3938 //adjust the count and add a constant to add a segment
3939 static const int ExtraSegCount = 2;
3940 static const int FinalizerListSeg = NUMBERGENERATIONS+1;
3941 static const int CriticalFinalizerListSeg = NUMBERGENERATIONS;
3942 //Does not correspond to a segment
3943 static const int FreeList = NUMBERGENERATIONS+ExtraSegCount;
3945 PTR_PTR_Object m_FillPointers[NUMBERGENERATIONS+ExtraSegCount];
3946 PTR_PTR_Object m_Array;
3947 PTR_PTR_Object m_EndArray;
3948 size_t m_PromotedCount;
3950 VOLATILE(int32_t) lock;
3952 EEThreadId lockowner_threadid;
3956 void MoveItem (Object** fromIndex,
3957 unsigned int fromSeg,
3958 unsigned int toSeg);
3960 inline PTR_PTR_Object& SegQueue (unsigned int Seg)
3962 return (Seg ? m_FillPointers [Seg-1] : m_Array);
3964 inline PTR_PTR_Object& SegQueueLimit (unsigned int Seg)
3966 return m_FillPointers [Seg];
3969 BOOL IsSegEmpty ( unsigned int i)
3971 ASSERT ( (int)i < FreeList);
3972 return (SegQueueLimit(i) == SegQueue (i));
3976 BOOL FinalizeSegForAppDomain (void *pDomain,
3977 BOOL fRunFinalizers,
3983 void EnterFinalizeLock();
3984 void LeaveFinalizeLock();
3985 bool RegisterForFinalization (int gen, Object* obj, size_t size=0);
3986 Object* GetNextFinalizableObject (BOOL only_non_critical=FALSE);
3987 BOOL ScanForFinalization (promote_func* fn, int gen,BOOL mark_only_p, gc_heap* hp);
3988 void RelocateFinalizationData (int gen, gc_heap* hp);
3989 void WalkFReachableObjects (fq_walk_fn fn);
3990 void GcScanRoots (promote_func* fn, int hn, ScanContext *pSC);
3991 void UpdatePromotedGenerations (int gen, BOOL gen_0_empty_p);
3992 size_t GetPromotedCount();
3994 //Methods used by the shutdown code to call every finalizer
3995 void SetSegForShutDown(BOOL fHasLock);
3996 size_t GetNumberFinalizableObjects();
3997 void DiscardNonCriticalObjects();
3999 //Methods used by the app domain unloading call to finalize objects in an app domain
4000 bool FinalizeAppDomain (void *pDomain, bool fRunFinalizers);
4002 void CheckFinalizerObjects();
4005 class CFinalizeStaticAsserts {
4006 static_assert(dac_finalize_queue::ExtraSegCount == CFinalize::ExtraSegCount, "ExtraSegCount mismatch");
4007 static_assert(offsetof(dac_finalize_queue, m_FillPointers) == offsetof(CFinalize, m_FillPointers), "CFinalize layout mismatch");
4009 #endif // FEATURE_PREMORTEM_FINALIZATION
4012 size_t& dd_begin_data_size (dynamic_data* inst)
4014 return inst->begin_data_size;
4017 size_t& dd_survived_size (dynamic_data* inst)
4019 return inst->survived_size;
4021 #if defined (RESPECT_LARGE_ALIGNMENT) || defined (FEATURE_STRUCTALIGN)
4023 size_t& dd_num_npinned_plugs(dynamic_data* inst)
4025 return inst->num_npinned_plugs;
4027 #endif //RESPECT_LARGE_ALIGNMENT || FEATURE_STRUCTALIGN
4029 size_t& dd_pinned_survived_size (dynamic_data* inst)
4031 return inst->pinned_survived_size;
4034 size_t& dd_added_pinned_size (dynamic_data* inst)
4036 return inst->added_pinned_size;
4039 size_t& dd_artificial_pinned_survived_size (dynamic_data* inst)
4041 return inst->artificial_pinned_survived_size;
4045 size_t& dd_padding_size (dynamic_data* inst)
4047 return inst->padding_size;
4049 #endif //SHORT_PLUGS
4051 size_t& dd_current_size (dynamic_data* inst)
4053 return inst->current_size;
4056 float& dd_surv (dynamic_data* inst)
4061 size_t& dd_freach_previous_promotion (dynamic_data* inst)
4063 return inst->freach_previous_promotion;
4066 size_t& dd_desired_allocation (dynamic_data* inst)
4068 return inst->desired_allocation;
4071 size_t& dd_collection_count (dynamic_data* inst)
4073 return inst->collection_count;
4076 size_t& dd_promoted_size (dynamic_data* inst)
4078 return inst->promoted_size;
4081 float& dd_limit (dynamic_data* inst)
4083 return inst->sdata->limit;
4086 float& dd_max_limit (dynamic_data* inst)
4088 return inst->sdata->max_limit;
4091 size_t& dd_max_size (dynamic_data* inst)
4093 return inst->sdata->max_size;
4096 size_t& dd_min_size (dynamic_data* inst)
4098 return inst->min_size;
4101 ptrdiff_t& dd_new_allocation (dynamic_data* inst)
4103 return inst->new_allocation;
4106 ptrdiff_t& dd_gc_new_allocation (dynamic_data* inst)
4108 return inst->gc_new_allocation;
4111 size_t& dd_fragmentation_limit (dynamic_data* inst)
4113 return inst->sdata->fragmentation_limit;
4116 float& dd_fragmentation_burden_limit (dynamic_data* inst)
4118 return inst->sdata->fragmentation_burden_limit;
4121 float dd_v_fragmentation_burden_limit (dynamic_data* inst)
4123 return (min (2*dd_fragmentation_burden_limit (inst), 0.75f));
4126 size_t& dd_fragmentation (dynamic_data* inst)
4128 return inst->fragmentation;
4131 size_t& dd_gc_clock (dynamic_data* inst)
4133 return inst->gc_clock;
4136 size_t& dd_time_clock (dynamic_data* inst)
4138 return inst->time_clock;
4142 size_t& dd_gc_clock_interval (dynamic_data* inst)
4144 return inst->sdata->gc_clock;
4147 size_t& dd_time_clock_interval (dynamic_data* inst)
4149 return inst->sdata->time_clock;
4153 size_t& dd_gc_elapsed_time (dynamic_data* inst)
4155 return inst->gc_elapsed_time;
4159 float& dd_gc_speed (dynamic_data* inst)
4161 return inst->gc_speed;
4165 alloc_context* generation_alloc_context (generation* inst)
4167 return &(inst->allocation_context);
4171 uint8_t*& generation_allocation_start (generation* inst)
4173 return inst->allocation_start;
4176 uint8_t*& generation_allocation_pointer (generation* inst)
4178 return inst->allocation_context.alloc_ptr;
4181 uint8_t*& generation_allocation_limit (generation* inst)
4183 return inst->allocation_context.alloc_limit;
4186 allocator* generation_allocator (generation* inst)
4188 return &inst->free_list_allocator;
4192 PTR_heap_segment& generation_start_segment (generation* inst)
4194 return inst->start_segment;
4197 heap_segment*& generation_allocation_segment (generation* inst)
4199 return inst->allocation_segment;
4202 uint8_t*& generation_plan_allocation_start (generation* inst)
4204 return inst->plan_allocation_start;
4207 size_t& generation_plan_allocation_start_size (generation* inst)
4209 return inst->plan_allocation_start_size;
4212 uint8_t*& generation_allocation_context_start_region (generation* inst)
4214 return inst->allocation_context_start_region;
4217 size_t& generation_free_list_space (generation* inst)
4219 return inst->free_list_space;
4222 size_t& generation_free_obj_space (generation* inst)
4224 return inst->free_obj_space;
4227 size_t& generation_allocation_size (generation* inst)
4229 return inst->allocation_size;
4233 size_t& generation_pinned_allocated (generation* inst)
4235 return inst->pinned_allocated;
4238 size_t& generation_pinned_allocation_sweep_size (generation* inst)
4240 return inst->pinned_allocation_sweep_size;
4243 size_t& generation_pinned_allocation_compact_size (generation* inst)
4245 return inst->pinned_allocation_compact_size;
4248 size_t& generation_free_list_allocated (generation* inst)
4250 return inst->free_list_allocated;
4253 size_t& generation_end_seg_allocated (generation* inst)
4255 return inst->end_seg_allocated;
4258 BOOL& generation_allocate_end_seg_p (generation* inst)
4260 return inst->allocate_end_seg_p;
4263 size_t& generation_condemned_allocated (generation* inst)
4265 return inst->condemned_allocated;
4267 #ifdef FREE_USAGE_STATS
4269 size_t& generation_pinned_free_obj_space (generation* inst)
4271 return inst->pinned_free_obj_space;
4274 size_t& generation_allocated_in_pinned_free (generation* inst)
4276 return inst->allocated_in_pinned_free;
4279 size_t& generation_allocated_since_last_pin (generation* inst)
4281 return inst->allocated_since_last_pin;
4283 #endif //FREE_USAGE_STATS
4285 float generation_allocator_efficiency (generation* inst)
4287 if ((generation_free_list_allocated (inst) + generation_free_obj_space (inst)) != 0)
4289 return ((float) (generation_free_list_allocated (inst)) / (float)(generation_free_list_allocated (inst) + generation_free_obj_space (inst)));
4295 size_t generation_unusable_fragmentation (generation* inst)
4297 return (size_t)(generation_free_obj_space (inst) +
4298 (1.0f-generation_allocator_efficiency(inst))*generation_free_list_space (inst));
4301 #define plug_skew sizeof(ObjHeader)
4302 // We always use USE_PADDING_TAIL when fitting so items on the free list should be
4303 // twice the min_obj_size.
4304 #define min_free_list (2*min_obj_size)
4307 uint8_t * skew[plug_skew / sizeof(uint8_t *)];
4317 //Note that these encode the fact that plug_skew is a multiple of uint8_t*.
4318 // Each of new field is prepended to the prior struct.
4320 struct plug_and_pair
4326 struct plug_and_reloc
4340 int lr; //for clearing the entire pair in one instruction
4345 struct gap_reloc_pair
4352 #define min_pre_pin_obj_size (sizeof (gap_reloc_pair) + min_obj_size)
4354 struct DECLSPEC_ALIGN(8) aligned_plug_and_gap
4356 plug_and_gap plugandgap;
4359 struct loh_obj_and_pad
4365 struct loh_padding_obj
4372 #define loh_padding_obj_size (sizeof(loh_padding_obj))
4375 #define heap_segment_flags_readonly 1
4376 #define heap_segment_flags_inrange 2
4377 #define heap_segment_flags_unmappable 4
4378 #define heap_segment_flags_loh 8
4379 #ifdef BACKGROUND_GC
4380 #define heap_segment_flags_swept 16
4381 #define heap_segment_flags_decommitted 32
4382 #define heap_segment_flags_ma_committed 64
4383 // for segments whose mark array is only partially committed.
4384 #define heap_segment_flags_ma_pcommitted 128
4385 #define heap_segment_flags_loh_delete 256
4386 #endif //BACKGROUND_GC
4388 //need to be careful to keep enough pad items to fit a relocation node
4389 //padded to QuadWord before the plug_skew
4400 PTR_heap_segment next;
4401 uint8_t* background_allocated;
4402 #ifdef MULTIPLE_HEAPS
4404 #endif //MULTIPLE_HEAPS
4405 uint8_t* plan_allocated;
4406 uint8_t* saved_bg_allocated;
4409 // Disable this warning - we intentionally want __declspec(align()) to insert padding for us
4410 #pragma warning(disable:4324) // structure was padded due to __declspec(align())
4412 aligned_plug_and_gap padandplug;
4414 #pragma warning(default:4324) // structure was padded due to __declspec(align())
4418 static_assert(offsetof(dac_heap_segment, allocated) == offsetof(heap_segment, allocated), "DAC heap segment layout mismatch");
4419 static_assert(offsetof(dac_heap_segment, committed) == offsetof(heap_segment, committed), "DAC heap segment layout mismatch");
4420 static_assert(offsetof(dac_heap_segment, reserved) == offsetof(heap_segment, reserved), "DAC heap segment layout mismatch");
4421 static_assert(offsetof(dac_heap_segment, used) == offsetof(heap_segment, used), "DAC heap segment layout mismatch");
4422 static_assert(offsetof(dac_heap_segment, mem) == offsetof(heap_segment, mem), "DAC heap segment layout mismatch");
4423 static_assert(offsetof(dac_heap_segment, flags) == offsetof(heap_segment, flags), "DAC heap segment layout mismatch");
4424 static_assert(offsetof(dac_heap_segment, next) == offsetof(heap_segment, next), "DAC heap segment layout mismatch");
4425 static_assert(offsetof(dac_heap_segment, background_allocated) == offsetof(heap_segment, background_allocated), "DAC heap segment layout mismatch");
4426 #ifdef MULTIPLE_HEAPS
4427 static_assert(offsetof(dac_heap_segment, heap) == offsetof(heap_segment, heap), "DAC heap segment layout mismatch");
4428 #endif // MULTIPLE_HEAPS
4431 uint8_t*& heap_segment_reserved (heap_segment* inst)
4433 return inst->reserved;
4436 uint8_t*& heap_segment_committed (heap_segment* inst)
4438 return inst->committed;
4441 uint8_t*& heap_segment_used (heap_segment* inst)
4446 uint8_t*& heap_segment_allocated (heap_segment* inst)
4448 return inst->allocated;
4452 BOOL heap_segment_read_only_p (heap_segment* inst)
4454 return ((inst->flags & heap_segment_flags_readonly) != 0);
4458 BOOL heap_segment_in_range_p (heap_segment* inst)
4460 return (!(inst->flags & heap_segment_flags_readonly) ||
4461 ((inst->flags & heap_segment_flags_inrange) != 0));
4465 BOOL heap_segment_unmappable_p (heap_segment* inst)
4467 return (!(inst->flags & heap_segment_flags_readonly) ||
4468 ((inst->flags & heap_segment_flags_unmappable) != 0));
4472 BOOL heap_segment_loh_p (heap_segment * inst)
4474 return !!(inst->flags & heap_segment_flags_loh);
4477 #ifdef BACKGROUND_GC
4479 BOOL heap_segment_decommitted_p (heap_segment * inst)
4481 return !!(inst->flags & heap_segment_flags_decommitted);
4483 #endif //BACKGROUND_GC
4486 PTR_heap_segment & heap_segment_next (heap_segment* inst)
4491 uint8_t*& heap_segment_mem (heap_segment* inst)
4496 uint8_t*& heap_segment_plan_allocated (heap_segment* inst)
4498 return inst->plan_allocated;
4501 #ifdef BACKGROUND_GC
4503 uint8_t*& heap_segment_background_allocated (heap_segment* inst)
4505 return inst->background_allocated;
4508 uint8_t*& heap_segment_saved_bg_allocated (heap_segment* inst)
4510 return inst->saved_bg_allocated;
4512 #endif //BACKGROUND_GC
4514 #ifdef MULTIPLE_HEAPS
4516 gc_heap*& heap_segment_heap (heap_segment* inst)
4520 #endif //MULTIPLE_HEAPS
4523 generation* gc_heap::generation_of (int n)
4525 assert (((n <= max_generation+1) && (n >= 0)));
4526 return &generation_table [ n ];
4530 dynamic_data* gc_heap::dynamic_data_of (int gen_number)
4532 return &dynamic_data_table [ gen_number ];
4535 #define GC_PAGE_SIZE 0x1000
4537 #define card_word_width ((size_t)32)
4540 // The value of card_size is determined empirically according to the average size of an object
4541 // In the code we also rely on the assumption that one card_table entry (uint32_t) covers an entire os page
4544 #define card_size ((size_t)(2*GC_PAGE_SIZE/card_word_width))
4546 #define card_size ((size_t)(GC_PAGE_SIZE/card_word_width))
4550 size_t card_word (size_t card)
4552 return card / card_word_width;
4556 unsigned card_bit (size_t card)
4558 return (unsigned)(card % card_word_width);
4562 size_t gcard_of (uint8_t* object)
4564 return (size_t)(object) / card_size;