1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
3 // See the LICENSE file in the project root for more information.
9 // GC automatically manages memory allocated by managed code.
10 // The design doc for GC can be found at Documentation/botr/garbage-collection.md
12 // This file includes both the code for GC and the allocator. The most common
13 // case for a GC to be triggered is from the allocator code. See
14 // code:#try_allocate_more_space where it calls GarbageCollectGeneration.
16 // Entry points for the allocator are GCHeap::Alloc* which are called by the
17 // allocation helpers in gcscan.cpp
21 #include "softwarewritewatch.h"
25 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
26 BOOL bgc_heap_walk_for_etw_p = FALSE;
27 #endif //BACKGROUND_GC && FEATURE_EVENT_TRACE
29 #if defined(FEATURE_REDHAWK)
30 #define MAYBE_UNUSED_VAR(v) v = v
32 #define MAYBE_UNUSED_VAR(v)
33 #endif // FEATURE_REDHAWK
35 #define MAX_PTR ((uint8_t*)(~(ptrdiff_t)0))
38 #define partial_size_th 100
39 #define num_partial_refs 64
41 #define partial_size_th 100
42 #define num_partial_refs 32
45 #define demotion_plug_len_th (6*1024*1024)
48 #define MARK_STACK_INITIAL_LENGTH 1024
50 #define MARK_STACK_INITIAL_LENGTH 128
53 #define LOH_PIN_QUEUE_LENGTH 100
54 #define LOH_PIN_DECAY 10
57 // Right now we support maximum 1024 procs - meaning that we will create at most
58 // that many GC threads and GC heaps.
59 #define MAX_SUPPORTED_CPUS 1024
61 #define MAX_SUPPORTED_CPUS 64
64 #ifdef GC_CONFIG_DRIVEN
65 int compact_ratio = 0;
66 #endif //GC_CONFIG_DRIVEN
68 // See comments in reset_memory.
69 BOOL reset_mm_p = TRUE;
71 bool g_fFinalizerRunOnShutDown = false;
74 bool g_built_with_svr_gc = true;
76 bool g_built_with_svr_gc = false;
77 #endif // FEATURE_SVR_GC
79 #if defined(BUILDENV_DEBUG)
80 uint8_t g_build_variant = 0;
81 #elif defined(BUILDENV_CHECKED)
82 uint8_t g_build_variant = 1;
84 uint8_t g_build_variant = 2;
85 #endif // defined(BUILDENV_DEBUG)
87 VOLATILE(int32_t) g_no_gc_lock = -1;
89 #if defined (TRACE_GC) && !defined (DACCESS_COMPILE)
90 const char * const allocation_state_str[] = {
96 "try_fit_new_seg_after_cg",
100 "try_free_full_seg_in_bgc",
101 "try_free_after_bgc",
104 "acquire_seg_after_cg",
105 "acquire_seg_after_bgc",
106 "check_and_wait_for_bgc",
107 "trigger_full_compact_gc",
108 "trigger_ephemeral_gc",
109 "trigger_2nd_ephemeral_gc",
112 #endif //TRACE_GC && !DACCESS_COMPILE
114 // Keep this in sync with the definition of gc_reason
115 #if (defined(DT_LOG) || defined(TRACE_GC)) && !defined (DACCESS_COMPILE)
116 static const char* const str_gc_reasons[] =
131 static const char* const str_gc_pause_modes[] =
136 "sustained_low_latency",
139 #endif // defined(DT_LOG) || defined(TRACE_GC)
142 BOOL is_induced (gc_reason reason)
144 return ((reason == reason_induced) ||
145 (reason == reason_induced_noforce) ||
146 (reason == reason_lowmemory) ||
147 (reason == reason_lowmemory_blocking) ||
148 (reason == reason_induced_compacting));
152 BOOL is_induced_blocking (gc_reason reason)
154 return ((reason == reason_induced) ||
155 (reason == reason_lowmemory_blocking) ||
156 (reason == reason_induced_compacting));
159 #ifndef DACCESS_COMPILE
162 size_t GetHighPrecisionTimeStamp()
164 int64_t ts = GCToOSInterface::QueryPerformanceCounter();
166 return (size_t)(ts / (qpf / 1000));
172 // There is a current and a prior copy of the statistics. This allows us to display deltas per reporting
173 // interval, as well as running totals. The 'min' and 'max' values require special treatment. They are
174 // Reset (zeroed) in the current statistics when we begin a new interval and they are updated via a
175 // comparison with the global min/max.
176 GCStatistics g_GCStatistics;
177 GCStatistics g_LastGCStatistics;
179 char* GCStatistics::logFileName = NULL;
180 FILE* GCStatistics::logFile = NULL;
182 void GCStatistics::AddGCStats(const gc_mechanisms& settings, size_t timeInMSec)
185 if (settings.concurrent)
187 bgc.Accumulate((uint32_t)timeInMSec*1000);
190 else if (settings.background_p)
192 fgc.Accumulate((uint32_t)timeInMSec*1000);
194 if (settings.compaction)
196 assert(settings.condemned_generation < max_generation);
197 cntFGCGen[settings.condemned_generation]++;
200 #endif // BACKGROUND_GC
202 ngc.Accumulate((uint32_t)timeInMSec*1000);
204 if (settings.compaction)
206 cntNGCGen[settings.condemned_generation]++;
209 if (is_induced (settings.reason))
210 cntReasons[(int)reason_induced]++;
212 else if (settings.stress_induced)
213 cntReasons[(int)reason_gcstress]++;
214 #endif // STRESS_HEAP
216 cntReasons[(int)settings.reason]++;
219 if (settings.concurrent || !settings.background_p)
221 #endif // BACKGROUND_GC
225 #endif // BACKGROUND_GC
228 void GCStatistics::Initialize()
230 LIMITED_METHOD_CONTRACT;
231 // for efficiency sake we're taking a dependency on the layout of a C++ object
232 // with a vtable. protect against violations of our premise:
233 static_assert(offsetof(GCStatistics, cntDisplay) == sizeof(void*),
234 "The first field of GCStatistics follows the pointer sized vtable");
236 int podOffs = offsetof(GCStatistics, cntDisplay); // offset of the first POD field
237 memset((uint8_t*)(&g_GCStatistics)+podOffs, 0, sizeof(g_GCStatistics)-podOffs);
238 memset((uint8_t*)(&g_LastGCStatistics)+podOffs, 0, sizeof(g_LastGCStatistics)-podOffs);
241 void GCStatistics::DisplayAndUpdate()
243 LIMITED_METHOD_CONTRACT;
245 if (logFileName == NULL || logFile == NULL)
250 fprintf(logFile, "\nGCMix **** Initialize *****\n\n");
252 fprintf(logFile, "GCMix **** Summary ***** %d\n", cntDisplay);
254 // NGC summary (total, timing info)
255 ngc.DisplayAndUpdate(logFile, "NGC ", &g_LastGCStatistics.ngc, cntNGC, g_LastGCStatistics.cntNGC, msec);
257 // FGC summary (total, timing info)
258 fgc.DisplayAndUpdate(logFile, "FGC ", &g_LastGCStatistics.fgc, cntFGC, g_LastGCStatistics.cntFGC, msec);
261 bgc.DisplayAndUpdate(logFile, "BGC ", &g_LastGCStatistics.bgc, cntBGC, g_LastGCStatistics.cntBGC, msec);
263 // NGC/FGC break out by generation & compacting vs. sweeping
264 fprintf(logFile, "NGC ");
265 for (int i = max_generation; i >= 0; --i)
266 fprintf(logFile, "gen%d %d (%d). ", i, cntNGCGen[i]-g_LastGCStatistics.cntNGCGen[i], cntNGCGen[i]);
267 fprintf(logFile, "\n");
269 fprintf(logFile, "FGC ");
270 for (int i = max_generation-1; i >= 0; --i)
271 fprintf(logFile, "gen%d %d (%d). ", i, cntFGCGen[i]-g_LastGCStatistics.cntFGCGen[i], cntFGCGen[i]);
272 fprintf(logFile, "\n");
274 // Compacting vs. Sweeping break out
275 int _cntSweep = cntNGC-cntCompactNGC;
276 int _cntLastSweep = g_LastGCStatistics.cntNGC-g_LastGCStatistics.cntCompactNGC;
277 fprintf(logFile, "NGC Sweeping %d (%d) Compacting %d (%d)\n",
278 _cntSweep - _cntLastSweep, _cntSweep,
279 cntCompactNGC - g_LastGCStatistics.cntCompactNGC, cntCompactNGC);
281 _cntSweep = cntFGC-cntCompactFGC;
282 _cntLastSweep = g_LastGCStatistics.cntFGC-g_LastGCStatistics.cntCompactFGC;
283 fprintf(logFile, "FGC Sweeping %d (%d) Compacting %d (%d)\n",
284 _cntSweep - _cntLastSweep, _cntSweep,
285 cntCompactFGC - g_LastGCStatistics.cntCompactFGC, cntCompactFGC);
289 for (int reason=(int)reason_alloc_soh; reason <= (int)reason_gcstress; ++reason)
291 if (cntReasons[reason] != 0)
292 fprintf(logFile, "%s %d (%d). ", str_gc_reasons[reason],
293 cntReasons[reason]-g_LastGCStatistics.cntReasons[reason], cntReasons[reason]);
296 fprintf(logFile, "\n\n");
298 // flush the log file...
302 g_LastGCStatistics = *this;
312 size_t round_up_power2 (size_t size)
314 // Get the 0-based index of the most-significant bit in size-1.
315 // If the call failed (because size-1 is zero), size must be 1,
316 // so return 1 (because 1 rounds up to itself).
317 DWORD highest_set_bit_index;
324 &highest_set_bit_index, size - 1)) { return 1; }
326 // The size == 0 case (which would have overflowed to SIZE_MAX when decremented)
327 // is handled below by relying on the fact that highest_set_bit_index is the maximum value
328 // (31 or 63, depending on sizeof(size_t)) and left-shifting a value >= 2 by that
329 // number of bits shifts in zeros from the right, resulting in an output of zero.
330 return static_cast<size_t>(2) << highest_set_bit_index;
334 size_t round_down_power2 (size_t size)
336 // Get the 0-based index of the most-significant bit in size.
337 // If the call failed, size must be zero so return zero.
338 DWORD highest_set_bit_index;
345 &highest_set_bit_index, size)) { return 0; }
347 // Left-shift 1 by highest_set_bit_index to get back a value containing only
348 // the most-significant set bit of size, i.e. size rounded down
349 // to the next power-of-two value.
350 return static_cast<size_t>(1) << highest_set_bit_index;
353 // Get the 0-based index of the most-significant bit in the value.
354 // Returns -1 if the input value is zero (i.e. has no set bits).
356 int index_of_highest_set_bit (size_t value)
358 // Get the 0-based index of the most-significant bit in the value.
359 // If the call failed (because value is zero), return -1.
360 DWORD highest_set_bit_index;
367 &highest_set_bit_index, value)) ? -1 : static_cast<int>(highest_set_bit_index);
371 int relative_index_power2_plug (size_t power2)
373 int index = index_of_highest_set_bit (power2);
374 assert (index <= MAX_INDEX_POWER2);
376 return ((index < MIN_INDEX_POWER2) ? 0 : (index - MIN_INDEX_POWER2));
380 int relative_index_power2_free_space (size_t power2)
382 int index = index_of_highest_set_bit (power2);
383 assert (index <= MAX_INDEX_POWER2);
385 return ((index < MIN_INDEX_POWER2) ? -1 : (index - MIN_INDEX_POWER2));
389 uint32_t bgc_alloc_spin_count = 140;
390 uint32_t bgc_alloc_spin_count_loh = 16;
391 uint32_t bgc_alloc_spin = 2;
395 void c_write (uint32_t& place, uint32_t value)
397 Interlocked::Exchange (&place, value);
401 #ifndef DACCESS_COMPILE
402 // If every heap's gen2 or gen3 size is less than this threshold we will do a blocking GC.
403 const size_t bgc_min_per_heap = 4*1024*1024;
405 int gc_heap::gchist_index = 0;
406 gc_mechanisms_store gc_heap::gchist[max_history_count];
408 #ifndef MULTIPLE_HEAPS
409 size_t gc_heap::total_promoted_bytes = 0;
410 VOLATILE(bgc_state) gc_heap::current_bgc_state = bgc_not_in_process;
411 int gc_heap::gchist_index_per_heap = 0;
412 gc_heap::gc_history gc_heap::gchist_per_heap[max_history_count];
413 #endif //MULTIPLE_HEAPS
415 void gc_heap::add_to_history_per_heap()
418 gc_history* current_hist = &gchist_per_heap[gchist_index_per_heap];
419 current_hist->gc_index = settings.gc_index;
420 current_hist->current_bgc_state = current_bgc_state;
421 size_t elapsed = dd_gc_elapsed_time (dynamic_data_of (0));
422 current_hist->gc_time_ms = (uint32_t)elapsed;
423 current_hist->gc_efficiency = (elapsed ? (total_promoted_bytes / elapsed) : total_promoted_bytes);
424 current_hist->eph_low = generation_allocation_start (generation_of (max_generation-1));
425 current_hist->gen0_start = generation_allocation_start (generation_of (0));
426 current_hist->eph_high = heap_segment_allocated (ephemeral_heap_segment);
428 current_hist->bgc_lowest = background_saved_lowest_address;
429 current_hist->bgc_highest = background_saved_highest_address;
430 #endif //BACKGROUND_GC
431 current_hist->fgc_lowest = lowest_address;
432 current_hist->fgc_highest = highest_address;
433 current_hist->g_lowest = g_gc_lowest_address;
434 current_hist->g_highest = g_gc_highest_address;
436 gchist_index_per_heap++;
437 if (gchist_index_per_heap == max_history_count)
439 gchist_index_per_heap = 0;
444 void gc_heap::add_to_history()
447 gc_mechanisms_store* current_settings = &gchist[gchist_index];
448 current_settings->store (&settings);
451 if (gchist_index == max_history_count)
458 #endif //DACCESS_COMPILE
459 #endif //BACKGROUND_GC
461 #if defined(TRACE_GC) && !defined(DACCESS_COMPILE)
462 BOOL gc_log_on = TRUE;
464 size_t gc_log_file_size = 0;
466 size_t gc_buffer_index = 0;
467 size_t max_gc_buffers = 0;
469 static CLRCriticalSection gc_log_lock;
471 // we keep this much in a buffer and only flush when the buffer is full
472 #define gc_log_buffer_size (1024*1024)
473 uint8_t* gc_log_buffer = 0;
474 size_t gc_log_buffer_offset = 0;
476 void log_va_msg(const char *fmt, va_list args)
480 const int BUFFERSIZE = 512;
481 static char rgchBuffer[BUFFERSIZE];
482 char * pBuffer = &rgchBuffer[0];
485 int buffer_start = 1;
486 int pid_len = sprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, "[%5d]", (uint32_t)GCToOSInterface::GetCurrentThreadIdForLogging());
487 buffer_start += pid_len;
488 memset(&pBuffer[buffer_start], '-', BUFFERSIZE - buffer_start);
489 int msg_len = _vsnprintf_s(&pBuffer[buffer_start], BUFFERSIZE - buffer_start, _TRUNCATE, fmt, args );
492 msg_len = BUFFERSIZE - buffer_start;
495 msg_len += buffer_start;
497 if ((gc_log_buffer_offset + msg_len) > (gc_log_buffer_size - 12))
500 memset (index_str, '-', 8);
501 sprintf_s (index_str, _countof(index_str), "%d", (int)gc_buffer_index);
502 gc_log_buffer[gc_log_buffer_offset] = '\n';
503 memcpy (gc_log_buffer + (gc_log_buffer_offset + 1), index_str, 8);
506 if (gc_buffer_index > max_gc_buffers)
508 fseek (gc_log, 0, SEEK_SET);
511 fwrite(gc_log_buffer, gc_log_buffer_size, 1, gc_log);
513 memset (gc_log_buffer, '*', gc_log_buffer_size);
514 gc_log_buffer_offset = 0;
517 memcpy (gc_log_buffer + gc_log_buffer_offset, pBuffer, msg_len);
518 gc_log_buffer_offset += msg_len;
523 void GCLog (const char *fmt, ... )
525 if (gc_log_on && (gc_log != NULL))
529 log_va_msg (fmt, args);
533 #endif // TRACE_GC && !DACCESS_COMPILE
535 #if defined(GC_CONFIG_DRIVEN) && !defined(DACCESS_COMPILE)
537 BOOL gc_config_log_on = FALSE;
538 FILE* gc_config_log = NULL;
540 // we keep this much in a buffer and only flush when the buffer is full
541 #define gc_config_log_buffer_size (1*1024) // TEMP
542 uint8_t* gc_config_log_buffer = 0;
543 size_t gc_config_log_buffer_offset = 0;
545 // For config since we log so little we keep the whole history. Also it's only
546 // ever logged by one thread so no need to synchronize.
547 void log_va_msg_config(const char *fmt, va_list args)
549 const int BUFFERSIZE = 256;
550 static char rgchBuffer[BUFFERSIZE];
551 char * pBuffer = &rgchBuffer[0];
554 int buffer_start = 1;
555 int msg_len = _vsnprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, _TRUNCATE, fmt, args );
556 assert (msg_len != -1);
557 msg_len += buffer_start;
559 if ((gc_config_log_buffer_offset + msg_len) > gc_config_log_buffer_size)
561 fwrite(gc_config_log_buffer, gc_config_log_buffer_offset, 1, gc_config_log);
562 fflush(gc_config_log);
563 gc_config_log_buffer_offset = 0;
566 memcpy (gc_config_log_buffer + gc_config_log_buffer_offset, pBuffer, msg_len);
567 gc_config_log_buffer_offset += msg_len;
570 void GCLogConfig (const char *fmt, ... )
572 if (gc_config_log_on && (gc_config_log != NULL))
575 va_start( args, fmt );
576 log_va_msg_config (fmt, args);
579 #endif // GC_CONFIG_DRIVEN && !DACCESS_COMPILE
581 #ifdef SYNCHRONIZATION_STATS
583 // Number of GCs have we done since we last logged.
584 static unsigned int gc_count_during_log;
585 // In ms. This is how often we print out stats.
586 static const unsigned int log_interval = 5000;
587 // Time (in ms) when we start a new log interval.
588 static unsigned int log_start_tick;
589 static unsigned int gc_lock_contended;
590 static int64_t log_start_hires;
591 // Cycles accumulated in SuspendEE during log_interval.
592 static uint64_t suspend_ee_during_log;
593 // Cycles accumulated in RestartEE during log_interval.
594 static uint64_t restart_ee_during_log;
595 static uint64_t gc_during_log;
597 #endif //SYNCHRONIZATION_STATS
600 init_sync_log_stats()
602 #ifdef SYNCHRONIZATION_STATS
603 if (gc_count_during_log == 0)
605 gc_heap::init_sync_stats();
606 suspend_ee_during_log = 0;
607 restart_ee_during_log = 0;
609 gc_lock_contended = 0;
611 log_start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
612 log_start_hires = GCToOSInterface::QueryPerformanceCounter();
614 gc_count_during_log++;
615 #endif //SYNCHRONIZATION_STATS
619 process_sync_log_stats()
621 #ifdef SYNCHRONIZATION_STATS
623 unsigned int log_elapsed = GCToOSInterface::GetLowPrecisionTimeStamp() - log_start_tick;
625 if (log_elapsed > log_interval)
627 uint64_t total = GCToOSInterface::QueryPerformanceCounter() - log_start_hires;
628 // Print out the cycles we spent on average in each suspend and restart.
629 printf("\n_________________________________________________________________________________\n"
630 "Past %d(s): #%3d GCs; Total gc_lock contended: %8u; GC: %12u\n"
631 "SuspendEE: %8u; RestartEE: %8u GC %.3f%%\n",
635 (unsigned int)(gc_during_log / gc_count_during_log),
636 (unsigned int)(suspend_ee_during_log / gc_count_during_log),
637 (unsigned int)(restart_ee_during_log / gc_count_during_log),
638 (double)(100.0f * gc_during_log / total));
639 gc_heap::print_sync_stats(gc_count_during_log);
641 gc_count_during_log = 0;
643 #endif //SYNCHRONIZATION_STATS
646 #ifdef MULTIPLE_HEAPS
650 gc_join_init_cpu_mapping = 0,
652 gc_join_generation_determined = 2,
653 gc_join_begin_mark_phase = 3,
654 gc_join_scan_dependent_handles = 4,
655 gc_join_rescan_dependent_handles = 5,
656 gc_join_scan_sizedref_done = 6,
657 gc_join_null_dead_short_weak = 7,
658 gc_join_scan_finalization = 8,
659 gc_join_null_dead_long_weak = 9,
660 gc_join_null_dead_syncblk = 10,
661 gc_join_decide_on_compaction = 11,
662 gc_join_rearrange_segs_compaction = 12,
663 gc_join_adjust_handle_age_compact = 13,
664 gc_join_adjust_handle_age_sweep = 14,
665 gc_join_begin_relocate_phase = 15,
666 gc_join_relocate_phase_done = 16,
667 gc_join_verify_objects_done = 17,
668 gc_join_start_bgc = 18,
669 gc_join_restart_ee = 19,
670 gc_join_concurrent_overflow = 20,
671 gc_join_suspend_ee = 21,
672 gc_join_bgc_after_ephemeral = 22,
673 gc_join_allow_fgc = 23,
674 gc_join_bgc_sweep = 24,
675 gc_join_suspend_ee_verify = 25,
676 gc_join_restart_ee_verify = 26,
677 gc_join_set_state_free = 27,
678 gc_r_join_update_card_bundle = 28,
679 gc_join_after_absorb = 29,
680 gc_join_verify_copy_table = 30,
681 gc_join_after_reset = 31,
682 gc_join_after_ephemeral_sweep = 32,
683 gc_join_after_profiler_heap_walk = 33,
684 gc_join_minimal_gc = 34,
685 gc_join_after_commit_soh_no_gc = 35,
686 gc_join_expand_loh_no_gc = 36,
687 gc_join_final_no_gc = 37,
688 gc_join_disable_software_write_watch = 38,
694 join_flavor_server_gc = 0,
698 #define first_thread_arrived 2
699 struct DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE) join_structure
701 // Shared non volatile keep on separate line to prevent eviction
704 // Keep polling/wait structures on separate line write once per join
705 DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE)
706 GCEvent joined_event[3]; // the last event in the array is only used for first_thread_arrived.
707 Volatile<int> lock_color;
708 VOLATILE(BOOL) wait_done;
709 VOLATILE(BOOL) joined_p;
711 // Keep volatile counted locks on separate cache line write many per join
712 DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE)
713 VOLATILE(int32_t) join_lock;
714 VOLATILE(int32_t) r_join_lock;
723 type_first_r_join = 3,
735 join_heap_restart = 100,
736 join_heap_r_restart = 200
748 join_structure join_struct;
751 gc_join_flavor flavor;
754 uint64_t start[MAX_SUPPORTED_CPUS], end[MAX_SUPPORTED_CPUS], start_seq;
755 // remember join id and last thread to arrive so restart can use these
757 // we want to print statistics every 10 seconds - this is to remember the start of the 10 sec interval
759 // counters for joins, in 1000's of clock cycles
760 uint64_t elapsed_total[gc_join_max], wake_total[gc_join_max], seq_loss_total[gc_join_max], par_loss_total[gc_join_max], in_join_total[gc_join_max];
764 BOOL init (int n_th, gc_join_flavor f)
766 dprintf (JOIN_LOG, ("Initializing join structure"));
767 join_struct.n_threads = n_th;
768 join_struct.lock_color = 0;
769 for (int i = 0; i < 3; i++)
771 if (!join_struct.joined_event[i].IsValid())
773 join_struct.joined_p = FALSE;
774 dprintf (JOIN_LOG, ("Creating join event %d", i));
775 // TODO - changing this to a non OS event
776 // because this is also used by BGC threads which are
777 // managed threads and WaitEx does not allow you to wait
778 // for an OS event on a managed thread.
779 // But we are not sure if this plays well in the hosting
781 //join_struct.joined_event[i].CreateOSManualEventNoThrow(FALSE);
782 if (!join_struct.joined_event[i].CreateManualEventNoThrow(FALSE))
786 join_struct.join_lock = join_struct.n_threads;
787 join_struct.r_join_lock = join_struct.n_threads;
788 join_struct.wait_done = FALSE;
792 start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
800 dprintf (JOIN_LOG, ("Destroying join structure"));
801 for (int i = 0; i < 3; i++)
803 if (join_struct.joined_event[i].IsValid())
804 join_struct.joined_event[i].CloseEvent();
808 inline void fire_event (int heap, join_time time, join_type type, int join_id)
810 FIRE_EVENT(GCJoin_V2, heap, time, type, join_id);
813 void join (gc_heap* gch, int join_id)
816 // parallel execution ends here
817 end[gch->heap_number] = get_ts();
820 assert (!join_struct.joined_p);
821 int color = join_struct.lock_color.LoadWithoutBarrier();
823 if (Interlocked::Decrement(&join_struct.join_lock) != 0)
825 dprintf (JOIN_LOG, ("join%d(%d): Join() Waiting...join_lock is now %d",
826 flavor, join_id, (int32_t)(join_struct.join_lock)));
828 fire_event (gch->heap_number, time_start, type_join, join_id);
830 //busy wait around the color
831 if (color == join_struct.lock_color.LoadWithoutBarrier())
834 int spin_count = 4096 * (gc_heap::n_heaps - 1);
835 for (int j = 0; j < spin_count; j++)
837 if (color != join_struct.lock_color.LoadWithoutBarrier())
841 YieldProcessor(); // indicate to the processor that we are spinning
844 // we've spun, and if color still hasn't changed, fall into hard wait
845 if (color == join_struct.lock_color.LoadWithoutBarrier())
847 dprintf (JOIN_LOG, ("join%d(%d): Join() hard wait on reset event %d, join_lock is now %d",
848 flavor, join_id, color, (int32_t)(join_struct.join_lock)));
850 //Thread* current_thread = GCToEEInterface::GetThread();
851 //BOOL cooperative_mode = gc_heap::enable_preemptive (current_thread);
852 uint32_t dwJoinWait = join_struct.joined_event[color].Wait(INFINITE, FALSE);
853 //gc_heap::disable_preemptive (current_thread, cooperative_mode);
855 if (dwJoinWait != WAIT_OBJECT_0)
857 STRESS_LOG1 (LF_GC, LL_FATALERROR, "joined event wait failed with code: %Ix", dwJoinWait);
862 // avoid race due to the thread about to reset the event (occasionally) being preempted before ResetEvent()
863 if (color == join_struct.lock_color.LoadWithoutBarrier())
868 dprintf (JOIN_LOG, ("join%d(%d): Join() done, join_lock is %d",
869 flavor, join_id, (int32_t)(join_struct.join_lock)));
872 fire_event (gch->heap_number, time_end, type_join, join_id);
875 // parallel execution starts here
876 start[gch->heap_number] = get_ts();
877 Interlocked::ExchangeAdd(&in_join_total[join_id], (start[gch->heap_number] - end[gch->heap_number]));
882 fire_event (gch->heap_number, time_start, type_last_join, join_id);
884 join_struct.joined_p = TRUE;
885 dprintf (JOIN_LOG, ("join%d(%d): Last thread to complete the join, setting id", flavor, join_id));
886 join_struct.joined_event[!color].Reset();
888 // this one is alone so it can proceed
890 // remember the join id, the last thread arriving, the start of the sequential phase,
891 // and keep track of the cycles spent waiting in the join
892 thd = gch->heap_number;
893 start_seq = get_ts();
894 Interlocked::ExchangeAdd(&in_join_total[join_id], (start_seq - end[gch->heap_number]));
899 // Reverse join - first thread gets here does the work; other threads will only proceed
900 // after the work is done.
901 // Note that you cannot call this twice in a row on the same thread. Plus there's no
902 // need to call it twice in row - you should just merge the work.
903 BOOL r_join (gc_heap* gch, int join_id)
906 if (join_struct.n_threads == 1)
911 if (Interlocked::CompareExchange(&join_struct.r_join_lock, 0, join_struct.n_threads) == 0)
913 if (!join_struct.wait_done)
915 dprintf (JOIN_LOG, ("r_join() Waiting..."));
917 fire_event (gch->heap_number, time_start, type_join, join_id);
919 //busy wait around the color
920 if (!join_struct.wait_done)
923 int spin_count = 2 * 4096 * (gc_heap::n_heaps - 1);
924 for (int j = 0; j < spin_count; j++)
926 if (join_struct.wait_done)
930 YieldProcessor(); // indicate to the processor that we are spinning
933 // we've spun, and if color still hasn't changed, fall into hard wait
934 if (!join_struct.wait_done)
936 dprintf (JOIN_LOG, ("Join() hard wait on reset event %d", first_thread_arrived));
937 uint32_t dwJoinWait = join_struct.joined_event[first_thread_arrived].Wait(INFINITE, FALSE);
938 if (dwJoinWait != WAIT_OBJECT_0)
940 STRESS_LOG1 (LF_GC, LL_FATALERROR, "joined event wait failed with code: %Ix", dwJoinWait);
945 // avoid race due to the thread about to reset the event (occasionally) being preempted before ResetEvent()
946 if (!join_struct.wait_done)
951 dprintf (JOIN_LOG, ("r_join() done"));
954 fire_event (gch->heap_number, time_end, type_join, join_id);
961 fire_event (gch->heap_number, time_start, type_first_r_join, join_id);
969 return GCToOSInterface::QueryPerformanceCounter();
972 void start_ts (gc_heap* gch)
974 // parallel execution ends here
975 start[gch->heap_number] = get_ts();
982 uint64_t elapsed_seq = get_ts() - start_seq;
983 uint64_t max = 0, sum = 0, wake = 0;
984 uint64_t min_ts = start[0];
985 for (int i = 1; i < join_struct.n_threads; i++)
987 if(min_ts > start[i]) min_ts = start[i];
990 for (int i = 0; i < join_struct.n_threads; i++)
992 uint64_t wake_delay = start[i] - min_ts;
993 uint64_t elapsed = end[i] - start[i];
999 uint64_t seq_loss = (join_struct.n_threads - 1)*elapsed_seq;
1000 uint64_t par_loss = join_struct.n_threads*max - sum;
1001 double efficiency = 0.0;
1003 efficiency = sum*100.0/(join_struct.n_threads*max);
1005 const double ts_scale = 1e-6;
1007 // enable this printf to get statistics on each individual join as it occurs
1008 // printf("join #%3d seq_loss = %5g par_loss = %5g efficiency = %3.0f%%\n", join_id, ts_scale*seq_loss, ts_scale*par_loss, efficiency);
1010 elapsed_total[id] += sum;
1011 wake_total[id] += wake;
1012 seq_loss_total[id] += seq_loss;
1013 par_loss_total[id] += par_loss;
1015 // every 10 seconds, print a summary of the time spent in each type of join
1016 if (GCToOSInterface::GetLowPrecisionTimeStamp() - start_tick > 10*1000)
1018 printf("**** summary *****\n");
1019 for (int i = 0; i < 16; i++)
1021 printf("join #%3d elapsed_total = %8g wake_loss = %8g seq_loss = %8g par_loss = %8g in_join_total = %8g\n",
1023 ts_scale*elapsed_total[i],
1024 ts_scale*wake_total[i],
1025 ts_scale*seq_loss_total[i],
1026 ts_scale*par_loss_total[i],
1027 ts_scale*in_join_total[i]);
1028 elapsed_total[i] = wake_total[i] = seq_loss_total[i] = par_loss_total[i] = in_join_total[i] = 0;
1030 start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
1034 fire_event (join_heap_restart, time_start, type_restart, -1);
1035 assert (join_struct.joined_p);
1036 join_struct.joined_p = FALSE;
1037 join_struct.join_lock = join_struct.n_threads;
1038 dprintf (JOIN_LOG, ("join%d(%d): Restarting from join: join_lock is %d", flavor, id, (int32_t)(join_struct.join_lock)));
1039 // printf("restart from join #%d at cycle %u from start of gc\n", join_id, GetCycleCount32() - gc_start);
1040 int color = join_struct.lock_color.LoadWithoutBarrier();
1041 join_struct.lock_color = !color;
1042 join_struct.joined_event[color].Set();
1044 // printf("Set joined_event %d\n", !join_struct.lock_color);
1046 fire_event (join_heap_restart, time_end, type_restart, -1);
1049 start[thd] = get_ts();
1055 dprintf (JOIN_LOG, ("join%d(%d): joined, join_lock is %d", flavor, id, (int32_t)(join_struct.join_lock)));
1056 return join_struct.joined_p;
1061 if (join_struct.n_threads != 1)
1063 fire_event (join_heap_r_restart, time_start, type_restart, -1);
1064 join_struct.wait_done = TRUE;
1065 join_struct.joined_event[first_thread_arrived].Set();
1066 fire_event (join_heap_r_restart, time_end, type_restart, -1);
1072 if (join_struct.n_threads != 1)
1074 join_struct.r_join_lock = join_struct.n_threads;
1075 join_struct.wait_done = FALSE;
1076 join_struct.joined_event[first_thread_arrived].Reset();
1083 #ifdef BACKGROUND_GC
1085 #endif //BACKGROUND_GC
1087 #endif //MULTIPLE_HEAPS
1089 #define spin_and_switch(count_to_spin, expr) \
1091 for (int j = 0; j < count_to_spin; j++) \
1101 GCToOSInterface::YieldThread(0); \
1105 #ifndef DACCESS_COMPILE
1106 #ifdef BACKGROUND_GC
1108 #define max_pending_allocs 64
1110 class exclusive_sync
1112 // TODO - verify that this is the right syntax for Volatile.
1113 VOLATILE(uint8_t*) rwp_object;
1114 VOLATILE(int32_t) needs_checking;
1118 uint8_t cache_separator[HS_CACHE_LINE_SIZE - sizeof (int) - sizeof (int32_t)];
1120 // TODO - perhaps each object should be on its own cache line...
1121 VOLATILE(uint8_t*) alloc_objects[max_pending_allocs];
1123 int find_free_index ()
1125 for (int i = 0; i < max_pending_allocs; i++)
1127 if (alloc_objects [i] == (uint8_t*)0)
1139 spin_count = 32 * (g_num_processors - 1);
1142 for (int i = 0; i < max_pending_allocs; i++)
1144 alloc_objects [i] = (uint8_t*)0;
1150 for (int i = 0; i < max_pending_allocs; i++)
1152 if (alloc_objects [i] != (uint8_t*)0)
1154 GCToOSInterface::DebugBreak();
1159 void bgc_mark_set (uint8_t* obj)
1161 dprintf (3, ("cm: probing %Ix", obj));
1163 if (Interlocked::CompareExchange(&needs_checking, 1, 0) == 0)
1165 // If we spend too much time spending all the allocs,
1166 // consider adding a high water mark and scan up
1167 // to that; we'll need to interlock in done when
1168 // we update the high watermark.
1169 for (int i = 0; i < max_pending_allocs; i++)
1171 if (obj == alloc_objects[i])
1174 dprintf (3, ("cm: will spin"));
1175 spin_and_switch (spin_count, (obj != alloc_objects[i]));
1182 dprintf (3, ("cm: set %Ix", obj));
1187 spin_and_switch (spin_count, (needs_checking == 0));
1192 int loh_alloc_set (uint8_t* obj)
1194 if (!gc_heap::cm_in_progress)
1200 dprintf (3, ("loh alloc: probing %Ix", obj));
1202 if (Interlocked::CompareExchange(&needs_checking, 1, 0) == 0)
1204 if (obj == rwp_object)
1207 spin_and_switch (spin_count, (obj != rwp_object));
1212 int cookie = find_free_index();
1216 alloc_objects[cookie] = obj;
1220 // GCToOSInterface::DebugBreak();
1223 dprintf (3, ("loh alloc: set %Ix at %d", obj, cookie));
1229 dprintf (3, ("loh alloc: setting %Ix will spin to acquire a free index", obj));
1230 spin_and_switch (spin_count, (find_free_index () != -1));
1237 dprintf (3, ("loh alloc: will spin on checking %Ix", obj));
1238 spin_and_switch (spin_count, (needs_checking == 0));
1243 void bgc_mark_done ()
1245 dprintf (3, ("cm: release lock on %Ix", (uint8_t *)rwp_object));
1249 void loh_alloc_done_with_index (int index)
1251 dprintf (3, ("loh alloc: release lock on %Ix based on %d", (uint8_t *)alloc_objects[index], index));
1252 assert ((index >= 0) && (index < max_pending_allocs));
1253 alloc_objects[index] = (uint8_t*)0;
1256 void loh_alloc_done (uint8_t* obj)
1258 #ifdef BACKGROUND_GC
1259 if (!gc_heap::cm_in_progress)
1264 for (int i = 0; i < max_pending_allocs; i++)
1266 if (alloc_objects [i] == obj)
1268 dprintf (3, ("loh alloc: release lock on %Ix at %d", (uint8_t *)alloc_objects[i], i));
1269 alloc_objects[i] = (uint8_t*)0;
1273 #endif //BACKGROUND_GC
1277 // Note that this class was written assuming just synchronization between
1278 // one background GC thread and multiple user threads that might request
1279 // an FGC - it does not take into account what kind of locks the multiple
1280 // user threads might be holding at the time (eg, there could only be one
1281 // user thread requesting an FGC because it needs to take gc_lock first)
1282 // so you'll see checks that may not be necessary if you take those conditions
1283 // into consideration.
1285 // With the introduction of Server Background GC we no longer use this
1286 // class to do synchronization between FGCs and BGC.
1287 class recursive_gc_sync
1289 static VOLATILE(int32_t) foreground_request_count;//initial state 0
1290 static VOLATILE(BOOL) gc_background_running; //initial state FALSE
1291 static VOLATILE(int32_t) foreground_count; // initial state 0;
1292 static VOLATILE(uint32_t) foreground_gate; // initial state FALSE;
1293 static GCEvent foreground_complete;//Auto Reset
1294 static GCEvent foreground_allowed;//Auto Reset
1296 static void begin_background();
1297 static void end_background();
1298 static void begin_foreground();
1299 static void end_foreground();
1300 BOOL allow_foreground ();
1302 static void shutdown();
1303 static BOOL background_running_p() {return gc_background_running;}
1306 VOLATILE(int32_t) recursive_gc_sync::foreground_request_count = 0;//initial state 0
1307 VOLATILE(int32_t) recursive_gc_sync::foreground_count = 0; // initial state 0;
1308 VOLATILE(BOOL) recursive_gc_sync::gc_background_running = FALSE; //initial state FALSE
1309 VOLATILE(uint32_t) recursive_gc_sync::foreground_gate = 0;
1310 GCEvent recursive_gc_sync::foreground_complete;//Auto Reset
1311 GCEvent recursive_gc_sync::foreground_allowed;//Manual Reset
1313 BOOL recursive_gc_sync::init ()
1315 foreground_request_count = 0;
1316 foreground_count = 0;
1317 gc_background_running = FALSE;
1318 foreground_gate = 0;
1320 if (!foreground_complete.CreateOSAutoEventNoThrow(FALSE))
1324 if (!foreground_allowed.CreateManualEventNoThrow(FALSE))
1336 void recursive_gc_sync::shutdown()
1338 if (foreground_complete.IsValid())
1339 foreground_complete.CloseEvent();
1340 if (foreground_allowed.IsValid())
1341 foreground_allowed.CloseEvent();
1344 void recursive_gc_sync::begin_background()
1346 dprintf (2, ("begin background"));
1347 foreground_request_count = 1;
1348 foreground_count = 1;
1349 foreground_allowed.Reset();
1350 gc_background_running = TRUE;
1352 void recursive_gc_sync::end_background()
1354 dprintf (2, ("end background"));
1355 gc_background_running = FALSE;
1356 foreground_gate = 1;
1357 foreground_allowed.Set();
1360 void recursive_gc_sync::begin_foreground()
1362 dprintf (2, ("begin_foreground"));
1364 bool cooperative_mode = false;
1365 if (gc_background_running)
1367 gc_heap::fire_alloc_wait_event_begin (awr_fgc_wait_for_bgc);
1368 gc_heap::alloc_wait_event_p = TRUE;
1372 Interlocked::Increment (&foreground_request_count);
1375 dprintf(2, ("Waiting sync gc point"));
1376 assert (foreground_allowed.IsValid());
1377 assert (foreground_complete.IsValid());
1379 cooperative_mode = gc_heap::enable_preemptive ();
1381 foreground_allowed.Wait(INFINITE, FALSE);
1383 dprintf(2, ("Waiting sync gc point is done"));
1385 gc_heap::disable_preemptive (cooperative_mode);
1387 if (foreground_gate)
1389 Interlocked::Increment (&foreground_count);
1390 dprintf (2, ("foreground_count: %d", (int32_t)foreground_count));
1391 if (foreground_gate)
1393 gc_heap::settings.concurrent = FALSE;
1404 goto try_again_no_inc;
1409 void recursive_gc_sync::end_foreground()
1411 dprintf (2, ("end_foreground"));
1412 if (gc_background_running)
1414 Interlocked::Decrement (&foreground_request_count);
1415 dprintf (2, ("foreground_count before decrement: %d", (int32_t)foreground_count));
1416 if (Interlocked::Decrement (&foreground_count) == 0)
1418 //c_write ((BOOL*)&foreground_gate, 0);
1419 // TODO - couldn't make the syntax work with Volatile<T>
1420 foreground_gate = 0;
1421 if (foreground_count == 0)
1423 foreground_allowed.Reset ();
1424 dprintf(2, ("setting foreground complete event"));
1425 foreground_complete.Set();
1432 BOOL recursive_gc_sync::allow_foreground()
1434 assert (gc_heap::settings.concurrent);
1435 dprintf (100, ("enter allow_foreground, f_req_count: %d, f_count: %d",
1436 (int32_t)foreground_request_count, (int32_t)foreground_count));
1438 BOOL did_fgc = FALSE;
1440 //if we have suspended the EE, just return because
1441 //some thread could be waiting on this to proceed.
1442 if (!GCHeap::GcInProgress)
1444 //TODO BACKGROUND_GC This is to stress the concurrency between
1445 //background and foreground
1446 // gc_heap::disallow_new_allocation (0);
1448 //GCToOSInterface::YieldThread(0);
1451 if (foreground_request_count != 0)
1453 //foreground wants to run
1454 //save the important settings
1455 //TODO BACKGROUND_GC be more selective about the important settings.
1456 gc_mechanisms saved_settings = gc_heap::settings;
1460 //c_write ((BOOL*)&foreground_gate, 1);
1461 // TODO - couldn't make the syntax work with Volatile<T>
1462 foreground_gate = 1;
1463 foreground_allowed.Set ();
1464 foreground_complete.Wait (INFINITE, FALSE);
1465 }while (/*foreground_request_count ||*/ foreground_gate);
1467 assert (!foreground_gate);
1469 //restore the important settings
1470 gc_heap::settings = saved_settings;
1471 GCHeap::GcCondemnedGeneration = gc_heap::settings.condemned_generation;
1472 //the background GC shouldn't be using gc_high and gc_low
1473 //gc_low = lowest_address;
1474 //gc_high = highest_address;
1477 //TODO BACKGROUND_GC This is to stress the concurrency between
1478 //background and foreground
1479 // gc_heap::allow_new_allocation (0);
1483 dprintf (100, ("leave allow_foreground"));
1484 assert (gc_heap::settings.concurrent);
1488 #endif //BACKGROUND_GC
1489 #endif //DACCESS_COMPILE
1492 #if defined(COUNT_CYCLES)
1494 #pragma warning(disable:4035)
1498 unsigned GetCycleCount32() // enough for about 40 seconds
1506 #pragma warning(default:4035)
1508 #endif //COUNT_CYCLES
1511 int mark_time, plan_time, sweep_time, reloc_time, compact_time;
1514 #ifndef MULTIPLE_HEAPS
1516 #endif // MULTIPLE_HEAPS
1518 void reset_memory (uint8_t* o, size_t sizeo);
1522 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1523 static bool virtual_alloc_hardware_write_watch = false;
1524 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1526 static bool hardware_write_watch_capability = false;
1528 #ifndef DACCESS_COMPILE
1530 //check if the write watch APIs are supported.
1532 void hardware_write_watch_api_supported()
1534 if (GCToOSInterface::SupportsWriteWatch())
1536 hardware_write_watch_capability = true;
1537 dprintf (2, ("WriteWatch supported"));
1541 dprintf (2,("WriteWatch not supported"));
1545 #endif //!DACCESS_COMPILE
1547 inline bool can_use_hardware_write_watch()
1549 return hardware_write_watch_capability;
1552 inline bool can_use_write_watch_for_gc_heap()
1554 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1556 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1557 return can_use_hardware_write_watch();
1558 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1561 inline bool can_use_write_watch_for_card_table()
1563 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
1566 return can_use_hardware_write_watch();
1571 #define mem_reserve (MEM_RESERVE)
1572 #endif //WRITE_WATCH
1574 //check if the low memory notification is supported
1576 #ifndef DACCESS_COMPILE
1578 void WaitLongerNoInstru (int i)
1580 // every 8th attempt:
1581 bool bToggleGC = GCToEEInterface::EnablePreemptiveGC();
1583 // if we're waiting for gc to finish, we should block immediately
1584 if (g_fSuspensionPending == 0)
1586 if (g_num_processors > 1)
1588 YieldProcessor(); // indicate to the processor that we are spining
1590 GCToOSInterface::YieldThread (0);
1592 GCToOSInterface::Sleep (5);
1595 GCToOSInterface::Sleep (5);
1598 // If CLR is hosted, a thread may reach here while it is in preemptive GC mode,
1599 // or it has no Thread object, in order to force a task to yield, or to triger a GC.
1600 // It is important that the thread is going to wait for GC. Otherwise the thread
1601 // is in a tight loop. If the thread has high priority, the perf is going to be very BAD.
1605 // In debug builds, all enter_spin_lock operations go through this code. If a GC has
1606 // started, it is important to block until the GC thread calls set_gc_done (since it is
1607 // guaranteed to have cleared g_TrapReturningThreads by this point). This avoids livelock
1608 // conditions which can otherwise occur if threads are allowed to spin in this function
1609 // (and therefore starve the GC thread) between the point when the GC thread sets the
1610 // WaitForGC event and the point when the GC thread clears g_TrapReturningThreads.
1611 if (gc_heap::gc_started)
1613 gc_heap::wait_for_gc_done();
1616 GCToEEInterface::DisablePreemptiveGC();
1618 else if (g_fSuspensionPending > 0)
1620 g_theGCHeap->WaitUntilGCComplete();
1625 static void safe_switch_to_thread()
1627 bool cooperative_mode = gc_heap::enable_preemptive();
1629 GCToOSInterface::YieldThread(0);
1631 gc_heap::disable_preemptive(cooperative_mode);
1635 // We need the following methods to have volatile arguments, so that they can accept
1636 // raw pointers in addition to the results of the & operator on Volatile<T>.
1639 static void enter_spin_lock_noinstru (RAW_KEYWORD(volatile) int32_t* lock)
1643 if (Interlocked::CompareExchange(lock, 0, -1) >= 0)
1646 while (VolatileLoad(lock) >= 0)
1648 if ((++i & 7) && !IsGCInProgress())
1650 if (g_num_processors > 1)
1652 #ifndef MULTIPLE_HEAPS
1653 int spin_count = 1024 * g_num_processors;
1654 #else //!MULTIPLE_HEAPS
1655 int spin_count = 32 * g_num_processors;
1656 #endif //!MULTIPLE_HEAPS
1657 for (int j = 0; j < spin_count; j++)
1659 if (VolatileLoad(lock) < 0 || IsGCInProgress())
1661 YieldProcessor(); // indicate to the processor that we are spining
1663 if (VolatileLoad(lock) >= 0 && !IsGCInProgress())
1665 safe_switch_to_thread();
1670 safe_switch_to_thread();
1675 WaitLongerNoInstru(i);
1683 static BOOL try_enter_spin_lock_noinstru(RAW_KEYWORD(volatile) int32_t* lock)
1685 return (Interlocked::CompareExchange(&*lock, 0, -1) < 0);
1689 static void leave_spin_lock_noinstru (RAW_KEYWORD(volatile) int32_t* lock)
1691 VolatileStore<int32_t>((int32_t*)lock, -1);
1697 static void enter_spin_lock(GCSpinLock *pSpinLock)
1699 enter_spin_lock_noinstru(&pSpinLock->lock);
1700 assert (pSpinLock->holding_thread == (Thread*)-1);
1701 pSpinLock->holding_thread = GCToEEInterface::GetThread();
1705 static BOOL try_enter_spin_lock(GCSpinLock *pSpinLock)
1707 BOOL ret = try_enter_spin_lock_noinstru(&pSpinLock->lock);
1709 pSpinLock->holding_thread = GCToEEInterface::GetThread();
1714 static void leave_spin_lock(GCSpinLock *pSpinLock)
1716 bool gc_thread_p = GCToEEInterface::WasCurrentThreadCreatedByGC();
1717 // _ASSERTE((pSpinLock->holding_thread == GCToEEInterface::GetThread()) || gc_thread_p || pSpinLock->released_by_gc_p);
1718 pSpinLock->released_by_gc_p = gc_thread_p;
1719 pSpinLock->holding_thread = (Thread*) -1;
1720 if (pSpinLock->lock != -1)
1721 leave_spin_lock_noinstru(&pSpinLock->lock);
1724 #define ASSERT_HOLDING_SPIN_LOCK(pSpinLock) \
1725 _ASSERTE((pSpinLock)->holding_thread == GCToEEInterface::GetThread());
1727 #define ASSERT_NOT_HOLDING_SPIN_LOCK(pSpinLock) \
1728 _ASSERTE((pSpinLock)->holding_thread != GCToEEInterface::GetThread());
1732 //In the concurrent version, the Enable/DisablePreemptiveGC is optional because
1733 //the gc thread call WaitLonger.
1734 void WaitLonger (int i
1735 #ifdef SYNCHRONIZATION_STATS
1736 , GCSpinLock* spin_lock
1737 #endif //SYNCHRONIZATION_STATS
1740 #ifdef SYNCHRONIZATION_STATS
1741 (spin_lock->num_wait_longer)++;
1742 #endif //SYNCHRONIZATION_STATS
1744 // every 8th attempt:
1745 bool bToggleGC = GCToEEInterface::EnablePreemptiveGC();
1748 // if we're waiting for gc to finish, we should block immediately
1749 if (!gc_heap::gc_started)
1751 #ifdef SYNCHRONIZATION_STATS
1752 (spin_lock->num_switch_thread_w)++;
1753 #endif //SYNCHRONIZATION_STATS
1754 if (g_num_processors > 1)
1756 YieldProcessor(); // indicate to the processor that we are spining
1758 GCToOSInterface::YieldThread (0);
1760 GCToOSInterface::Sleep (5);
1763 GCToOSInterface::Sleep (5);
1766 // If CLR is hosted, a thread may reach here while it is in preemptive GC mode,
1767 // or it has no Thread object, in order to force a task to yield, or to triger a GC.
1768 // It is important that the thread is going to wait for GC. Otherwise the thread
1769 // is in a tight loop. If the thread has high priority, the perf is going to be very BAD.
1770 if (gc_heap::gc_started)
1772 gc_heap::wait_for_gc_done();
1777 #ifdef SYNCHRONIZATION_STATS
1778 (spin_lock->num_disable_preemptive_w)++;
1779 #endif //SYNCHRONIZATION_STATS
1780 GCToEEInterface::DisablePreemptiveGC();
1785 static void enter_spin_lock (GCSpinLock* spin_lock)
1789 if (Interlocked::CompareExchange(&spin_lock->lock, 0, -1) >= 0)
1792 while (spin_lock->lock >= 0)
1794 if ((++i & 7) && !gc_heap::gc_started)
1796 if (g_num_processors > 1)
1798 #ifndef MULTIPLE_HEAPS
1799 int spin_count = 1024 * g_num_processors;
1800 #else //!MULTIPLE_HEAPS
1801 int spin_count = 32 * g_num_processors;
1802 #endif //!MULTIPLE_HEAPS
1803 for (int j = 0; j < spin_count; j++)
1805 if (spin_lock->lock < 0 || gc_heap::gc_started)
1807 YieldProcessor(); // indicate to the processor that we are spining
1809 if (spin_lock->lock >= 0 && !gc_heap::gc_started)
1811 #ifdef SYNCHRONIZATION_STATS
1812 (spin_lock->num_switch_thread)++;
1813 #endif //SYNCHRONIZATION_STATS
1814 bool cooperative_mode = gc_heap::enable_preemptive ();
1816 GCToOSInterface::YieldThread(0);
1818 gc_heap::disable_preemptive (cooperative_mode);
1822 GCToOSInterface::YieldThread(0);
1827 #ifdef SYNCHRONIZATION_STATS
1829 #endif //SYNCHRONIZATION_STATS
1837 inline BOOL try_enter_spin_lock(GCSpinLock* spin_lock)
1839 return (Interlocked::CompareExchange(&spin_lock->lock, 0, -1) < 0);
1843 static void leave_spin_lock (GCSpinLock * spin_lock)
1845 spin_lock->lock = -1;
1848 #define ASSERT_HOLDING_SPIN_LOCK(pSpinLock)
1852 bool gc_heap::enable_preemptive ()
1854 return GCToEEInterface::EnablePreemptiveGC();
1857 void gc_heap::disable_preemptive (bool restore_cooperative)
1859 if (restore_cooperative)
1861 GCToEEInterface::DisablePreemptiveGC();
1865 #endif // !DACCESS_COMPILE
1867 typedef void ** PTR_PTR;
1868 //This function clears a piece of memory
1869 // size has to be Dword aligned
1872 void memclr ( uint8_t* mem, size_t size)
1874 dprintf (3, ("MEMCLR: %Ix, %d", mem, size));
1875 assert ((size & (sizeof(PTR_PTR)-1)) == 0);
1876 assert (sizeof(PTR_PTR) == DATA_ALIGNMENT);
1879 // The compiler will recognize this pattern and replace it with memset call. We can as well just call
1880 // memset directly to make it obvious what's going on.
1881 PTR_PTR m = (PTR_PTR) mem;
1882 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
1886 memset (mem, 0, size);
1889 void memcopy (uint8_t* dmem, uint8_t* smem, size_t size)
1891 const size_t sz4ptr = sizeof(PTR_PTR)*4;
1892 const size_t sz2ptr = sizeof(PTR_PTR)*2;
1893 const size_t sz1ptr = sizeof(PTR_PTR)*1;
1895 // size must be a multiple of the pointer size
1896 assert ((size & (sizeof (PTR_PTR)-1)) == 0);
1897 assert (sizeof(PTR_PTR) == DATA_ALIGNMENT);
1899 // copy in groups of four pointer sized things at a time
1904 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1905 ((PTR_PTR)dmem)[1] = ((PTR_PTR)smem)[1];
1906 ((PTR_PTR)dmem)[2] = ((PTR_PTR)smem)[2];
1907 ((PTR_PTR)dmem)[3] = ((PTR_PTR)smem)[3];
1911 while ((size -= sz4ptr) >= sz4ptr);
1914 // still two pointer sized things or more left to copy?
1917 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1918 ((PTR_PTR)dmem)[1] = ((PTR_PTR)smem)[1];
1923 // still one pointer sized thing left to copy?
1926 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1934 ptrdiff_t round_down (ptrdiff_t add, int pitch)
1936 return ((add / pitch) * pitch);
1939 #if defined(FEATURE_STRUCTALIGN) && defined(RESPECT_LARGE_ALIGNMENT)
1940 // FEATURE_STRUCTALIGN allows the compiler to dictate the alignment,
1941 // i.e, if a larger alignment matters or is beneficial, the compiler
1942 // generated info tells us so. RESPECT_LARGE_ALIGNMENT is just the
1943 // converse - it's a heuristic for the GC to use a larger alignment.
1944 #error FEATURE_STRUCTALIGN should imply !RESPECT_LARGE_ALIGNMENT
1947 #if defined(FEATURE_STRUCTALIGN) && defined(FEATURE_LOH_COMPACTION)
1948 #error FEATURE_STRUCTALIGN and FEATURE_LOH_COMPACTION are mutually exclusive
1951 #if defined(GROWABLE_SEG_MAPPING_TABLE) && !defined(SEG_MAPPING_TABLE)
1952 #error if GROWABLE_SEG_MAPPING_TABLE is defined, SEG_MAPPING_TABLE must be defined
1956 BOOL same_large_alignment_p (uint8_t* p1, uint8_t* p2)
1958 #ifdef RESPECT_LARGE_ALIGNMENT
1959 return ((((size_t)p1 ^ (size_t)p2) & 7) == 0);
1961 UNREFERENCED_PARAMETER(p1);
1962 UNREFERENCED_PARAMETER(p2);
1964 #endif //RESPECT_LARGE_ALIGNMENT
1968 size_t switch_alignment_size (BOOL already_padded_p)
1970 if (already_padded_p)
1971 return DATA_ALIGNMENT;
1973 return (Align (min_obj_size) +((Align (min_obj_size)&DATA_ALIGNMENT)^DATA_ALIGNMENT));
1977 #ifdef FEATURE_STRUCTALIGN
1978 void set_node_aligninfo (uint8_t *node, int requiredAlignment, ptrdiff_t pad);
1979 void clear_node_aligninfo (uint8_t *node);
1980 #else // FEATURE_STRUCTALIGN
1981 #define node_realigned(node) (((plug_and_reloc*)(node))[-1].reloc & 1)
1982 void set_node_realigned (uint8_t* node);
1983 void clear_node_realigned(uint8_t* node);
1984 #endif // FEATURE_STRUCTALIGN
1987 size_t AlignQword (size_t nbytes)
1989 #ifdef FEATURE_STRUCTALIGN
1990 // This function is used to align everything on the large object
1991 // heap to an 8-byte boundary, to reduce the number of unaligned
1992 // accesses to (say) arrays of doubles. With FEATURE_STRUCTALIGN,
1993 // the compiler dictates the optimal alignment instead of having
1994 // a heuristic in the GC.
1995 return Align (nbytes);
1996 #else // FEATURE_STRUCTALIGN
1997 return (nbytes + 7) & ~7;
1998 #endif // FEATURE_STRUCTALIGN
2002 BOOL Aligned (size_t n)
2004 return (n & ALIGNCONST) == 0;
2007 #define OBJECT_ALIGNMENT_OFFSET (sizeof(MethodTable *))
2009 #ifdef FEATURE_STRUCTALIGN
2010 #define MAX_STRUCTALIGN OS_PAGE_SIZE
2011 #else // FEATURE_STRUCTALIGN
2012 #define MAX_STRUCTALIGN 0
2013 #endif // FEATURE_STRUCTALIGN
2015 #ifdef FEATURE_STRUCTALIGN
2017 ptrdiff_t AdjustmentForMinPadSize(ptrdiff_t pad, int requiredAlignment)
2019 // The resulting alignpad must be either 0 or at least min_obj_size.
2020 // Note that by computing the following difference on unsigned types,
2021 // we can do the range check 0 < alignpad < min_obj_size with a
2022 // single conditional branch.
2023 if ((size_t)(pad - DATA_ALIGNMENT) < Align (min_obj_size) - DATA_ALIGNMENT)
2025 return requiredAlignment;
2031 uint8_t* StructAlign (uint8_t* origPtr, int requiredAlignment, ptrdiff_t alignmentOffset=OBJECT_ALIGNMENT_OFFSET)
2033 // required alignment must be a power of two
2034 _ASSERTE(((size_t)origPtr & ALIGNCONST) == 0);
2035 _ASSERTE(((requiredAlignment - 1) & requiredAlignment) == 0);
2036 _ASSERTE(requiredAlignment >= sizeof(void *));
2037 _ASSERTE(requiredAlignment <= MAX_STRUCTALIGN);
2039 // When this method is invoked for individual objects (i.e., alignmentOffset
2040 // is just the size of the PostHeader), what needs to be aligned when
2041 // we're done is the pointer to the payload of the object (which means
2042 // the actual resulting object pointer is typically not aligned).
2044 uint8_t* result = (uint8_t*)Align ((size_t)origPtr + alignmentOffset, requiredAlignment-1) - alignmentOffset;
2045 ptrdiff_t alignpad = result - origPtr;
2047 return result + AdjustmentForMinPadSize (alignpad, requiredAlignment);
2051 ptrdiff_t ComputeStructAlignPad (uint8_t* plug, int requiredAlignment, size_t alignmentOffset=OBJECT_ALIGNMENT_OFFSET)
2053 return StructAlign (plug, requiredAlignment, alignmentOffset) - plug;
2056 BOOL IsStructAligned (uint8_t *ptr, int requiredAlignment)
2058 return StructAlign (ptr, requiredAlignment) == ptr;
2062 ptrdiff_t ComputeMaxStructAlignPad (int requiredAlignment)
2064 if (requiredAlignment == DATA_ALIGNMENT)
2066 // Since a non-zero alignment padding cannot be less than min_obj_size (so we can fit the
2067 // alignment padding object), the worst-case alignment padding is correspondingly larger
2068 // than the required alignment.
2069 return requiredAlignment + Align (min_obj_size) - DATA_ALIGNMENT;
2073 ptrdiff_t ComputeMaxStructAlignPadLarge (int requiredAlignment)
2075 if (requiredAlignment <= get_alignment_constant (TRUE)+1)
2077 // This is the same as ComputeMaxStructAlignPad, except that in addition to leaving space
2078 // for padding before the actual object, it also leaves space for filling a gap after the
2079 // actual object. This is needed on the large object heap, as the outer allocation functions
2080 // don't operate on an allocation context (which would have left space for the final gap).
2081 return requiredAlignment + Align (min_obj_size) * 2 - DATA_ALIGNMENT;
2084 uint8_t* gc_heap::pad_for_alignment (uint8_t* newAlloc, int requiredAlignment, size_t size, alloc_context* acontext)
2086 uint8_t* alignedPtr = StructAlign (newAlloc, requiredAlignment);
2087 if (alignedPtr != newAlloc) {
2088 make_unused_array (newAlloc, alignedPtr - newAlloc);
2090 acontext->alloc_ptr = alignedPtr + Align (size);
2094 uint8_t* gc_heap::pad_for_alignment_large (uint8_t* newAlloc, int requiredAlignment, size_t size)
2096 uint8_t* alignedPtr = StructAlign (newAlloc, requiredAlignment);
2097 if (alignedPtr != newAlloc) {
2098 make_unused_array (newAlloc, alignedPtr - newAlloc);
2100 if (alignedPtr < newAlloc + ComputeMaxStructAlignPadLarge (requiredAlignment)) {
2101 make_unused_array (alignedPtr + AlignQword (size), newAlloc + ComputeMaxStructAlignPadLarge (requiredAlignment) - alignedPtr);
2105 #else // FEATURE_STRUCTALIGN
2106 #define ComputeMaxStructAlignPad(requiredAlignment) 0
2107 #define ComputeMaxStructAlignPadLarge(requiredAlignment) 0
2108 #endif // FEATURE_STRUCTALIGN
2110 //CLR_SIZE is the max amount of bytes from gen0 that is set to 0 in one chunk
2112 #define CLR_SIZE ((size_t)(8*1024))
2114 #define CLR_SIZE ((size_t)(8*1024))
2117 #define END_SPACE_AFTER_GC (LARGE_OBJECT_SIZE + MAX_STRUCTALIGN)
2119 #ifdef BACKGROUND_GC
2120 #define SEGMENT_INITIAL_COMMIT (2*OS_PAGE_SIZE)
2122 #define SEGMENT_INITIAL_COMMIT (OS_PAGE_SIZE)
2123 #endif //BACKGROUND_GC
2129 #define INITIAL_ALLOC ((size_t)((size_t)4*1024*1024*1024))
2130 #define LHEAP_ALLOC ((size_t)(1024*1024*256))
2134 #define INITIAL_ALLOC ((size_t)(1024*1024*64))
2135 #define LHEAP_ALLOC ((size_t)(1024*1024*32))
2143 #define INITIAL_ALLOC ((size_t)(1024*1024*256))
2144 #define LHEAP_ALLOC ((size_t)(1024*1024*128))
2148 #define INITIAL_ALLOC ((size_t)(1024*1024*16))
2149 #define LHEAP_ALLOC ((size_t)(1024*1024*16))
2155 //amount in bytes of the etw allocation tick
2156 const size_t etw_allocation_tick = 100*1024;
2158 const size_t low_latency_alloc = 256*1024;
2160 const size_t fgn_check_quantum = 2*1024*1024;
2163 const int max_snoop_level = 128;
2168 //threshold of heap size to turn on card bundles.
2169 #define SH_TH_CARD_BUNDLE (40*1024*1024)
2170 #define MH_TH_CARD_BUNDLE (180*1024*1024)
2171 #endif //CARD_BUNDLE
2173 #define GC_EPHEMERAL_DECOMMIT_TIMEOUT 5000
2176 size_t align_on_page (size_t add)
2178 return ((add + OS_PAGE_SIZE - 1) & ~((size_t)OS_PAGE_SIZE - 1));
2182 uint8_t* align_on_page (uint8_t* add)
2184 return (uint8_t*)align_on_page ((size_t) add);
2188 size_t align_lower_page (size_t add)
2190 return (add & ~((size_t)OS_PAGE_SIZE - 1));
2194 uint8_t* align_lower_page (uint8_t* add)
2196 return (uint8_t*)align_lower_page ((size_t)add);
2200 size_t align_write_watch_lower_page (size_t add)
2202 return (add & ~(WRITE_WATCH_UNIT_SIZE - 1));
2206 uint8_t* align_write_watch_lower_page (uint8_t* add)
2208 return (uint8_t*)align_lower_page ((size_t)add);
2213 BOOL power_of_two_p (size_t integer)
2215 return !(integer & (integer-1));
2219 BOOL oddp (size_t integer)
2221 return (integer & 1) != 0;
2224 // we only ever use this for WORDs.
2225 size_t logcount (size_t word)
2227 //counts the number of high bits in a 16 bit word.
2228 assert (word < 0x10000);
2230 count = (word & 0x5555) + ( (word >> 1 ) & 0x5555);
2231 count = (count & 0x3333) + ( (count >> 2) & 0x3333);
2232 count = (count & 0x0F0F) + ( (count >> 4) & 0x0F0F);
2233 count = (count & 0x00FF) + ( (count >> 8) & 0x00FF);
2237 #ifndef DACCESS_COMPILE
2239 void stomp_write_barrier_resize(bool is_runtime_suspended, bool requires_upper_bounds_check)
2241 WriteBarrierParameters args = {};
2242 args.operation = WriteBarrierOp::StompResize;
2243 args.is_runtime_suspended = is_runtime_suspended;
2244 args.requires_upper_bounds_check = requires_upper_bounds_check;
2246 args.card_table = g_gc_card_table;
2247 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
2248 args.card_bundle_table = g_gc_card_bundle_table;
2251 args.lowest_address = g_gc_lowest_address;
2252 args.highest_address = g_gc_highest_address;
2254 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
2255 if (SoftwareWriteWatch::IsEnabledForGCHeap())
2257 args.write_watch_table = g_gc_sw_ww_table;
2259 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
2261 GCToEEInterface::StompWriteBarrier(&args);
2264 void stomp_write_barrier_ephemeral(uint8_t* ephemeral_low, uint8_t* ephemeral_high)
2266 WriteBarrierParameters args = {};
2267 args.operation = WriteBarrierOp::StompEphemeral;
2268 args.is_runtime_suspended = true;
2269 args.ephemeral_low = ephemeral_low;
2270 args.ephemeral_high = ephemeral_high;
2271 GCToEEInterface::StompWriteBarrier(&args);
2274 void stomp_write_barrier_initialize(uint8_t* ephemeral_low, uint8_t* ephemeral_high)
2276 WriteBarrierParameters args = {};
2277 args.operation = WriteBarrierOp::Initialize;
2278 args.is_runtime_suspended = true;
2279 args.requires_upper_bounds_check = false;
2280 args.card_table = g_gc_card_table;
2282 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
2283 args.card_bundle_table = g_gc_card_bundle_table;
2286 args.lowest_address = g_gc_lowest_address;
2287 args.highest_address = g_gc_highest_address;
2288 args.ephemeral_low = ephemeral_low;
2289 args.ephemeral_high = ephemeral_high;
2290 GCToEEInterface::StompWriteBarrier(&args);
2293 #endif // DACCESS_COMPILE
2295 //extract the low bits [0,low[ of a uint32_t
2296 #define lowbits(wrd, bits) ((wrd) & ((1 << (bits))-1))
2297 //extract the high bits [high, 32] of a uint32_t
2298 #define highbits(wrd, bits) ((wrd) & ~((1 << (bits))-1))
2300 // Things we need to manually initialize:
2301 // gen0 min_size - based on cache
2302 // gen0/1 max_size - based on segment size
2303 static static_data static_data_table[latency_level_last - latency_level_first + 1][NUMBERGENERATIONS] =
2305 // latency_level_memory_footprint
2308 {0, 0, 40000, 0.5f, 9.0f, 20.0f, 1000, 1},
2310 {163840, 0, 80000, 0.5f, 2.0f, 7.0f, 10000, 10},
2312 {256*1024, SSIZE_T_MAX, 200000, 0.25f, 1.2f, 1.8f, 100000, 100},
2314 {3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0}
2317 // latency_level_balanced
2321 #ifdef MULTIPLE_HEAPS
2325 #endif //MULTIPLE_HEAPS
2328 {9*32*1024, 0, 80000, 0.5f, 2.0f, 7.0f, 10000, 10},
2330 {256*1024, SSIZE_T_MAX, 200000, 0.25f, 1.2f, 1.8f, 100000, 100},
2332 {3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0}
2339 class CObjectHeader;
2343 class c_synchronize;
2345 #ifdef FEATURE_PREMORTEM_FINALIZATION
2346 #ifndef DACCESS_COMPILE
2348 HRESULT AllocateCFinalize(CFinalize **pCFinalize);
2349 #endif //!DACCESS_COMPILE
2350 #endif // FEATURE_PREMORTEM_FINALIZATION
2352 uint8_t* tree_search (uint8_t* tree, uint8_t* old_address);
2355 #ifdef USE_INTROSORT
2356 #define _sort introsort::sort
2357 #else //USE_INTROSORT
2358 #define _sort qsort1
2359 void qsort1(uint8_t** low, uint8_t** high, unsigned int depth);
2360 #endif //USE_INTROSORT
2362 void* virtual_alloc (size_t size);
2363 void virtual_free (void* add, size_t size);
2365 /* per heap static initialization */
2367 #ifndef MULTIPLE_HEAPS
2368 uint32_t* gc_heap::mark_array;
2369 #endif //MULTIPLE_HEAPS
2373 uint8_t** gc_heap::g_mark_list;
2375 #ifdef PARALLEL_MARK_LIST_SORT
2376 uint8_t** gc_heap::g_mark_list_copy;
2377 #endif //PARALLEL_MARK_LIST_SORT
2379 size_t gc_heap::mark_list_size;
2382 #ifdef SEG_MAPPING_TABLE
2383 seg_mapping* seg_mapping_table;
2384 #endif //SEG_MAPPING_TABLE
2386 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
2387 sorted_table* gc_heap::seg_table;
2388 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
2390 #ifdef MULTIPLE_HEAPS
2391 GCEvent gc_heap::ee_suspend_event;
2392 size_t gc_heap::min_balance_threshold = 0;
2393 #endif //MULTIPLE_HEAPS
2395 VOLATILE(BOOL) gc_heap::gc_started;
2397 #ifdef MULTIPLE_HEAPS
2399 GCEvent gc_heap::gc_start_event;
2401 bool gc_heap::gc_thread_no_affinitize_p = false;
2403 int gc_heap::n_heaps;
2405 gc_heap** gc_heap::g_heaps;
2407 size_t* gc_heap::g_promoted;
2410 int* gc_heap::g_mark_stack_busy;
2414 #ifdef BACKGROUND_GC
2415 size_t* gc_heap::g_bpromoted;
2416 #endif //BACKGROUND_GC
2418 #else //MULTIPLE_HEAPS
2420 size_t gc_heap::g_promoted;
2422 #ifdef BACKGROUND_GC
2423 size_t gc_heap::g_bpromoted;
2424 #endif //BACKGROUND_GC
2426 #endif //MULTIPLE_HEAPS
2428 size_t gc_heap::reserved_memory = 0;
2429 size_t gc_heap::reserved_memory_limit = 0;
2430 BOOL gc_heap::g_low_memory_status;
2432 #ifndef DACCESS_COMPILE
2433 static gc_reason gc_trigger_reason = reason_empty;
2434 #endif //DACCESS_COMPILE
2436 gc_latency_level gc_heap::latency_level = latency_level_default;
2438 gc_mechanisms gc_heap::settings;
2440 gc_history_global gc_heap::gc_data_global;
2442 size_t gc_heap::gc_last_ephemeral_decommit_time = 0;
2444 size_t gc_heap::gc_gen0_desired_high;
2447 double gc_heap::short_plugs_pad_ratio = 0;
2448 #endif //SHORT_PLUGS
2451 #define MAX_ALLOWED_MEM_LOAD 85
2453 // consider putting this in dynamic data -
2454 // we may want different values for workstation
2456 #define MIN_YOUNGEST_GEN_DESIRED (16*1024*1024)
2458 size_t gc_heap::youngest_gen_desired_th;
2461 uint32_t gc_heap::last_gc_memory_load = 0;
2463 size_t gc_heap::last_gc_heap_size = 0;
2465 size_t gc_heap::last_gc_fragmentation = 0;
2467 uint64_t gc_heap::mem_one_percent = 0;
2469 uint32_t gc_heap::high_memory_load_th = 0;
2471 uint64_t gc_heap::total_physical_mem = 0;
2473 uint64_t gc_heap::entry_available_physical_mem = 0;
2475 #ifdef BACKGROUND_GC
2476 GCEvent gc_heap::bgc_start_event;
2478 gc_mechanisms gc_heap::saved_bgc_settings;
2480 GCEvent gc_heap::background_gc_done_event;
2482 GCEvent gc_heap::ee_proceed_event;
2484 bool gc_heap::gc_can_use_concurrent = false;
2486 bool gc_heap::temp_disable_concurrent_p = false;
2488 uint32_t gc_heap::cm_in_progress = FALSE;
2490 BOOL gc_heap::dont_restart_ee_p = FALSE;
2492 BOOL gc_heap::keep_bgc_threads_p = FALSE;
2494 GCEvent gc_heap::bgc_threads_sync_event;
2496 BOOL gc_heap::do_ephemeral_gc_p = FALSE;
2498 BOOL gc_heap::do_concurrent_p = FALSE;
2500 size_t gc_heap::ephemeral_fgc_counts[max_generation];
2502 BOOL gc_heap::alloc_wait_event_p = FALSE;
2504 VOLATILE(c_gc_state) gc_heap::current_c_gc_state = c_gc_state_free;
2506 #endif //BACKGROUND_GC
2508 #ifndef MULTIPLE_HEAPS
2509 #ifdef SPINLOCK_HISTORY
2510 int gc_heap::spinlock_info_index = 0;
2511 spinlock_info gc_heap::last_spinlock_info[max_saved_spinlock_info + 8];
2512 #endif //SPINLOCK_HISTORY
2514 size_t gc_heap::fgn_last_alloc = 0;
2516 int gc_heap::generation_skip_ratio = 100;
2518 uint64_t gc_heap::loh_alloc_since_cg = 0;
2520 BOOL gc_heap::elevation_requested = FALSE;
2522 BOOL gc_heap::last_gc_before_oom = FALSE;
2524 #ifdef BACKGROUND_GC
2525 uint8_t* gc_heap::background_saved_lowest_address = 0;
2526 uint8_t* gc_heap::background_saved_highest_address = 0;
2527 uint8_t* gc_heap::next_sweep_obj = 0;
2528 uint8_t* gc_heap::current_sweep_pos = 0;
2529 exclusive_sync* gc_heap::bgc_alloc_lock;
2530 #endif //BACKGROUND_GC
2532 oom_history gc_heap::oom_info;
2534 fgm_history gc_heap::fgm_result;
2536 BOOL gc_heap::ro_segments_in_range;
2538 size_t gc_heap::gen0_big_free_spaces = 0;
2540 uint8_t* gc_heap::ephemeral_low;
2542 uint8_t* gc_heap::ephemeral_high;
2544 uint8_t* gc_heap::lowest_address;
2546 uint8_t* gc_heap::highest_address;
2548 BOOL gc_heap::ephemeral_promotion;
2550 uint8_t* gc_heap::saved_ephemeral_plan_start[NUMBERGENERATIONS-1];
2551 size_t gc_heap::saved_ephemeral_plan_start_size[NUMBERGENERATIONS-1];
2553 short* gc_heap::brick_table;
2555 uint32_t* gc_heap::card_table;
2558 uint32_t* gc_heap::card_bundle_table;
2559 #endif //CARD_BUNDLE
2561 uint8_t* gc_heap::gc_low;
2563 uint8_t* gc_heap::gc_high;
2565 uint8_t* gc_heap::demotion_low;
2567 uint8_t* gc_heap::demotion_high;
2569 BOOL gc_heap::demote_gen1_p = TRUE;
2571 uint8_t* gc_heap::last_gen1_pin_end;
2573 gen_to_condemn_tuning gc_heap::gen_to_condemn_reasons;
2575 size_t gc_heap::etw_allocation_running_amount[2];
2577 int gc_heap::gc_policy = 0;
2579 size_t gc_heap::allocation_running_time;
2581 size_t gc_heap::allocation_running_amount;
2583 heap_segment* gc_heap::ephemeral_heap_segment = 0;
2585 BOOL gc_heap::blocking_collection = FALSE;
2587 heap_segment* gc_heap::freeable_large_heap_segment = 0;
2589 size_t gc_heap::time_bgc_last = 0;
2591 size_t gc_heap::mark_stack_tos = 0;
2593 size_t gc_heap::mark_stack_bos = 0;
2595 size_t gc_heap::mark_stack_array_length = 0;
2597 mark* gc_heap::mark_stack_array = 0;
2599 BOOL gc_heap::verify_pinned_queue_p = FALSE;
2601 uint8_t* gc_heap::oldest_pinned_plug = 0;
2603 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
2604 size_t gc_heap::num_pinned_objects = 0;
2605 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
2607 #ifdef FEATURE_LOH_COMPACTION
2608 size_t gc_heap::loh_pinned_queue_tos = 0;
2610 size_t gc_heap::loh_pinned_queue_bos = 0;
2612 size_t gc_heap::loh_pinned_queue_length = 0;
2614 mark* gc_heap::loh_pinned_queue = 0;
2616 BOOL gc_heap::loh_compacted_p = FALSE;
2617 #endif //FEATURE_LOH_COMPACTION
2619 #ifdef BACKGROUND_GC
2621 EEThreadId gc_heap::bgc_thread_id;
2623 uint8_t* gc_heap::background_written_addresses [array_size+2];
2625 heap_segment* gc_heap::freeable_small_heap_segment = 0;
2627 size_t gc_heap::bgc_overflow_count = 0;
2629 size_t gc_heap::bgc_begin_loh_size = 0;
2630 size_t gc_heap::end_loh_size = 0;
2632 uint32_t gc_heap::bgc_alloc_spin_loh = 0;
2634 size_t gc_heap::bgc_loh_size_increased = 0;
2636 size_t gc_heap::bgc_loh_allocated_in_free = 0;
2638 size_t gc_heap::background_soh_alloc_count = 0;
2640 size_t gc_heap::background_loh_alloc_count = 0;
2642 uint8_t** gc_heap::background_mark_stack_tos = 0;
2644 uint8_t** gc_heap::background_mark_stack_array = 0;
2646 size_t gc_heap::background_mark_stack_array_length = 0;
2648 uint8_t* gc_heap::background_min_overflow_address =0;
2650 uint8_t* gc_heap::background_max_overflow_address =0;
2652 BOOL gc_heap::processed_soh_overflow_p = FALSE;
2654 uint8_t* gc_heap::background_min_soh_overflow_address =0;
2656 uint8_t* gc_heap::background_max_soh_overflow_address =0;
2658 heap_segment* gc_heap::saved_sweep_ephemeral_seg = 0;
2660 uint8_t* gc_heap::saved_sweep_ephemeral_start = 0;
2662 heap_segment* gc_heap::saved_overflow_ephemeral_seg = 0;
2664 Thread* gc_heap::bgc_thread = 0;
2666 BOOL gc_heap::expanded_in_fgc = FALSE;
2668 uint8_t** gc_heap::c_mark_list = 0;
2670 size_t gc_heap::c_mark_list_length = 0;
2672 size_t gc_heap::c_mark_list_index = 0;
2674 gc_history_per_heap gc_heap::bgc_data_per_heap;
2676 BOOL gc_heap::bgc_thread_running;
2678 CLRCriticalSection gc_heap::bgc_threads_timeout_cs;
2680 GCEvent gc_heap::gc_lh_block_event;
2682 #endif //BACKGROUND_GC
2685 uint8_t** gc_heap::mark_list;
2686 uint8_t** gc_heap::mark_list_index;
2687 uint8_t** gc_heap::mark_list_end;
2691 snoop_stats_data gc_heap::snoop_stat;
2692 #endif //SNOOP_STATS
2694 uint8_t* gc_heap::min_overflow_address = MAX_PTR;
2696 uint8_t* gc_heap::max_overflow_address = 0;
2698 uint8_t* gc_heap::shigh = 0;
2700 uint8_t* gc_heap::slow = MAX_PTR;
2702 size_t gc_heap::ordered_free_space_indices[MAX_NUM_BUCKETS];
2704 size_t gc_heap::saved_ordered_free_space_indices[MAX_NUM_BUCKETS];
2706 size_t gc_heap::ordered_plug_indices[MAX_NUM_BUCKETS];
2708 size_t gc_heap::saved_ordered_plug_indices[MAX_NUM_BUCKETS];
2710 BOOL gc_heap::ordered_plug_indices_init = FALSE;
2712 BOOL gc_heap::use_bestfit = FALSE;
2714 uint8_t* gc_heap::bestfit_first_pin = 0;
2716 BOOL gc_heap::commit_end_of_seg = FALSE;
2718 size_t gc_heap::max_free_space_items = 0;
2720 size_t gc_heap::free_space_buckets = 0;
2722 size_t gc_heap::free_space_items = 0;
2724 int gc_heap::trimmed_free_space_index = 0;
2726 size_t gc_heap::total_ephemeral_plugs = 0;
2728 seg_free_spaces* gc_heap::bestfit_seg = 0;
2730 size_t gc_heap::total_ephemeral_size = 0;
2734 size_t gc_heap::internal_root_array_length = initial_internal_roots;
2736 uint8_t** gc_heap::internal_root_array = 0;
2738 size_t gc_heap::internal_root_array_index = 0;
2740 BOOL gc_heap::heap_analyze_success = TRUE;
2742 uint8_t* gc_heap::current_obj = 0;
2743 size_t gc_heap::current_obj_size = 0;
2745 #endif //HEAP_ANALYZE
2747 #ifdef GC_CONFIG_DRIVEN
2748 size_t gc_heap::interesting_data_per_gc[max_idp_count];
2749 //size_t gc_heap::interesting_data_per_heap[max_idp_count];
2750 //size_t gc_heap::interesting_mechanisms_per_heap[max_im_count];
2751 #endif //GC_CONFIG_DRIVEN
2752 #endif //MULTIPLE_HEAPS
2754 no_gc_region_info gc_heap::current_no_gc_region_info;
2755 BOOL gc_heap::proceed_with_gc_p = FALSE;
2756 GCSpinLock gc_heap::gc_lock;
2758 size_t gc_heap::eph_gen_starts_size = 0;
2759 heap_segment* gc_heap::segment_standby_list;
2760 size_t gc_heap::last_gc_index = 0;
2761 #ifdef SEG_MAPPING_TABLE
2762 size_t gc_heap::min_segment_size = 0;
2763 size_t gc_heap::min_segment_size_shr = 0;
2764 #endif //SEG_MAPPING_TABLE
2765 size_t gc_heap::soh_segment_size = 0;
2766 size_t gc_heap::min_loh_segment_size = 0;
2767 size_t gc_heap::segment_info_size = 0;
2769 #ifdef GC_CONFIG_DRIVEN
2770 size_t gc_heap::time_init = 0;
2771 size_t gc_heap::time_since_init = 0;
2772 size_t gc_heap::compact_or_sweep_gcs[2];
2773 #endif //GC_CONFIG_DRIVEN
2775 #ifdef FEATURE_LOH_COMPACTION
2776 BOOL gc_heap::loh_compaction_always_p = FALSE;
2777 gc_loh_compaction_mode gc_heap::loh_compaction_mode = loh_compaction_default;
2778 int gc_heap::loh_pinned_queue_decay = LOH_PIN_DECAY;
2780 #endif //FEATURE_LOH_COMPACTION
2782 GCEvent gc_heap::full_gc_approach_event;
2784 GCEvent gc_heap::full_gc_end_event;
2786 uint32_t gc_heap::fgn_maxgen_percent = 0;
2788 uint32_t gc_heap::fgn_loh_percent = 0;
2790 #ifdef BACKGROUND_GC
2791 BOOL gc_heap::fgn_last_gc_was_concurrent = FALSE;
2792 #endif //BACKGROUND_GC
2794 VOLATILE(bool) gc_heap::full_gc_approach_event_set;
2796 size_t gc_heap::full_gc_counts[gc_type_max];
2798 BOOL gc_heap::should_expand_in_full_gc = FALSE;
2801 BOOL gc_heap::heap_analyze_enabled = FALSE;
2802 #endif //HEAP_ANALYZE
2804 #ifndef MULTIPLE_HEAPS
2806 alloc_list gc_heap::loh_alloc_list [NUM_LOH_ALIST-1];
2807 alloc_list gc_heap::gen2_alloc_list[NUM_GEN2_ALIST-1];
2809 dynamic_data gc_heap::dynamic_data_table [NUMBERGENERATIONS+1];
2810 gc_history_per_heap gc_heap::gc_data_per_heap;
2811 size_t gc_heap::maxgen_pinned_compact_before_advance = 0;
2813 uint8_t* gc_heap::alloc_allocated = 0;
2815 size_t gc_heap::allocation_quantum = CLR_SIZE;
2817 GCSpinLock gc_heap::more_space_lock;
2819 #ifdef SYNCHRONIZATION_STATS
2820 unsigned int gc_heap::good_suspension = 0;
2821 unsigned int gc_heap::bad_suspension = 0;
2822 uint64_t gc_heap::total_msl_acquire = 0;
2823 unsigned int gc_heap::num_msl_acquired = 0;
2824 unsigned int gc_heap::num_high_msl_acquire = 0;
2825 unsigned int gc_heap::num_low_msl_acquire = 0;
2826 #endif //SYNCHRONIZATION_STATS
2828 size_t gc_heap::alloc_contexts_used = 0;
2829 size_t gc_heap::soh_allocation_no_gc = 0;
2830 size_t gc_heap::loh_allocation_no_gc = 0;
2831 bool gc_heap::no_gc_oom_p = false;
2832 heap_segment* gc_heap::saved_loh_segment_no_gc = 0;
2834 #endif //MULTIPLE_HEAPS
2836 #ifndef MULTIPLE_HEAPS
2838 BOOL gc_heap::gen0_bricks_cleared = FALSE;
2841 int gc_heap::gen0_must_clear_bricks = 0;
2842 #endif //FFIND_OBJECT
2844 #ifdef FEATURE_PREMORTEM_FINALIZATION
2845 CFinalize* gc_heap::finalize_queue = 0;
2846 #endif // FEATURE_PREMORTEM_FINALIZATION
2848 generation gc_heap::generation_table [NUMBERGENERATIONS + 1];
2850 size_t gc_heap::interesting_data_per_heap[max_idp_count];
2852 size_t gc_heap::compact_reasons_per_heap[max_compact_reasons_count];
2854 size_t gc_heap::expand_mechanisms_per_heap[max_expand_mechanisms_count];
2856 size_t gc_heap::interesting_mechanism_bits_per_heap[max_gc_mechanism_bits_count];
2858 #endif // MULTIPLE_HEAPS
2860 /* end of per heap static initialization */
2862 /* end of static initialization */
2864 #ifndef DACCESS_COMPILE
2866 void gen_to_condemn_tuning::print (int heap_num)
2869 dprintf (DT_LOG_0, ("condemned reasons (%d %d)", condemn_reasons_gen, condemn_reasons_condition));
2870 dprintf (DT_LOG_0, ("%s", record_condemn_reasons_gen_header));
2871 gc_condemn_reason_gen r_gen;
2872 for (int i = 0; i < gcrg_max; i++)
2874 r_gen = (gc_condemn_reason_gen)(i);
2875 str_reasons_gen[i * 2] = get_gen_char (get_gen (r_gen));
2877 dprintf (DT_LOG_0, ("[%2d]%s", heap_num, str_reasons_gen));
2879 dprintf (DT_LOG_0, ("%s", record_condemn_reasons_condition_header));
2880 gc_condemn_reason_condition r_condition;
2881 for (int i = 0; i < gcrc_max; i++)
2883 r_condition = (gc_condemn_reason_condition)(i);
2884 str_reasons_condition[i * 2] = get_condition_char (get_condition (r_condition));
2887 dprintf (DT_LOG_0, ("[%2d]%s", heap_num, str_reasons_condition));
2889 UNREFERENCED_PARAMETER(heap_num);
2893 void gc_generation_data::print (int heap_num, int gen_num)
2895 #if defined(SIMPLE_DPRINTF) && defined(DT_LOG)
2896 dprintf (DT_LOG_0, ("[%2d]gen%d beg %Id fl %Id fo %Id end %Id fl %Id fo %Id in %Id p %Id np %Id alloc %Id",
2899 free_list_space_before, free_obj_space_before,
2901 free_list_space_after, free_obj_space_after,
2902 in, pinned_surv, npinned_surv,
2905 UNREFERENCED_PARAMETER(heap_num);
2906 UNREFERENCED_PARAMETER(gen_num);
2907 #endif //SIMPLE_DPRINTF && DT_LOG
2910 void gc_history_per_heap::set_mechanism (gc_mechanism_per_heap mechanism_per_heap, uint32_t value)
2912 uint32_t* mechanism = &mechanisms[mechanism_per_heap];
2914 *mechanism |= mechanism_mask;
2915 *mechanism |= (1 << value);
2918 gc_mechanism_descr* descr = &gc_mechanisms_descr[mechanism_per_heap];
2919 dprintf (DT_LOG_0, ("setting %s: %s",
2921 (descr->descr)[value]));
2925 void gc_history_per_heap::print()
2927 #if defined(SIMPLE_DPRINTF) && defined(DT_LOG)
2928 for (int i = 0; i < (sizeof (gen_data)/sizeof (gc_generation_data)); i++)
2930 gen_data[i].print (heap_index, i);
2933 dprintf (DT_LOG_0, ("fla %Id flr %Id esa %Id ca %Id pa %Id paa %Id, rfle %d, ec %Id",
2934 maxgen_size_info.free_list_allocated,
2935 maxgen_size_info.free_list_rejected,
2936 maxgen_size_info.end_seg_allocated,
2937 maxgen_size_info.condemned_allocated,
2938 maxgen_size_info.pinned_allocated,
2939 maxgen_size_info.pinned_allocated_advance,
2940 maxgen_size_info.running_free_list_efficiency,
2941 extra_gen0_committed));
2944 gc_mechanism_descr* descr = 0;
2946 for (int i = 0; i < max_mechanism_per_heap; i++)
2948 mechanism = get_mechanism ((gc_mechanism_per_heap)i);
2952 descr = &gc_mechanisms_descr[(gc_mechanism_per_heap)i];
2953 dprintf (DT_LOG_0, ("[%2d]%s%s",
2956 (descr->descr)[mechanism]));
2959 #endif //SIMPLE_DPRINTF && DT_LOG
2962 void gc_history_global::print()
2965 char str_settings[64];
2966 memset (str_settings, '|', sizeof (char) * 64);
2967 str_settings[max_global_mechanisms_count*2] = 0;
2969 for (int i = 0; i < max_global_mechanisms_count; i++)
2971 str_settings[i * 2] = (get_mechanism_p ((gc_global_mechanism_p)i) ? 'Y' : 'N');
2974 dprintf (DT_LOG_0, ("[hp]|c|p|o|d|b|e|"));
2976 dprintf (DT_LOG_0, ("%4d|%s", num_heaps, str_settings));
2977 dprintf (DT_LOG_0, ("Condemned gen%d(reason: %s; mode: %s), youngest budget %Id(%d), memload %d",
2978 condemned_generation,
2979 str_gc_reasons[reason],
2980 str_gc_pause_modes[pause_mode],
2981 final_youngest_desired,
2982 gen0_reduction_count,
2987 void gc_heap::fire_per_heap_hist_event (gc_history_per_heap* current_gc_data_per_heap, int heap_num)
2989 maxgen_size_increase* maxgen_size_info = &(current_gc_data_per_heap->maxgen_size_info);
2990 FIRE_EVENT(GCPerHeapHistory_V3,
2991 (void *)(maxgen_size_info->free_list_allocated),
2992 (void *)(maxgen_size_info->free_list_rejected),
2993 (void *)(maxgen_size_info->end_seg_allocated),
2994 (void *)(maxgen_size_info->condemned_allocated),
2995 (void *)(maxgen_size_info->pinned_allocated),
2996 (void *)(maxgen_size_info->pinned_allocated_advance),
2997 maxgen_size_info->running_free_list_efficiency,
2998 current_gc_data_per_heap->gen_to_condemn_reasons.get_reasons0(),
2999 current_gc_data_per_heap->gen_to_condemn_reasons.get_reasons1(),
3000 current_gc_data_per_heap->mechanisms[gc_heap_compact],
3001 current_gc_data_per_heap->mechanisms[gc_heap_expand],
3002 current_gc_data_per_heap->heap_index,
3003 (void *)(current_gc_data_per_heap->extra_gen0_committed),
3004 (max_generation + 2),
3005 (uint32_t)(sizeof (gc_generation_data)),
3006 (void *)&(current_gc_data_per_heap->gen_data[0]));
3008 current_gc_data_per_heap->print();
3009 current_gc_data_per_heap->gen_to_condemn_reasons.print (heap_num);
3012 void gc_heap::fire_pevents()
3015 settings.record (&gc_data_global);
3016 gc_data_global.print();
3018 FIRE_EVENT(GCGlobalHeapHistory_V2, gc_data_global.final_youngest_desired,
3019 gc_data_global.num_heaps,
3020 gc_data_global.condemned_generation,
3021 gc_data_global.gen0_reduction_count,
3022 gc_data_global.reason,
3023 gc_data_global.global_mechanims_p,
3024 gc_data_global.pause_mode,
3025 gc_data_global.mem_pressure);
3027 #ifdef MULTIPLE_HEAPS
3028 for (int i = 0; i < gc_heap::n_heaps; i++)
3030 gc_heap* hp = gc_heap::g_heaps[i];
3031 gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap();
3032 fire_per_heap_hist_event (current_gc_data_per_heap, hp->heap_number);
3035 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
3036 fire_per_heap_hist_event (current_gc_data_per_heap, heap_number);
3042 gc_heap::dt_low_ephemeral_space_p (gc_tuning_point tp)
3048 case tuning_deciding_condemned_gen:
3049 case tuning_deciding_compaction:
3050 case tuning_deciding_expansion:
3051 case tuning_deciding_full_gc:
3053 ret = (!ephemeral_gen_fit_p (tp));
3056 case tuning_deciding_promote_ephemeral:
3058 size_t new_gen0size = approximate_new_allocation();
3059 ptrdiff_t plan_ephemeral_size = total_ephemeral_size;
3061 dprintf (GTC_LOG, ("h%d: plan eph size is %Id, new gen0 is %Id",
3062 heap_number, plan_ephemeral_size, new_gen0size));
3064 // If we were in no_gc_region we could have allocated a larger than normal segment,
3065 // and the next seg we allocate will be a normal sized seg so if we can't fit the new
3066 // ephemeral generations there, do an ephemeral promotion.
3067 ret = ((soh_segment_size - segment_info_size) < (plan_ephemeral_size + new_gen0size));
3079 gc_heap::dt_high_frag_p (gc_tuning_point tp,
3087 case tuning_deciding_condemned_gen:
3089 dynamic_data* dd = dynamic_data_of (gen_number);
3090 float fragmentation_burden = 0;
3094 ret = (dd_fragmentation (dynamic_data_of (max_generation)) >= dd_max_size(dd));
3095 dprintf (GTC_LOG, ("h%d: frag is %Id, max size is %Id",
3096 heap_number, dd_fragmentation (dd), dd_max_size(dd)));
3100 #ifndef MULTIPLE_HEAPS
3101 if (gen_number == max_generation)
3103 float frag_ratio = (float)(dd_fragmentation (dynamic_data_of (max_generation))) / (float)generation_size (max_generation);
3104 if (frag_ratio > 0.65)
3106 dprintf (GTC_LOG, ("g2 FR: %d%%", (int)(frag_ratio*100)));
3110 #endif //!MULTIPLE_HEAPS
3111 size_t fr = generation_unusable_fragmentation (generation_of (gen_number));
3112 ret = (fr > dd_fragmentation_limit(dd));
3115 fragmentation_burden = (float)fr / generation_size (gen_number);
3116 ret = (fragmentation_burden > dd_v_fragmentation_burden_limit (dd));
3118 dprintf (GTC_LOG, ("h%d: gen%d, frag is %Id, alloc effi: %d%%, unusable frag is %Id, ratio is %d",
3119 heap_number, gen_number, dd_fragmentation (dd),
3120 (int)(100*generation_allocator_efficiency (generation_of (gen_number))),
3121 fr, (int)(fragmentation_burden*100)));
3133 gc_heap::dt_estimate_reclaim_space_p (gc_tuning_point tp, int gen_number)
3139 case tuning_deciding_condemned_gen:
3141 if (gen_number == max_generation)
3143 dynamic_data* dd = dynamic_data_of (gen_number);
3144 size_t maxgen_allocated = (dd_desired_allocation (dd) - dd_new_allocation (dd));
3145 size_t maxgen_total_size = maxgen_allocated + dd_current_size (dd);
3146 size_t est_maxgen_surv = (size_t)((float) (maxgen_total_size) * dd_surv (dd));
3147 size_t est_maxgen_free = maxgen_total_size - est_maxgen_surv + dd_fragmentation (dd);
3149 dprintf (GTC_LOG, ("h%d: Total gen2 size: %Id, est gen2 dead space: %Id (s: %d, allocated: %Id), frag: %Id",
3153 (int)(dd_surv (dd) * 100),
3155 dd_fragmentation (dd)));
3157 uint32_t num_heaps = 1;
3159 #ifdef MULTIPLE_HEAPS
3160 num_heaps = gc_heap::n_heaps;
3161 #endif //MULTIPLE_HEAPS
3163 size_t min_frag_th = min_reclaim_fragmentation_threshold (num_heaps);
3164 dprintf (GTC_LOG, ("h%d, min frag is %Id", heap_number, min_frag_th));
3165 ret = (est_maxgen_free >= min_frag_th);
3181 // DTREVIEW: Right now we only estimate gen2 fragmentation.
3182 // on 64-bit though we should consider gen1 or even gen0 fragmentatioin as
3185 gc_heap::dt_estimate_high_frag_p (gc_tuning_point tp, int gen_number, uint64_t available_mem)
3191 case tuning_deciding_condemned_gen:
3193 if (gen_number == max_generation)
3195 dynamic_data* dd = dynamic_data_of (gen_number);
3196 float est_frag_ratio = 0;
3197 if (dd_current_size (dd) == 0)
3201 else if ((dd_fragmentation (dd) == 0) || (dd_fragmentation (dd) + dd_current_size (dd) == 0))
3207 est_frag_ratio = (float)dd_fragmentation (dd) / (float)(dd_fragmentation (dd) + dd_current_size (dd));
3210 size_t est_frag = (dd_fragmentation (dd) + (size_t)((dd_desired_allocation (dd) - dd_new_allocation (dd)) * est_frag_ratio));
3211 dprintf (GTC_LOG, ("h%d: gen%d: current_size is %Id, frag is %Id, est_frag_ratio is %d%%, estimated frag is %Id",
3214 dd_current_size (dd),
3215 dd_fragmentation (dd),
3216 (int)(est_frag_ratio*100),
3219 uint32_t num_heaps = 1;
3221 #ifdef MULTIPLE_HEAPS
3222 num_heaps = gc_heap::n_heaps;
3223 #endif //MULTIPLE_HEAPS
3224 uint64_t min_frag_th = min_high_fragmentation_threshold(available_mem, num_heaps);
3225 //dprintf (GTC_LOG, ("h%d, min frag is %I64d", heap_number, min_frag_th));
3226 ret = (est_frag >= min_frag_th);
3243 gc_heap::dt_low_card_table_efficiency_p (gc_tuning_point tp)
3249 case tuning_deciding_condemned_gen:
3251 /* promote into max-generation if the card table has too many
3252 * generation faults besides the n -> 0
3254 ret = (generation_skip_ratio < 30);
3266 in_range_for_segment(uint8_t* add, heap_segment* seg)
3268 return ((add >= heap_segment_mem (seg)) && (add < heap_segment_reserved (seg)));
3271 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
3272 // The array we allocate is organized as follows:
3273 // 0th element is the address of the last array we allocated.
3274 // starting from the 1st element are the segment addresses, that's
3275 // what buckets() returns.
3288 bk* buckets() { return (slots + 1); }
3289 uint8_t*& last_slot (bk* arr) { return arr[0].add; }
3292 static sorted_table* make_sorted_table ();
3293 BOOL insert (uint8_t* add, size_t val);;
3294 size_t lookup (uint8_t*& add);
3295 void remove (uint8_t* add);
3297 void delete_sorted_table();
3298 void delete_old_slots();
3299 void enqueue_old_slot(bk* sl);
3300 BOOL ensure_space_for_insert();
3304 sorted_table::make_sorted_table ()
3308 // allocate one more bk to store the older slot address.
3309 sorted_table* res = (sorted_table*)new char [sizeof (sorted_table) + (size + 1) * sizeof (bk)];
3313 res->slots = (bk*)(res + 1);
3320 sorted_table::delete_sorted_table()
3322 if (slots != (bk*)(this+1))
3330 sorted_table::delete_old_slots()
3332 uint8_t* sl = (uint8_t*)old_slots;
3336 sl = last_slot ((bk*)sl);
3342 sorted_table::enqueue_old_slot(bk* sl)
3344 last_slot (sl) = (uint8_t*)old_slots;
3350 sorted_table::lookup (uint8_t*& add)
3352 ptrdiff_t high = (count-1);
3356 bk* buck = buckets();
3359 mid = ((low + high)/2);
3361 if (buck[ti].add > add)
3363 if ((ti > 0) && (buck[ti-1].add <= add))
3365 add = buck[ti-1].add;
3366 return buck[ti - 1].val;
3372 if (buck[ti+1].add > add)
3375 return buck[ti].val;
3385 sorted_table::ensure_space_for_insert()
3389 size = (size * 3)/2;
3390 assert((size * sizeof (bk)) > 0);
3391 bk* res = (bk*)new (nothrow) char [(size + 1) * sizeof (bk)];
3396 last_slot (res) = 0;
3397 memcpy (((bk*)res + 1), buckets(), count * sizeof (bk));
3398 bk* last_old_slots = slots;
3400 if (last_old_slots != (bk*)(this + 1))
3401 enqueue_old_slot (last_old_slots);
3407 sorted_table::insert (uint8_t* add, size_t val)
3409 //grow if no more room
3410 assert (count < size);
3413 ptrdiff_t high = (count-1);
3417 bk* buck = buckets();
3420 mid = ((low + high)/2);
3422 if (buck[ti].add > add)
3424 if ((ti == 0) || (buck[ti-1].add <= add))
3426 // found insertion point
3427 for (ptrdiff_t k = count; k > ti;k--)
3429 buck [k] = buck [k-1];
3440 if (buck[ti+1].add > add)
3442 //found the insertion point
3443 for (ptrdiff_t k = count; k > ti+1;k--)
3445 buck [k] = buck [k-1];
3447 buck[ti+1].add = add;
3448 buck[ti+1].val = val;
3460 sorted_table::remove (uint8_t* add)
3462 ptrdiff_t high = (count-1);
3466 bk* buck = buckets();
3469 mid = ((low + high)/2);
3471 if (buck[ti].add > add)
3473 if (buck[ti-1].add <= add)
3475 // found the guy to remove
3476 for (ptrdiff_t k = ti; k < count; k++)
3477 buck[k-1] = buck[k];
3485 if (buck[ti+1].add > add)
3487 // found the guy to remove
3488 for (ptrdiff_t k = ti+1; k < count; k++)
3489 buck[k-1] = buck[k];
3500 sorted_table::clear()
3503 buckets()[0].add = MAX_PTR;
3505 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
3507 #ifdef SEG_MAPPING_TABLE
3508 #ifdef GROWABLE_SEG_MAPPING_TABLE
3510 uint8_t* align_on_segment (uint8_t* add)
3512 return (uint8_t*)((size_t)(add + (gc_heap::min_segment_size - 1)) & ~(gc_heap::min_segment_size - 1));
3516 uint8_t* align_lower_segment (uint8_t* add)
3518 return (uint8_t*)((size_t)(add) & ~(gc_heap::min_segment_size - 1));
3521 size_t size_seg_mapping_table_of (uint8_t* from, uint8_t* end)
3523 from = align_lower_segment (from);
3524 end = align_on_segment (end);
3525 dprintf (1, ("from: %Ix, end: %Ix, size: %Ix", from, end, sizeof (seg_mapping)*(((size_t)(end - from) >> gc_heap::min_segment_size_shr))));
3526 return sizeof (seg_mapping)*((size_t)(end - from) >> gc_heap::min_segment_size_shr);
3529 // for seg_mapping_table we want it to start from a pointer sized address.
3531 size_t align_for_seg_mapping_table (size_t size)
3533 return ((size + (sizeof (uint8_t*) - 1)) &~ (sizeof (uint8_t*) - 1));
3537 size_t seg_mapping_word_of (uint8_t* add)
3539 return (size_t)add >> gc_heap::min_segment_size_shr;
3541 #else //GROWABLE_SEG_MAPPING_TABLE
3542 BOOL seg_mapping_table_init()
3545 uint64_t total_address_space = (uint64_t)8*1024*1024*1024*1024;
3547 uint64_t total_address_space = (uint64_t)4*1024*1024*1024;
3550 size_t num_entries = (size_t)(total_address_space >> gc_heap::min_segment_size_shr);
3551 seg_mapping_table = new seg_mapping[num_entries];
3553 if (seg_mapping_table)
3555 memset (seg_mapping_table, 0, num_entries * sizeof (seg_mapping));
3556 dprintf (1, ("created %d entries for heap mapping (%Id bytes)",
3557 num_entries, (num_entries * sizeof (seg_mapping))));
3562 dprintf (1, ("failed to create %d entries for heap mapping (%Id bytes)",
3563 num_entries, (num_entries * sizeof (seg_mapping))));
3567 #endif //GROWABLE_SEG_MAPPING_TABLE
3569 #ifdef FEATURE_BASICFREEZE
3571 size_t ro_seg_begin_index (heap_segment* seg)
3573 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3574 begin_index = max (begin_index, (size_t)g_gc_lowest_address >> gc_heap::min_segment_size_shr);
3579 size_t ro_seg_end_index (heap_segment* seg)
3581 size_t end_index = (size_t)(heap_segment_reserved (seg) - 1) >> gc_heap::min_segment_size_shr;
3582 end_index = min (end_index, (size_t)g_gc_highest_address >> gc_heap::min_segment_size_shr);
3586 void seg_mapping_table_add_ro_segment (heap_segment* seg)
3588 #ifdef GROWABLE_SEG_MAPPING_TABLE
3589 if ((heap_segment_reserved (seg) <= g_gc_lowest_address) || (heap_segment_mem (seg) >= g_gc_highest_address))
3591 #endif //GROWABLE_SEG_MAPPING_TABLE
3593 for (size_t entry_index = ro_seg_begin_index (seg); entry_index <= ro_seg_end_index (seg); entry_index++)
3594 seg_mapping_table[entry_index].seg1 = (heap_segment*)((size_t)seg_mapping_table[entry_index].seg1 | ro_in_entry);
3597 void seg_mapping_table_remove_ro_segment (heap_segment* seg)
3599 UNREFERENCED_PARAMETER(seg);
3601 // POSSIBLE PERF TODO: right now we are not doing anything because we can't simply remove the flag. If it proves
3602 // to be a perf problem, we can search in the current ro segs and see if any lands in this range and only
3603 // remove the flag if none lands in this range.
3607 heap_segment* ro_segment_lookup (uint8_t* o)
3609 uint8_t* ro_seg_start = o;
3610 heap_segment* seg = (heap_segment*)gc_heap::seg_table->lookup (ro_seg_start);
3612 if (ro_seg_start && in_range_for_segment (o, seg))
3618 #endif //FEATURE_BASICFREEZE
3620 void gc_heap::seg_mapping_table_add_segment (heap_segment* seg, gc_heap* hp)
3622 size_t seg_end = (size_t)(heap_segment_reserved (seg) - 1);
3623 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3624 seg_mapping* begin_entry = &seg_mapping_table[begin_index];
3625 size_t end_index = seg_end >> gc_heap::min_segment_size_shr;
3626 seg_mapping* end_entry = &seg_mapping_table[end_index];
3628 dprintf (1, ("adding seg %Ix(%d)-%Ix(%d)",
3629 seg, begin_index, heap_segment_reserved (seg), end_index));
3631 dprintf (1, ("before add: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3632 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3633 end_index, (seg_mapping_table[end_index].boundary + 1)));
3635 #ifdef MULTIPLE_HEAPS
3636 #ifdef SIMPLE_DPRINTF
3637 dprintf (1, ("begin %d: h0: %Ix(%d), h1: %Ix(%d); end %d: h0: %Ix(%d), h1: %Ix(%d)",
3638 begin_index, (uint8_t*)(begin_entry->h0), (begin_entry->h0 ? begin_entry->h0->heap_number : -1),
3639 (uint8_t*)(begin_entry->h1), (begin_entry->h1 ? begin_entry->h1->heap_number : -1),
3640 end_index, (uint8_t*)(end_entry->h0), (end_entry->h0 ? end_entry->h0->heap_number : -1),
3641 (uint8_t*)(end_entry->h1), (end_entry->h1 ? end_entry->h1->heap_number : -1)));
3642 #endif //SIMPLE_DPRINTF
3643 assert (end_entry->boundary == 0);
3644 assert (end_entry->h0 == 0);
3646 assert (begin_entry->h1 == 0);
3647 begin_entry->h1 = hp;
3649 UNREFERENCED_PARAMETER(hp);
3650 #endif //MULTIPLE_HEAPS
3652 end_entry->boundary = (uint8_t*)seg_end;
3654 dprintf (1, ("set entry %d seg1 and %d seg0 to %Ix", begin_index, end_index, seg));
3655 assert ((begin_entry->seg1 == 0) || ((size_t)(begin_entry->seg1) == ro_in_entry));
3656 begin_entry->seg1 = (heap_segment*)((size_t)(begin_entry->seg1) | (size_t)seg);
3657 end_entry->seg0 = seg;
3659 // for every entry inbetween we need to set its heap too.
3660 for (size_t entry_index = (begin_index + 1); entry_index <= (end_index - 1); entry_index++)
3662 assert (seg_mapping_table[entry_index].boundary == 0);
3663 #ifdef MULTIPLE_HEAPS
3664 assert (seg_mapping_table[entry_index].h0 == 0);
3665 seg_mapping_table[entry_index].h1 = hp;
3666 #endif //MULTIPLE_HEAPS
3667 seg_mapping_table[entry_index].seg1 = seg;
3670 dprintf (1, ("after add: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3671 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3672 end_index, (seg_mapping_table[end_index].boundary + 1)));
3673 #if defined(MULTIPLE_HEAPS) && defined(SIMPLE_DPRINTF)
3674 dprintf (1, ("begin %d: h0: %Ix(%d), h1: %Ix(%d); end: %d h0: %Ix(%d), h1: %Ix(%d)",
3675 begin_index, (uint8_t*)(begin_entry->h0), (begin_entry->h0 ? begin_entry->h0->heap_number : -1),
3676 (uint8_t*)(begin_entry->h1), (begin_entry->h1 ? begin_entry->h1->heap_number : -1),
3677 end_index, (uint8_t*)(end_entry->h0), (end_entry->h0 ? end_entry->h0->heap_number : -1),
3678 (uint8_t*)(end_entry->h1), (end_entry->h1 ? end_entry->h1->heap_number : -1)));
3679 #endif //MULTIPLE_HEAPS && SIMPLE_DPRINTF
3682 void gc_heap::seg_mapping_table_remove_segment (heap_segment* seg)
3684 size_t seg_end = (size_t)(heap_segment_reserved (seg) - 1);
3685 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3686 seg_mapping* begin_entry = &seg_mapping_table[begin_index];
3687 size_t end_index = seg_end >> gc_heap::min_segment_size_shr;
3688 seg_mapping* end_entry = &seg_mapping_table[end_index];
3689 dprintf (1, ("removing seg %Ix(%d)-%Ix(%d)",
3690 seg, begin_index, heap_segment_reserved (seg), end_index));
3692 assert (end_entry->boundary == (uint8_t*)seg_end);
3693 end_entry->boundary = 0;
3695 #ifdef MULTIPLE_HEAPS
3696 gc_heap* hp = heap_segment_heap (seg);
3697 assert (end_entry->h0 == hp);
3699 assert (begin_entry->h1 == hp);
3700 begin_entry->h1 = 0;
3701 #endif //MULTIPLE_HEAPS
3703 assert (begin_entry->seg1 != 0);
3704 begin_entry->seg1 = (heap_segment*)((size_t)(begin_entry->seg1) & ro_in_entry);
3705 end_entry->seg0 = 0;
3707 // for every entry inbetween we need to reset its heap too.
3708 for (size_t entry_index = (begin_index + 1); entry_index <= (end_index - 1); entry_index++)
3710 assert (seg_mapping_table[entry_index].boundary == 0);
3711 #ifdef MULTIPLE_HEAPS
3712 assert (seg_mapping_table[entry_index].h0 == 0);
3713 assert (seg_mapping_table[entry_index].h1 == hp);
3714 seg_mapping_table[entry_index].h1 = 0;
3715 #endif //MULTIPLE_HEAPS
3716 seg_mapping_table[entry_index].seg1 = 0;
3719 dprintf (1, ("after remove: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3720 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3721 end_index, (seg_mapping_table[end_index].boundary + 1)));
3722 #ifdef MULTIPLE_HEAPS
3723 dprintf (1, ("begin %d: h0: %Ix, h1: %Ix; end: %d h0: %Ix, h1: %Ix",
3724 begin_index, (uint8_t*)(begin_entry->h0), (uint8_t*)(begin_entry->h1),
3725 end_index, (uint8_t*)(end_entry->h0), (uint8_t*)(end_entry->h1)));
3726 #endif //MULTIPLE_HEAPS
3729 #ifdef MULTIPLE_HEAPS
3731 gc_heap* seg_mapping_table_heap_of_worker (uint8_t* o)
3733 size_t index = (size_t)o >> gc_heap::min_segment_size_shr;
3734 seg_mapping* entry = &seg_mapping_table[index];
3736 gc_heap* hp = ((o > entry->boundary) ? entry->h1 : entry->h0);
3738 dprintf (2, ("checking obj %Ix, index is %Id, entry: boundry: %Ix, h0: %Ix, seg0: %Ix, h1: %Ix, seg1: %Ix",
3739 o, index, (entry->boundary + 1),
3740 (uint8_t*)(entry->h0), (uint8_t*)(entry->seg0),
3741 (uint8_t*)(entry->h1), (uint8_t*)(entry->seg1)));
3744 heap_segment* seg = ((o > entry->boundary) ? entry->seg1 : entry->seg0);
3745 #ifdef FEATURE_BASICFREEZE
3746 if ((size_t)seg & ro_in_entry)
3747 seg = (heap_segment*)((size_t)seg & ~ro_in_entry);
3748 #endif //FEATURE_BASICFREEZE
3752 if (in_range_for_segment (o, seg))
3754 dprintf (2, ("obj %Ix belongs to segment %Ix(-%Ix)", o, seg, (uint8_t*)heap_segment_allocated (seg)));
3758 dprintf (2, ("found seg %Ix(-%Ix) for obj %Ix, but it's not on the seg",
3759 seg, (uint8_t*)heap_segment_allocated (seg), o));
3764 dprintf (2, ("could not find obj %Ix in any existing segments", o));
3771 gc_heap* seg_mapping_table_heap_of (uint8_t* o)
3773 #ifdef GROWABLE_SEG_MAPPING_TABLE
3774 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3776 #endif //GROWABLE_SEG_MAPPING_TABLE
3778 return seg_mapping_table_heap_of_worker (o);
3781 gc_heap* seg_mapping_table_heap_of_gc (uint8_t* o)
3783 #if defined(FEATURE_BASICFREEZE) && defined(GROWABLE_SEG_MAPPING_TABLE)
3784 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3786 #endif //FEATURE_BASICFREEZE || GROWABLE_SEG_MAPPING_TABLE
3788 return seg_mapping_table_heap_of_worker (o);
3790 #endif //MULTIPLE_HEAPS
3792 // Only returns a valid seg if we can actually find o on the seg.
3793 heap_segment* seg_mapping_table_segment_of (uint8_t* o)
3795 #if defined(FEATURE_BASICFREEZE) && defined(GROWABLE_SEG_MAPPING_TABLE)
3796 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3797 #ifdef FEATURE_BASICFREEZE
3798 return ro_segment_lookup (o);
3801 #endif //FEATURE_BASICFREEZE
3802 #endif //FEATURE_BASICFREEZE || GROWABLE_SEG_MAPPING_TABLE
3804 size_t index = (size_t)o >> gc_heap::min_segment_size_shr;
3805 seg_mapping* entry = &seg_mapping_table[index];
3807 dprintf (2, ("checking obj %Ix, index is %Id, entry: boundry: %Ix, seg0: %Ix, seg1: %Ix",
3808 o, index, (entry->boundary + 1),
3809 (uint8_t*)(entry->seg0), (uint8_t*)(entry->seg1)));
3811 heap_segment* seg = ((o > entry->boundary) ? entry->seg1 : entry->seg0);
3812 #ifdef FEATURE_BASICFREEZE
3813 if ((size_t)seg & ro_in_entry)
3814 seg = (heap_segment*)((size_t)seg & ~ro_in_entry);
3815 #endif //FEATURE_BASICFREEZE
3819 if (in_range_for_segment (o, seg))
3821 dprintf (2, ("obj %Ix belongs to segment %Ix(-%Ix)", o, (uint8_t*)heap_segment_mem(seg), (uint8_t*)heap_segment_reserved(seg)));
3825 dprintf (2, ("found seg %Ix(-%Ix) for obj %Ix, but it's not on the seg, setting it to 0",
3826 (uint8_t*)heap_segment_mem(seg), (uint8_t*)heap_segment_reserved(seg), o));
3832 dprintf (2, ("could not find obj %Ix in any existing segments", o));
3835 #ifdef FEATURE_BASICFREEZE
3836 // TODO: This was originally written assuming that the seg_mapping_table would always contain entries for ro
3837 // segments whenever the ro segment falls into the [g_gc_lowest_address,g_gc_highest_address) range. I.e., it had an
3838 // extra "&& (size_t)(entry->seg1) & ro_in_entry" expression. However, at the moment, grow_brick_card_table does
3839 // not correctly go through the ro segments and add them back to the seg_mapping_table when the [lowest,highest)
3840 // range changes. We should probably go ahead and modify grow_brick_card_table and put back the
3841 // "&& (size_t)(entry->seg1) & ro_in_entry" here.
3844 seg = ro_segment_lookup (o);
3845 if (seg && !in_range_for_segment (o, seg))
3848 #endif //FEATURE_BASICFREEZE
3852 #endif //SEG_MAPPING_TABLE
3854 size_t gcard_of ( uint8_t*);
3856 #define memref(i) *(uint8_t**)(i)
3859 #define GC_MARKED (size_t)0x1
3860 #define slot(i, j) ((uint8_t**)(i))[j+1]
3862 #define free_object_base_size (plug_skew + sizeof(ArrayBase))
3864 class CObjectHeader : public Object
3868 #if defined(FEATURE_REDHAWK) || defined(BUILD_AS_STANDALONE)
3869 // The GC expects the following methods that are provided by the Object class in the CLR but not provided
3870 // by Redhawk's version of Object.
3871 uint32_t GetNumComponents()
3873 return ((ArrayBase *)this)->GetNumComponents();
3876 void Validate(BOOL bDeep=TRUE, BOOL bVerifyNextHeader = TRUE)
3878 UNREFERENCED_PARAMETER(bVerifyNextHeader);
3883 MethodTable * pMT = GetMethodTable();
3885 _ASSERTE(pMT->SanityCheck());
3887 bool noRangeChecks =
3888 (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_NO_RANGE_CHECKS) == GCConfig::HEAPVERIFY_NO_RANGE_CHECKS;
3890 BOOL fSmallObjectHeapPtr = FALSE, fLargeObjectHeapPtr = FALSE;
3893 fSmallObjectHeapPtr = g_theGCHeap->IsHeapPointer(this, TRUE);
3894 if (!fSmallObjectHeapPtr)
3895 fLargeObjectHeapPtr = g_theGCHeap->IsHeapPointer(this);
3897 _ASSERTE(fSmallObjectHeapPtr || fLargeObjectHeapPtr);
3900 #ifdef FEATURE_STRUCTALIGN
3901 _ASSERTE(IsStructAligned((uint8_t *)this, GetMethodTable()->GetBaseAlignment()));
3902 #endif // FEATURE_STRUCTALIGN
3904 #ifdef FEATURE_64BIT_ALIGNMENT
3905 if (pMT->RequiresAlign8())
3907 _ASSERTE((((size_t)this) & 0x7) == (pMT->IsValueType() ? 4U : 0U));
3909 #endif // FEATURE_64BIT_ALIGNMENT
3912 if (bDeep && (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC))
3913 g_theGCHeap->ValidateObjectMember(this);
3915 if (fSmallObjectHeapPtr)
3917 #ifdef FEATURE_BASICFREEZE
3918 _ASSERTE(!g_theGCHeap->IsLargeObject(pMT) || g_theGCHeap->IsInFrozenSegment(this));
3920 _ASSERTE(!g_theGCHeap->IsLargeObject(pMT));
3925 void ValidatePromote(ScanContext *sc, uint32_t flags)
3927 UNREFERENCED_PARAMETER(sc);
3928 UNREFERENCED_PARAMETER(flags);
3933 void ValidateHeap(Object *from, BOOL bDeep)
3935 UNREFERENCED_PARAMETER(from);
3937 Validate(bDeep, FALSE);
3940 ADIndex GetAppDomainIndex()
3942 return (ADIndex)RH_DEFAULT_DOMAIN_ID;
3944 #endif //FEATURE_REDHAWK
3948 // Header Status Information
3951 MethodTable *GetMethodTable() const
3953 return( (MethodTable *) (((size_t) RawGetMethodTable()) & (~(GC_MARKED))));
3958 RawSetMethodTable((MethodTable *) (((size_t) RawGetMethodTable()) | GC_MARKED));
3961 BOOL IsMarked() const
3963 return !!(((size_t)RawGetMethodTable()) & GC_MARKED);
3968 assert (!(gc_heap::settings.concurrent));
3969 GetHeader()->SetGCBit();
3972 BOOL IsPinned() const
3974 return !!((((CObjectHeader*)this)->GetHeader()->GetBits()) & BIT_SBLK_GC_RESERVE);
3979 RawSetMethodTable( GetMethodTable() );
3982 CGCDesc *GetSlotMap ()
3984 assert (GetMethodTable()->ContainsPointers());
3985 return CGCDesc::GetCGCDescFromMT(GetMethodTable());
3988 void SetFree(size_t size)
3990 assert (size >= free_object_base_size);
3992 assert (g_gc_pFreeObjectMethodTable->GetBaseSize() == free_object_base_size);
3993 assert (g_gc_pFreeObjectMethodTable->RawGetComponentSize() == 1);
3995 RawSetMethodTable( g_gc_pFreeObjectMethodTable );
3997 size_t* numComponentsPtr = (size_t*) &((uint8_t*) this)[ArrayBase::GetOffsetOfNumComponents()];
3998 *numComponentsPtr = size - free_object_base_size;
4000 //This introduces a bug in the free list management.
4001 //((void**) this)[-1] = 0; // clear the sync block,
4002 assert (*numComponentsPtr >= 0);
4003 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
4004 memset (((uint8_t*)this)+sizeof(ArrayBase), 0xcc, *numComponentsPtr);
4005 #endif //VERIFY_HEAP
4010 size_t size = free_object_base_size - plug_skew;
4012 // since we only need to clear 2 ptr size, we do it manually
4013 PTR_PTR m = (PTR_PTR) this;
4014 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
4018 BOOL IsFree () const
4020 return (GetMethodTable() == g_gc_pFreeObjectMethodTable);
4023 #ifdef FEATURE_STRUCTALIGN
4024 int GetRequiredAlignment () const
4026 return GetMethodTable()->GetRequiredAlignment();
4028 #endif // FEATURE_STRUCTALIGN
4030 BOOL ContainsPointers() const
4032 return GetMethodTable()->ContainsPointers();
4035 #ifdef COLLECTIBLE_CLASS
4036 BOOL Collectible() const
4038 return GetMethodTable()->Collectible();
4041 FORCEINLINE BOOL ContainsPointersOrCollectible() const
4043 MethodTable *pMethodTable = GetMethodTable();
4044 return (pMethodTable->ContainsPointers() || pMethodTable->Collectible());
4046 #endif //COLLECTIBLE_CLASS
4048 Object* GetObjectBase() const
4050 return (Object*) this;
4054 #define header(i) ((CObjectHeader*)(i))
4056 #define free_list_slot(x) ((uint8_t**)(x))[2]
4057 #define free_list_undo(x) ((uint8_t**)(x))[-1]
4058 #define UNDO_EMPTY ((uint8_t*)1)
4062 void set_plug_padded (uint8_t* node)
4064 header(node)->SetMarked();
4067 void clear_plug_padded (uint8_t* node)
4069 header(node)->ClearMarked();
4072 BOOL is_plug_padded (uint8_t* node)
4074 return header(node)->IsMarked();
4077 inline void set_plug_padded (uint8_t* node){}
4078 inline void clear_plug_padded (uint8_t* node){}
4080 BOOL is_plug_padded (uint8_t* node){return FALSE;}
4081 #endif //SHORT_PLUGS
4084 inline size_t unused_array_size(uint8_t * p)
4086 assert(((CObjectHeader*)p)->IsFree());
4088 size_t* numComponentsPtr = (size_t*)(p + ArrayBase::GetOffsetOfNumComponents());
4089 return free_object_base_size + *numComponentsPtr;
4092 heap_segment* heap_segment_rw (heap_segment* ns)
4094 if ((ns == 0) || !heap_segment_read_only_p (ns))
4102 ns = heap_segment_next (ns);
4103 } while ((ns != 0) && heap_segment_read_only_p (ns));
4108 //returns the next non ro segment.
4109 heap_segment* heap_segment_next_rw (heap_segment* seg)
4111 heap_segment* ns = heap_segment_next (seg);
4112 return heap_segment_rw (ns);
4115 // returns the segment before seg.
4116 heap_segment* heap_segment_prev_rw (heap_segment* begin, heap_segment* seg)
4118 assert (begin != 0);
4119 heap_segment* prev = begin;
4120 heap_segment* current = heap_segment_next_rw (begin);
4122 while (current && current != seg)
4125 current = heap_segment_next_rw (current);
4138 // returns the segment before seg.
4139 heap_segment* heap_segment_prev (heap_segment* begin, heap_segment* seg)
4141 assert (begin != 0);
4142 heap_segment* prev = begin;
4143 heap_segment* current = heap_segment_next (begin);
4145 while (current && current != seg)
4148 current = heap_segment_next (current);
4161 heap_segment* heap_segment_in_range (heap_segment* ns)
4163 if ((ns == 0) || heap_segment_in_range_p (ns))
4171 ns = heap_segment_next (ns);
4172 } while ((ns != 0) && !heap_segment_in_range_p (ns));
4177 heap_segment* heap_segment_next_in_range (heap_segment* seg)
4179 heap_segment* ns = heap_segment_next (seg);
4180 return heap_segment_in_range (ns);
4185 uint8_t* memory_base;
4190 imemory_data *initial_memory;
4191 imemory_data *initial_normal_heap; // points into initial_memory_array
4192 imemory_data *initial_large_heap; // points into initial_memory_array
4194 size_t block_size_normal;
4195 size_t block_size_large;
4197 size_t block_count; // # of blocks in each
4198 size_t current_block_normal;
4199 size_t current_block_large;
4208 size_t allocation_pattern;
4209 } initial_memory_details;
4211 initial_memory_details memory_details;
4213 BOOL reserve_initial_memory (size_t normal_size, size_t large_size, size_t num_heaps)
4215 BOOL reserve_success = FALSE;
4217 // should only be called once
4218 assert (memory_details.initial_memory == 0);
4220 memory_details.initial_memory = new (nothrow) imemory_data[num_heaps*2];
4221 if (memory_details.initial_memory == 0)
4223 dprintf (2, ("failed to reserve %Id bytes for imemory_data", num_heaps*2*sizeof(imemory_data)));
4227 memory_details.initial_normal_heap = memory_details.initial_memory;
4228 memory_details.initial_large_heap = memory_details.initial_memory + num_heaps;
4229 memory_details.block_size_normal = normal_size;
4230 memory_details.block_size_large = large_size;
4231 memory_details.block_count = num_heaps;
4233 memory_details.current_block_normal = 0;
4234 memory_details.current_block_large = 0;
4236 g_gc_lowest_address = MAX_PTR;
4237 g_gc_highest_address = 0;
4239 if (((size_t)MAX_PTR - large_size) < normal_size)
4241 // we are already overflowing with just one heap.
4242 dprintf (2, ("0x%Ix + 0x%Ix already overflow", normal_size, large_size));
4246 if (((size_t)MAX_PTR / memory_details.block_count) < (normal_size + large_size))
4248 dprintf (2, ("(0x%Ix + 0x%Ix)*0x%Ix overflow", normal_size, large_size, memory_details.block_count));
4252 size_t requestedMemory = memory_details.block_count * (normal_size + large_size);
4254 uint8_t* allatonce_block = (uint8_t*)virtual_alloc (requestedMemory);
4255 if (allatonce_block)
4257 g_gc_lowest_address = allatonce_block;
4258 g_gc_highest_address = allatonce_block + (memory_details.block_count * (large_size + normal_size));
4259 memory_details.allocation_pattern = initial_memory_details::ALLATONCE;
4261 for(size_t i = 0; i < memory_details.block_count; i++)
4263 memory_details.initial_normal_heap[i].memory_base = allatonce_block + (i*normal_size);
4264 memory_details.initial_large_heap[i].memory_base = allatonce_block +
4265 (memory_details.block_count*normal_size) + (i*large_size);
4266 reserve_success = TRUE;
4271 // try to allocate 2 blocks
4274 b1 = (uint8_t*)virtual_alloc (memory_details.block_count * normal_size);
4277 b2 = (uint8_t*)virtual_alloc (memory_details.block_count * large_size);
4280 memory_details.allocation_pattern = initial_memory_details::TWO_STAGE;
4281 g_gc_lowest_address = min(b1,b2);
4282 g_gc_highest_address = max(b1 + memory_details.block_count*normal_size,
4283 b2 + memory_details.block_count*large_size);
4284 for(size_t i = 0; i < memory_details.block_count; i++)
4286 memory_details.initial_normal_heap[i].memory_base = b1 + (i*normal_size);
4287 memory_details.initial_large_heap[i].memory_base = b2 + (i*large_size);
4288 reserve_success = TRUE;
4293 // b2 allocation failed, we'll go on to try allocating each block.
4294 // We could preserve the b1 alloc, but code complexity increases
4295 virtual_free (b1, memory_details.block_count * normal_size);
4299 if ((b2==NULL) && ( memory_details.block_count > 1))
4301 memory_details.allocation_pattern = initial_memory_details::EACH_BLOCK;
4303 imemory_data *current_block = memory_details.initial_memory;
4304 for(size_t i = 0; i < (memory_details.block_count*2); i++, current_block++)
4306 size_t block_size = ((i < memory_details.block_count) ?
4307 memory_details.block_size_normal :
4308 memory_details.block_size_large);
4309 current_block->memory_base =
4310 (uint8_t*)virtual_alloc (block_size);
4311 if (current_block->memory_base == 0)
4313 // Free the blocks that we've allocated so far
4314 current_block = memory_details.initial_memory;
4315 for(size_t j = 0; j < i; j++, current_block++){
4316 if (current_block->memory_base != 0){
4317 block_size = ((j < memory_details.block_count) ?
4318 memory_details.block_size_normal :
4319 memory_details.block_size_large);
4320 virtual_free (current_block->memory_base , block_size);
4323 reserve_success = FALSE;
4328 if (current_block->memory_base < g_gc_lowest_address)
4329 g_gc_lowest_address = current_block->memory_base;
4330 if (((uint8_t *) current_block->memory_base + block_size) > g_gc_highest_address)
4331 g_gc_highest_address = (current_block->memory_base + block_size);
4333 reserve_success = TRUE;
4338 return reserve_success;
4341 void destroy_initial_memory()
4343 if (memory_details.initial_memory != NULL)
4345 if (memory_details.allocation_pattern == initial_memory_details::ALLATONCE)
4347 virtual_free(memory_details.initial_memory[0].memory_base,
4348 memory_details.block_count*(memory_details.block_size_normal +
4349 memory_details.block_size_large));
4351 else if (memory_details.allocation_pattern == initial_memory_details::TWO_STAGE)
4353 virtual_free (memory_details.initial_normal_heap[0].memory_base,
4354 memory_details.block_count*memory_details.block_size_normal);
4356 virtual_free (memory_details.initial_large_heap[0].memory_base,
4357 memory_details.block_count*memory_details.block_size_large);
4361 assert (memory_details.allocation_pattern == initial_memory_details::EACH_BLOCK);
4362 imemory_data *current_block = memory_details.initial_memory;
4363 for(size_t i = 0; i < (memory_details.block_count*2); i++, current_block++)
4365 size_t block_size = (i < memory_details.block_count) ? memory_details.block_size_normal :
4366 memory_details.block_size_large;
4367 if (current_block->memory_base != NULL)
4369 virtual_free (current_block->memory_base, block_size);
4374 delete [] memory_details.initial_memory;
4375 memory_details.initial_memory = NULL;
4376 memory_details.initial_normal_heap = NULL;
4377 memory_details.initial_large_heap = NULL;
4381 void* next_initial_memory (size_t size)
4383 assert ((size == memory_details.block_size_normal) || (size == memory_details.block_size_large));
4386 if ((size != memory_details.block_size_normal) ||
4387 ((memory_details.current_block_normal == memory_details.block_count) &&
4388 (memory_details.block_size_normal == memory_details.block_size_large)))
4390 // If the block sizes are the same, flow block requests from normal to large
4391 assert (memory_details.current_block_large < memory_details.block_count);
4392 assert (memory_details.initial_large_heap != 0);
4394 res = memory_details.initial_large_heap[memory_details.current_block_large].memory_base;
4395 memory_details.current_block_large++;
4399 assert (memory_details.current_block_normal < memory_details.block_count);
4400 assert (memory_details.initial_normal_heap != NULL);
4402 res = memory_details.initial_normal_heap[memory_details.current_block_normal].memory_base;
4403 memory_details.current_block_normal++;
4409 heap_segment* get_initial_segment (size_t size, int h_number)
4411 void* mem = next_initial_memory (size);
4412 heap_segment* res = gc_heap::make_heap_segment ((uint8_t*)mem, size , h_number);
4417 void* virtual_alloc (size_t size)
4419 size_t requested_size = size;
4421 if ((gc_heap::reserved_memory_limit - gc_heap::reserved_memory) < requested_size)
4423 gc_heap::reserved_memory_limit =
4424 GCScan::AskForMoreReservedMemory (gc_heap::reserved_memory_limit, requested_size);
4425 if ((gc_heap::reserved_memory_limit - gc_heap::reserved_memory) < requested_size)
4431 uint32_t flags = VirtualReserveFlags::None;
4432 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
4433 if (virtual_alloc_hardware_write_watch)
4435 flags = VirtualReserveFlags::WriteWatch;
4437 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
4438 void* prgmem = GCToOSInterface::VirtualReserve (requested_size, card_size * card_word_width, flags);
4439 void *aligned_mem = prgmem;
4441 // We don't want (prgmem + size) to be right at the end of the address space
4442 // because we'd have to worry about that everytime we do (address + size).
4443 // We also want to make sure that we leave LARGE_OBJECT_SIZE at the end
4444 // so we allocate a small object we don't need to worry about overflow there
4445 // when we do alloc_ptr+size.
4448 uint8_t* end_mem = (uint8_t*)prgmem + requested_size;
4450 if ((end_mem == 0) || ((size_t)(MAX_PTR - end_mem) <= END_SPACE_AFTER_GC))
4452 GCToOSInterface::VirtualRelease (prgmem, requested_size);
4453 dprintf (2, ("Virtual Alloc size %Id returned memory right against 4GB [%Ix, %Ix[ - discarding",
4454 requested_size, (size_t)prgmem, (size_t)((uint8_t*)prgmem+requested_size)));
4462 gc_heap::reserved_memory += requested_size;
4465 dprintf (2, ("Virtual Alloc size %Id: [%Ix, %Ix[",
4466 requested_size, (size_t)prgmem, (size_t)((uint8_t*)prgmem+requested_size)));
4471 void virtual_free (void* add, size_t size)
4473 GCToOSInterface::VirtualRelease (add, size);
4474 gc_heap::reserved_memory -= size;
4475 dprintf (2, ("Virtual Free size %Id: [%Ix, %Ix[",
4476 size, (size_t)add, (size_t)((uint8_t*)add+size)));
4479 static size_t get_valid_segment_size (BOOL large_seg=FALSE)
4481 size_t seg_size, initial_seg_size;
4485 initial_seg_size = INITIAL_ALLOC;
4486 seg_size = static_cast<size_t>(GCConfig::GetSegmentSize());
4490 initial_seg_size = LHEAP_ALLOC;
4491 seg_size = static_cast<size_t>(GCConfig::GetSegmentSize()) / 2;
4494 #ifdef MULTIPLE_HEAPS
4499 if (g_num_processors > 4)
4500 initial_seg_size /= 2;
4501 if (g_num_processors > 8)
4502 initial_seg_size /= 2;
4504 #endif //MULTIPLE_HEAPS
4506 // if seg_size is small but not 0 (0 is default if config not set)
4507 // then set the segment to the minimum size
4508 if (!g_theGCHeap->IsValidSegmentSize(seg_size))
4510 // if requested size is between 1 byte and 4MB, use min
4511 if ((seg_size >> 1) && !(seg_size >> 22))
4512 seg_size = 1024*1024*4;
4514 seg_size = initial_seg_size;
4517 #ifdef SEG_MAPPING_TABLE
4519 seg_size = round_up_power2 (seg_size);
4521 seg_size = round_down_power2 (seg_size);
4523 #endif //SEG_MAPPING_TABLE
4529 gc_heap::compute_new_ephemeral_size()
4531 int eph_gen_max = max_generation - 1 - (settings.promotion ? 1 : 0);
4532 size_t padding_size = 0;
4534 for (int i = 0; i <= eph_gen_max; i++)
4536 dynamic_data* dd = dynamic_data_of (i);
4537 total_ephemeral_size += (dd_survived_size (dd) - dd_pinned_survived_size (dd));
4538 #ifdef RESPECT_LARGE_ALIGNMENT
4539 total_ephemeral_size += dd_num_npinned_plugs (dd) * switch_alignment_size (FALSE);
4540 #endif //RESPECT_LARGE_ALIGNMENT
4541 #ifdef FEATURE_STRUCTALIGN
4542 total_ephemeral_size += dd_num_npinned_plugs (dd) * MAX_STRUCTALIGN;
4543 #endif //FEATURE_STRUCTALIGN
4546 padding_size += dd_padding_size (dd);
4547 #endif //SHORT_PLUGS
4550 total_ephemeral_size += eph_gen_starts_size;
4552 #ifdef RESPECT_LARGE_ALIGNMENT
4553 size_t planned_ephemeral_size = heap_segment_plan_allocated (ephemeral_heap_segment) -
4554 generation_plan_allocation_start (generation_of (max_generation-1));
4555 total_ephemeral_size = min (total_ephemeral_size, planned_ephemeral_size);
4556 #endif //RESPECT_LARGE_ALIGNMENT
4559 total_ephemeral_size = Align ((size_t)((double)total_ephemeral_size * short_plugs_pad_ratio) + 1);
4560 total_ephemeral_size += Align (DESIRED_PLUG_LENGTH);
4561 #endif //SHORT_PLUGS
4563 dprintf (3, ("total ephemeral size is %Ix, padding %Ix(%Ix)",
4564 total_ephemeral_size,
4565 padding_size, (total_ephemeral_size - padding_size)));
4569 #pragma warning(disable:4706) // "assignment within conditional expression" is intentional in this function.
4573 gc_heap::soh_get_segment_to_expand()
4575 size_t size = soh_segment_size;
4577 ordered_plug_indices_init = FALSE;
4578 use_bestfit = FALSE;
4580 //compute the size of the new ephemeral heap segment.
4581 compute_new_ephemeral_size();
4583 if ((settings.pause_mode != pause_low_latency) &&
4584 (settings.pause_mode != pause_no_gc)
4585 #ifdef BACKGROUND_GC
4586 && (!recursive_gc_sync::background_running_p())
4587 #endif //BACKGROUND_GC
4590 allocator* gen_alloc = ((settings.condemned_generation == max_generation) ? 0 :
4591 generation_allocator (generation_of (max_generation)));
4592 dprintf (2, ("(gen%d)soh_get_segment_to_expand", settings.condemned_generation));
4594 // try to find one in the gen 2 segment list, search backwards because the first segments
4595 // tend to be more compact than the later ones.
4596 heap_segment* fseg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
4598 PREFIX_ASSUME(fseg != NULL);
4600 #ifdef SEG_REUSE_STATS
4602 #endif //SEG_REUSE_STATS
4604 heap_segment* seg = ephemeral_heap_segment;
4605 while ((seg = heap_segment_prev_rw (fseg, seg)) && (seg != fseg))
4607 #ifdef SEG_REUSE_STATS
4609 #endif //SEG_REUSE_STATS
4611 if (can_expand_into_p (seg, size/3, total_ephemeral_size, gen_alloc))
4613 get_gc_data_per_heap()->set_mechanism (gc_heap_expand,
4614 (use_bestfit ? expand_reuse_bestfit : expand_reuse_normal));
4615 if (settings.condemned_generation == max_generation)
4619 build_ordered_free_spaces (seg);
4620 dprintf (GTC_LOG, ("can use best fit"));
4623 #ifdef SEG_REUSE_STATS
4624 dprintf (SEG_REUSE_LOG_0, ("(gen%d)soh_get_segment_to_expand: found seg #%d to reuse",
4625 settings.condemned_generation, try_reuse));
4626 #endif //SEG_REUSE_STATS
4627 dprintf (GTC_LOG, ("max_gen: Found existing segment to expand into %Ix", (size_t)seg));
4632 #ifdef SEG_REUSE_STATS
4633 dprintf (SEG_REUSE_LOG_0, ("(gen%d)soh_get_segment_to_expand: found seg #%d to reuse - returning",
4634 settings.condemned_generation, try_reuse));
4635 #endif //SEG_REUSE_STATS
4636 dprintf (GTC_LOG, ("max_gen-1: Found existing segment to expand into %Ix", (size_t)seg));
4638 // If we return 0 here, the allocator will think since we are short on end
4639 // of seg we neeed to trigger a full compacting GC. So if sustained low latency
4640 // is set we should acquire a new seg instead, that way we wouldn't be short.
4641 // The real solution, of course, is to actually implement seg reuse in gen1.
4642 if (settings.pause_mode != pause_sustained_low_latency)
4644 dprintf (GTC_LOG, ("max_gen-1: SustainedLowLatency is set, acquire a new seg"));
4645 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, expand_next_full_gc);
4653 heap_segment* result = get_segment (size, FALSE);
4657 #ifdef BACKGROUND_GC
4658 if (current_c_gc_state == c_gc_state_planning)
4660 // When we expand heap during bgc sweep, we set the seg to be swept so
4661 // we'll always look at cards for objects on the new segment.
4662 result->flags |= heap_segment_flags_swept;
4664 #endif //BACKGROUND_GC
4666 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(result),
4667 (size_t)(heap_segment_reserved (result) - heap_segment_mem(result)),
4668 gc_etw_segment_small_object_heap);
4671 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, (result ? expand_new_seg : expand_no_memory));
4675 dprintf (2, ("h%d: failed to allocate a new segment!", heap_number));
4679 #ifdef MULTIPLE_HEAPS
4680 heap_segment_heap (result) = this;
4681 #endif //MULTIPLE_HEAPS
4684 dprintf (GTC_LOG, ("(gen%d)creating new segment %Ix", settings.condemned_generation, result));
4689 #pragma warning(default:4706)
4692 //returns 0 in case of allocation failure
4694 gc_heap::get_segment (size_t size, BOOL loh_p)
4696 heap_segment* result = 0;
4698 if (segment_standby_list != 0)
4700 result = segment_standby_list;
4701 heap_segment* last = 0;
4704 size_t hs = (size_t)(heap_segment_reserved (result) - (uint8_t*)result);
4705 if ((hs >= size) && ((hs / 2) < size))
4707 dprintf (2, ("Hoarded segment %Ix found", (size_t) result));
4710 heap_segment_next (last) = heap_segment_next (result);
4714 segment_standby_list = heap_segment_next (result);
4721 result = heap_segment_next (result);
4728 init_heap_segment (result);
4729 #ifdef BACKGROUND_GC
4730 if (should_commit_mark_array())
4732 dprintf (GC_TABLE_LOG, ("hoarded seg %Ix, mark_array is %Ix", result, mark_array));
4733 if (!commit_mark_array_new_seg (__this, result))
4735 dprintf (GC_TABLE_LOG, ("failed to commit mark array for hoarded seg"));
4736 // If we can't use it we need to thread it back.
4737 if (segment_standby_list != 0)
4739 heap_segment_next (result) = segment_standby_list;
4740 segment_standby_list = result;
4744 segment_standby_list = result;
4750 #endif //BACKGROUND_GC
4752 #ifdef SEG_MAPPING_TABLE
4754 seg_mapping_table_add_segment (result, __this);
4755 #endif //SEG_MAPPING_TABLE
4760 #ifndef SEG_MAPPING_TABLE
4761 if (!seg_table->ensure_space_for_insert ())
4763 #endif //SEG_MAPPING_TABLE
4764 void* mem = virtual_alloc (size);
4767 fgm_result.set_fgm (fgm_reserve_segment, size, loh_p);
4771 result = gc_heap::make_heap_segment ((uint8_t*)mem, size, heap_number);
4777 if (mem < g_gc_lowest_address)
4779 start = (uint8_t*)mem;
4783 start = (uint8_t*)g_gc_lowest_address;
4786 if (((uint8_t*)mem + size) > g_gc_highest_address)
4788 end = (uint8_t*)mem + size;
4792 end = (uint8_t*)g_gc_highest_address;
4795 if (gc_heap::grow_brick_card_tables (start, end, size, result, __this, loh_p) != 0)
4797 virtual_free (mem, size);
4803 fgm_result.set_fgm (fgm_commit_segment_beg, SEGMENT_INITIAL_COMMIT, loh_p);
4804 virtual_free (mem, size);
4809 #ifdef SEG_MAPPING_TABLE
4810 seg_mapping_table_add_segment (result, __this);
4811 #else //SEG_MAPPING_TABLE
4812 gc_heap::seg_table->insert ((uint8_t*)result, delta);
4813 #endif //SEG_MAPPING_TABLE
4817 #ifdef BACKGROUND_GC
4820 ::record_changed_seg ((uint8_t*)result, heap_segment_reserved (result),
4821 settings.gc_index, current_bgc_state,
4823 bgc_verify_mark_array_cleared (result);
4825 #endif //BACKGROUND_GC
4827 dprintf (GC_TABLE_LOG, ("h%d: new seg: %Ix-%Ix (%Id)", heap_number, result, ((uint8_t*)result + size), size));
4831 void release_segment (heap_segment* sg)
4833 ptrdiff_t delta = 0;
4834 FIRE_EVENT(GCFreeSegment_V1, heap_segment_mem(sg));
4835 virtual_free (sg, (uint8_t*)heap_segment_reserved (sg)-(uint8_t*)sg);
4838 heap_segment* gc_heap::get_segment_for_loh (size_t size
4839 #ifdef MULTIPLE_HEAPS
4841 #endif //MULTIPLE_HEAPS
4844 #ifndef MULTIPLE_HEAPS
4846 #endif //MULTIPLE_HEAPS
4847 heap_segment* res = hp->get_segment (size, TRUE);
4850 #ifdef MULTIPLE_HEAPS
4851 heap_segment_heap (res) = hp;
4852 #endif //MULTIPLE_HEAPS
4853 res->flags |= heap_segment_flags_loh;
4855 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(res), (size_t)(heap_segment_reserved (res) - heap_segment_mem(res)), gc_etw_segment_large_object_heap);
4857 GCToEEInterface::DiagUpdateGenerationBounds();
4859 #ifdef MULTIPLE_HEAPS
4860 hp->thread_loh_segment (res);
4862 thread_loh_segment (res);
4863 #endif //MULTIPLE_HEAPS
4869 void gc_heap::thread_loh_segment (heap_segment* new_seg)
4871 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
4873 while (heap_segment_next_rw (seg))
4874 seg = heap_segment_next_rw (seg);
4875 heap_segment_next (seg) = new_seg;
4879 gc_heap::get_large_segment (size_t size, BOOL* did_full_compact_gc)
4881 *did_full_compact_gc = FALSE;
4882 size_t last_full_compact_gc_count = get_full_compact_gc_count();
4884 //access to get_segment needs to be serialized
4885 add_saved_spinlock_info (me_release, mt_get_large_seg);
4887 dprintf (SPINLOCK_LOG, ("[%d]Seg: Lmsl", heap_number));
4888 leave_spin_lock (&more_space_lock);
4889 enter_spin_lock (&gc_heap::gc_lock);
4890 dprintf (SPINLOCK_LOG, ("[%d]Seg: Egc", heap_number));
4891 // if a GC happened between here and before we ask for a segment in
4892 // get_large_segment, we need to count that GC.
4893 size_t current_full_compact_gc_count = get_full_compact_gc_count();
4895 if (current_full_compact_gc_count > last_full_compact_gc_count)
4897 *did_full_compact_gc = TRUE;
4900 #ifdef BACKGROUND_GC
4901 while (current_c_gc_state == c_gc_state_planning)
4903 dprintf (3, ("lh state planning, waiting to get a large seg"));
4905 dprintf (SPINLOCK_LOG, ("[%d]Seg: P, Lgc", heap_number));
4906 leave_spin_lock (&gc_lock);
4907 background_gc_wait_lh (awr_get_loh_seg);
4908 enter_spin_lock (&gc_lock);
4909 dprintf (SPINLOCK_LOG, ("[%d]Seg: P, Egc", heap_number));
4911 assert ((current_c_gc_state == c_gc_state_free) ||
4912 (current_c_gc_state == c_gc_state_marking));
4913 #endif //BACKGROUND_GC
4915 heap_segment* res = get_segment_for_loh (size
4916 #ifdef MULTIPLE_HEAPS
4918 #endif //MULTIPLE_HEAPS
4921 dprintf (SPINLOCK_LOG, ("[%d]Seg: A Lgc", heap_number));
4922 leave_spin_lock (&gc_heap::gc_lock);
4923 enter_spin_lock (&more_space_lock);
4924 dprintf (SPINLOCK_LOG, ("[%d]Seg: A Emsl", heap_number));
4925 add_saved_spinlock_info (me_acquire, mt_get_large_seg);
4927 #ifdef BACKGROUND_GC
4928 wait_for_background_planning (awr_get_loh_seg);
4929 #endif //BACKGROUND_GC
4935 BOOL gc_heap::unprotect_segment (heap_segment* seg)
4937 uint8_t* start = align_lower_page (heap_segment_mem (seg));
4938 ptrdiff_t region_size = heap_segment_allocated (seg) - start;
4940 if (region_size != 0 )
4942 dprintf (3, ("unprotecting segment %Ix:", (size_t)seg));
4944 BOOL status = GCToOSInterface::VirtualUnprotect (start, region_size);
4952 #ifdef MULTIPLE_HEAPS
4955 #pragma warning(disable:4035)
4956 static ptrdiff_t get_cycle_count()
4960 #pragma warning(default:4035)
4961 #elif defined(__GNUC__)
4962 static ptrdiff_t get_cycle_count()
4966 __asm__ __volatile__
4967 ("rdtsc":"=a" (cycles), "=d" (cyclesHi));
4971 #error Unknown compiler
4973 #elif defined(_TARGET_AMD64_)
4975 extern "C" uint64_t __rdtsc();
4976 #pragma intrinsic(__rdtsc)
4977 static ptrdiff_t get_cycle_count()
4979 return (ptrdiff_t)__rdtsc();
4981 #elif defined(__clang__)
4982 static ptrdiff_t get_cycle_count()
4986 __asm__ __volatile__
4987 ("rdtsc":"=a" (cycles), "=d" (cyclesHi));
4988 return (cyclesHi << 32) | cycles;
4991 extern "C" ptrdiff_t get_cycle_count(void);
4993 #elif defined(_TARGET_ARM_)
4994 static ptrdiff_t get_cycle_count()
4996 // @ARMTODO: cycle counter is not exposed to user mode by CoreARM. For now (until we can show this
4997 // makes a difference on the ARM configurations on which we'll run) just return 0. This will result in
4998 // all buffer access times being reported as equal in access_time().
5001 #elif defined(_TARGET_ARM64_)
5002 static ptrdiff_t get_cycle_count()
5004 // @ARM64TODO: cycle counter is not exposed to user mode by CoreARM. For now (until we can show this
5005 // makes a difference on the ARM configurations on which we'll run) just return 0. This will result in
5006 // all buffer access times being reported as equal in access_time().
5010 #error NYI platform: get_cycle_count
5011 #endif //_TARGET_X86_
5016 static uint8_t* sniff_buffer;
5017 static unsigned n_sniff_buffers;
5018 static unsigned cur_sniff_index;
5020 static uint16_t proc_no_to_heap_no[MAX_SUPPORTED_CPUS];
5021 static uint16_t heap_no_to_proc_no[MAX_SUPPORTED_CPUS];
5022 static uint16_t heap_no_to_numa_node[MAX_SUPPORTED_CPUS];
5023 static uint16_t heap_no_to_cpu_group[MAX_SUPPORTED_CPUS];
5024 static uint16_t heap_no_to_group_proc[MAX_SUPPORTED_CPUS];
5025 static uint16_t numa_node_to_heap_map[MAX_SUPPORTED_CPUS+4];
5027 static int access_time(uint8_t *sniff_buffer, int heap_number, unsigned sniff_index, unsigned n_sniff_buffers)
5029 ptrdiff_t start_cycles = get_cycle_count();
5030 uint8_t sniff = sniff_buffer[(1 + heap_number*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE];
5031 assert (sniff == 0);
5032 ptrdiff_t elapsed_cycles = get_cycle_count() - start_cycles;
5033 // add sniff here just to defeat the optimizer
5034 elapsed_cycles += sniff;
5035 return (int) elapsed_cycles;
5039 static BOOL init(int n_heaps)
5041 assert (sniff_buffer == NULL && n_sniff_buffers == 0);
5042 if (!GCToOSInterface::CanGetCurrentProcessorNumber())
5044 n_sniff_buffers = n_heaps*2+1;
5045 size_t n_cache_lines = 1 + n_heaps * n_sniff_buffers + 1;
5046 size_t sniff_buf_size = n_cache_lines * HS_CACHE_LINE_SIZE;
5047 if (sniff_buf_size / HS_CACHE_LINE_SIZE != n_cache_lines) // check for overlow
5052 sniff_buffer = new (nothrow) uint8_t[sniff_buf_size];
5053 if (sniff_buffer == 0)
5055 memset(sniff_buffer, 0, sniff_buf_size*sizeof(uint8_t));
5058 //can not enable gc numa aware, force all heaps to be in
5059 //one numa node by filling the array with all 0s
5060 if (!GCToOSInterface::CanEnableGCNumaAware())
5061 memset(heap_no_to_numa_node, 0, sizeof (heap_no_to_numa_node));
5066 static void init_cpu_mapping(gc_heap * /*heap*/, int heap_number)
5068 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5070 uint32_t proc_no = GCToOSInterface::GetCurrentProcessorNumber() % gc_heap::n_heaps;
5071 // We can safely cast heap_number to a uint16_t 'cause GetCurrentProcessCpuCount
5072 // only returns up to MAX_SUPPORTED_CPUS procs right now. We only ever create at most
5073 // MAX_SUPPORTED_CPUS GC threads.
5074 proc_no_to_heap_no[proc_no] = (uint16_t)heap_number;
5078 static void mark_heap(int heap_number)
5080 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5083 for (unsigned sniff_index = 0; sniff_index < n_sniff_buffers; sniff_index++)
5084 sniff_buffer[(1 + heap_number*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE] &= 1;
5087 static int select_heap(alloc_context* acontext, int /*hint*/)
5089 UNREFERENCED_PARAMETER(acontext); // only referenced by dprintf
5091 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5092 return proc_no_to_heap_no[GCToOSInterface::GetCurrentProcessorNumber() % gc_heap::n_heaps];
5094 unsigned sniff_index = Interlocked::Increment(&cur_sniff_index);
5095 sniff_index %= n_sniff_buffers;
5098 int best_access_time = 1000*1000*1000;
5099 int second_best_access_time = best_access_time;
5101 uint8_t *l_sniff_buffer = sniff_buffer;
5102 unsigned l_n_sniff_buffers = n_sniff_buffers;
5103 for (int heap_number = 0; heap_number < gc_heap::n_heaps; heap_number++)
5105 int this_access_time = access_time(l_sniff_buffer, heap_number, sniff_index, l_n_sniff_buffers);
5106 if (this_access_time < best_access_time)
5108 second_best_access_time = best_access_time;
5109 best_access_time = this_access_time;
5110 best_heap = heap_number;
5112 else if (this_access_time < second_best_access_time)
5114 second_best_access_time = this_access_time;
5118 if (best_access_time*2 < second_best_access_time)
5120 sniff_buffer[(1 + best_heap*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE] &= 1;
5122 dprintf (3, ("select_heap yields crisp %d for context %p\n", best_heap, (void *)acontext));
5126 dprintf (3, ("select_heap yields vague %d for context %p\n", best_heap, (void *)acontext ));
5132 static bool can_find_heap_fast()
5134 return GCToOSInterface::CanGetCurrentProcessorNumber();
5137 static uint16_t find_proc_no_from_heap_no(int heap_number)
5139 return heap_no_to_proc_no[heap_number];
5142 static void set_proc_no_for_heap(int heap_number, uint16_t proc_no)
5144 heap_no_to_proc_no[heap_number] = proc_no;
5147 static uint16_t find_numa_node_from_heap_no(int heap_number)
5149 return heap_no_to_numa_node[heap_number];
5152 static void set_numa_node_for_heap(int heap_number, uint16_t numa_node)
5154 heap_no_to_numa_node[heap_number] = numa_node;
5157 static uint16_t find_cpu_group_from_heap_no(int heap_number)
5159 return heap_no_to_cpu_group[heap_number];
5162 static void set_cpu_group_for_heap(int heap_number, uint16_t group_number)
5164 heap_no_to_cpu_group[heap_number] = group_number;
5167 static uint16_t find_group_proc_from_heap_no(int heap_number)
5169 return heap_no_to_group_proc[heap_number];
5172 static void set_group_proc_for_heap(int heap_number, uint16_t group_proc)
5174 heap_no_to_group_proc[heap_number] = group_proc;
5177 static void init_numa_node_to_heap_map(int nheaps)
5178 { // called right after GCHeap::Init() for each heap is finished
5179 // when numa is not enabled, heap_no_to_numa_node[] are all filled
5180 // with 0s during initialization, and will be treated as one node
5181 numa_node_to_heap_map[0] = 0;
5184 for (int i=1; i < nheaps; i++)
5186 if (heap_no_to_numa_node[i] != heap_no_to_numa_node[i-1])
5187 numa_node_to_heap_map[node_index++] = (uint16_t)i;
5189 numa_node_to_heap_map[node_index] = (uint16_t)nheaps; //mark the end with nheaps
5192 static void get_heap_range_for_heap(int hn, int* start, int* end)
5193 { // 1-tier/no numa case: heap_no_to_numa_node[] all zeros,
5194 // and treated as in one node. thus: start=0, end=n_heaps
5195 uint16_t numa_node = heap_no_to_numa_node[hn];
5196 *start = (int)numa_node_to_heap_map[numa_node];
5197 *end = (int)(numa_node_to_heap_map[numa_node+1]);
5200 uint8_t* heap_select::sniff_buffer;
5201 unsigned heap_select::n_sniff_buffers;
5202 unsigned heap_select::cur_sniff_index;
5203 uint16_t heap_select::proc_no_to_heap_no[MAX_SUPPORTED_CPUS];
5204 uint16_t heap_select::heap_no_to_proc_no[MAX_SUPPORTED_CPUS];
5205 uint16_t heap_select::heap_no_to_numa_node[MAX_SUPPORTED_CPUS];
5206 uint16_t heap_select::heap_no_to_cpu_group[MAX_SUPPORTED_CPUS];
5207 uint16_t heap_select::heap_no_to_group_proc[MAX_SUPPORTED_CPUS];
5208 uint16_t heap_select::numa_node_to_heap_map[MAX_SUPPORTED_CPUS+4];
5210 BOOL gc_heap::create_thread_support (unsigned number_of_heaps)
5213 if (!gc_start_event.CreateOSManualEventNoThrow (FALSE))
5217 if (!ee_suspend_event.CreateOSAutoEventNoThrow (FALSE))
5221 if (!gc_t_join.init (number_of_heaps, join_flavor_server_gc))
5232 destroy_thread_support();
5238 void gc_heap::destroy_thread_support ()
5240 if (ee_suspend_event.IsValid())
5242 ee_suspend_event.CloseEvent();
5244 if (gc_start_event.IsValid())
5246 gc_start_event.CloseEvent();
5250 void set_thread_group_affinity_for_heap(int heap_number, GCThreadAffinity* affinity)
5252 affinity->Group = GCThreadAffinity::None;
5253 affinity->Processor = GCThreadAffinity::None;
5256 GCToOSInterface::GetGroupForProcessor((uint16_t)heap_number, &gn, &gpn);
5259 for (uintptr_t mask = 1; mask !=0; mask <<=1)
5261 if (bit_number == gpn)
5263 dprintf(3, ("using processor group %d, mask %Ix for heap %d\n", gn, mask, heap_number));
5264 affinity->Processor = gpn;
5265 affinity->Group = gn;
5266 heap_select::set_cpu_group_for_heap(heap_number, gn);
5267 heap_select::set_group_proc_for_heap(heap_number, gpn);
5268 if (GCToOSInterface::CanEnableGCNumaAware())
5270 PROCESSOR_NUMBER proc_no;
5272 proc_no.Number = (uint8_t)gpn;
5273 proc_no.Reserved = 0;
5275 uint16_t node_no = 0;
5276 if (GCToOSInterface::GetNumaProcessorNode(&proc_no, &node_no))
5277 heap_select::set_numa_node_for_heap(heap_number, node_no);
5280 { // no numa setting, each cpu group is treated as a node
5281 heap_select::set_numa_node_for_heap(heap_number, gn);
5289 void set_thread_affinity_mask_for_heap(int heap_number, GCThreadAffinity* affinity)
5291 affinity->Group = GCThreadAffinity::None;
5292 affinity->Processor = GCThreadAffinity::None;
5294 uintptr_t pmask, smask;
5295 if (GCToOSInterface::GetCurrentProcessAffinityMask(&pmask, &smask))
5299 uint8_t proc_number = 0;
5300 for (uintptr_t mask = 1; mask != 0; mask <<= 1)
5302 if ((mask & pmask) != 0)
5304 if (bit_number == heap_number)
5306 dprintf (3, ("Using processor %d for heap %d", proc_number, heap_number));
5307 affinity->Processor = proc_number;
5308 heap_select::set_proc_no_for_heap(heap_number, proc_number);
5309 if (GCToOSInterface::CanEnableGCNumaAware())
5311 uint16_t node_no = 0;
5312 PROCESSOR_NUMBER proc_no;
5314 proc_no.Number = (uint8_t)proc_number;
5315 proc_no.Reserved = 0;
5316 if (GCToOSInterface::GetNumaProcessorNode(&proc_no, &node_no))
5318 heap_select::set_numa_node_for_heap(heap_number, node_no);
5330 bool gc_heap::create_gc_thread ()
5332 dprintf (3, ("Creating gc thread\n"));
5333 return GCToEEInterface::CreateThread(gc_thread_stub, this, false, ".NET Server GC");
5337 #pragma warning(disable:4715) //IA64 xcompiler recognizes that without the 'break;' the while(1) will never end and therefore not return a value for that code path
5339 void gc_heap::gc_thread_function ()
5341 assert (gc_done_event.IsValid());
5342 assert (gc_start_event.IsValid());
5343 dprintf (3, ("gc thread started"));
5345 heap_select::init_cpu_mapping(this, heap_number);
5349 assert (!gc_t_join.joined());
5351 if (heap_number == 0)
5353 gc_heap::ee_suspend_event.Wait(INFINITE, FALSE);
5355 BEGIN_TIMING(suspend_ee_during_log);
5356 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC);
5357 END_TIMING(suspend_ee_during_log);
5359 proceed_with_gc_p = TRUE;
5361 if (!should_proceed_with_gc())
5363 update_collection_counts_for_no_gc();
5364 proceed_with_gc_p = FALSE;
5368 settings.init_mechanisms();
5369 gc_start_event.Set();
5371 dprintf (3, ("%d gc thread waiting...", heap_number));
5375 gc_start_event.Wait(INFINITE, FALSE);
5376 dprintf (3, ("%d gc thread waiting... Done", heap_number));
5379 assert ((heap_number == 0) || proceed_with_gc_p);
5381 if (proceed_with_gc_p)
5382 garbage_collect (GCHeap::GcCondemnedGeneration);
5384 if (heap_number == 0)
5386 if (proceed_with_gc_p && (!settings.concurrent))
5391 #ifdef BACKGROUND_GC
5392 recover_bgc_settings();
5393 #endif //BACKGROUND_GC
5395 #ifdef MULTIPLE_HEAPS
5396 for (int i = 0; i < gc_heap::n_heaps; i++)
5398 gc_heap* hp = gc_heap::g_heaps[i];
5399 hp->add_saved_spinlock_info (me_release, mt_block_gc);
5400 dprintf (SPINLOCK_LOG, ("[%d]GC Lmsl", i));
5401 leave_spin_lock(&hp->more_space_lock);
5403 #endif //MULTIPLE_HEAPS
5405 gc_heap::gc_started = FALSE;
5407 BEGIN_TIMING(restart_ee_during_log);
5408 GCToEEInterface::RestartEE(TRUE);
5409 END_TIMING(restart_ee_during_log);
5410 process_sync_log_stats();
5412 dprintf (SPINLOCK_LOG, ("GC Lgc"));
5413 leave_spin_lock (&gc_heap::gc_lock);
5415 gc_heap::internal_gc_done = true;
5417 if (proceed_with_gc_p)
5421 // If we didn't actually do a GC, it means we didn't wait up the other threads,
5422 // we still need to set the gc_done_event for those threads.
5423 for (int i = 0; i < gc_heap::n_heaps; i++)
5425 gc_heap* hp = gc_heap::g_heaps[i];
5432 int spin_count = 32 * (gc_heap::n_heaps - 1);
5434 // wait until RestartEE has progressed to a stage where we can restart user threads
5435 while (!gc_heap::internal_gc_done && !GCHeap::SafeToRestartManagedThreads())
5437 spin_and_switch (spin_count, (gc_heap::internal_gc_done || GCHeap::SafeToRestartManagedThreads()));
5444 #pragma warning(default:4715) //IA64 xcompiler recognizes that without the 'break;' the while(1) will never end and therefore not return a value for that code path
5447 #endif //MULTIPLE_HEAPS
5449 bool virtual_alloc_commit_for_heap(void* addr, size_t size, int h_number)
5451 #if defined(MULTIPLE_HEAPS) && !defined(FEATURE_REDHAWK)
5452 // Currently there is no way for us to specific the numa node to allocate on via hosting interfaces to
5453 // a host. This will need to be added later.
5454 #if !defined(FEATURE_CORECLR)
5455 if (!CLRMemoryHosted())
5458 if (GCToOSInterface::CanEnableGCNumaAware())
5460 uint32_t numa_node = heap_select::find_numa_node_from_heap_no(h_number);
5461 if (GCToOSInterface::VirtualCommit(addr, size, numa_node))
5466 UNREFERENCED_PARAMETER(h_number);
5469 //numa aware not enabled, or call failed --> fallback to VirtualCommit()
5470 return GCToOSInterface::VirtualCommit(addr, size);
5473 #ifndef SEG_MAPPING_TABLE
5475 heap_segment* gc_heap::segment_of (uint8_t* add, ptrdiff_t& delta, BOOL verify_p)
5477 uint8_t* sadd = add;
5478 heap_segment* hs = 0;
5479 heap_segment* hs1 = 0;
5480 if (!((add >= g_gc_lowest_address) && (add < g_gc_highest_address)))
5485 //repeat in case there is a concurrent insertion in the table.
5490 seg_table->lookup (sadd);
5491 hs1 = (heap_segment*)sadd;
5492 } while (hs1 && !in_range_for_segment (add, hs1) && (hs != hs1));
5497 (verify_p && (add > heap_segment_reserved ((heap_segment*)(sadd + delta)))))
5501 #endif //SEG_MAPPING_TABLE
5509 // If we want to save space we can have a pool of plug_and_gap's instead of
5510 // always having 2 allocated for each pinned plug.
5511 gap_reloc_pair saved_pre_plug;
5512 // If we decide to not compact, we need to restore the original values.
5513 gap_reloc_pair saved_pre_plug_reloc;
5515 gap_reloc_pair saved_post_plug;
5517 // Supposedly Pinned objects cannot have references but we are seeing some from pinvoke
5518 // frames. Also if it's an artificially pinned plug created by us, it can certainly
5520 // We know these cases will be rare so we can optimize this to be only allocated on decommand.
5521 gap_reloc_pair saved_post_plug_reloc;
5523 // We need to calculate this after we are done with plan phase and before compact
5524 // phase because compact phase will change the bricks so relocate_address will no
5526 uint8_t* saved_pre_plug_info_reloc_start;
5528 // We need to save this because we will have no way to calculate it, unlike the
5529 // pre plug info start which is right before this plug.
5530 uint8_t* saved_post_plug_info_start;
5533 uint8_t* allocation_context_start_region;
5534 #endif //SHORT_PLUGS
5536 // How the bits in these bytes are organized:
5538 // bit to indicate whether it's a short obj | 3 bits for refs in this short obj | 2 unused bits | bit to indicate if it's collectible | last bit
5539 // last bit indicates if there's pre or post info associated with this plug. If it's not set all other bits will be 0.
5544 // We are seeing this is getting corrupted for a PP with a NP after.
5545 // Save it when we first set it and make sure it doesn't change.
5546 gap_reloc_pair saved_post_plug_debug;
5549 size_t get_max_short_bits()
5551 return (sizeof (gap_reloc_pair) / sizeof (uint8_t*));
5555 size_t get_pre_short_start_bit ()
5557 return (sizeof (saved_pre_p) * 8 - 1 - (sizeof (gap_reloc_pair) / sizeof (uint8_t*)));
5562 return (saved_pre_p & (1 << (sizeof (saved_pre_p) * 8 - 1)));
5565 void set_pre_short()
5567 saved_pre_p |= (1 << (sizeof (saved_pre_p) * 8 - 1));
5570 void set_pre_short_bit (size_t bit)
5572 saved_pre_p |= 1 << (get_pre_short_start_bit() + bit);
5575 BOOL pre_short_bit_p (size_t bit)
5577 return (saved_pre_p & (1 << (get_pre_short_start_bit() + bit)));
5580 #ifdef COLLECTIBLE_CLASS
5581 void set_pre_short_collectible()
5586 BOOL pre_short_collectible_p()
5588 return (saved_pre_p & 2);
5590 #endif //COLLECTIBLE_CLASS
5593 size_t get_post_short_start_bit ()
5595 return (sizeof (saved_post_p) * 8 - 1 - (sizeof (gap_reloc_pair) / sizeof (uint8_t*)));
5600 return (saved_post_p & (1 << (sizeof (saved_post_p) * 8 - 1)));
5603 void set_post_short()
5605 saved_post_p |= (1 << (sizeof (saved_post_p) * 8 - 1));
5608 void set_post_short_bit (size_t bit)
5610 saved_post_p |= 1 << (get_post_short_start_bit() + bit);
5613 BOOL post_short_bit_p (size_t bit)
5615 return (saved_post_p & (1 << (get_post_short_start_bit() + bit)));
5618 #ifdef COLLECTIBLE_CLASS
5619 void set_post_short_collectible()
5624 BOOL post_short_collectible_p()
5626 return (saved_post_p & 2);
5628 #endif //COLLECTIBLE_CLASS
5630 uint8_t* get_plug_address() { return first; }
5632 BOOL has_pre_plug_info() { return saved_pre_p; }
5633 BOOL has_post_plug_info() { return saved_post_p; }
5635 gap_reloc_pair* get_pre_plug_reloc_info() { return &saved_pre_plug_reloc; }
5636 gap_reloc_pair* get_post_plug_reloc_info() { return &saved_post_plug_reloc; }
5637 void set_pre_plug_info_reloc_start (uint8_t* reloc) { saved_pre_plug_info_reloc_start = reloc; }
5638 uint8_t* get_post_plug_info_start() { return saved_post_plug_info_start; }
5640 // We need to temporarily recover the shortened plugs for compact phase so we can
5641 // copy over the whole plug and their related info (mark bits/cards). But we will
5642 // need to set the artificial gap back so compact phase can keep reading the plug info.
5643 // We also need to recover the saved info because we'll need to recover it later.
5645 // So we would call swap_p*_plug_and_saved once to recover the object info; then call
5646 // it again to recover the artificial gap.
5647 void swap_pre_plug_and_saved()
5649 gap_reloc_pair temp;
5650 memcpy (&temp, (first - sizeof (plug_and_gap)), sizeof (temp));
5651 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug_reloc, sizeof (saved_pre_plug_reloc));
5652 saved_pre_plug_reloc = temp;
5655 void swap_post_plug_and_saved()
5657 gap_reloc_pair temp;
5658 memcpy (&temp, saved_post_plug_info_start, sizeof (temp));
5659 memcpy (saved_post_plug_info_start, &saved_post_plug_reloc, sizeof (saved_post_plug_reloc));
5660 saved_post_plug_reloc = temp;
5663 void swap_pre_plug_and_saved_for_profiler()
5665 gap_reloc_pair temp;
5666 memcpy (&temp, (first - sizeof (plug_and_gap)), sizeof (temp));
5667 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug, sizeof (saved_pre_plug));
5668 saved_pre_plug = temp;
5671 void swap_post_plug_and_saved_for_profiler()
5673 gap_reloc_pair temp;
5674 memcpy (&temp, saved_post_plug_info_start, sizeof (temp));
5675 memcpy (saved_post_plug_info_start, &saved_post_plug, sizeof (saved_post_plug));
5676 saved_post_plug = temp;
5679 // We should think about whether it's really necessary to have to copy back the pre plug
5680 // info since it was already copied during compacting plugs. But if a plug doesn't move
5681 // by >= 3 ptr size (the size of gap_reloc_pair), it means we'd have to recover pre plug info.
5682 void recover_plug_info()
5686 if (gc_heap::settings.compaction)
5688 dprintf (3, ("%Ix: REC Pre: %Ix-%Ix",
5690 &saved_pre_plug_reloc,
5691 saved_pre_plug_info_reloc_start));
5692 memcpy (saved_pre_plug_info_reloc_start, &saved_pre_plug_reloc, sizeof (saved_pre_plug_reloc));
5696 dprintf (3, ("%Ix: REC Pre: %Ix-%Ix",
5699 (first - sizeof (plug_and_gap))));
5700 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug, sizeof (saved_pre_plug));
5706 if (gc_heap::settings.compaction)
5708 dprintf (3, ("%Ix: REC Post: %Ix-%Ix",
5710 &saved_post_plug_reloc,
5711 saved_post_plug_info_start));
5712 memcpy (saved_post_plug_info_start, &saved_post_plug_reloc, sizeof (saved_post_plug_reloc));
5716 dprintf (3, ("%Ix: REC Post: %Ix-%Ix",
5719 saved_post_plug_info_start));
5720 memcpy (saved_post_plug_info_start, &saved_post_plug, sizeof (saved_post_plug));
5727 void gc_mechanisms::init_mechanisms()
5729 condemned_generation = 0;
5730 promotion = FALSE;//TRUE;
5732 #ifdef FEATURE_LOH_COMPACTION
5733 loh_compaction = gc_heap::should_compact_loh();
5735 loh_compaction = FALSE;
5736 #endif //FEATURE_LOH_COMPACTION
5737 heap_expansion = FALSE;
5740 elevation_reduced = FALSE;
5741 found_finalizers = FALSE;
5742 #ifdef BACKGROUND_GC
5743 background_p = recursive_gc_sync::background_running_p() != FALSE;
5744 allocations_allowed = TRUE;
5745 #endif //BACKGROUND_GC
5747 entry_memory_load = 0;
5748 exit_memory_load = 0;
5751 stress_induced = FALSE;
5752 #endif // STRESS_HEAP
5755 void gc_mechanisms::first_init()
5758 gen0_reduction_count = 0;
5759 should_lock_elevation = FALSE;
5760 elevation_locked_count = 0;
5761 reason = reason_empty;
5762 #ifdef BACKGROUND_GC
5763 pause_mode = gc_heap::gc_can_use_concurrent ? pause_interactive : pause_batch;
5765 int debug_pause_mode = static_cast<int>(GCConfig::GetLatencyMode());
5766 if (debug_pause_mode >= 0)
5768 assert (debug_pause_mode <= pause_sustained_low_latency);
5769 pause_mode = (gc_pause_mode)debug_pause_mode;
5772 #else //BACKGROUND_GC
5773 pause_mode = pause_batch;
5774 #endif //BACKGROUND_GC
5779 void gc_mechanisms::record (gc_history_global* history)
5781 #ifdef MULTIPLE_HEAPS
5782 history->num_heaps = gc_heap::n_heaps;
5784 history->num_heaps = 1;
5785 #endif //MULTIPLE_HEAPS
5787 history->condemned_generation = condemned_generation;
5788 history->gen0_reduction_count = gen0_reduction_count;
5789 history->reason = reason;
5790 history->pause_mode = (int)pause_mode;
5791 history->mem_pressure = entry_memory_load;
5792 history->global_mechanims_p = 0;
5794 // start setting the boolean values.
5796 history->set_mechanism_p (global_concurrent);
5799 history->set_mechanism_p (global_compaction);
5802 history->set_mechanism_p (global_promotion);
5805 history->set_mechanism_p (global_demotion);
5808 history->set_mechanism_p (global_card_bundles);
5810 if (elevation_reduced)
5811 history->set_mechanism_p (global_elevation);
5814 /**********************************
5815 called at the beginning of GC to fix the allocated size to
5816 what is really allocated, or to turn the free area into an unused object
5817 It needs to be called after all of the other allocation contexts have been
5818 fixed since it relies on alloc_allocated.
5819 ********************************/
5821 //for_gc_p indicates that the work is being done for GC,
5822 //as opposed to concurrent heap verification
5823 void gc_heap::fix_youngest_allocation_area (BOOL for_gc_p)
5825 UNREFERENCED_PARAMETER(for_gc_p);
5827 // The gen 0 alloc context is never used for allocation in the allocator path. It's
5828 // still used in the allocation path during GCs.
5829 assert (generation_allocation_pointer (youngest_generation) == nullptr);
5830 assert (generation_allocation_limit (youngest_generation) == nullptr);
5831 heap_segment_allocated (ephemeral_heap_segment) = alloc_allocated;
5834 void gc_heap::fix_large_allocation_area (BOOL for_gc_p)
5836 UNREFERENCED_PARAMETER(for_gc_p);
5839 alloc_context* acontext =
5841 generation_alloc_context (large_object_generation);
5842 assert (acontext->alloc_ptr == 0);
5843 assert (acontext->alloc_limit == 0);
5845 dprintf (3, ("Large object alloc context: ptr: %Ix, limit %Ix",
5846 (size_t)acontext->alloc_ptr, (size_t)acontext->alloc_limit));
5847 fix_allocation_context (acontext, FALSE, get_alignment_constant (FALSE));
5850 acontext->alloc_ptr = 0;
5851 acontext->alloc_limit = acontext->alloc_ptr;
5856 //for_gc_p indicates that the work is being done for GC,
5857 //as opposed to concurrent heap verification
5858 void gc_heap::fix_allocation_context (alloc_context* acontext, BOOL for_gc_p,
5861 dprintf (3, ("Fixing allocation context %Ix: ptr: %Ix, limit: %Ix",
5863 (size_t)acontext->alloc_ptr, (size_t)acontext->alloc_limit));
5865 if (((size_t)(alloc_allocated - acontext->alloc_limit) > Align (min_obj_size, align_const)) ||
5868 uint8_t* point = acontext->alloc_ptr;
5871 size_t size = (acontext->alloc_limit - acontext->alloc_ptr);
5872 // the allocation area was from the free list
5873 // it was shortened by Align (min_obj_size) to make room for
5874 // at least the shortest unused object
5875 size += Align (min_obj_size, align_const);
5876 assert ((size >= Align (min_obj_size)));
5878 dprintf(3,("Making unused area [%Ix, %Ix[", (size_t)point,
5879 (size_t)point + size ));
5880 make_unused_array (point, size);
5884 generation_free_obj_space (generation_of (0)) += size;
5885 alloc_contexts_used ++;
5891 alloc_allocated = acontext->alloc_ptr;
5892 assert (heap_segment_allocated (ephemeral_heap_segment) <=
5893 heap_segment_committed (ephemeral_heap_segment));
5894 alloc_contexts_used ++;
5899 // We need to update the alloc_bytes to reflect the portion that we have not used
5900 acontext->alloc_bytes -= (acontext->alloc_limit - acontext->alloc_ptr);
5901 acontext->alloc_ptr = 0;
5902 acontext->alloc_limit = acontext->alloc_ptr;
5906 //used by the heap verification for concurrent gc.
5907 //it nulls out the words set by fix_allocation_context for heap_verification
5908 void repair_allocation (gc_alloc_context* acontext, void*)
5910 uint8_t* point = acontext->alloc_ptr;
5914 dprintf (3, ("Clearing [%Ix, %Ix[", (size_t)acontext->alloc_ptr,
5915 (size_t)acontext->alloc_limit+Align(min_obj_size)));
5916 memclr (acontext->alloc_ptr - plug_skew,
5917 (acontext->alloc_limit - acontext->alloc_ptr)+Align (min_obj_size));
5921 void void_allocation (gc_alloc_context* acontext, void*)
5923 uint8_t* point = acontext->alloc_ptr;
5927 dprintf (3, ("Void [%Ix, %Ix[", (size_t)acontext->alloc_ptr,
5928 (size_t)acontext->alloc_limit+Align(min_obj_size)));
5929 acontext->alloc_ptr = 0;
5930 acontext->alloc_limit = acontext->alloc_ptr;
5934 void gc_heap::repair_allocation_contexts (BOOL repair_p)
5936 GCToEEInterface::GcEnumAllocContexts (repair_p ? repair_allocation : void_allocation, NULL);
5939 struct fix_alloc_context_args
5945 void fix_alloc_context(gc_alloc_context* acontext, void* param)
5947 fix_alloc_context_args* args = (fix_alloc_context_args*)param;
5948 g_theGCHeap->FixAllocContext(acontext, false, (void*)(size_t)(args->for_gc_p), args->heap);
5951 void gc_heap::fix_allocation_contexts(BOOL for_gc_p)
5953 fix_alloc_context_args args;
5954 args.for_gc_p = for_gc_p;
5957 GCToEEInterface::GcEnumAllocContexts(fix_alloc_context, &args);
5958 fix_youngest_allocation_area(for_gc_p);
5959 fix_large_allocation_area(for_gc_p);
5962 void gc_heap::fix_older_allocation_area (generation* older_gen)
5964 heap_segment* older_gen_seg = generation_allocation_segment (older_gen);
5965 if (generation_allocation_limit (older_gen) !=
5966 heap_segment_plan_allocated (older_gen_seg))
5968 uint8_t* point = generation_allocation_pointer (older_gen);
5970 size_t size = (generation_allocation_limit (older_gen) -
5971 generation_allocation_pointer (older_gen));
5974 assert ((size >= Align (min_obj_size)));
5975 dprintf(3,("Making unused area [%Ix, %Ix[", (size_t)point, (size_t)point+size));
5976 make_unused_array (point, size);
5981 assert (older_gen_seg != ephemeral_heap_segment);
5982 heap_segment_plan_allocated (older_gen_seg) =
5983 generation_allocation_pointer (older_gen);
5984 generation_allocation_limit (older_gen) =
5985 generation_allocation_pointer (older_gen);
5989 void gc_heap::set_allocation_heap_segment (generation* gen)
5991 uint8_t* p = generation_allocation_start (gen);
5993 heap_segment* seg = generation_allocation_segment (gen);
5994 if (in_range_for_segment (p, seg))
5997 // try ephemeral heap segment in case of heap expansion
5998 seg = ephemeral_heap_segment;
5999 if (!in_range_for_segment (p, seg))
6001 seg = heap_segment_rw (generation_start_segment (gen));
6003 PREFIX_ASSUME(seg != NULL);
6005 while (!in_range_for_segment (p, seg))
6007 seg = heap_segment_next_rw (seg);
6008 PREFIX_ASSUME(seg != NULL);
6012 generation_allocation_segment (gen) = seg;
6015 void gc_heap::reset_allocation_pointers (generation* gen, uint8_t* start)
6018 assert (Align ((size_t)start) == (size_t)start);
6019 generation_allocation_start (gen) = start;
6020 generation_allocation_pointer (gen) = 0;//start + Align (min_obj_size);
6021 generation_allocation_limit (gen) = 0;//generation_allocation_pointer (gen);
6022 set_allocation_heap_segment (gen);
6025 #ifdef BACKGROUND_GC
6026 //TODO BACKGROUND_GC this is for test only
6028 gc_heap::disallow_new_allocation (int gen_number)
6030 UNREFERENCED_PARAMETER(gen_number);
6031 settings.allocations_allowed = FALSE;
6034 gc_heap::allow_new_allocation (int gen_number)
6036 UNREFERENCED_PARAMETER(gen_number);
6037 settings.allocations_allowed = TRUE;
6040 #endif //BACKGROUND_GC
6042 bool gc_heap::new_allocation_allowed (int gen_number)
6044 #ifdef BACKGROUND_GC
6045 //TODO BACKGROUND_GC this is for test only
6046 if (!settings.allocations_allowed)
6048 dprintf (2, ("new allocation not allowed"));
6051 #endif //BACKGROUND_GC
6053 if (dd_new_allocation (dynamic_data_of (gen_number)) < 0)
6055 if (gen_number != 0)
6057 // For LOH we will give it more budget before we try a GC.
6058 if (settings.concurrent)
6060 dynamic_data* dd2 = dynamic_data_of (max_generation + 1 );
6062 if (dd_new_allocation (dd2) <= (ptrdiff_t)(-2 * dd_desired_allocation (dd2)))
6070 #ifndef MULTIPLE_HEAPS
6071 else if ((settings.pause_mode != pause_no_gc) && (gen_number == 0))
6073 dprintf (3, ("evaluating allocation rate"));
6074 dynamic_data* dd0 = dynamic_data_of (0);
6075 if ((allocation_running_amount - dd_new_allocation (dd0)) >
6078 uint32_t ctime = GCToOSInterface::GetLowPrecisionTimeStamp();
6079 if ((ctime - allocation_running_time) > 1000)
6081 dprintf (2, (">1s since last gen0 gc"));
6086 allocation_running_amount = dd_new_allocation (dd0);
6090 #endif //MULTIPLE_HEAPS
6095 ptrdiff_t gc_heap::get_desired_allocation (int gen_number)
6097 return dd_desired_allocation (dynamic_data_of (gen_number));
6101 ptrdiff_t gc_heap::get_new_allocation (int gen_number)
6103 return dd_new_allocation (dynamic_data_of (gen_number));
6106 //return the amount allocated so far in gen_number
6108 ptrdiff_t gc_heap::get_allocation (int gen_number)
6110 dynamic_data* dd = dynamic_data_of (gen_number);
6112 return dd_desired_allocation (dd) - dd_new_allocation (dd);
6116 BOOL grow_mark_stack (mark*& m, size_t& len, size_t init_len)
6118 size_t new_size = max (init_len, 2*len);
6119 mark* tmp = new (nothrow) mark [new_size];
6122 memcpy (tmp, m, len * sizeof (mark));
6130 dprintf (1, ("Failed to allocate %Id bytes for mark stack", (len * sizeof (mark))));
6136 uint8_t* pinned_plug (mark* m)
6142 size_t& pinned_len (mark* m)
6148 void set_new_pin_info (mark* m, uint8_t* pin_free_space_start)
6150 m->len = pinned_plug (m) - pin_free_space_start;
6152 m->allocation_context_start_region = pin_free_space_start;
6153 #endif //SHORT_PLUGS
6158 uint8_t*& pin_allocation_context_start_region (mark* m)
6160 return m->allocation_context_start_region;
6163 uint8_t* get_plug_start_in_saved (uint8_t* old_loc, mark* pinned_plug_entry)
6165 uint8_t* saved_pre_plug_info = (uint8_t*)(pinned_plug_entry->get_pre_plug_reloc_info());
6166 uint8_t* plug_start_in_saved = saved_pre_plug_info + (old_loc - (pinned_plug (pinned_plug_entry) - sizeof (plug_and_gap)));
6167 //dprintf (1, ("detected a very short plug: %Ix before PP %Ix, pad %Ix",
6168 // old_loc, pinned_plug (pinned_plug_entry), plug_start_in_saved));
6169 dprintf (1, ("EP: %Ix(%Ix), %Ix", old_loc, pinned_plug (pinned_plug_entry), plug_start_in_saved));
6170 return plug_start_in_saved;
6174 void set_padding_in_expand (uint8_t* old_loc,
6175 BOOL set_padding_on_saved_p,
6176 mark* pinned_plug_entry)
6178 if (set_padding_on_saved_p)
6180 set_plug_padded (get_plug_start_in_saved (old_loc, pinned_plug_entry));
6184 set_plug_padded (old_loc);
6189 void clear_padding_in_expand (uint8_t* old_loc,
6190 BOOL set_padding_on_saved_p,
6191 mark* pinned_plug_entry)
6193 if (set_padding_on_saved_p)
6195 clear_plug_padded (get_plug_start_in_saved (old_loc, pinned_plug_entry));
6199 clear_plug_padded (old_loc);
6202 #endif //SHORT_PLUGS
6204 void gc_heap::reset_pinned_queue()
6210 void gc_heap::reset_pinned_queue_bos()
6215 // last_pinned_plug is only for asserting purpose.
6216 void gc_heap::merge_with_last_pinned_plug (uint8_t* last_pinned_plug, size_t plug_size)
6218 if (last_pinned_plug)
6220 mark& last_m = mark_stack_array[mark_stack_tos - 1];
6221 assert (last_pinned_plug == last_m.first);
6222 if (last_m.saved_post_p)
6224 last_m.saved_post_p = FALSE;
6225 dprintf (3, ("setting last plug %Ix post to false", last_m.first));
6226 // We need to recover what the gap has overwritten.
6227 memcpy ((last_m.first + last_m.len - sizeof (plug_and_gap)), &(last_m.saved_post_plug), sizeof (gap_reloc_pair));
6229 last_m.len += plug_size;
6230 dprintf (3, ("recovered the last part of plug %Ix, setting its plug size to %Ix", last_m.first, last_m.len));
6234 void gc_heap::set_allocator_next_pin (uint8_t* alloc_pointer, uint8_t*& alloc_limit)
6236 dprintf (3, ("sanp: ptr: %Ix, limit: %Ix", alloc_pointer, alloc_limit));
6237 dprintf (3, ("oldest %Id: %Ix", mark_stack_bos, pinned_plug (oldest_pin())));
6238 if (!(pinned_plug_que_empty_p()))
6240 mark* oldest_entry = oldest_pin();
6241 uint8_t* plug = pinned_plug (oldest_entry);
6242 if ((plug >= alloc_pointer) && (plug < alloc_limit))
6244 alloc_limit = pinned_plug (oldest_entry);
6245 dprintf (3, ("now setting alloc context: %Ix->%Ix(%Id)",
6246 alloc_pointer, alloc_limit, (alloc_limit - alloc_pointer)));
6251 void gc_heap::set_allocator_next_pin (generation* gen)
6253 dprintf (3, ("SANP: gen%d, ptr; %Ix, limit: %Ix", gen->gen_num, generation_allocation_pointer (gen), generation_allocation_limit (gen)));
6254 if (!(pinned_plug_que_empty_p()))
6256 mark* oldest_entry = oldest_pin();
6257 uint8_t* plug = pinned_plug (oldest_entry);
6258 if ((plug >= generation_allocation_pointer (gen)) &&
6259 (plug < generation_allocation_limit (gen)))
6261 generation_allocation_limit (gen) = pinned_plug (oldest_entry);
6262 dprintf (3, ("SANP: get next pin free space in gen%d for alloc: %Ix->%Ix(%Id)",
6264 generation_allocation_pointer (gen), generation_allocation_limit (gen),
6265 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
6268 assert (!((plug < generation_allocation_pointer (gen)) &&
6269 (plug >= heap_segment_mem (generation_allocation_segment (gen)))));
6273 // After we set the info, we increase tos.
6274 void gc_heap::set_pinned_info (uint8_t* last_pinned_plug, size_t plug_len, uint8_t* alloc_pointer, uint8_t*& alloc_limit)
6276 UNREFERENCED_PARAMETER(last_pinned_plug);
6278 mark& m = mark_stack_array[mark_stack_tos];
6279 assert (m.first == last_pinned_plug);
6283 set_allocator_next_pin (alloc_pointer, alloc_limit);
6286 // After we set the info, we increase tos.
6287 void gc_heap::set_pinned_info (uint8_t* last_pinned_plug, size_t plug_len, generation* gen)
6289 UNREFERENCED_PARAMETER(last_pinned_plug);
6291 mark& m = mark_stack_array[mark_stack_tos];
6292 assert (m.first == last_pinned_plug);
6297 // Why are we checking here? gen is never 0.
6300 set_allocator_next_pin (gen);
6304 size_t gc_heap::deque_pinned_plug ()
6306 dprintf (3, ("dequed: %Id", mark_stack_bos));
6307 size_t m = mark_stack_bos;
6313 mark* gc_heap::pinned_plug_of (size_t bos)
6315 return &mark_stack_array [ bos ];
6319 mark* gc_heap::oldest_pin ()
6321 return pinned_plug_of (mark_stack_bos);
6325 BOOL gc_heap::pinned_plug_que_empty_p ()
6327 return (mark_stack_bos == mark_stack_tos);
6331 mark* gc_heap::before_oldest_pin()
6333 if (mark_stack_bos >= 1)
6334 return pinned_plug_of (mark_stack_bos-1);
6340 BOOL gc_heap::ephemeral_pointer_p (uint8_t* o)
6342 return ((o >= ephemeral_low) && (o < ephemeral_high));
6347 int& gc_heap::mark_stack_busy()
6349 return g_mark_stack_busy [(heap_number+2)*HS_CACHE_LINE_SIZE/sizeof(int)];
6353 void gc_heap::make_mark_stack (mark* arr)
6355 reset_pinned_queue();
6356 mark_stack_array = arr;
6357 mark_stack_array_length = MARK_STACK_INITIAL_LENGTH;
6359 mark_stack_busy() = 0;
6363 #ifdef BACKGROUND_GC
6365 size_t& gc_heap::bpromoted_bytes(int thread)
6367 #ifdef MULTIPLE_HEAPS
6368 return g_bpromoted [thread*16];
6369 #else //MULTIPLE_HEAPS
6370 UNREFERENCED_PARAMETER(thread);
6372 #endif //MULTIPLE_HEAPS
6375 void gc_heap::make_background_mark_stack (uint8_t** arr)
6377 background_mark_stack_array = arr;
6378 background_mark_stack_array_length = MARK_STACK_INITIAL_LENGTH;
6379 background_mark_stack_tos = arr;
6382 void gc_heap::make_c_mark_list (uint8_t** arr)
6385 c_mark_list_index = 0;
6386 c_mark_list_length = 1 + (OS_PAGE_SIZE / MIN_OBJECT_SIZE);
6388 #endif //BACKGROUND_GC
6393 // The card bundle keeps track of groups of card words.
6394 static const size_t card_bundle_word_width = 32;
6396 // How do we express the fact that 32 bits (card_word_width) is one uint32_t?
6397 static const size_t card_bundle_size = (size_t)(GC_PAGE_SIZE / (sizeof(uint32_t)*card_bundle_word_width));
6400 size_t card_bundle_word (size_t cardb)
6402 return cardb / card_bundle_word_width;
6406 uint32_t card_bundle_bit (size_t cardb)
6408 return (uint32_t)(cardb % card_bundle_word_width);
6411 size_t align_cardw_on_bundle (size_t cardw)
6413 return ((size_t)(cardw + card_bundle_size - 1) & ~(card_bundle_size - 1 ));
6416 // Get the card bundle representing a card word
6417 size_t cardw_card_bundle (size_t cardw)
6419 return cardw / card_bundle_size;
6422 // Get the first card word in a card bundle
6423 size_t card_bundle_cardw (size_t cardb)
6425 return cardb * card_bundle_size;
6428 // Clear the specified card bundle
6429 void gc_heap::card_bundle_clear (size_t cardb)
6431 card_bundle_table [card_bundle_word (cardb)] &= ~(1 << card_bundle_bit (cardb));
6432 dprintf (1,("Cleared card bundle %Ix [%Ix, %Ix[", cardb, (size_t)card_bundle_cardw (cardb),
6433 (size_t)card_bundle_cardw (cardb+1)));
6436 void gc_heap::card_bundle_set (size_t cardb)
6438 if (!card_bundle_set_p (cardb))
6440 card_bundle_table [card_bundle_word (cardb)] |= (1 << card_bundle_bit (cardb));
6444 // Set the card bundle bits between start_cardb and end_cardb
6445 void gc_heap::card_bundles_set (size_t start_cardb, size_t end_cardb)
6447 if (start_cardb == end_cardb)
6449 card_bundle_set(start_cardb);
6453 size_t start_word = card_bundle_word (start_cardb);
6454 size_t end_word = card_bundle_word (end_cardb);
6456 if (start_word < end_word)
6458 // Set the partial words
6459 card_bundle_table [start_word] |= highbits (~0u, card_bundle_bit (start_cardb));
6461 if (card_bundle_bit (end_cardb))
6462 card_bundle_table [end_word] |= lowbits (~0u, card_bundle_bit (end_cardb));
6464 // Set the full words
6465 for (size_t i = start_word + 1; i < end_word; i++)
6466 card_bundle_table [i] = ~0u;
6470 card_bundle_table [start_word] |= (highbits (~0u, card_bundle_bit (start_cardb)) &
6471 lowbits (~0u, card_bundle_bit (end_cardb)));
6475 // Indicates whether the specified bundle is set.
6476 BOOL gc_heap::card_bundle_set_p (size_t cardb)
6478 return (card_bundle_table[card_bundle_word(cardb)] & (1 << card_bundle_bit (cardb)));
6481 // Returns the size (in bytes) of a card bundle representing the region from 'from' to 'end'
6482 size_t size_card_bundle_of (uint8_t* from, uint8_t* end)
6484 // Number of heap bytes represented by a card bundle word
6485 size_t cbw_span = card_size * card_word_width * card_bundle_size * card_bundle_word_width;
6487 // Align the start of the region down
6488 from = (uint8_t*)((size_t)from & ~(cbw_span - 1));
6490 // Align the end of the region up
6491 end = (uint8_t*)((size_t)(end + (cbw_span - 1)) & ~(cbw_span - 1));
6493 // Make sure they're really aligned
6494 assert (((size_t)from & (cbw_span - 1)) == 0);
6495 assert (((size_t)end & (cbw_span - 1)) == 0);
6497 return ((end - from) / cbw_span) * sizeof (uint32_t);
6500 // Takes a pointer to a card bundle table and an address, and returns a pointer that represents
6501 // where a theoretical card bundle table that represents every address (starting from 0) would
6502 // start if the bundle word representing the address were to be located at the pointer passed in.
6503 // The returned 'translated' pointer makes it convenient/fast to calculate where the card bundle
6504 // for a given address is using a simple shift operation on the address.
6505 uint32_t* translate_card_bundle_table (uint32_t* cb, uint8_t* lowest_address)
6507 // The number of bytes of heap memory represented by a card bundle word
6508 const size_t heap_bytes_for_bundle_word = card_size * card_word_width * card_bundle_size * card_bundle_word_width;
6510 // Each card bundle word is 32 bits
6511 return (uint32_t*)((uint8_t*)cb - (((size_t)lowest_address / heap_bytes_for_bundle_word) * sizeof (uint32_t)));
6514 void gc_heap::enable_card_bundles ()
6516 if (can_use_write_watch_for_card_table() && (!card_bundles_enabled()))
6518 dprintf (1, ("Enabling card bundles"));
6520 // We initially set all of the card bundles
6521 card_bundles_set (cardw_card_bundle (card_word (card_of (lowest_address))),
6522 cardw_card_bundle (align_cardw_on_bundle (card_word (card_of (highest_address)))));
6523 settings.card_bundles = TRUE;
6527 BOOL gc_heap::card_bundles_enabled ()
6529 return settings.card_bundles;
6532 #endif // CARD_BUNDLE
6534 #if defined (_TARGET_AMD64_)
6535 #define brick_size ((size_t)4096)
6537 #define brick_size ((size_t)2048)
6538 #endif //_TARGET_AMD64_
6541 size_t gc_heap::brick_of (uint8_t* add)
6543 return (size_t)(add - lowest_address) / brick_size;
6547 uint8_t* gc_heap::brick_address (size_t brick)
6549 return lowest_address + (brick_size * brick);
6553 void gc_heap::clear_brick_table (uint8_t* from, uint8_t* end)
6555 for (size_t i = brick_of (from);i < brick_of (end); i++)
6559 //codes for the brick entries:
6560 //entry == 0 -> not assigned
6561 //entry >0 offset is entry-1
6562 //entry <0 jump back entry bricks
6566 void gc_heap::set_brick (size_t index, ptrdiff_t val)
6572 assert (val < 32767);
6574 brick_table [index] = (short)val+1;
6576 brick_table [index] = (short)val;
6580 int gc_heap::get_brick_entry (size_t index)
6582 #ifdef MULTIPLE_HEAPS
6583 return VolatileLoadWithoutBarrier(&brick_table [index]);
6585 return brick_table[index];
6591 uint8_t* align_on_brick (uint8_t* add)
6593 return (uint8_t*)((size_t)(add + brick_size - 1) & ~(brick_size - 1));
6597 uint8_t* align_lower_brick (uint8_t* add)
6599 return (uint8_t*)(((size_t)add) & ~(brick_size - 1));
6602 size_t size_brick_of (uint8_t* from, uint8_t* end)
6604 assert (((size_t)from & (brick_size-1)) == 0);
6605 assert (((size_t)end & (brick_size-1)) == 0);
6607 return ((end - from) / brick_size) * sizeof (short);
6611 uint8_t* gc_heap::card_address (size_t card)
6613 return (uint8_t*) (card_size * card);
6617 size_t gc_heap::card_of ( uint8_t* object)
6619 return (size_t)(object) / card_size;
6623 size_t gc_heap::card_to_brick (size_t card)
6625 return brick_of (card_address (card));
6629 uint8_t* align_on_card (uint8_t* add)
6631 return (uint8_t*)((size_t)(add + card_size - 1) & ~(card_size - 1 ));
6634 uint8_t* align_on_card_word (uint8_t* add)
6636 return (uint8_t*) ((size_t)(add + (card_size*card_word_width)-1) & ~(card_size*card_word_width - 1));
6640 uint8_t* align_lower_card (uint8_t* add)
6642 return (uint8_t*)((size_t)add & ~(card_size-1));
6646 void gc_heap::clear_card (size_t card)
6648 card_table [card_word (card)] =
6649 (card_table [card_word (card)] & ~(1 << card_bit (card)));
6650 dprintf (3,("Cleared card %Ix [%Ix, %Ix[", card, (size_t)card_address (card),
6651 (size_t)card_address (card+1)));
6655 void gc_heap::set_card (size_t card)
6657 size_t word = card_word (card);
6658 card_table[word] = (card_table [word] | (1 << card_bit (card)));
6660 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
6661 // Also set the card bundle that corresponds to the card
6662 size_t bundle_to_set = cardw_card_bundle(word);
6664 card_bundle_set(bundle_to_set);
6666 dprintf (3,("Set card %Ix [%Ix, %Ix[ and bundle %Ix", card, (size_t)card_address (card), (size_t)card_address (card+1), bundle_to_set));
6667 assert(card_bundle_set_p(bundle_to_set) != 0);
6672 BOOL gc_heap::card_set_p (size_t card)
6674 return ( card_table [ card_word (card) ] & (1 << card_bit (card)));
6677 // Returns the number of DWORDs in the card table that cover the
6678 // range of addresses [from, end[.
6679 size_t count_card_of (uint8_t* from, uint8_t* end)
6681 return card_word (gcard_of (end - 1)) - card_word (gcard_of (from)) + 1;
6684 // Returns the number of bytes to allocate for a card table
6685 // that covers the range of addresses [from, end[.
6686 size_t size_card_of (uint8_t* from, uint8_t* end)
6688 return count_card_of (from, end) * sizeof(uint32_t);
6691 // We don't store seg_mapping_table in card_table_info because there's only always one view.
6692 class card_table_info
6696 uint8_t* lowest_address;
6697 uint8_t* highest_address;
6701 uint32_t* card_bundle_table;
6702 #endif //CARD_BUNDLE
6704 // mark_array is always at the end of the data structure because we
6705 // want to be able to make one commit call for everything before it.
6707 uint32_t* mark_array;
6711 uint32_t* next_card_table;
6714 //These are accessors on untranslated cardtable
6716 unsigned& card_table_refcount (uint32_t* c_table)
6718 return *(unsigned*)((char*)c_table - sizeof (card_table_info));
6722 uint8_t*& card_table_lowest_address (uint32_t* c_table)
6724 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->lowest_address;
6727 uint32_t* translate_card_table (uint32_t* ct)
6729 return (uint32_t*)((uint8_t*)ct - card_word (gcard_of (card_table_lowest_address (ct))) * sizeof(uint32_t));
6733 uint8_t*& card_table_highest_address (uint32_t* c_table)
6735 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->highest_address;
6739 short*& card_table_brick_table (uint32_t* c_table)
6741 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->brick_table;
6745 // Get the card bundle table for the specified card table.
6747 uint32_t*& card_table_card_bundle_table (uint32_t* c_table)
6749 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->card_bundle_table;
6751 #endif //CARD_BUNDLE
6754 /* Support for mark_array */
6757 uint32_t*& card_table_mark_array (uint32_t* c_table)
6759 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->mark_array;
6763 #define mark_bit_pitch ((size_t)16)
6765 #define mark_bit_pitch ((size_t)8)
6767 #define mark_word_width ((size_t)32)
6768 #define mark_word_size (mark_word_width * mark_bit_pitch)
6771 uint8_t* align_on_mark_bit (uint8_t* add)
6773 return (uint8_t*)((size_t)(add + (mark_bit_pitch - 1)) & ~(mark_bit_pitch - 1));
6777 uint8_t* align_lower_mark_bit (uint8_t* add)
6779 return (uint8_t*)((size_t)(add) & ~(mark_bit_pitch - 1));
6783 BOOL is_aligned_on_mark_word (uint8_t* add)
6785 return ((size_t)add == ((size_t)(add) & ~(mark_word_size - 1)));
6789 uint8_t* align_on_mark_word (uint8_t* add)
6791 return (uint8_t*)((size_t)(add + mark_word_size - 1) & ~(mark_word_size - 1));
6795 uint8_t* align_lower_mark_word (uint8_t* add)
6797 return (uint8_t*)((size_t)(add) & ~(mark_word_size - 1));
6801 size_t mark_bit_of (uint8_t* add)
6803 return ((size_t)add / mark_bit_pitch);
6807 unsigned int mark_bit_bit (size_t mark_bit)
6809 return (unsigned int)(mark_bit % mark_word_width);
6813 size_t mark_bit_word (size_t mark_bit)
6815 return (mark_bit / mark_word_width);
6819 size_t mark_word_of (uint8_t* add)
6821 return ((size_t)add) / mark_word_size;
6824 uint8_t* mark_word_address (size_t wd)
6826 return (uint8_t*)(wd*mark_word_size);
6829 uint8_t* mark_bit_address (size_t mark_bit)
6831 return (uint8_t*)(mark_bit*mark_bit_pitch);
6835 size_t mark_bit_bit_of (uint8_t* add)
6837 return (((size_t)add / mark_bit_pitch) % mark_word_width);
6841 unsigned int gc_heap::mark_array_marked(uint8_t* add)
6843 return mark_array [mark_word_of (add)] & (1 << mark_bit_bit_of (add));
6847 BOOL gc_heap::is_mark_bit_set (uint8_t* add)
6849 return (mark_array [mark_word_of (add)] & (1 << mark_bit_bit_of (add)));
6853 void gc_heap::mark_array_set_marked (uint8_t* add)
6855 size_t index = mark_word_of (add);
6856 uint32_t val = (1 << mark_bit_bit_of (add));
6857 #ifdef MULTIPLE_HEAPS
6858 Interlocked::Or (&(mark_array [index]), val);
6860 mark_array [index] |= val;
6865 void gc_heap::mark_array_clear_marked (uint8_t* add)
6867 mark_array [mark_word_of (add)] &= ~(1 << mark_bit_bit_of (add));
6870 size_t size_mark_array_of (uint8_t* from, uint8_t* end)
6872 assert (((size_t)from & ((mark_word_size)-1)) == 0);
6873 assert (((size_t)end & ((mark_word_size)-1)) == 0);
6874 return sizeof (uint32_t)*(((end - from) / mark_word_size));
6877 //In order to eliminate the lowest_address in the mark array
6878 //computations (mark_word_of, etc) mark_array is offset
6879 // according to the lowest_address.
6880 uint32_t* translate_mark_array (uint32_t* ma)
6882 return (uint32_t*)((uint8_t*)ma - size_mark_array_of (0, g_gc_lowest_address));
6885 // from and end must be page aligned addresses.
6886 void gc_heap::clear_mark_array (uint8_t* from, uint8_t* end, BOOL check_only/*=TRUE*/
6887 #ifdef FEATURE_BASICFREEZE
6888 , BOOL read_only/*=FALSE*/
6889 #endif // FEATURE_BASICFREEZE
6892 if(!gc_can_use_concurrent)
6895 #ifdef FEATURE_BASICFREEZE
6897 #endif // FEATURE_BASICFREEZE
6899 assert (from == align_on_mark_word (from));
6901 assert (end == align_on_mark_word (end));
6903 #ifdef BACKGROUND_GC
6904 uint8_t* current_lowest_address = background_saved_lowest_address;
6905 uint8_t* current_highest_address = background_saved_highest_address;
6907 uint8_t* current_lowest_address = lowest_address;
6908 uint8_t* current_highest_address = highest_address;
6909 #endif //BACKGROUND_GC
6911 //there is a possibility of the addresses to be
6912 //outside of the covered range because of a newly allocated
6913 //large object segment
6914 if ((end <= current_highest_address) && (from >= current_lowest_address))
6916 size_t beg_word = mark_word_of (align_on_mark_word (from));
6917 MAYBE_UNUSED_VAR(beg_word);
6918 //align end word to make sure to cover the address
6919 size_t end_word = mark_word_of (align_on_mark_word (end));
6920 MAYBE_UNUSED_VAR(end_word);
6921 dprintf (3, ("Calling clearing mark array [%Ix, %Ix[ for addresses [%Ix, %Ix[(%s)",
6922 (size_t)mark_word_address (beg_word),
6923 (size_t)mark_word_address (end_word),
6924 (size_t)from, (size_t)end,
6925 (check_only ? "check_only" : "clear")));
6929 while (op < mark_word_address (beg_word))
6931 mark_array_clear_marked (op);
6932 op += mark_bit_pitch;
6935 memset (&mark_array[beg_word], 0, (end_word - beg_word)*sizeof (uint32_t));
6940 //Beware, it is assumed that the mark array word straddling
6941 //start has been cleared before
6942 //verify that the array is empty.
6943 size_t markw = mark_word_of (align_on_mark_word (from));
6944 size_t markw_end = mark_word_of (align_on_mark_word (end));
6945 while (markw < markw_end)
6947 assert (!(mark_array [markw]));
6950 uint8_t* p = mark_word_address (markw_end);
6953 assert (!(mark_array_marked (p)));
6962 //These work on untranslated card tables
6964 uint32_t*& card_table_next (uint32_t* c_table)
6966 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->next_card_table;
6970 size_t& card_table_size (uint32_t* c_table)
6972 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->size;
6975 void own_card_table (uint32_t* c_table)
6977 card_table_refcount (c_table) += 1;
6980 void destroy_card_table (uint32_t* c_table);
6982 void delete_next_card_table (uint32_t* c_table)
6984 uint32_t* n_table = card_table_next (c_table);
6987 if (card_table_next (n_table))
6989 delete_next_card_table (n_table);
6991 if (card_table_refcount (n_table) == 0)
6993 destroy_card_table (n_table);
6994 card_table_next (c_table) = 0;
6999 void release_card_table (uint32_t* c_table)
7001 assert (card_table_refcount (c_table) >0);
7002 card_table_refcount (c_table) -= 1;
7003 if (card_table_refcount (c_table) == 0)
7005 delete_next_card_table (c_table);
7006 if (card_table_next (c_table) == 0)
7008 destroy_card_table (c_table);
7009 // sever the link from the parent
7010 if (&g_gc_card_table[card_word (gcard_of(g_gc_lowest_address))] == c_table)
7012 g_gc_card_table = 0;
7014 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7015 g_gc_card_bundle_table = 0;
7017 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7018 SoftwareWriteWatch::StaticClose();
7019 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7023 uint32_t* p_table = &g_gc_card_table[card_word (gcard_of(g_gc_lowest_address))];
7026 while (p_table && (card_table_next (p_table) != c_table))
7027 p_table = card_table_next (p_table);
7028 card_table_next (p_table) = 0;
7035 void destroy_card_table (uint32_t* c_table)
7037 // delete (uint32_t*)&card_table_refcount(c_table);
7039 GCToOSInterface::VirtualRelease (&card_table_refcount(c_table), card_table_size(c_table));
7040 dprintf (2, ("Table Virtual Free : %Ix", (size_t)&card_table_refcount(c_table)));
7043 uint32_t* gc_heap::make_card_table (uint8_t* start, uint8_t* end)
7045 assert (g_gc_lowest_address == start);
7046 assert (g_gc_highest_address == end);
7048 uint32_t virtual_reserve_flags = VirtualReserveFlags::None;
7050 size_t bs = size_brick_of (start, end);
7051 size_t cs = size_card_of (start, end);
7053 size_t ms = (gc_can_use_concurrent ?
7054 size_mark_array_of (start, end) :
7063 if (can_use_write_watch_for_card_table())
7065 cb = size_card_bundle_of (g_gc_lowest_address, g_gc_highest_address);
7066 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7067 // If we're not manually managing the card bundles, we will need to use OS write
7068 // watch APIs over this region to track changes.
7069 virtual_reserve_flags |= VirtualReserveFlags::WriteWatch;
7072 #endif //CARD_BUNDLE
7075 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7076 size_t sw_ww_table_offset = 0;
7077 if (gc_can_use_concurrent)
7079 size_t sw_ww_size_before_table = sizeof(card_table_info) + cs + bs + cb;
7080 sw_ww_table_offset = SoftwareWriteWatch::GetTableStartByteOffset(sw_ww_size_before_table);
7081 wws = sw_ww_table_offset - sw_ww_size_before_table + SoftwareWriteWatch::GetTableByteSize(start, end);
7083 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7085 #ifdef GROWABLE_SEG_MAPPING_TABLE
7086 size_t st = size_seg_mapping_table_of (g_gc_lowest_address, g_gc_highest_address);
7087 size_t st_table_offset = sizeof(card_table_info) + cs + bs + cb + wws;
7088 size_t st_table_offset_aligned = align_for_seg_mapping_table (st_table_offset);
7090 st += (st_table_offset_aligned - st_table_offset);
7091 #else //GROWABLE_SEG_MAPPING_TABLE
7093 #endif //GROWABLE_SEG_MAPPING_TABLE
7095 // it is impossible for alloc_size to overflow due bounds on each of
7097 size_t alloc_size = sizeof (uint8_t)*(sizeof(card_table_info) + cs + bs + cb + wws + st + ms);
7098 uint8_t* mem = (uint8_t*)GCToOSInterface::VirtualReserve (alloc_size, 0, virtual_reserve_flags);
7103 dprintf (2, ("Init - Card table alloc for %Id bytes: [%Ix, %Ix[",
7104 alloc_size, (size_t)mem, (size_t)(mem+alloc_size)));
7106 // mark array will be committed separately (per segment).
7107 size_t commit_size = alloc_size - ms;
7109 if (!GCToOSInterface::VirtualCommit (mem, commit_size))
7111 dprintf (2, ("Card table commit failed"));
7112 GCToOSInterface::VirtualRelease (mem, alloc_size);
7116 // initialize the ref count
7117 uint32_t* ct = (uint32_t*)(mem+sizeof (card_table_info));
7118 card_table_refcount (ct) = 0;
7119 card_table_lowest_address (ct) = start;
7120 card_table_highest_address (ct) = end;
7121 card_table_brick_table (ct) = (short*)((uint8_t*)ct + cs);
7122 card_table_size (ct) = alloc_size;
7123 card_table_next (ct) = 0;
7126 card_table_card_bundle_table (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs);
7128 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7129 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), g_gc_lowest_address);
7132 #endif //CARD_BUNDLE
7134 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7135 if (gc_can_use_concurrent)
7137 SoftwareWriteWatch::InitializeUntranslatedTable(mem + sw_ww_table_offset, start);
7139 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7141 #ifdef GROWABLE_SEG_MAPPING_TABLE
7142 seg_mapping_table = (seg_mapping*)(mem + st_table_offset_aligned);
7143 seg_mapping_table = (seg_mapping*)((uint8_t*)seg_mapping_table -
7144 size_seg_mapping_table_of (0, (align_lower_segment (g_gc_lowest_address))));
7145 #endif //GROWABLE_SEG_MAPPING_TABLE
7148 if (gc_can_use_concurrent)
7149 card_table_mark_array (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs + cb + wws + st);
7151 card_table_mark_array (ct) = NULL;
7154 return translate_card_table(ct);
7157 void gc_heap::set_fgm_result (failure_get_memory f, size_t s, BOOL loh_p)
7159 #ifdef MULTIPLE_HEAPS
7160 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
7162 gc_heap* hp = gc_heap::g_heaps [hn];
7163 hp->fgm_result.set_fgm (f, s, loh_p);
7165 #else //MULTIPLE_HEAPS
7166 fgm_result.set_fgm (f, s, loh_p);
7167 #endif //MULTIPLE_HEAPS
7170 //returns 0 for success, -1 otherwise
7171 // We are doing all the decommitting here because we want to make sure we have
7172 // enough memory to do so - if we do this during copy_brick_card_table and
7173 // and fail to decommit it would make the failure case very complicated to
7174 // handle. This way we can waste some decommit if we call this multiple
7175 // times before the next FGC but it's easier to handle the failure case.
7176 int gc_heap::grow_brick_card_tables (uint8_t* start,
7179 heap_segment* new_seg,
7183 uint8_t* la = g_gc_lowest_address;
7184 uint8_t* ha = g_gc_highest_address;
7185 uint8_t* saved_g_lowest_address = min (start, g_gc_lowest_address);
7186 uint8_t* saved_g_highest_address = max (end, g_gc_highest_address);
7187 seg_mapping* new_seg_mapping_table = nullptr;
7188 #ifdef BACKGROUND_GC
7189 // This value is only for logging purpose - it's not necessarily exactly what we
7190 // would commit for mark array but close enough for diagnostics purpose.
7191 size_t logging_ma_commit_size = size_mark_array_of (0, (uint8_t*)size);
7192 #endif //BACKGROUND_GC
7194 // See if the address is already covered
7195 if ((la != saved_g_lowest_address ) || (ha != saved_g_highest_address))
7198 //modify the higest address so the span covered
7199 //is twice the previous one.
7200 uint8_t* top = (uint8_t*)0 + Align (GCToOSInterface::GetVirtualMemoryLimit());
7201 // On non-Windows systems, we get only an approximate value that can possibly be
7202 // slightly lower than the saved_g_highest_address.
7203 // In such case, we set the top to the saved_g_highest_address so that the
7204 // card and brick tables always cover the whole new range.
7205 if (top < saved_g_highest_address)
7207 top = saved_g_highest_address;
7211 if (ps > (uint64_t)200*1024*1024*1024)
7212 ps += (uint64_t)100*1024*1024*1024;
7217 if (saved_g_lowest_address < g_gc_lowest_address)
7219 if (ps > (size_t)g_gc_lowest_address)
7220 saved_g_lowest_address = (uint8_t*)(size_t)OS_PAGE_SIZE;
7223 assert (((size_t)g_gc_lowest_address - ps) >= OS_PAGE_SIZE);
7224 saved_g_lowest_address = min (saved_g_lowest_address, (g_gc_lowest_address - ps));
7228 if (saved_g_highest_address > g_gc_highest_address)
7230 saved_g_highest_address = max ((saved_g_lowest_address + ps), saved_g_highest_address);
7231 if (saved_g_highest_address > top)
7232 saved_g_highest_address = top;
7235 dprintf (GC_TABLE_LOG, ("Growing card table [%Ix, %Ix[",
7236 (size_t)saved_g_lowest_address,
7237 (size_t)saved_g_highest_address));
7239 bool write_barrier_updated = false;
7240 uint32_t virtual_reserve_flags = VirtualReserveFlags::None;
7241 uint32_t* saved_g_card_table = g_gc_card_table;
7243 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7244 uint32_t* saved_g_card_bundle_table = g_gc_card_bundle_table;
7248 uint32_t* translated_ct = 0;
7251 size_t cs = size_card_of (saved_g_lowest_address, saved_g_highest_address);
7252 size_t bs = size_brick_of (saved_g_lowest_address, saved_g_highest_address);
7255 size_t ms = (gc_heap::gc_can_use_concurrent ?
7256 size_mark_array_of (saved_g_lowest_address, saved_g_highest_address) :
7265 if (can_use_write_watch_for_card_table())
7267 cb = size_card_bundle_of (saved_g_lowest_address, saved_g_highest_address);
7269 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7270 // If we're not manually managing the card bundles, we will need to use OS write
7271 // watch APIs over this region to track changes.
7272 virtual_reserve_flags |= VirtualReserveFlags::WriteWatch;
7275 #endif //CARD_BUNDLE
7278 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7279 size_t sw_ww_table_offset = 0;
7280 if (gc_can_use_concurrent)
7282 size_t sw_ww_size_before_table = sizeof(card_table_info) + cs + bs + cb;
7283 sw_ww_table_offset = SoftwareWriteWatch::GetTableStartByteOffset(sw_ww_size_before_table);
7285 sw_ww_table_offset -
7286 sw_ww_size_before_table +
7287 SoftwareWriteWatch::GetTableByteSize(saved_g_lowest_address, saved_g_highest_address);
7289 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7291 #ifdef GROWABLE_SEG_MAPPING_TABLE
7292 size_t st = size_seg_mapping_table_of (saved_g_lowest_address, saved_g_highest_address);
7293 size_t st_table_offset = sizeof(card_table_info) + cs + bs + cb + wws;
7294 size_t st_table_offset_aligned = align_for_seg_mapping_table (st_table_offset);
7295 st += (st_table_offset_aligned - st_table_offset);
7296 #else //GROWABLE_SEG_MAPPING_TABLE
7298 #endif //GROWABLE_SEG_MAPPING_TABLE
7300 // it is impossible for alloc_size to overflow due bounds on each of
7302 size_t alloc_size = sizeof (uint8_t)*(sizeof(card_table_info) + cs + bs + cb + wws + st + ms);
7303 dprintf (GC_TABLE_LOG, ("card table: %Id; brick table: %Id; card bundle: %Id; sw ww table: %Id; seg table: %Id; mark array: %Id",
7304 cs, bs, cb, wws, st, ms));
7306 uint8_t* mem = (uint8_t*)GCToOSInterface::VirtualReserve (alloc_size, 0, virtual_reserve_flags);
7310 set_fgm_result (fgm_grow_table, alloc_size, loh_p);
7314 dprintf (GC_TABLE_LOG, ("Table alloc for %Id bytes: [%Ix, %Ix[",
7315 alloc_size, (size_t)mem, (size_t)((uint8_t*)mem+alloc_size)));
7318 // mark array will be committed separately (per segment).
7319 size_t commit_size = alloc_size - ms;
7321 if (!GCToOSInterface::VirtualCommit (mem, commit_size))
7323 dprintf (GC_TABLE_LOG, ("Table commit failed"));
7324 set_fgm_result (fgm_commit_table, commit_size, loh_p);
7329 ct = (uint32_t*)(mem + sizeof (card_table_info));
7330 card_table_refcount (ct) = 0;
7331 card_table_lowest_address (ct) = saved_g_lowest_address;
7332 card_table_highest_address (ct) = saved_g_highest_address;
7333 card_table_next (ct) = &g_gc_card_table[card_word (gcard_of (la))];
7335 //clear the card table
7337 memclr ((uint8_t*)ct,
7338 (((saved_g_highest_address - saved_g_lowest_address)*sizeof (uint32_t) /
7339 (card_size * card_word_width))
7340 + sizeof (uint32_t)));
7343 bt = (short*)((uint8_t*)ct + cs);
7345 // No initialization needed, will be done in copy_brick_card
7347 card_table_brick_table (ct) = bt;
7350 card_table_card_bundle_table (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs);
7351 //set all bundle to look at all of the cards
7352 memset(card_table_card_bundle_table (ct), 0xFF, cb);
7353 #endif //CARD_BUNDLE
7355 #ifdef GROWABLE_SEG_MAPPING_TABLE
7357 new_seg_mapping_table = (seg_mapping*)(mem + st_table_offset_aligned);
7358 new_seg_mapping_table = (seg_mapping*)((uint8_t*)new_seg_mapping_table -
7359 size_seg_mapping_table_of (0, (align_lower_segment (saved_g_lowest_address))));
7360 memcpy(&new_seg_mapping_table[seg_mapping_word_of(g_gc_lowest_address)],
7361 &seg_mapping_table[seg_mapping_word_of(g_gc_lowest_address)],
7362 size_seg_mapping_table_of(g_gc_lowest_address, g_gc_highest_address));
7364 // new_seg_mapping_table gets assigned to seg_mapping_table at the bottom of this function,
7365 // not here. The reason for this is that, if we fail at mark array committing (OOM) and we've
7366 // already switched seg_mapping_table to point to the new mapping table, we'll decommit it and
7367 // run into trouble. By not assigning here, we're making sure that we will not change seg_mapping_table
7368 // if an OOM occurs.
7370 #endif //GROWABLE_SEG_MAPPING_TABLE
7373 if(gc_can_use_concurrent)
7374 card_table_mark_array (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs + cb + wws + st);
7376 card_table_mark_array (ct) = NULL;
7379 translated_ct = translate_card_table (ct);
7381 dprintf (GC_TABLE_LOG, ("card table: %Ix(translated: %Ix), seg map: %Ix, mark array: %Ix",
7382 (size_t)ct, (size_t)translated_ct, (size_t)new_seg_mapping_table, (size_t)card_table_mark_array (ct)));
7384 #ifdef BACKGROUND_GC
7385 if (hp->should_commit_mark_array())
7387 dprintf (GC_TABLE_LOG, ("new low: %Ix, new high: %Ix, latest mark array is %Ix(translate: %Ix)",
7388 saved_g_lowest_address, saved_g_highest_address,
7389 card_table_mark_array (ct),
7390 translate_mark_array (card_table_mark_array (ct))));
7391 uint32_t* new_mark_array = (uint32_t*)((uint8_t*)card_table_mark_array (ct) - size_mark_array_of (0, saved_g_lowest_address));
7392 if (!commit_new_mark_array_global (new_mark_array))
7394 dprintf (GC_TABLE_LOG, ("failed to commit portions in the mark array for existing segments"));
7395 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7399 if (!commit_mark_array_new_seg (hp, new_seg, translated_ct, saved_g_lowest_address))
7401 dprintf (GC_TABLE_LOG, ("failed to commit mark array for the new seg"));
7402 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7408 clear_commit_flag_global();
7410 #endif //BACKGROUND_GC
7412 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7413 if (gc_can_use_concurrent)
7415 // The current design of software write watch requires that the runtime is suspended during resize. Suspending
7416 // on resize is preferred because it is a far less frequent operation than GetWriteWatch() / ResetWriteWatch().
7417 // Suspending here allows copying dirty state from the old table into the new table, and not have to merge old
7418 // table info lazily as done for card tables.
7420 // Either this thread was the thread that did the suspension which means we are suspended; or this is called
7421 // from a GC thread which means we are in a blocking GC and also suspended.
7422 bool is_runtime_suspended = GCToEEInterface::IsGCThread();
7423 if (!is_runtime_suspended)
7425 // Note on points where the runtime is suspended anywhere in this function. Upon an attempt to suspend the
7426 // runtime, a different thread may suspend first, causing this thread to block at the point of the suspend call.
7427 // So, at any suspend point, externally visible state needs to be consistent, as code that depends on that state
7428 // may run while this thread is blocked. This includes updates to g_gc_card_table, g_gc_lowest_address, and
7429 // g_gc_highest_address.
7433 g_gc_card_table = translated_ct;
7435 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7436 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), saved_g_lowest_address);
7439 SoftwareWriteWatch::SetResizedUntranslatedTable(
7440 mem + sw_ww_table_offset,
7441 saved_g_lowest_address,
7442 saved_g_highest_address);
7444 // Since the runtime is already suspended, update the write barrier here as well.
7445 // This passes a bool telling whether we need to switch to the post
7446 // grow version of the write barrier. This test tells us if the new
7447 // segment was allocated at a lower address than the old, requiring
7448 // that we start doing an upper bounds check in the write barrier.
7449 g_gc_lowest_address = saved_g_lowest_address;
7450 g_gc_highest_address = saved_g_highest_address;
7451 stomp_write_barrier_resize(true, la != saved_g_lowest_address);
7452 write_barrier_updated = true;
7454 if (!is_runtime_suspended)
7460 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7462 g_gc_card_table = translated_ct;
7464 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7465 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), saved_g_lowest_address);
7469 seg_mapping_table = new_seg_mapping_table;
7471 GCToOSInterface::FlushProcessWriteBuffers();
7472 g_gc_lowest_address = saved_g_lowest_address;
7473 g_gc_highest_address = saved_g_highest_address;
7475 if (!write_barrier_updated)
7477 // This passes a bool telling whether we need to switch to the post
7478 // grow version of the write barrier. This test tells us if the new
7479 // segment was allocated at a lower address than the old, requiring
7480 // that we start doing an upper bounds check in the write barrier.
7481 // This will also suspend the runtime if the write barrier type needs
7482 // to be changed, so we are doing this after all global state has
7483 // been updated. See the comment above suspend_EE() above for more
7485 stomp_write_barrier_resize(GCToEEInterface::IsGCThread(), la != saved_g_lowest_address);
7492 //cleanup mess and return -1;
7496 assert(g_gc_card_table == saved_g_card_table);
7498 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7499 assert(g_gc_card_bundle_table == saved_g_card_bundle_table);
7502 //delete (uint32_t*)((uint8_t*)ct - sizeof(card_table_info));
7503 if (!GCToOSInterface::VirtualRelease (mem, alloc_size))
7505 dprintf (GC_TABLE_LOG, ("GCToOSInterface::VirtualRelease failed"));
7506 assert (!"release failed");
7514 #ifdef BACKGROUND_GC
7515 if (hp->should_commit_mark_array())
7517 dprintf (GC_TABLE_LOG, ("in range new seg %Ix, mark_array is %Ix", new_seg, hp->mark_array));
7518 if (!commit_mark_array_new_seg (hp, new_seg))
7520 dprintf (GC_TABLE_LOG, ("failed to commit mark array for the new seg in range"));
7521 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7525 #endif //BACKGROUND_GC
7531 //copy all of the arrays managed by the card table for a page aligned range
7532 void gc_heap::copy_brick_card_range (uint8_t* la, uint32_t* old_card_table,
7533 short* old_brick_table,
7535 uint8_t* start, uint8_t* end)
7537 ptrdiff_t brick_offset = brick_of (start) - brick_of (la);
7540 dprintf (2, ("copying tables for range [%Ix %Ix[", (size_t)start, (size_t)end));
7543 short* brick_start = &brick_table [brick_of (start)];
7544 if (old_brick_table)
7546 // segments are always on page boundaries
7547 memcpy (brick_start, &old_brick_table[brick_offset],
7548 size_brick_of (start, end));
7553 // This is a large heap, just clear the brick table
7556 uint32_t* old_ct = &old_card_table[card_word (card_of (la))];
7558 #ifdef BACKGROUND_GC
7559 UNREFERENCED_PARAMETER(seg);
7560 if (recursive_gc_sync::background_running_p())
7562 uint32_t* old_mark_array = card_table_mark_array (old_ct);
7564 // We don't need to go through all the card tables here because
7565 // we only need to copy from the GC version of the mark array - when we
7566 // mark (even in allocate_large_object) we always use that mark array.
7567 if ((card_table_highest_address (old_ct) >= start) &&
7568 (card_table_lowest_address (old_ct) <= end))
7570 if ((background_saved_highest_address >= start) &&
7571 (background_saved_lowest_address <= end))
7573 //copy the mark bits
7574 // segments are always on page boundaries
7575 uint8_t* m_start = max (background_saved_lowest_address, start);
7576 uint8_t* m_end = min (background_saved_highest_address, end);
7577 memcpy (&mark_array[mark_word_of (m_start)],
7578 &old_mark_array[mark_word_of (m_start) - mark_word_of (la)],
7579 size_mark_array_of (m_start, m_end));
7584 //only large segments can be out of range
7585 assert (old_brick_table == 0);
7588 #else //BACKGROUND_GC
7590 clear_mark_array (start, heap_segment_committed(seg));
7591 #endif //BACKGROUND_GC
7594 // n way merge with all of the card table ever used in between
7595 uint32_t* ct = card_table_next (&card_table[card_word (card_of(lowest_address))]);
7598 while (card_table_next (old_ct) != ct)
7600 //copy if old card table contained [start, end[
7601 if ((card_table_highest_address (ct) >= end) &&
7602 (card_table_lowest_address (ct) <= start))
7604 // or the card_tables
7606 size_t start_word = card_word (card_of (start));
7608 uint32_t* dest = &card_table[start_word];
7609 uint32_t* src = &((translate_card_table (ct))[start_word]);
7610 ptrdiff_t count = count_card_of (start, end);
7611 for (int x = 0; x < count; x++)
7615 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7618 card_bundle_set(cardw_card_bundle(start_word+x));
7626 ct = card_table_next (ct);
7630 //initialize all of the arrays managed by the card table for a page aligned range when an existing ro segment becomes in range
7631 void gc_heap::init_brick_card_range (heap_segment* seg)
7633 dprintf (2, ("initialising tables for range [%Ix %Ix[",
7634 (size_t)heap_segment_mem (seg),
7635 (size_t)heap_segment_allocated (seg)));
7637 // initialize the brick table
7638 for (size_t b = brick_of (heap_segment_mem (seg));
7639 b < brick_of (align_on_brick (heap_segment_allocated (seg)));
7646 if (recursive_gc_sync::background_running_p() && (seg->flags & heap_segment_flags_ma_committed))
7649 clear_mark_array (heap_segment_mem (seg), heap_segment_committed(seg));
7653 clear_card_for_addresses (heap_segment_mem (seg),
7654 heap_segment_allocated (seg));
7657 void gc_heap::copy_brick_card_table()
7659 uint8_t* la = lowest_address;
7660 uint8_t* ha = highest_address;
7661 MAYBE_UNUSED_VAR(ha);
7662 uint32_t* old_card_table = card_table;
7663 short* old_brick_table = brick_table;
7665 assert (la == card_table_lowest_address (&old_card_table[card_word (card_of (la))]));
7666 assert (ha == card_table_highest_address (&old_card_table[card_word (card_of (la))]));
7668 /* todo: Need a global lock for this */
7669 uint32_t* ct = &g_gc_card_table[card_word (gcard_of (g_gc_lowest_address))];
7670 own_card_table (ct);
7671 card_table = translate_card_table (ct);
7672 /* End of global lock */
7673 highest_address = card_table_highest_address (ct);
7674 lowest_address = card_table_lowest_address (ct);
7676 brick_table = card_table_brick_table (ct);
7679 if (gc_can_use_concurrent)
7681 mark_array = translate_mark_array (card_table_mark_array (ct));
7682 assert (mark_word_of (g_gc_highest_address) ==
7683 mark_word_of (align_on_mark_word (g_gc_highest_address)));
7690 #if defined(MARK_ARRAY) && defined(_DEBUG)
7691 #ifdef GROWABLE_SEG_MAPPING_TABLE
7692 size_t st = size_seg_mapping_table_of (g_gc_lowest_address, g_gc_highest_address);
7693 #else //GROWABLE_SEG_MAPPING_TABLE
7695 #endif //GROWABLE_SEG_MAPPING_TABLE
7696 #endif //MARK_ARRAY && _DEBUG
7697 card_bundle_table = translate_card_bundle_table (card_table_card_bundle_table (ct), g_gc_lowest_address);
7699 // Ensure that the word that represents g_gc_lowest_address in the translated table is located at the
7700 // start of the untranslated table.
7701 assert (&card_bundle_table [card_bundle_word (cardw_card_bundle (card_word (card_of (g_gc_lowest_address))))] ==
7702 card_table_card_bundle_table (ct));
7704 //set the card table if we are in a heap growth scenario
7705 if (card_bundles_enabled())
7707 card_bundles_set (cardw_card_bundle (card_word (card_of (lowest_address))),
7708 cardw_card_bundle (align_cardw_on_bundle (card_word (card_of (highest_address)))));
7710 //check if we need to turn on card_bundles.
7711 #ifdef MULTIPLE_HEAPS
7712 // use INT64 arithmetic here because of possible overflow on 32p
7713 uint64_t th = (uint64_t)MH_TH_CARD_BUNDLE*gc_heap::n_heaps;
7715 // use INT64 arithmetic here because of possible overflow on 32p
7716 uint64_t th = (uint64_t)SH_TH_CARD_BUNDLE;
7717 #endif //MULTIPLE_HEAPS
7718 if (reserved_memory >= th)
7720 enable_card_bundles();
7723 #endif //CARD_BUNDLE
7725 // for each of the segments and heaps, copy the brick table and
7726 // or the card table
7727 heap_segment* seg = generation_start_segment (generation_of (max_generation));
7730 if (heap_segment_read_only_p (seg) && !heap_segment_in_range_p (seg))
7732 //check if it became in range
7733 if ((heap_segment_reserved (seg) > lowest_address) &&
7734 (heap_segment_mem (seg) < highest_address))
7736 set_ro_segment_in_range (seg);
7742 uint8_t* end = align_on_page (heap_segment_allocated (seg));
7743 copy_brick_card_range (la, old_card_table,
7746 align_lower_page (heap_segment_mem (seg)),
7749 seg = heap_segment_next (seg);
7752 seg = generation_start_segment (large_object_generation);
7755 if (heap_segment_read_only_p (seg) && !heap_segment_in_range_p (seg))
7757 //check if it became in range
7758 if ((heap_segment_reserved (seg) > lowest_address) &&
7759 (heap_segment_mem (seg) < highest_address))
7761 set_ro_segment_in_range (seg);
7766 uint8_t* end = align_on_page (heap_segment_allocated (seg));
7767 copy_brick_card_range (la, old_card_table,
7770 align_lower_page (heap_segment_mem (seg)),
7773 seg = heap_segment_next (seg);
7776 release_card_table (&old_card_table[card_word (card_of(la))]);
7779 #ifdef FEATURE_BASICFREEZE
7780 BOOL gc_heap::insert_ro_segment (heap_segment* seg)
7782 enter_spin_lock (&gc_heap::gc_lock);
7784 if (!gc_heap::seg_table->ensure_space_for_insert ()
7785 || (should_commit_mark_array() && !commit_mark_array_new_seg(__this, seg)))
7787 leave_spin_lock(&gc_heap::gc_lock);
7791 //insert at the head of the segment list
7792 generation* gen2 = generation_of (max_generation);
7793 heap_segment* oldhead = generation_start_segment (gen2);
7794 heap_segment_next (seg) = oldhead;
7795 generation_start_segment (gen2) = seg;
7797 seg_table->insert (heap_segment_mem(seg), (size_t)seg);
7799 #ifdef SEG_MAPPING_TABLE
7800 seg_mapping_table_add_ro_segment (seg);
7801 #endif //SEG_MAPPING_TABLE
7804 if ((heap_segment_reserved (seg) > lowest_address) &&
7805 (heap_segment_mem (seg) < highest_address))
7807 set_ro_segment_in_range (seg);
7810 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(seg), (size_t)(heap_segment_reserved (seg) - heap_segment_mem(seg)), gc_etw_segment_read_only_heap);
7812 leave_spin_lock (&gc_heap::gc_lock);
7816 // No one is calling this function right now. If this is getting called we need
7817 // to take care of decommitting the mark array for it - we will need to remember
7818 // which portion of the mark array was committed and only decommit that.
7819 void gc_heap::remove_ro_segment (heap_segment* seg)
7821 //clear the mark bits so a new segment allocated in its place will have a clear mark bits
7823 if (gc_can_use_concurrent)
7825 clear_mark_array (align_lower_mark_word (max (heap_segment_mem (seg), lowest_address)),
7826 align_on_card_word (min (heap_segment_allocated (seg), highest_address)),
7827 false); // read_only segments need the mark clear
7831 enter_spin_lock (&gc_heap::gc_lock);
7833 seg_table->remove ((uint8_t*)seg);
7835 #ifdef SEG_MAPPING_TABLE
7836 seg_mapping_table_remove_ro_segment (seg);
7837 #endif //SEG_MAPPING_TABLE
7839 // Locate segment (and previous segment) in the list.
7840 generation* gen2 = generation_of (max_generation);
7841 heap_segment* curr_seg = generation_start_segment (gen2);
7842 heap_segment* prev_seg = NULL;
7844 while (curr_seg && curr_seg != seg)
7846 prev_seg = curr_seg;
7847 curr_seg = heap_segment_next (curr_seg);
7849 assert (curr_seg == seg);
7851 // Patch previous segment (or list head if there is none) to skip the removed segment.
7853 heap_segment_next (prev_seg) = heap_segment_next (curr_seg);
7855 generation_start_segment (gen2) = heap_segment_next (curr_seg);
7857 leave_spin_lock (&gc_heap::gc_lock);
7859 #endif //FEATURE_BASICFREEZE
7861 BOOL gc_heap::set_ro_segment_in_range (heap_segment* seg)
7864 seg->flags |= heap_segment_flags_inrange;
7865 // init_brick_card_range (seg);
7866 ro_segments_in_range = TRUE;
7867 //right now, segments aren't protected
7868 //unprotect_segment (seg);
7874 uint8_t** make_mark_list (size_t size)
7876 uint8_t** mark_list = new (nothrow) uint8_t* [size];
7880 #define swap(a,b){uint8_t* t; t = a; a = b; b = t;}
7882 void verify_qsort_array (uint8_t* *low, uint8_t* *high)
7886 for (i = low+1; i <= high; i++)
7895 #ifndef USE_INTROSORT
7896 void qsort1( uint8_t* *low, uint8_t* *high, unsigned int depth)
7898 if (((low + 16) >= high) || (depth > 100))
7902 for (i = low+1; i <= high; i++)
7905 for (j=i;j >low && val<*(j-1);j--)
7914 uint8_t *pivot, **left, **right;
7916 //sort low middle and high
7917 if (*(low+((high-low)/2)) < *low)
7918 swap (*(low+((high-low)/2)), *low);
7921 if (*high < *(low+((high-low)/2)))
7922 swap (*(low+((high-low)/2)), *high);
7924 swap (*(low+((high-low)/2)), *(high-1));
7926 left = low; right = high-1;
7928 while (*(--right) > pivot);
7929 while (*(++left) < pivot);
7932 swap(*left, *right);
7937 swap (*left, *(high-1));
7938 qsort1(low, left-1, depth+1);
7939 qsort1(left+1, high, depth+1);
7942 #endif //USE_INTROSORT
7943 void rqsort1( uint8_t* *low, uint8_t* *high)
7945 if ((low + 16) >= high)
7949 for (i = low+1; i <= high; i++)
7952 for (j=i;j >low && val>*(j-1);j--)
7961 uint8_t *pivot, **left, **right;
7963 //sort low middle and high
7964 if (*(low+((high-low)/2)) > *low)
7965 swap (*(low+((high-low)/2)), *low);
7968 if (*high > *(low+((high-low)/2)))
7969 swap (*(low+((high-low)/2)), *high);
7971 swap (*(low+((high-low)/2)), *(high-1));
7973 left = low; right = high-1;
7975 while (*(--right) < pivot);
7976 while (*(++left) > pivot);
7979 swap(*left, *right);
7984 swap (*left, *(high-1));
7985 rqsort1(low, left-1);
7986 rqsort1(left+1, high);
7990 #ifdef USE_INTROSORT
7995 static const int size_threshold = 64;
7996 static const int max_depth = 100;
7999 inline static void swap_elements(uint8_t** i,uint8_t** j)
8007 static void sort (uint8_t** begin, uint8_t** end, int ignored)
8010 introsort_loop (begin, end, max_depth);
8011 insertionsort (begin, end);
8016 static void introsort_loop (uint8_t** lo, uint8_t** hi, int depth_limit)
8018 while (hi-lo >= size_threshold)
8020 if (depth_limit == 0)
8025 uint8_t** p=median_partition (lo, hi);
8026 depth_limit=depth_limit-1;
8027 introsort_loop (p, hi, depth_limit);
8032 static uint8_t** median_partition (uint8_t** low, uint8_t** high)
8034 uint8_t *pivot, **left, **right;
8036 //sort low middle and high
8037 if (*(low+((high-low)/2)) < *low)
8038 swap_elements ((low+((high-low)/2)), low);
8040 swap_elements (low, high);
8041 if (*high < *(low+((high-low)/2)))
8042 swap_elements ((low+((high-low)/2)), high);
8044 swap_elements ((low+((high-low)/2)), (high-1));
8046 left = low; right = high-1;
8048 while (*(--right) > pivot);
8049 while (*(++left) < pivot);
8052 swap_elements(left, right);
8057 swap_elements (left, (high-1));
8062 static void insertionsort (uint8_t** lo, uint8_t** hi)
8064 for (uint8_t** i=lo+1; i <= hi; i++)
8068 while((j > lo) && (t <*(j-1)))
8077 static void heapsort (uint8_t** lo, uint8_t** hi)
8079 size_t n = hi - lo + 1;
8080 for (size_t i=n / 2; i >= 1; i--)
8084 for (size_t i = n; i > 1; i--)
8086 swap_elements (lo, lo + i - 1);
8087 downheap(1, i - 1, lo);
8091 static void downheap (size_t i, size_t n, uint8_t** lo)
8093 uint8_t* d = *(lo + i - 1);
8098 if (child < n && *(lo + child - 1)<(*(lo + child)))
8102 if (!(d<*(lo + child - 1)))
8106 *(lo + i - 1) = *(lo + child - 1);
8114 #endif //USE_INTROSORT
8116 #ifdef MULTIPLE_HEAPS
8117 #ifdef PARALLEL_MARK_LIST_SORT
8118 void gc_heap::sort_mark_list()
8120 // if this heap had a mark list overflow, we don't do anything
8121 if (mark_list_index > mark_list_end)
8123 // printf("sort_mark_list: overflow on heap %d\n", heap_number);
8127 // if any other heap had a mark list overflow, we fake one too,
8128 // so we don't use an incomplete mark list by mistake
8129 for (int i = 0; i < n_heaps; i++)
8131 if (g_heaps[i]->mark_list_index > g_heaps[i]->mark_list_end)
8133 mark_list_index = mark_list_end + 1;
8134 // printf("sort_mark_list: overflow on heap %d\n", i);
8139 // unsigned long start = GetCycleCount32();
8141 dprintf (3, ("Sorting mark lists"));
8142 if (mark_list_index > mark_list)
8143 _sort (mark_list, mark_list_index - 1, 0);
8145 // printf("first phase of sort_mark_list for heap %d took %u cycles to sort %u entries\n", this->heap_number, GetCycleCount32() - start, mark_list_index - mark_list);
8146 // start = GetCycleCount32();
8148 // first set the pieces for all heaps to empty
8150 for (heap_num = 0; heap_num < n_heaps; heap_num++)
8152 mark_list_piece_start[heap_num] = NULL;
8153 mark_list_piece_end[heap_num] = NULL;
8156 uint8_t** x = mark_list;
8158 // predicate means: x is still within the mark list, and within the bounds of this heap
8159 #define predicate(x) (((x) < mark_list_index) && (*(x) < heap->ephemeral_high))
8162 while (x < mark_list_index)
8165 // find the heap x points into - searching cyclically from the last heap,
8166 // because in many cases the right heap is the next one or comes soon after
8167 int last_heap_num = heap_num;
8168 MAYBE_UNUSED_VAR(last_heap_num);
8172 if (heap_num >= n_heaps)
8174 assert(heap_num != last_heap_num); // we should always find the heap - infinite loop if not!
8175 heap = g_heaps[heap_num];
8177 while (!(*x >= heap->ephemeral_low && *x < heap->ephemeral_high));
8179 // x is the start of the mark list piece for this heap
8180 mark_list_piece_start[heap_num] = x;
8182 // to find the end of the mark list piece for this heap, find the first x
8183 // that has !predicate(x), i.e. that is either not in this heap, or beyond the end of the list
8186 // let's see if we get lucky and the whole rest belongs to this piece
8187 if (predicate(mark_list_index-1))
8189 x = mark_list_index;
8190 mark_list_piece_end[heap_num] = x;
8194 // we play a variant of binary search to find the point sooner.
8195 // the first loop advances by increasing steps until the predicate turns false.
8196 // then we retreat the last step, and the second loop advances by decreasing steps, keeping the predicate true.
8201 uint8_t** temp_x = x;
8208 while (predicate(x));
8209 // we know that only the last step was wrong, so we undo it
8213 // loop invariant - predicate holds at x, but not x + inc
8214 assert (predicate(x) && !(((x + inc) > x) && predicate(x + inc)));
8216 if (((x + inc) > x) && predicate(x + inc))
8222 // the termination condition and the loop invariant together imply this:
8223 assert(predicate(x) && !predicate(x + inc) && (inc == 1));
8224 // so the spot we're looking for is one further
8227 mark_list_piece_end[heap_num] = x;
8232 // printf("second phase of sort_mark_list for heap %d took %u cycles\n", this->heap_number, GetCycleCount32() - start);
8235 void gc_heap::append_to_mark_list(uint8_t **start, uint8_t **end)
8237 size_t slots_needed = end - start;
8238 size_t slots_available = mark_list_end + 1 - mark_list_index;
8239 size_t slots_to_copy = min(slots_needed, slots_available);
8240 memcpy(mark_list_index, start, slots_to_copy*sizeof(*start));
8241 mark_list_index += slots_to_copy;
8242 // printf("heap %d: appended %Id slots to mark_list\n", heap_number, slots_to_copy);
8245 void gc_heap::merge_mark_lists()
8247 uint8_t** source[MAX_SUPPORTED_CPUS];
8248 uint8_t** source_end[MAX_SUPPORTED_CPUS];
8249 int source_heap[MAX_SUPPORTED_CPUS];
8250 int source_count = 0;
8252 // in case of mark list overflow, don't bother
8253 if (mark_list_index > mark_list_end)
8255 // printf("merge_mark_lists: overflow\n");
8259 dprintf(3, ("merge_mark_lists: heap_number = %d starts out with %Id entries", heap_number, mark_list_index - mark_list));
8260 // unsigned long start = GetCycleCount32();
8261 for (int i = 0; i < n_heaps; i++)
8263 gc_heap* heap = g_heaps[i];
8264 if (heap->mark_list_piece_start[heap_number] < heap->mark_list_piece_end[heap_number])
8266 source[source_count] = heap->mark_list_piece_start[heap_number];
8267 source_end[source_count] = heap->mark_list_piece_end[heap_number];
8268 source_heap[source_count] = i;
8269 if (source_count < MAX_SUPPORTED_CPUS)
8273 // printf("first phase of merge_mark_lists for heap %d took %u cycles\n", heap_number, GetCycleCount32() - start);
8275 dprintf(3, ("heap_number = %d has %d sources\n", heap_number, source_count));
8276 #if defined(_DEBUG) || defined(TRACE_GC)
8277 for (int j = 0; j < source_count; j++)
8279 dprintf(3, ("heap_number = %d ", heap_number));
8280 dprintf(3, (" source from heap %d = %Ix .. %Ix (%Id entries)",
8281 (size_t)(source_heap[j]), (size_t)(source[j][0]), (size_t)(source_end[j][-1]), (size_t)(source_end[j] - source[j])));
8282 // the sources should all be sorted
8283 for (uint8_t **x = source[j]; x < source_end[j] - 1; x++)
8287 dprintf(3, ("oops, mark_list from source %d for heap %d isn't sorted\n", j, heap_number));
8292 #endif //_DEBUG || TRACE_GC
8294 // start = GetCycleCount32();
8296 mark_list = &g_mark_list_copy [heap_number*mark_list_size];
8297 mark_list_index = mark_list;
8298 mark_list_end = &mark_list [mark_list_size-1];
8299 int piece_count = 0;
8300 if (source_count == 0)
8304 else if (source_count == 1)
8306 mark_list = source[0];
8307 mark_list_index = source_end[0];
8308 mark_list_end = mark_list_index;
8313 while (source_count > 1)
8315 // find the lowest and second lowest value in the sources we're merging from
8316 int lowest_source = 0;
8317 uint8_t *lowest = *source[0];
8318 uint8_t *second_lowest = *source[1];
8319 for (int i = 1; i < source_count; i++)
8321 if (lowest > *source[i])
8323 second_lowest = lowest;
8324 lowest = *source[i];
8327 else if (second_lowest > *source[i])
8329 second_lowest = *source[i];
8333 // find the point in the lowest source where it either runs out or is not <= second_lowest anymore
8335 // let's first try to get lucky and see if the whole source is <= second_lowest -- this is actually quite common
8337 if (source_end[lowest_source][-1] <= second_lowest)
8338 x = source_end[lowest_source];
8341 // use linear search to find the end -- could also use binary search as in sort_mark_list,
8342 // but saw no improvement doing that
8343 for (x = source[lowest_source]; x < source_end[lowest_source] && *x <= second_lowest; x++)
8347 // blast this piece to the mark list
8348 append_to_mark_list(source[lowest_source], x);
8351 source[lowest_source] = x;
8353 // check whether this source is now exhausted
8354 if (x >= source_end[lowest_source])
8356 // if it's not the source with the highest index, copy the source with the highest index
8357 // over it so the non-empty sources are always at the beginning
8358 if (lowest_source < source_count-1)
8360 source[lowest_source] = source[source_count-1];
8361 source_end[lowest_source] = source_end[source_count-1];
8366 // we're left with just one source that we copy
8367 append_to_mark_list(source[0], source_end[0]);
8371 // printf("second phase of merge_mark_lists for heap %d took %u cycles to merge %d pieces\n", heap_number, GetCycleCount32() - start, piece_count);
8373 #if defined(_DEBUG) || defined(TRACE_GC)
8374 // the final mark list must be sorted
8375 for (uint8_t **x = mark_list; x < mark_list_index - 1; x++)
8379 dprintf(3, ("oops, mark_list for heap %d isn't sorted at the end of merge_mark_lists", heap_number));
8383 #endif //defined(_DEBUG) || defined(TRACE_GC)
8385 #else //PARALLEL_MARK_LIST_SORT
8386 void gc_heap::combine_mark_lists()
8388 dprintf (3, ("Combining mark lists"));
8389 //verify if a heap has overflowed its mark list
8390 BOOL use_mark_list = TRUE;
8391 for (int i = 0; i < n_heaps; i++)
8393 if (g_heaps [i]->mark_list_index > g_heaps [i]->mark_list_end)
8395 use_mark_list = FALSE;
8402 dprintf (3, ("Using mark list"));
8403 //compact the gaps out of the mark list
8405 uint8_t** current_gap = g_heaps [gn]->mark_list_index;
8406 uint8_t** current_gap_end = g_heaps[gn]->mark_list_end + 1;
8407 uint8_t** dst_last = current_gap-1;
8409 int srcn = n_heaps-1;
8410 gc_heap* srch = g_heaps [srcn];
8411 uint8_t** src = srch->mark_list_index - 1;
8412 uint8_t** src_beg = srch->mark_list;
8414 while (current_gap <= src)
8416 while ((gn < n_heaps-1) && (current_gap >= current_gap_end))
8418 //go to the next gap
8420 dprintf (3, ("Going to the next gap %d", gn));
8421 assert (gn < n_heaps);
8422 current_gap = g_heaps [gn]->mark_list_index;
8423 current_gap_end = g_heaps[gn]->mark_list_end + 1;
8424 assert ((gn == (n_heaps-1)) || (current_gap_end == g_heaps[gn+1]->mark_list));
8426 while ((srcn > 0) && (src < src_beg))
8428 //go to the previous source
8430 dprintf (3, ("going to the previous source %d", srcn));
8432 gc_heap* srch = g_heaps [srcn];
8433 src = srch->mark_list_index - 1;
8434 src_beg = srch->mark_list;
8436 if (current_gap < src)
8438 dst_last = current_gap;
8439 *current_gap++ = *src--;
8442 dprintf (3, ("src: %Ix dst_last: %Ix", (size_t)src, (size_t)dst_last));
8444 uint8_t** end_of_list = max (src, dst_last);
8446 //sort the resulting compacted list
8447 assert (end_of_list < &g_mark_list [n_heaps*mark_list_size]);
8448 if (end_of_list > &g_mark_list[0])
8449 _sort (&g_mark_list[0], end_of_list, 0);
8450 //adjust the mark_list to the begining of the resulting mark list.
8451 for (int i = 0; i < n_heaps; i++)
8453 g_heaps [i]->mark_list = g_mark_list;
8454 g_heaps [i]->mark_list_index = end_of_list + 1;
8455 g_heaps [i]->mark_list_end = end_of_list + 1;
8460 uint8_t** end_of_list = g_mark_list;
8461 //adjust the mark_list to the begining of the resulting mark list.
8462 //put the index beyond the end to turn off mark list processing
8463 for (int i = 0; i < n_heaps; i++)
8465 g_heaps [i]->mark_list = g_mark_list;
8466 g_heaps [i]->mark_list_index = end_of_list + 1;
8467 g_heaps [i]->mark_list_end = end_of_list;
8471 #endif // PARALLEL_MARK_LIST_SORT
8472 #endif //MULTIPLE_HEAPS
8475 class seg_free_spaces
8477 struct seg_free_space
8483 struct free_space_bucket
8485 seg_free_space* free_space;
8486 ptrdiff_t count_add; // Assigned when we first contruct the array.
8487 ptrdiff_t count_fit; // How many items left when we are fitting plugs.
8490 void move_bucket (int old_power2, int new_power2)
8492 // PREFAST warning 22015: old_power2 could be negative
8493 assert (old_power2 >= 0);
8494 assert (old_power2 >= new_power2);
8496 if (old_power2 == new_power2)
8501 seg_free_space* src_index = free_space_buckets[old_power2].free_space;
8502 for (int i = old_power2; i > new_power2; i--)
8504 seg_free_space** dest = &(free_space_buckets[i].free_space);
8507 seg_free_space* dest_index = free_space_buckets[i - 1].free_space;
8508 if (i > (new_power2 + 1))
8510 seg_free_space temp = *src_index;
8511 *src_index = *dest_index;
8514 src_index = dest_index;
8517 free_space_buckets[old_power2].count_fit--;
8518 free_space_buckets[new_power2].count_fit++;
8523 void dump_free_space (seg_free_space* item)
8530 mark* m = (mark*)(item->start);
8531 len = pinned_len (m);
8532 addr = pinned_plug (m) - len;
8536 heap_segment* seg = (heap_segment*)(item->start);
8537 addr = heap_segment_plan_allocated (seg);
8538 len = heap_segment_committed (seg) - addr;
8541 dprintf (SEG_REUSE_LOG_1, ("[%d]0x%Ix %Id", heap_num, addr, len));
8546 seg_free_space* item = NULL;
8549 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------\nnow the free spaces look like:", heap_num));
8550 for (i = 0; i < (free_space_bucket_count - 1); i++)
8552 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces for 2^%d bucket:", heap_num, (base_power2 + i)));
8553 dprintf (SEG_REUSE_LOG_1, ("[%d]%s %s", heap_num, "start", "len"));
8554 item = free_space_buckets[i].free_space;
8555 while (item < free_space_buckets[i + 1].free_space)
8557 dump_free_space (item);
8560 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------", heap_num));
8563 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces for 2^%d bucket:", heap_num, (base_power2 + i)));
8564 dprintf (SEG_REUSE_LOG_1, ("[%d]%s %s", heap_num, "start", "len"));
8565 item = free_space_buckets[i].free_space;
8567 while (item <= &seg_free_space_array[free_space_item_count - 1])
8569 dump_free_space (item);
8572 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------", heap_num));
8577 free_space_bucket* free_space_buckets;
8578 seg_free_space* seg_free_space_array;
8579 ptrdiff_t free_space_bucket_count;
8580 ptrdiff_t free_space_item_count;
8584 BOOL has_end_of_seg;
8589 seg_free_spaces (int h_number)
8591 heap_num = h_number;
8596 size_t total_prealloc_size =
8597 MAX_NUM_BUCKETS * sizeof (free_space_bucket) +
8598 MAX_NUM_FREE_SPACES * sizeof (seg_free_space);
8600 free_space_buckets = (free_space_bucket*) new (nothrow) uint8_t[total_prealloc_size];
8602 return (!!free_space_buckets);
8605 // We take the ordered free space array we got from the 1st pass,
8606 // and feed the portion that we decided to use to this method, ie,
8607 // the largest item_count free spaces.
8608 void add_buckets (int base, size_t* ordered_free_spaces, int bucket_count, size_t item_count)
8610 assert (free_space_buckets);
8611 assert (item_count <= (size_t)MAX_PTR);
8613 free_space_bucket_count = bucket_count;
8614 free_space_item_count = item_count;
8617 has_end_of_seg = FALSE;
8620 ptrdiff_t total_item_count = 0;
8623 seg_free_space_array = (seg_free_space*)(free_space_buckets + free_space_bucket_count);
8625 for (i = 0; i < (ptrdiff_t)item_count; i++)
8627 seg_free_space_array[i].start = 0;
8628 seg_free_space_array[i].is_plug = FALSE;
8631 for (i = 0; i < bucket_count; i++)
8633 free_space_buckets[i].count_add = ordered_free_spaces[i];
8634 free_space_buckets[i].count_fit = ordered_free_spaces[i];
8635 free_space_buckets[i].free_space = &seg_free_space_array[total_item_count];
8636 total_item_count += free_space_buckets[i].count_add;
8639 assert (total_item_count == (ptrdiff_t)item_count);
8642 // If we are adding a free space before a plug we pass the
8643 // mark stack position so we can update the length; we could
8644 // also be adding the free space after the last plug in which
8645 // case start is the segment which we'll need to update the
8646 // heap_segment_plan_allocated.
8647 void add (void* start, BOOL plug_p, BOOL first_p)
8649 size_t size = (plug_p ?
8650 pinned_len ((mark*)start) :
8651 (heap_segment_committed ((heap_segment*)start) -
8652 heap_segment_plan_allocated ((heap_segment*)start)));
8656 dprintf (SEG_REUSE_LOG_1, ("[%d]Adding a free space before plug: %Id", heap_num, size));
8660 dprintf (SEG_REUSE_LOG_1, ("[%d]Adding a free space at end of seg: %Id", heap_num, size));
8662 has_end_of_seg = TRUE;
8668 size_t eph_gen_starts = gc_heap::eph_gen_starts_size;
8669 size -= eph_gen_starts;
8672 mark* m = (mark*)(start);
8673 pinned_len (m) -= eph_gen_starts;
8677 heap_segment* seg = (heap_segment*)start;
8678 heap_segment_plan_allocated (seg) += eph_gen_starts;
8682 int bucket_power2 = index_of_highest_set_bit (size);
8683 if (bucket_power2 < base_power2)
8688 free_space_bucket* bucket = &free_space_buckets[bucket_power2 - base_power2];
8690 seg_free_space* bucket_free_space = bucket->free_space;
8691 assert (plug_p || (!plug_p && bucket->count_add));
8693 if (bucket->count_add == 0)
8695 dprintf (SEG_REUSE_LOG_1, ("[%d]Already have enough of 2^%d", heap_num, bucket_power2));
8699 ptrdiff_t index = bucket->count_add - 1;
8701 dprintf (SEG_REUSE_LOG_1, ("[%d]Building free spaces: adding %Ix; len: %Id (2^%d)",
8704 (pinned_plug ((mark*)start) - pinned_len ((mark*)start)) :
8705 heap_segment_plan_allocated ((heap_segment*)start)),
8711 bucket_free_space[index].is_plug = TRUE;
8714 bucket_free_space[index].start = start;
8715 bucket->count_add--;
8720 // Do a consistency check after all free spaces are added.
8724 int end_of_seg_count = 0;
8726 for (i = 0; i < free_space_item_count; i++)
8728 assert (seg_free_space_array[i].start);
8729 if (!(seg_free_space_array[i].is_plug))
8737 assert (end_of_seg_count == 1);
8741 assert (end_of_seg_count == 0);
8744 for (i = 0; i < free_space_bucket_count; i++)
8746 assert (free_space_buckets[i].count_add == 0);
8752 uint8_t* fit (uint8_t* old_loc,
8754 BOOL set_padding_on_saved_p,
8755 mark* pinned_plug_entry,
8756 #endif //SHORT_PLUGS
8758 REQD_ALIGN_AND_OFFSET_DCL)
8763 assert (!is_plug_padded (old_loc));
8764 #endif //SHORT_PLUGS
8765 assert (!node_realigned (old_loc));
8768 size_t saved_plug_size = plug_size;
8770 #ifdef FEATURE_STRUCTALIGN
8771 // BARTOKTODO (4841): this code path is disabled (see can_fit_all_blocks_p) until we take alignment requirements into account
8772 _ASSERTE(requiredAlignment == DATA_ALIGNMENT && false);
8773 #endif // FEATURE_STRUCTALIGN
8774 // TODO: this is also not large alignment ready. We would need to consider alignment when chosing the
8777 size_t plug_size_to_fit = plug_size;
8779 int pad_in_front = (old_loc != 0) ? USE_PADDING_FRONT : 0;
8782 plug_size_to_fit += (pad_in_front ? Align(min_obj_size) : 0);
8783 #endif //SHORT_PLUGS
8785 int plug_power2 = index_of_highest_set_bit (round_up_power2 (plug_size_to_fit + Align(min_obj_size)));
8787 uint8_t* new_address = 0;
8789 if (plug_power2 < base_power2)
8791 plug_power2 = base_power2;
8794 int chosen_power2 = plug_power2 - base_power2;
8796 for (i = chosen_power2; i < free_space_bucket_count; i++)
8798 if (free_space_buckets[i].count_fit != 0)
8805 dprintf (SEG_REUSE_LOG_1, ("[%d]Fitting plug len %Id (2^%d) using 2^%d free space",
8809 (chosen_power2 + base_power2)));
8811 assert (i < free_space_bucket_count);
8813 seg_free_space* bucket_free_space = free_space_buckets[chosen_power2].free_space;
8814 ptrdiff_t free_space_count = free_space_buckets[chosen_power2].count_fit;
8815 size_t new_free_space_size = 0;
8816 BOOL can_fit = FALSE;
8819 for (i = 0; i < free_space_count; i++)
8821 size_t free_space_size = 0;
8824 BOOL short_plugs_padding_p = FALSE;
8825 #endif //SHORT_PLUGS
8826 BOOL realign_padding_p = FALSE;
8828 if (bucket_free_space[i].is_plug)
8830 mark* m = (mark*)(bucket_free_space[i].start);
8831 uint8_t* plug_free_space_start = pinned_plug (m) - pinned_len (m);
8834 if ((pad_in_front & USE_PADDING_FRONT) &&
8835 (((plug_free_space_start - pin_allocation_context_start_region (m))==0) ||
8836 ((plug_free_space_start - pin_allocation_context_start_region (m))>=DESIRED_PLUG_LENGTH)))
8838 pad = Align (min_obj_size);
8839 short_plugs_padding_p = TRUE;
8841 #endif //SHORT_PLUGS
8843 if (!((old_loc == 0) || same_large_alignment_p (old_loc, plug_free_space_start+pad)))
8845 pad += switch_alignment_size (pad != 0);
8846 realign_padding_p = TRUE;
8849 plug_size = saved_plug_size + pad;
8851 free_space_size = pinned_len (m);
8852 new_address = pinned_plug (m) - pinned_len (m);
8854 if (free_space_size >= (plug_size + Align (min_obj_size)) ||
8855 free_space_size == plug_size)
8857 new_free_space_size = free_space_size - plug_size;
8858 pinned_len (m) = new_free_space_size;
8859 #ifdef SIMPLE_DPRINTF
8860 dprintf (SEG_REUSE_LOG_0, ("[%d]FP: 0x%Ix->0x%Ix(%Ix)(%Ix), [0x%Ix (2^%d) -> [0x%Ix (2^%d)",
8867 index_of_highest_set_bit (free_space_size),
8868 (pinned_plug (m) - pinned_len (m)),
8869 index_of_highest_set_bit (new_free_space_size)));
8870 #endif //SIMPLE_DPRINTF
8873 if (short_plugs_padding_p)
8875 pin_allocation_context_start_region (m) = plug_free_space_start;
8876 set_padding_in_expand (old_loc, set_padding_on_saved_p, pinned_plug_entry);
8878 #endif //SHORT_PLUGS
8880 if (realign_padding_p)
8882 set_node_realigned (old_loc);
8890 heap_segment* seg = (heap_segment*)(bucket_free_space[i].start);
8891 free_space_size = heap_segment_committed (seg) - heap_segment_plan_allocated (seg);
8893 if (!((old_loc == 0) || same_large_alignment_p (old_loc, heap_segment_plan_allocated (seg))))
8895 pad = switch_alignment_size (FALSE);
8896 realign_padding_p = TRUE;
8899 plug_size = saved_plug_size + pad;
8901 if (free_space_size >= (plug_size + Align (min_obj_size)) ||
8902 free_space_size == plug_size)
8904 new_address = heap_segment_plan_allocated (seg);
8905 new_free_space_size = free_space_size - plug_size;
8906 heap_segment_plan_allocated (seg) = new_address + plug_size;
8907 #ifdef SIMPLE_DPRINTF
8908 dprintf (SEG_REUSE_LOG_0, ("[%d]FS: 0x%Ix-> 0x%Ix(%Ix) (2^%d) -> 0x%Ix (2^%d)",
8913 index_of_highest_set_bit (free_space_size),
8914 heap_segment_plan_allocated (seg),
8915 index_of_highest_set_bit (new_free_space_size)));
8916 #endif //SIMPLE_DPRINTF
8918 if (realign_padding_p)
8919 set_node_realigned (old_loc);
8933 assert (chosen_power2 == 0);
8943 assert ((chosen_power2 && (i == 0)) ||
8944 (!chosen_power2) && (i < free_space_count));
8947 int new_bucket_power2 = index_of_highest_set_bit (new_free_space_size);
8949 if (new_bucket_power2 < base_power2)
8951 new_bucket_power2 = base_power2;
8954 move_bucket (chosen_power2, new_bucket_power2 - base_power2);
8963 if (free_space_buckets)
8965 delete [] free_space_buckets;
8967 if (seg_free_space_array)
8969 delete [] seg_free_space_array;
8975 #define marked(i) header(i)->IsMarked()
8976 #define set_marked(i) header(i)->SetMarked()
8977 #define clear_marked(i) header(i)->ClearMarked()
8978 #define pinned(i) header(i)->IsPinned()
8979 #define set_pinned(i) header(i)->SetPinned()
8980 #define clear_pinned(i) header(i)->GetHeader()->ClrGCBit();
8982 inline size_t my_get_size (Object* ob)
8984 MethodTable* mT = header(ob)->GetMethodTable();
8985 return (mT->GetBaseSize() +
8986 (mT->HasComponentSize() ?
8987 ((size_t)((CObjectHeader*)ob)->GetNumComponents() * mT->RawGetComponentSize()) : 0));
8990 //#define size(i) header(i)->GetSize()
8991 #define size(i) my_get_size (header(i))
8993 #define contain_pointers(i) header(i)->ContainsPointers()
8994 #ifdef COLLECTIBLE_CLASS
8995 #define contain_pointers_or_collectible(i) header(i)->ContainsPointersOrCollectible()
8997 #define get_class_object(i) GCToEEInterface::GetLoaderAllocatorObjectForGC((Object *)i)
8998 #define is_collectible(i) method_table(i)->Collectible()
8999 #else //COLLECTIBLE_CLASS
9000 #define contain_pointers_or_collectible(i) header(i)->ContainsPointers()
9001 #endif //COLLECTIBLE_CLASS
9003 #if defined (MARK_ARRAY) && defined (BACKGROUND_GC)
9005 void gc_heap::seg_clear_mark_array_bits_soh (heap_segment* seg)
9007 uint8_t* range_beg = 0;
9008 uint8_t* range_end = 0;
9009 if (bgc_mark_array_range (seg, FALSE, &range_beg, &range_end))
9011 clear_mark_array (range_beg, align_on_mark_word (range_end), FALSE
9012 #ifdef FEATURE_BASICFREEZE
9014 #endif // FEATURE_BASICFREEZE
9019 void gc_heap::clear_batch_mark_array_bits (uint8_t* start, uint8_t* end)
9021 if ((start < background_saved_highest_address) &&
9022 (end > background_saved_lowest_address))
9024 start = max (start, background_saved_lowest_address);
9025 end = min (end, background_saved_highest_address);
9027 size_t start_mark_bit = mark_bit_of (start);
9028 size_t end_mark_bit = mark_bit_of (end);
9029 unsigned int startbit = mark_bit_bit (start_mark_bit);
9030 unsigned int endbit = mark_bit_bit (end_mark_bit);
9031 size_t startwrd = mark_bit_word (start_mark_bit);
9032 size_t endwrd = mark_bit_word (end_mark_bit);
9034 dprintf (3, ("Clearing all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
9035 (size_t)start, (size_t)start_mark_bit,
9036 (size_t)end, (size_t)end_mark_bit));
9038 unsigned int firstwrd = lowbits (~0, startbit);
9039 unsigned int lastwrd = highbits (~0, endbit);
9041 if (startwrd == endwrd)
9043 unsigned int wrd = firstwrd | lastwrd;
9044 mark_array[startwrd] &= wrd;
9048 // clear the first mark word.
9051 mark_array[startwrd] &= firstwrd;
9055 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
9057 mark_array[wrdtmp] = 0;
9060 // clear the last mark word.
9063 mark_array[endwrd] &= lastwrd;
9068 void gc_heap::bgc_clear_batch_mark_array_bits (uint8_t* start, uint8_t* end)
9070 if ((start < background_saved_highest_address) &&
9071 (end > background_saved_lowest_address))
9073 start = max (start, background_saved_lowest_address);
9074 end = min (end, background_saved_highest_address);
9076 clear_batch_mark_array_bits (start, end);
9080 void gc_heap::clear_mark_array_by_objects (uint8_t* from, uint8_t* end, BOOL loh_p)
9082 dprintf (3, ("clearing mark array bits by objects for addr [%Ix,[%Ix",
9084 int align_const = get_alignment_constant (!loh_p);
9090 uint8_t* next_o = o + Align (size (o), align_const);
9092 if (background_object_marked (o, TRUE))
9094 dprintf (3, ("%Ix was marked by bgc, is now cleared", o));
9100 #endif //MARK_ARRAY && BACKGROUND_GC
9103 BOOL gc_heap::is_mark_set (uint8_t* o)
9108 #if defined (_MSC_VER) && defined (_TARGET_X86_)
9109 #pragma optimize("y", on) // Small critical routines, don't put in EBP frame
9110 #endif //_MSC_VER && _TARGET_X86_
9112 // return the generation number of an object.
9113 // It is assumed that the object is valid.
9114 //Note that this will return max_generation for a LOH object
9115 int gc_heap::object_gennum (uint8_t* o)
9117 if (in_range_for_segment (o, ephemeral_heap_segment) &&
9118 (o >= generation_allocation_start (generation_of (max_generation-1))))
9120 // in an ephemeral generation.
9121 for ( int i = 0; i < max_generation-1; i++)
9123 if ((o >= generation_allocation_start (generation_of (i))))
9126 return max_generation-1;
9130 return max_generation;
9134 int gc_heap::object_gennum_plan (uint8_t* o)
9136 if (in_range_for_segment (o, ephemeral_heap_segment))
9138 for (int i = 0; i <= max_generation-1; i++)
9140 uint8_t* plan_start = generation_plan_allocation_start (generation_of (i));
9141 if (plan_start && (o >= plan_start))
9147 return max_generation;
9150 #if defined(_MSC_VER) && defined(_TARGET_X86_)
9151 #pragma optimize("", on) // Go back to command line default optimizations
9152 #endif //_MSC_VER && _TARGET_X86_
9154 heap_segment* gc_heap::make_heap_segment (uint8_t* new_pages, size_t size, int h_number)
9156 size_t initial_commit = SEGMENT_INITIAL_COMMIT;
9158 //Commit the first page
9159 if (!virtual_alloc_commit_for_heap (new_pages, initial_commit, h_number))
9164 //overlay the heap_segment
9165 heap_segment* new_segment = (heap_segment*)new_pages;
9167 uint8_t* start = new_pages + segment_info_size;
9168 heap_segment_mem (new_segment) = start;
9169 heap_segment_used (new_segment) = start;
9170 heap_segment_reserved (new_segment) = new_pages + size;
9171 heap_segment_committed (new_segment) = new_pages + initial_commit;
9172 init_heap_segment (new_segment);
9173 dprintf (2, ("Creating heap segment %Ix", (size_t)new_segment));
9177 void gc_heap::init_heap_segment (heap_segment* seg)
9180 heap_segment_next (seg) = 0;
9181 heap_segment_plan_allocated (seg) = heap_segment_mem (seg);
9182 heap_segment_allocated (seg) = heap_segment_mem (seg);
9183 #ifdef BACKGROUND_GC
9184 heap_segment_background_allocated (seg) = 0;
9185 heap_segment_saved_bg_allocated (seg) = 0;
9186 #endif //BACKGROUND_GC
9189 //Releases the segment to the OS.
9190 // this is always called on one thread only so calling seg_table->remove is fine.
9191 void gc_heap::delete_heap_segment (heap_segment* seg, BOOL consider_hoarding)
9193 if (!heap_segment_loh_p (seg))
9195 //cleanup the brick table back to the empty value
9196 clear_brick_table (heap_segment_mem (seg), heap_segment_reserved (seg));
9199 if (consider_hoarding)
9201 assert ((heap_segment_mem (seg) - (uint8_t*)seg) <= ptrdiff_t(2*OS_PAGE_SIZE));
9202 size_t ss = (size_t) (heap_segment_reserved (seg) - (uint8_t*)seg);
9203 //Don't keep the big ones.
9204 if (ss <= INITIAL_ALLOC)
9206 dprintf (2, ("Hoarding segment %Ix", (size_t)seg));
9207 #ifdef BACKGROUND_GC
9208 // We don't need to clear the decommitted flag because when this segment is used
9209 // for a new segment the flags will be cleared.
9210 if (!heap_segment_decommitted_p (seg))
9211 #endif //BACKGROUND_GC
9213 decommit_heap_segment (seg);
9216 #ifdef SEG_MAPPING_TABLE
9217 seg_mapping_table_remove_segment (seg);
9218 #endif //SEG_MAPPING_TABLE
9220 heap_segment_next (seg) = segment_standby_list;
9221 segment_standby_list = seg;
9228 dprintf (2, ("h%d: del seg: [%Ix, %Ix[",
9229 heap_number, (size_t)seg,
9230 (size_t)(heap_segment_reserved (seg))));
9232 #ifdef BACKGROUND_GC
9233 ::record_changed_seg ((uint8_t*)seg, heap_segment_reserved (seg),
9234 settings.gc_index, current_bgc_state,
9236 decommit_mark_array_by_seg (seg);
9237 #endif //BACKGROUND_GC
9239 #ifdef SEG_MAPPING_TABLE
9240 seg_mapping_table_remove_segment (seg);
9241 #else //SEG_MAPPING_TABLE
9242 seg_table->remove ((uint8_t*)seg);
9243 #endif //SEG_MAPPING_TABLE
9245 release_segment (seg);
9249 //resets the pages beyond allocates size so they won't be swapped out and back in
9251 void gc_heap::reset_heap_segment_pages (heap_segment* seg)
9253 size_t page_start = align_on_page ((size_t)heap_segment_allocated (seg));
9254 size_t size = (size_t)heap_segment_committed (seg) - page_start;
9256 GCToOSInterface::VirtualReset((void*)page_start, size, false /* unlock */);
9259 void gc_heap::decommit_heap_segment_pages (heap_segment* seg,
9262 uint8_t* page_start = align_on_page (heap_segment_allocated(seg));
9263 size_t size = heap_segment_committed (seg) - page_start;
9264 extra_space = align_on_page (extra_space);
9265 if (size >= max ((extra_space + 2*OS_PAGE_SIZE), 100*OS_PAGE_SIZE))
9267 page_start += max(extra_space, 32*OS_PAGE_SIZE);
9268 size -= max (extra_space, 32*OS_PAGE_SIZE);
9270 GCToOSInterface::VirtualDecommit (page_start, size);
9271 dprintf (3, ("Decommitting heap segment [%Ix, %Ix[(%d)",
9273 (size_t)(page_start + size),
9275 heap_segment_committed (seg) = page_start;
9276 if (heap_segment_used (seg) > heap_segment_committed (seg))
9278 heap_segment_used (seg) = heap_segment_committed (seg);
9283 //decommit all pages except one or 2
9284 void gc_heap::decommit_heap_segment (heap_segment* seg)
9286 uint8_t* page_start = align_on_page (heap_segment_mem (seg));
9288 dprintf (3, ("Decommitting heap segment %Ix", (size_t)seg));
9290 #ifdef BACKGROUND_GC
9291 page_start += OS_PAGE_SIZE;
9292 #endif //BACKGROUND_GC
9294 size_t size = heap_segment_committed (seg) - page_start;
9295 GCToOSInterface::VirtualDecommit (page_start, size);
9297 //re-init the segment object
9298 heap_segment_committed (seg) = page_start;
9299 if (heap_segment_used (seg) > heap_segment_committed (seg))
9301 heap_segment_used (seg) = heap_segment_committed (seg);
9305 void gc_heap::clear_gen0_bricks()
9307 if (!gen0_bricks_cleared)
9309 gen0_bricks_cleared = TRUE;
9310 //initialize brick table for gen 0
9311 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
9312 b < brick_of (align_on_brick
9313 (heap_segment_allocated (ephemeral_heap_segment)));
9321 #ifdef BACKGROUND_GC
9322 void gc_heap::rearrange_small_heap_segments()
9324 heap_segment* seg = freeable_small_heap_segment;
9327 heap_segment* next_seg = heap_segment_next (seg);
9328 // TODO: we need to consider hoarding here.
9329 delete_heap_segment (seg, FALSE);
9332 freeable_small_heap_segment = 0;
9334 #endif //BACKGROUND_GC
9336 void gc_heap::rearrange_large_heap_segments()
9338 dprintf (2, ("deleting empty large segments"));
9339 heap_segment* seg = freeable_large_heap_segment;
9342 heap_segment* next_seg = heap_segment_next (seg);
9343 delete_heap_segment (seg, GCConfig::GetRetainVM());
9346 freeable_large_heap_segment = 0;
9349 void gc_heap::rearrange_heap_segments(BOOL compacting)
9352 generation_start_segment (generation_of (max_generation));
9354 heap_segment* prev_seg = 0;
9355 heap_segment* next_seg = 0;
9358 next_seg = heap_segment_next (seg);
9360 //link ephemeral segment when expanding
9361 if ((next_seg == 0) && (seg != ephemeral_heap_segment))
9363 seg->next = ephemeral_heap_segment;
9364 next_seg = heap_segment_next (seg);
9367 //re-used expanded heap segment
9368 if ((seg == ephemeral_heap_segment) && next_seg)
9370 heap_segment_next (prev_seg) = next_seg;
9371 heap_segment_next (seg) = 0;
9375 uint8_t* end_segment = (compacting ?
9376 heap_segment_plan_allocated (seg) :
9377 heap_segment_allocated (seg));
9378 // check if the segment was reached by allocation
9379 if ((end_segment == heap_segment_mem (seg))&&
9380 !heap_segment_read_only_p (seg))
9382 //if not, unthread and delete
9384 assert (seg != ephemeral_heap_segment);
9385 heap_segment_next (prev_seg) = next_seg;
9386 delete_heap_segment (seg, GCConfig::GetRetainVM());
9388 dprintf (2, ("Deleting heap segment %Ix", (size_t)seg));
9392 if (!heap_segment_read_only_p (seg))
9396 heap_segment_allocated (seg) =
9397 heap_segment_plan_allocated (seg);
9400 // reset the pages between allocated and committed.
9401 if (seg != ephemeral_heap_segment)
9403 decommit_heap_segment_pages (seg, 0);
9417 uint8_t* g_addresses [array_size+2]; // to get around the bug in GetWriteWatch
9419 #ifdef TIME_WRITE_WATCH
9420 static unsigned int tot_cycles = 0;
9421 #endif //TIME_WRITE_WATCH
9425 inline void gc_heap::verify_card_bundle_bits_set(size_t first_card_word, size_t last_card_word)
9428 for (size_t x = cardw_card_bundle (first_card_word); x < cardw_card_bundle (last_card_word); x++)
9430 if (!card_bundle_set_p (x))
9432 assert (!"Card bundle not set");
9433 dprintf (3, ("Card bundle %Ix not set", x));
9439 // Verifies that any bundles that are not set represent only cards that are not set.
9440 inline void gc_heap::verify_card_bundles()
9443 size_t lowest_card = card_word (card_of (lowest_address));
9444 size_t highest_card = card_word (card_of (highest_address));
9445 size_t cardb = cardw_card_bundle (lowest_card);
9446 size_t end_cardb = cardw_card_bundle (align_cardw_on_bundle (highest_card));
9448 while (cardb < end_cardb)
9450 uint32_t* card_word = &card_table[max(card_bundle_cardw (cardb), lowest_card)];
9451 uint32_t* card_word_end = &card_table[min(card_bundle_cardw (cardb+1), highest_card)];
9453 if (card_bundle_set_p (cardb) == 0)
9455 // Verify that no card is set
9456 while (card_word < card_word_end)
9458 if (*card_word != 0)
9460 dprintf (3, ("gc: %d, Card word %Ix for address %Ix set, card_bundle %Ix clear",
9461 dd_collection_count (dynamic_data_of (0)),
9462 (size_t)(card_word-&card_table[0]),
9463 (size_t)(card_address ((size_t)(card_word-&card_table[0]) * card_word_width)), cardb));
9466 assert((*card_word)==0);
9476 // If card bundles are enabled, use write watch to find pages in the card table that have
9477 // been dirtied, and set the corresponding card bundle bits.
9478 void gc_heap::update_card_table_bundle()
9480 if (card_bundles_enabled())
9482 // The address of the card word containing the card representing the lowest heap address
9483 uint8_t* base_address = (uint8_t*)(&card_table[card_word (card_of (lowest_address))]);
9485 // The address of the card word containing the card representing the highest heap address
9486 uint8_t* high_address = (uint8_t*)(&card_table[card_word (card_of (highest_address))]);
9488 uint8_t* saved_base_address = base_address;
9489 uintptr_t bcount = array_size;
9490 size_t saved_region_size = align_on_page (high_address) - saved_base_address;
9494 size_t region_size = align_on_page (high_address) - base_address;
9496 dprintf (3,("Probing card table pages [%Ix, %Ix[", (size_t)base_address, (size_t)base_address+region_size));
9497 bool success = GCToOSInterface::GetWriteWatch(false /* resetState */,
9500 (void**)g_addresses,
9502 assert (success && "GetWriteWatch failed!");
9504 dprintf (3,("Found %d pages written", bcount));
9505 for (unsigned i = 0; i < bcount; i++)
9507 // Offset of the dirty page from the start of the card table (clamped to base_address)
9508 size_t bcardw = (uint32_t*)(max(g_addresses[i],base_address)) - &card_table[0];
9510 // Offset of the end of the page from the start of the card table (clamped to high addr)
9511 size_t ecardw = (uint32_t*)(min(g_addresses[i]+OS_PAGE_SIZE, high_address)) - &card_table[0];
9512 assert (bcardw >= card_word (card_of (g_gc_lowest_address)));
9514 // Set the card bundle bits representing the dirty card table page
9515 card_bundles_set (cardw_card_bundle (bcardw), cardw_card_bundle (align_cardw_on_bundle (ecardw)));
9516 dprintf (3,("Set Card bundle [%Ix, %Ix[", cardw_card_bundle (bcardw), cardw_card_bundle (align_cardw_on_bundle (ecardw))));
9518 verify_card_bundle_bits_set(bcardw, ecardw);
9521 if (bcount >= array_size)
9523 base_address = g_addresses [array_size-1] + OS_PAGE_SIZE;
9524 bcount = array_size;
9527 } while ((bcount >= array_size) && (base_address < high_address));
9529 // Now that we've updated the card bundle bits, reset the write-tracking state.
9530 GCToOSInterface::ResetWriteWatch (saved_base_address, saved_region_size);
9533 #endif //CARD_BUNDLE
9536 void gc_heap::reset_write_watch_for_gc_heap(void* base_address, size_t region_size)
9538 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9539 SoftwareWriteWatch::ClearDirty(base_address, region_size);
9540 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9541 GCToOSInterface::ResetWriteWatch(base_address, region_size);
9542 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9546 void gc_heap::get_write_watch_for_gc_heap(bool reset, void *base_address, size_t region_size, void** dirty_pages, uintptr_t* dirty_page_count_ref, bool is_runtime_suspended)
9548 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9549 SoftwareWriteWatch::GetDirty(base_address, region_size, dirty_pages, dirty_page_count_ref, reset, is_runtime_suspended);
9550 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9551 UNREFERENCED_PARAMETER(is_runtime_suspended);
9552 bool success = GCToOSInterface::GetWriteWatch(reset, base_address, region_size, dirty_pages, dirty_page_count_ref);
9554 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9557 const size_t ww_reset_quantum = 128*1024*1024;
9560 void gc_heap::switch_one_quantum()
9562 enable_preemptive ();
9563 GCToOSInterface::Sleep (1);
9564 disable_preemptive (true);
9567 void gc_heap::reset_ww_by_chunk (uint8_t* start_address, size_t total_reset_size)
9569 size_t reset_size = 0;
9570 size_t remaining_reset_size = 0;
9571 size_t next_reset_size = 0;
9573 while (reset_size != total_reset_size)
9575 remaining_reset_size = total_reset_size - reset_size;
9576 next_reset_size = ((remaining_reset_size >= ww_reset_quantum) ? ww_reset_quantum : remaining_reset_size);
9577 if (next_reset_size)
9579 reset_write_watch_for_gc_heap(start_address, next_reset_size);
9580 reset_size += next_reset_size;
9582 switch_one_quantum();
9586 assert (reset_size == total_reset_size);
9589 // This does a Sleep(1) for every reset ww_reset_quantum bytes of reset
9590 // we do concurrently.
9591 void gc_heap::switch_on_reset (BOOL concurrent_p, size_t* current_total_reset_size, size_t last_reset_size)
9595 *current_total_reset_size += last_reset_size;
9597 dprintf (2, ("reset %Id bytes so far", *current_total_reset_size));
9599 if (*current_total_reset_size > ww_reset_quantum)
9601 switch_one_quantum();
9603 *current_total_reset_size = 0;
9608 void gc_heap::reset_write_watch (BOOL concurrent_p)
9610 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9611 // Software write watch currently requires the runtime to be suspended during reset. See SoftwareWriteWatch::ClearDirty().
9612 assert(!concurrent_p);
9613 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9615 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
9617 PREFIX_ASSUME(seg != NULL);
9619 size_t reset_size = 0;
9620 size_t region_size = 0;
9622 dprintf (2, ("bgc lowest: %Ix, bgc highest: %Ix", background_saved_lowest_address, background_saved_highest_address));
9626 uint8_t* base_address = align_lower_page (heap_segment_mem (seg));
9627 base_address = max (base_address, background_saved_lowest_address);
9629 uint8_t* high_address = 0;
9630 high_address = ((seg == ephemeral_heap_segment) ? alloc_allocated : heap_segment_allocated (seg));
9631 high_address = min (high_address, background_saved_highest_address);
9633 if (base_address < high_address)
9635 region_size = high_address - base_address;
9637 #ifdef TIME_WRITE_WATCH
9638 unsigned int time_start = GetCycleCount32();
9639 #endif //TIME_WRITE_WATCH
9640 dprintf (3, ("h%d: soh ww: [%Ix(%Id)", heap_number, (size_t)base_address, region_size));
9641 //reset_ww_by_chunk (base_address, region_size);
9642 reset_write_watch_for_gc_heap(base_address, region_size);
9644 #ifdef TIME_WRITE_WATCH
9645 unsigned int time_stop = GetCycleCount32();
9646 tot_cycles += time_stop - time_start;
9647 printf ("ResetWriteWatch Duration: %d, total: %d\n",
9648 time_stop - time_start, tot_cycles);
9649 #endif //TIME_WRITE_WATCH
9651 switch_on_reset (concurrent_p, &reset_size, region_size);
9654 seg = heap_segment_next_rw (seg);
9656 concurrent_print_time_delta ("CRWW soh");
9659 //concurrent_print_time_delta ("CRW soh");
9661 seg = heap_segment_rw (generation_start_segment (large_object_generation));
9663 PREFIX_ASSUME(seg != NULL);
9667 uint8_t* base_address = align_lower_page (heap_segment_mem (seg));
9668 uint8_t* high_address = heap_segment_allocated (seg);
9670 base_address = max (base_address, background_saved_lowest_address);
9671 high_address = min (high_address, background_saved_highest_address);
9673 if (base_address < high_address)
9675 region_size = high_address - base_address;
9677 #ifdef TIME_WRITE_WATCH
9678 unsigned int time_start = GetCycleCount32();
9679 #endif //TIME_WRITE_WATCH
9680 dprintf (3, ("h%d: loh ww: [%Ix(%Id)", heap_number, (size_t)base_address, region_size));
9681 //reset_ww_by_chunk (base_address, region_size);
9682 reset_write_watch_for_gc_heap(base_address, region_size);
9684 #ifdef TIME_WRITE_WATCH
9685 unsigned int time_stop = GetCycleCount32();
9686 tot_cycles += time_stop - time_start;
9687 printf ("ResetWriteWatch Duration: %d, total: %d\n",
9688 time_stop - time_start, tot_cycles);
9689 #endif //TIME_WRITE_WATCH
9691 switch_on_reset (concurrent_p, &reset_size, region_size);
9694 seg = heap_segment_next_rw (seg);
9696 concurrent_print_time_delta ("CRWW loh");
9699 #ifdef DEBUG_WRITE_WATCH
9700 debug_write_watch = (uint8_t**)~0;
9701 #endif //DEBUG_WRITE_WATCH
9704 #endif //WRITE_WATCH
9706 #ifdef BACKGROUND_GC
9707 void gc_heap::restart_vm()
9709 //assert (generation_allocation_pointer (youngest_generation) == 0);
9710 dprintf (3, ("Restarting EE"));
9711 STRESS_LOG0(LF_GC, LL_INFO10000, "Concurrent GC: Retarting EE\n");
9712 ee_proceed_event.Set();
9716 void fire_alloc_wait_event (alloc_wait_reason awr, BOOL begin_p)
9718 if (awr != awr_ignored)
9722 FIRE_EVENT(BGCAllocWaitBegin, awr);
9726 FIRE_EVENT(BGCAllocWaitEnd, awr);
9732 void gc_heap::fire_alloc_wait_event_begin (alloc_wait_reason awr)
9734 fire_alloc_wait_event (awr, TRUE);
9738 void gc_heap::fire_alloc_wait_event_end (alloc_wait_reason awr)
9740 fire_alloc_wait_event (awr, FALSE);
9742 #endif //BACKGROUND_GC
9743 void gc_heap::make_generation (generation& gen, heap_segment* seg, uint8_t* start, uint8_t* pointer)
9745 gen.allocation_start = start;
9746 gen.allocation_context.alloc_ptr = pointer;
9747 gen.allocation_context.alloc_limit = pointer;
9748 gen.allocation_context.alloc_bytes = 0;
9749 gen.allocation_context.alloc_bytes_loh = 0;
9750 gen.allocation_context_start_region = pointer;
9751 gen.start_segment = seg;
9752 gen.allocation_segment = seg;
9753 gen.plan_allocation_start = 0;
9754 gen.free_list_space = 0;
9755 gen.pinned_allocated = 0;
9756 gen.free_list_allocated = 0;
9757 gen.end_seg_allocated = 0;
9758 gen.condemned_allocated = 0;
9759 gen.free_obj_space = 0;
9760 gen.allocation_size = 0;
9761 gen.pinned_allocation_sweep_size = 0;
9762 gen.pinned_allocation_compact_size = 0;
9763 gen.allocate_end_seg_p = FALSE;
9764 gen.free_list_allocator.clear();
9766 #ifdef FREE_USAGE_STATS
9767 memset (gen.gen_free_spaces, 0, sizeof (gen.gen_free_spaces));
9768 memset (gen.gen_current_pinned_free_spaces, 0, sizeof (gen.gen_current_pinned_free_spaces));
9769 memset (gen.gen_plugs, 0, sizeof (gen.gen_plugs));
9770 #endif //FREE_USAGE_STATS
9773 void gc_heap::adjust_ephemeral_limits ()
9775 ephemeral_low = generation_allocation_start (generation_of (max_generation - 1));
9776 ephemeral_high = heap_segment_reserved (ephemeral_heap_segment);
9778 dprintf (3, ("new ephemeral low: %Ix new ephemeral high: %Ix",
9779 (size_t)ephemeral_low, (size_t)ephemeral_high))
9781 #ifndef MULTIPLE_HEAPS
9782 // This updates the write barrier helpers with the new info.
9783 stomp_write_barrier_ephemeral(ephemeral_low, ephemeral_high);
9784 #endif // MULTIPLE_HEAPS
9787 #if defined(TRACE_GC) || defined(GC_CONFIG_DRIVEN)
9788 FILE* CreateLogFile(const GCConfigStringHolder& temp_logfile_name, bool is_config)
9792 if (!temp_logfile_name.Get())
9797 char logfile_name[MAX_LONGPATH+1];
9798 uint32_t pid = GCToOSInterface::GetCurrentProcessId();
9799 const char* suffix = is_config ? ".config.log" : ".log";
9800 _snprintf_s(logfile_name, MAX_LONGPATH+1, _TRUNCATE, "%s.%d%s", temp_logfile_name.Get(), pid, suffix);
9801 logFile = fopen(logfile_name, "wb");
9804 #endif //TRACE_GC || GC_CONFIG_DRIVEN
9806 HRESULT gc_heap::initialize_gc (size_t segment_size,
9808 #ifdef MULTIPLE_HEAPS
9809 ,unsigned number_of_heaps
9810 #endif //MULTIPLE_HEAPS
9814 if (GCConfig::GetLogEnabled())
9816 gc_log = CreateLogFile(GCConfig::GetLogFile(), false);
9821 // GCLogFileSize in MBs.
9822 gc_log_file_size = static_cast<size_t>(GCConfig::GetLogFileSize());
9824 if (gc_log_file_size <= 0 || gc_log_file_size > 500)
9830 gc_log_lock.Initialize();
9831 gc_log_buffer = new (nothrow) uint8_t [gc_log_buffer_size];
9838 memset (gc_log_buffer, '*', gc_log_buffer_size);
9840 max_gc_buffers = gc_log_file_size * 1024 * 1024 / gc_log_buffer_size;
9844 #ifdef GC_CONFIG_DRIVEN
9845 if (GCConfig::GetConfigLogEnabled())
9847 gc_config_log = CreateLogFile(GCConfig::GetConfigLogFile(), true);
9849 if (gc_config_log == NULL)
9852 gc_config_log_buffer = new (nothrow) uint8_t [gc_config_log_buffer_size];
9853 if (!gc_config_log_buffer)
9855 fclose(gc_config_log);
9859 compact_ratio = static_cast<int>(GCConfig::GetCompactRatio());
9861 // h# | GC | gen | C | EX | NF | BF | ML | DM || PreS | PostS | Merge | Conv | Pre | Post | PrPo | PreP | PostP |
9862 cprintf (("%2s | %6s | %1s | %1s | %2s | %2s | %2s | %2s | %2s || %5s | %5s | %5s | %5s | %5s | %5s | %5s | %5s | %5s |",
9866 "C", // compaction (empty means sweeping), 'M' means it was mandatory, 'W' means it was not
9867 "EX", // heap expansion
9869 "BF", // best fit (if it indicates neither NF nor BF it means it had to acquire a new seg.
9872 "PreS", // short object before pinned plug
9873 "PostS", // short object after pinned plug
9874 "Merge", // merged pinned plugs
9875 "Conv", // converted to pinned plug
9876 "Pre", // plug before pinned plug but not after
9877 "Post", // plug after pinned plug but not before
9878 "PrPo", // plug both before and after pinned plug
9879 "PreP", // pre short object padded
9880 "PostP" // post short object padded
9883 #endif //GC_CONFIG_DRIVEN
9886 GCConfigStringHolder logFileName = GCConfig::GetMixLogFile();
9887 if (logFileName.Get() != nullptr)
9889 GCStatistics::logFileName = _strdup(logFileName.Get());
9890 GCStatistics::logFile = fopen(GCStatistics::logFileName, "a");
9891 if (!GCStatistics::logFile)
9898 HRESULT hres = S_OK;
9901 hardware_write_watch_api_supported();
9902 #ifdef BACKGROUND_GC
9903 if (can_use_write_watch_for_gc_heap() && GCConfig::GetConcurrentGC())
9905 gc_can_use_concurrent = true;
9906 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9907 virtual_alloc_hardware_write_watch = true;
9908 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9912 gc_can_use_concurrent = false;
9914 #endif //BACKGROUND_GC
9915 #endif //WRITE_WATCH
9917 #ifdef BACKGROUND_GC
9918 // leave the first page to contain only segment info
9919 // because otherwise we could need to revisit the first page frequently in
9921 segment_info_size = OS_PAGE_SIZE;
9923 segment_info_size = Align (sizeof (heap_segment), get_alignment_constant (FALSE));
9924 #endif //BACKGROUND_GC
9926 reserved_memory = 0;
9927 unsigned block_count;
9928 #ifdef MULTIPLE_HEAPS
9929 reserved_memory_limit = (segment_size + heap_size) * number_of_heaps;
9930 block_count = number_of_heaps;
9931 n_heaps = number_of_heaps;
9932 #else //MULTIPLE_HEAPS
9933 reserved_memory_limit = segment_size + heap_size;
9935 #endif //MULTIPLE_HEAPS
9937 if (!reserve_initial_memory(segment_size,heap_size,block_count))
9938 return E_OUTOFMEMORY;
9941 //check if we need to turn on card_bundles.
9942 #ifdef MULTIPLE_HEAPS
9943 // use INT64 arithmetic here because of possible overflow on 32p
9944 uint64_t th = (uint64_t)MH_TH_CARD_BUNDLE*number_of_heaps;
9946 // use INT64 arithmetic here because of possible overflow on 32p
9947 uint64_t th = (uint64_t)SH_TH_CARD_BUNDLE;
9948 #endif //MULTIPLE_HEAPS
9950 if (can_use_write_watch_for_card_table() && reserved_memory >= th)
9952 settings.card_bundles = TRUE;
9956 settings.card_bundles = FALSE;
9958 #endif //CARD_BUNDLE
9960 settings.first_init();
9962 int latency_level_from_config = static_cast<int>(GCConfig::GetLatencyLevel());
9963 if (latency_level_from_config >= latency_level_first && latency_level_from_config <= latency_level_last)
9965 gc_heap::latency_level = static_cast<gc_latency_level>(latency_level_from_config);
9970 g_gc_card_table = make_card_table (g_gc_lowest_address, g_gc_highest_address);
9972 if (!g_gc_card_table)
9973 return E_OUTOFMEMORY;
9977 #ifdef MULTIPLE_HEAPS
9978 g_heaps = new (nothrow) gc_heap* [number_of_heaps];
9980 return E_OUTOFMEMORY;
9983 #pragma warning(push)
9984 #pragma warning(disable:22011) // Suppress PREFast warning about integer underflow/overflow
9986 g_promoted = new (nothrow) size_t [number_of_heaps*16];
9987 g_bpromoted = new (nothrow) size_t [number_of_heaps*16];
9989 g_mark_stack_busy = new (nothrow) int[(number_of_heaps+2)*HS_CACHE_LINE_SIZE/sizeof(int)];
9992 #pragma warning(pop)
9994 if (!g_promoted || !g_bpromoted)
9995 return E_OUTOFMEMORY;
9998 if (!g_mark_stack_busy)
9999 return E_OUTOFMEMORY;
10000 #endif //MH_SC_MARK
10002 if (!create_thread_support (number_of_heaps))
10003 return E_OUTOFMEMORY;
10005 if (!heap_select::init (number_of_heaps))
10006 return E_OUTOFMEMORY;
10008 #endif //MULTIPLE_HEAPS
10010 if (!init_semi_shared())
10018 //Initializes PER_HEAP_ISOLATED data members.
10020 gc_heap::init_semi_shared()
10024 // This is used for heap expansion - it's to fix exactly the start for gen 0
10025 // through (max_generation-1). When we expand the heap we allocate all these
10026 // gen starts at the beginning of the new ephemeral seg.
10027 eph_gen_starts_size = (Align (min_obj_size)) * max_generation;
10030 #ifdef MULTIPLE_HEAPS
10031 mark_list_size = min (150*1024, max (8192, soh_segment_size/(2*10*32)));
10032 g_mark_list = make_mark_list (mark_list_size*n_heaps);
10034 min_balance_threshold = alloc_quantum_balance_units * CLR_SIZE * 2;
10035 #ifdef PARALLEL_MARK_LIST_SORT
10036 g_mark_list_copy = make_mark_list (mark_list_size*n_heaps);
10037 if (!g_mark_list_copy)
10041 #endif //PARALLEL_MARK_LIST_SORT
10043 #else //MULTIPLE_HEAPS
10045 mark_list_size = max (8192, soh_segment_size/(64*32));
10046 g_mark_list = make_mark_list (mark_list_size);
10048 #endif //MULTIPLE_HEAPS
10050 dprintf (3, ("mark_list_size: %d", mark_list_size));
10058 #if defined(SEG_MAPPING_TABLE) && !defined(GROWABLE_SEG_MAPPING_TABLE)
10059 if (!seg_mapping_table_init())
10061 #endif //SEG_MAPPING_TABLE && !GROWABLE_SEG_MAPPING_TABLE
10063 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
10064 seg_table = sorted_table::make_sorted_table();
10068 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
10070 segment_standby_list = 0;
10072 if (!full_gc_approach_event.CreateManualEventNoThrow(FALSE))
10076 if (!full_gc_end_event.CreateManualEventNoThrow(FALSE))
10081 fgn_maxgen_percent = 0;
10082 fgn_loh_percent = 0;
10083 full_gc_approach_event_set = false;
10085 memset (full_gc_counts, 0, sizeof (full_gc_counts));
10088 should_expand_in_full_gc = FALSE;
10090 #ifdef FEATURE_LOH_COMPACTION
10091 loh_compaction_always_p = GCConfig::GetLOHCompactionMode() != 0;
10092 loh_compaction_mode = loh_compaction_default;
10093 #endif //FEATURE_LOH_COMPACTION
10095 #ifdef BACKGROUND_GC
10096 memset (ephemeral_fgc_counts, 0, sizeof (ephemeral_fgc_counts));
10097 bgc_alloc_spin_count = static_cast<uint32_t>(GCConfig::GetBGCSpinCount());
10098 bgc_alloc_spin = static_cast<uint32_t>(GCConfig::GetBGCSpin());
10101 int number_bgc_threads = 1;
10102 #ifdef MULTIPLE_HEAPS
10103 number_bgc_threads = n_heaps;
10104 #endif //MULTIPLE_HEAPS
10105 if (!create_bgc_threads_support (number_bgc_threads))
10110 #endif //BACKGROUND_GC
10112 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
10114 #ifdef GC_CONFIG_DRIVEN
10115 compact_or_sweep_gcs[0] = 0;
10116 compact_or_sweep_gcs[1] = 0;
10117 #endif //GC_CONFIG_DRIVEN
10120 short_plugs_pad_ratio = (double)DESIRED_PLUG_LENGTH / (double)(DESIRED_PLUG_LENGTH - Align (min_obj_size));
10121 #endif //SHORT_PLUGS
10129 if (full_gc_approach_event.IsValid())
10131 full_gc_approach_event.CloseEvent();
10133 if (full_gc_end_event.IsValid())
10135 full_gc_end_event.CloseEvent();
10142 gc_heap* gc_heap::make_gc_heap (
10143 #ifdef MULTIPLE_HEAPS
10146 #endif //MULTIPLE_HEAPS
10151 #ifdef MULTIPLE_HEAPS
10152 res = new (nothrow) gc_heap;
10156 res->vm_heap = vm_hp;
10157 res->alloc_context_count = 0;
10160 #ifdef PARALLEL_MARK_LIST_SORT
10161 res->mark_list_piece_start = new (nothrow) uint8_t**[n_heaps];
10162 if (!res->mark_list_piece_start)
10166 #pragma warning(push)
10167 #pragma warning(disable:22011) // Suppress PREFast warning about integer underflow/overflow
10168 #endif // _PREFAST_
10169 res->mark_list_piece_end = new (nothrow) uint8_t**[n_heaps + 32]; // +32 is padding to reduce false sharing
10171 #pragma warning(pop)
10172 #endif // _PREFAST_
10174 if (!res->mark_list_piece_end)
10176 #endif //PARALLEL_MARK_LIST_SORT
10180 #endif //MULTIPLE_HEAPS
10182 if (res->init_gc_heap (
10183 #ifdef MULTIPLE_HEAPS
10185 #else //MULTIPLE_HEAPS
10187 #endif //MULTIPLE_HEAPS
10193 #ifdef MULTIPLE_HEAPS
10196 return (gc_heap*)1;
10197 #endif //MULTIPLE_HEAPS
10201 gc_heap::wait_for_gc_done(int32_t timeOut)
10203 bool cooperative_mode = enable_preemptive ();
10205 uint32_t dwWaitResult = NOERROR;
10207 gc_heap* wait_heap = NULL;
10208 while (gc_heap::gc_started)
10210 #ifdef MULTIPLE_HEAPS
10211 wait_heap = GCHeap::GetHeap(heap_select::select_heap(NULL, 0))->pGenGCHeap;
10212 dprintf(2, ("waiting for the gc_done_event on heap %d", wait_heap->heap_number));
10213 #endif // MULTIPLE_HEAPS
10216 PREFIX_ASSUME(wait_heap != NULL);
10217 #endif // _PREFAST_
10219 dwWaitResult = wait_heap->gc_done_event.Wait(timeOut, FALSE);
10221 disable_preemptive (cooperative_mode);
10223 return dwWaitResult;
10227 gc_heap::set_gc_done()
10229 enter_gc_done_event_lock();
10230 if (!gc_done_event_set)
10232 gc_done_event_set = true;
10233 dprintf (2, ("heap %d: setting gc_done_event", heap_number));
10234 gc_done_event.Set();
10236 exit_gc_done_event_lock();
10240 gc_heap::reset_gc_done()
10242 enter_gc_done_event_lock();
10243 if (gc_done_event_set)
10245 gc_done_event_set = false;
10246 dprintf (2, ("heap %d: resetting gc_done_event", heap_number));
10247 gc_done_event.Reset();
10249 exit_gc_done_event_lock();
10253 gc_heap::enter_gc_done_event_lock()
10255 uint32_t dwSwitchCount = 0;
10258 if (Interlocked::CompareExchange(&gc_done_event_lock, 0, -1) >= 0)
10260 while (gc_done_event_lock >= 0)
10262 if (g_num_processors > 1)
10264 int spin_count = 32 * g_num_processors;
10265 for (int j = 0; j < spin_count; j++)
10267 if (gc_done_event_lock < 0)
10269 YieldProcessor(); // indicate to the processor that we are spining
10271 if (gc_done_event_lock >= 0)
10272 GCToOSInterface::YieldThread(++dwSwitchCount);
10275 GCToOSInterface::YieldThread(++dwSwitchCount);
10282 gc_heap::exit_gc_done_event_lock()
10284 gc_done_event_lock = -1;
10287 #ifndef MULTIPLE_HEAPS
10289 #ifdef RECORD_LOH_STATE
10290 int gc_heap::loh_state_index = 0;
10291 gc_heap::loh_state_info gc_heap::last_loh_states[max_saved_loh_states];
10292 #endif //RECORD_LOH_STATE
10294 VOLATILE(int32_t) gc_heap::gc_done_event_lock;
10295 VOLATILE(bool) gc_heap::gc_done_event_set;
10296 GCEvent gc_heap::gc_done_event;
10297 #endif //!MULTIPLE_HEAPS
10298 VOLATILE(bool) gc_heap::internal_gc_done;
10300 void gc_heap::add_saved_spinlock_info (
10301 msl_enter_state enter_state,
10302 msl_take_state take_state)
10305 #ifdef SPINLOCK_HISTORY
10306 spinlock_info* current = &last_spinlock_info[spinlock_info_index];
10308 current->enter_state = enter_state;
10309 current->take_state = take_state;
10310 current->thread_id.SetToCurrentThread();
10312 spinlock_info_index++;
10314 assert (spinlock_info_index <= max_saved_spinlock_info);
10316 if (spinlock_info_index >= max_saved_spinlock_info)
10318 spinlock_info_index = 0;
10321 MAYBE_UNUSED_VAR(enter_state);
10322 MAYBE_UNUSED_VAR(take_state);
10323 #endif //SPINLOCK_HISTORY
10327 gc_heap::init_gc_heap (int h_number)
10329 #ifdef MULTIPLE_HEAPS
10333 #ifdef SPINLOCK_HISTORY
10334 spinlock_info_index = 0;
10335 memset (last_spinlock_info, 0, sizeof(last_spinlock_info));
10336 #endif //SPINLOCK_HISTORY
10338 // initialize per heap members.
10339 ephemeral_low = (uint8_t*)1;
10341 ephemeral_high = MAX_PTR;
10343 ephemeral_heap_segment = 0;
10345 freeable_large_heap_segment = 0;
10347 condemned_generation_num = 0;
10349 blocking_collection = FALSE;
10351 generation_skip_ratio = 100;
10353 mark_stack_tos = 0;
10355 mark_stack_bos = 0;
10357 mark_stack_array_length = 0;
10359 mark_stack_array = 0;
10361 verify_pinned_queue_p = FALSE;
10363 loh_pinned_queue_tos = 0;
10365 loh_pinned_queue_bos = 0;
10367 loh_pinned_queue_length = 0;
10369 loh_pinned_queue_decay = LOH_PIN_DECAY;
10371 loh_pinned_queue = 0;
10373 min_overflow_address = MAX_PTR;
10375 max_overflow_address = 0;
10377 gen0_bricks_cleared = FALSE;
10379 gen0_must_clear_bricks = 0;
10381 allocation_quantum = CLR_SIZE;
10383 more_space_lock = gc_lock;
10385 ro_segments_in_range = FALSE;
10387 loh_alloc_since_cg = 0;
10389 new_heap_segment = NULL;
10391 #ifdef RECORD_LOH_STATE
10392 loh_state_index = 0;
10393 #endif //RECORD_LOH_STATE
10394 #endif //MULTIPLE_HEAPS
10396 #ifdef MULTIPLE_HEAPS
10397 if (h_number > n_heaps)
10399 assert (!"Number of heaps exceeded");
10403 heap_number = h_number;
10404 #endif //MULTIPLE_HEAPS
10406 memset (&oom_info, 0, sizeof (oom_info));
10407 memset (&fgm_result, 0, sizeof (fgm_result));
10408 if (!gc_done_event.CreateManualEventNoThrow(FALSE))
10412 gc_done_event_lock = -1;
10413 gc_done_event_set = false;
10415 #ifndef SEG_MAPPING_TABLE
10416 if (!gc_heap::seg_table->ensure_space_for_insert ())
10420 #endif //!SEG_MAPPING_TABLE
10422 heap_segment* seg = get_initial_segment (soh_segment_size, h_number);
10426 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(seg),
10427 (size_t)(heap_segment_reserved (seg) - heap_segment_mem(seg)),
10428 gc_etw_segment_small_object_heap);
10430 #ifdef SEG_MAPPING_TABLE
10431 seg_mapping_table_add_segment (seg, __this);
10432 #else //SEG_MAPPING_TABLE
10433 seg_table->insert ((uint8_t*)seg, sdelta);
10434 #endif //SEG_MAPPING_TABLE
10436 #ifdef MULTIPLE_HEAPS
10437 heap_segment_heap (seg) = this;
10438 #endif //MULTIPLE_HEAPS
10440 /* todo: Need a global lock for this */
10441 uint32_t* ct = &g_gc_card_table [card_word (card_of (g_gc_lowest_address))];
10442 own_card_table (ct);
10443 card_table = translate_card_table (ct);
10444 /* End of global lock */
10446 brick_table = card_table_brick_table (ct);
10447 highest_address = card_table_highest_address (ct);
10448 lowest_address = card_table_lowest_address (ct);
10451 card_bundle_table = translate_card_bundle_table (card_table_card_bundle_table (ct), g_gc_lowest_address);
10452 assert (&card_bundle_table [card_bundle_word (cardw_card_bundle (card_word (card_of (g_gc_lowest_address))))] ==
10453 card_table_card_bundle_table (ct));
10454 #endif //CARD_BUNDLE
10457 if (gc_can_use_concurrent)
10458 mark_array = translate_mark_array (card_table_mark_array (&g_gc_card_table[card_word (card_of (g_gc_lowest_address))]));
10461 #endif //MARK_ARRAY
10463 uint8_t* start = heap_segment_mem (seg);
10465 for (int i = 0; i < 1 + max_generation; i++)
10467 make_generation (generation_table [ (max_generation - i) ],
10469 generation_table [(max_generation - i)].gen_num = max_generation - i;
10470 start += Align (min_obj_size);
10473 heap_segment_allocated (seg) = start;
10474 alloc_allocated = start;
10475 heap_segment_used (seg) = start - plug_skew;
10477 ephemeral_heap_segment = seg;
10479 #ifndef SEG_MAPPING_TABLE
10480 if (!gc_heap::seg_table->ensure_space_for_insert ())
10484 #endif //!SEG_MAPPING_TABLE
10485 //Create the large segment generation
10486 heap_segment* lseg = get_initial_segment(min_loh_segment_size, h_number);
10489 lseg->flags |= heap_segment_flags_loh;
10491 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(lseg),
10492 (size_t)(heap_segment_reserved (lseg) - heap_segment_mem(lseg)),
10493 gc_etw_segment_large_object_heap);
10495 #ifdef SEG_MAPPING_TABLE
10496 seg_mapping_table_add_segment (lseg, __this);
10497 #else //SEG_MAPPING_TABLE
10498 seg_table->insert ((uint8_t*)lseg, sdelta);
10499 #endif //SEG_MAPPING_TABLE
10501 generation_table [max_generation].free_list_allocator = allocator(NUM_GEN2_ALIST, BASE_GEN2_ALIST, gen2_alloc_list);
10502 //assign the alloc_list for the large generation
10503 generation_table [max_generation+1].free_list_allocator = allocator(NUM_LOH_ALIST, BASE_LOH_ALIST, loh_alloc_list);
10504 generation_table [max_generation+1].gen_num = max_generation+1;
10505 make_generation (generation_table [max_generation+1],lseg, heap_segment_mem (lseg), 0);
10506 heap_segment_allocated (lseg) = heap_segment_mem (lseg) + Align (min_obj_size, get_alignment_constant (FALSE));
10507 heap_segment_used (lseg) = heap_segment_allocated (lseg) - plug_skew;
10509 for (int gen_num = 0; gen_num <= 1 + max_generation; gen_num++)
10511 generation* gen = generation_of (gen_num);
10512 make_unused_array (generation_allocation_start (gen), Align (min_obj_size));
10515 #ifdef MULTIPLE_HEAPS
10516 heap_segment_heap (lseg) = this;
10518 //initialize the alloc context heap
10519 generation_alloc_context (generation_of (0))->set_alloc_heap(vm_heap);
10521 //initialize the alloc context heap
10522 generation_alloc_context (generation_of (max_generation+1))->set_alloc_heap(vm_heap);
10524 #endif //MULTIPLE_HEAPS
10526 //Do this only once
10527 #ifdef MULTIPLE_HEAPS
10529 #endif //MULTIPLE_HEAPS
10531 #ifndef INTERIOR_POINTERS
10532 //set the brick_table for large objects
10533 //but default value is clearded
10534 //clear_brick_table ((uint8_t*)heap_segment_mem (lseg),
10535 // (uint8_t*)heap_segment_reserved (lseg));
10537 #else //INTERIOR_POINTERS
10539 //Because of the interior pointer business, we have to clear
10540 //the whole brick table
10541 //but the default value is cleared
10542 // clear_brick_table (lowest_address, highest_address);
10543 #endif //INTERIOR_POINTERS
10546 if (!init_dynamic_data())
10551 etw_allocation_running_amount[0] = 0;
10552 etw_allocation_running_amount[1] = 0;
10554 //needs to be done after the dynamic data has been initialized
10555 #ifndef MULTIPLE_HEAPS
10556 allocation_running_amount = dd_min_size (dynamic_data_of (0));
10557 #endif //!MULTIPLE_HEAPS
10559 fgn_last_alloc = dd_min_size (dynamic_data_of (0));
10561 mark* arr = new (nothrow) (mark [MARK_STACK_INITIAL_LENGTH]);
10565 make_mark_stack(arr);
10567 #ifdef BACKGROUND_GC
10568 freeable_small_heap_segment = 0;
10569 gchist_index_per_heap = 0;
10570 uint8_t** b_arr = new (nothrow) (uint8_t* [MARK_STACK_INITIAL_LENGTH]);
10574 make_background_mark_stack (b_arr);
10575 #endif //BACKGROUND_GC
10577 ephemeral_low = generation_allocation_start(generation_of(max_generation - 1));
10578 ephemeral_high = heap_segment_reserved(ephemeral_heap_segment);
10579 if (heap_number == 0)
10581 stomp_write_barrier_initialize(
10582 #ifdef MULTIPLE_HEAPS
10583 reinterpret_cast<uint8_t*>(1), reinterpret_cast<uint8_t*>(~0)
10585 ephemeral_low, ephemeral_high
10586 #endif //!MULTIPLE_HEAPS
10591 // why would we clear the mark array for this page? it should be cleared..
10592 // clear the first committed page
10593 //if(gc_can_use_concurrent)
10595 // clear_mark_array (align_lower_page (heap_segment_mem (seg)), heap_segment_committed (seg));
10597 #endif //MARK_ARRAY
10599 #ifdef MULTIPLE_HEAPS
10600 //register the heap in the heaps array
10602 if (!create_gc_thread ())
10605 g_heaps [heap_number] = this;
10607 #endif //MULTIPLE_HEAPS
10609 #ifdef FEATURE_PREMORTEM_FINALIZATION
10610 HRESULT hr = AllocateCFinalize(&finalize_queue);
10613 #endif // FEATURE_PREMORTEM_FINALIZATION
10615 max_free_space_items = MAX_NUM_FREE_SPACES;
10617 bestfit_seg = new (nothrow) seg_free_spaces (heap_number);
10624 if (!bestfit_seg->alloc())
10629 last_gc_before_oom = FALSE;
10631 #ifdef MULTIPLE_HEAPS
10633 #ifdef HEAP_ANALYZE
10635 heap_analyze_success = TRUE;
10637 internal_root_array = 0;
10639 internal_root_array_index = 0;
10641 internal_root_array_length = initial_internal_roots;
10645 current_obj_size = 0;
10647 #endif //HEAP_ANALYZE
10649 #endif // MULTIPLE_HEAPS
10651 #ifdef BACKGROUND_GC
10652 bgc_thread_id.Clear();
10654 if (!create_bgc_thread_support())
10659 bgc_alloc_lock = new (nothrow) exclusive_sync;
10660 if (!bgc_alloc_lock)
10665 bgc_alloc_lock->init();
10669 if (!recursive_gc_sync::init())
10673 bgc_thread_running = 0;
10675 bgc_threads_timeout_cs.Initialize();
10676 expanded_in_fgc = 0;
10677 current_bgc_state = bgc_not_in_process;
10678 background_soh_alloc_count = 0;
10679 background_loh_alloc_count = 0;
10680 bgc_overflow_count = 0;
10681 end_loh_size = dd_min_size (dynamic_data_of (max_generation + 1));
10682 #endif //BACKGROUND_GC
10684 #ifdef GC_CONFIG_DRIVEN
10685 memset (interesting_data_per_heap, 0, sizeof (interesting_data_per_heap));
10686 memset(compact_reasons_per_heap, 0, sizeof (compact_reasons_per_heap));
10687 memset(expand_mechanisms_per_heap, 0, sizeof (expand_mechanisms_per_heap));
10688 memset(interesting_mechanism_bits_per_heap, 0, sizeof (interesting_mechanism_bits_per_heap));
10689 #endif //GC_CONFIG_DRIVEN
10695 gc_heap::destroy_semi_shared()
10697 //TODO: will need to move this to per heap
10698 //#ifdef BACKGROUND_GC
10699 // if (c_mark_list)
10700 // delete c_mark_list;
10701 //#endif //BACKGROUND_GC
10705 delete g_mark_list;
10708 #if defined(SEG_MAPPING_TABLE) && !defined(GROWABLE_SEG_MAPPING_TABLE)
10709 if (seg_mapping_table)
10710 delete seg_mapping_table;
10711 #endif //SEG_MAPPING_TABLE && !GROWABLE_SEG_MAPPING_TABLE
10713 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
10714 //destroy the segment map
10715 seg_table->delete_sorted_table();
10716 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
10720 gc_heap::self_destroy()
10722 #ifdef BACKGROUND_GC
10724 #endif //BACKGROUND_GC
10726 if (gc_done_event.IsValid())
10728 gc_done_event.CloseEvent();
10731 // destroy every segment.
10732 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
10734 PREFIX_ASSUME(seg != NULL);
10736 heap_segment* next_seg;
10739 next_seg = heap_segment_next_rw (seg);
10740 delete_heap_segment (seg);
10744 seg = heap_segment_rw (generation_start_segment (generation_of (max_generation+1)));
10746 PREFIX_ASSUME(seg != NULL);
10750 next_seg = heap_segment_next_rw (seg);
10751 delete_heap_segment (seg);
10755 // get rid of the card table
10756 release_card_table (card_table);
10758 // destroy the mark stack
10759 delete mark_stack_array;
10761 #ifdef FEATURE_PREMORTEM_FINALIZATION
10762 if (finalize_queue)
10763 delete finalize_queue;
10764 #endif // FEATURE_PREMORTEM_FINALIZATION
10768 gc_heap::destroy_gc_heap(gc_heap* heap)
10770 heap->self_destroy();
10774 // Destroys resources owned by gc. It is assumed that a last GC has been performed and that
10775 // the finalizer queue has been drained.
10776 void gc_heap::shutdown_gc()
10778 destroy_semi_shared();
10780 #ifdef MULTIPLE_HEAPS
10781 //delete the heaps array
10783 destroy_thread_support();
10785 #endif //MULTIPLE_HEAPS
10786 //destroy seg_manager
10788 destroy_initial_memory();
10790 GCToOSInterface::Shutdown();
10794 BOOL gc_heap::size_fit_p (size_t size REQD_ALIGN_AND_OFFSET_DCL, uint8_t* alloc_pointer, uint8_t* alloc_limit,
10795 uint8_t* old_loc, int use_padding)
10797 BOOL already_padded = FALSE;
10799 if ((old_loc != 0) && (use_padding & USE_PADDING_FRONT))
10801 alloc_pointer = alloc_pointer + Align (min_obj_size);
10802 already_padded = TRUE;
10804 #endif //SHORT_PLUGS
10806 if (!((old_loc == 0) || same_large_alignment_p (old_loc, alloc_pointer)))
10807 size = size + switch_alignment_size (already_padded);
10809 #ifdef FEATURE_STRUCTALIGN
10810 alloc_pointer = StructAlign(alloc_pointer, requiredAlignment, alignmentOffset);
10811 #endif // FEATURE_STRUCTALIGN
10813 // in allocate_in_condemned_generation we can have this when we
10814 // set the alloc_limit to plan_allocated which could be less than
10816 if (alloc_limit < alloc_pointer)
10823 return (((size_t)(alloc_limit - alloc_pointer) >= (size + ((use_padding & USE_PADDING_TAIL)? Align(min_obj_size) : 0)))
10825 ||((!(use_padding & USE_PADDING_FRONT)) && ((alloc_pointer + size) == alloc_limit))
10826 #else //SHORT_PLUGS
10827 ||((alloc_pointer + size) == alloc_limit)
10828 #endif //SHORT_PLUGS
10833 assert (size == Align (min_obj_size));
10834 return ((size_t)(alloc_limit - alloc_pointer) >= size);
10839 BOOL gc_heap::a_size_fit_p (size_t size, uint8_t* alloc_pointer, uint8_t* alloc_limit,
10842 // We could have run into cases where this is true when alloc_allocated is the
10843 // the same as the seg committed.
10844 if (alloc_limit < alloc_pointer)
10849 return ((size_t)(alloc_limit - alloc_pointer) >= (size + Align(min_obj_size, align_const)));
10852 // Grow by committing more pages
10853 BOOL gc_heap::grow_heap_segment (heap_segment* seg, uint8_t* high_address)
10855 assert (high_address <= heap_segment_reserved (seg));
10857 //return 0 if we are at the end of the segment.
10858 if (align_on_page (high_address) > heap_segment_reserved (seg))
10861 if (high_address <= heap_segment_committed (seg))
10864 size_t c_size = align_on_page ((size_t)(high_address - heap_segment_committed (seg)));
10865 c_size = max (c_size, 16*OS_PAGE_SIZE);
10866 c_size = min (c_size, (size_t)(heap_segment_reserved (seg) - heap_segment_committed (seg)));
10871 STRESS_LOG2(LF_GC, LL_INFO10000,
10872 "Growing heap_segment: %Ix high address: %Ix\n",
10873 (size_t)seg, (size_t)high_address);
10875 dprintf(3, ("Growing segment allocation %Ix %Ix", (size_t)heap_segment_committed(seg),c_size));
10877 if (!virtual_alloc_commit_for_heap(heap_segment_committed (seg), c_size, heap_number))
10879 dprintf(3, ("Cannot grow heap segment"));
10883 #ifndef BACKGROUND_GC
10884 clear_mark_array (heap_segment_committed (seg),
10885 heap_segment_committed (seg)+c_size, TRUE);
10886 #endif //BACKGROUND_GC
10887 #endif //MARK_ARRAY
10888 heap_segment_committed (seg) += c_size;
10889 STRESS_LOG1(LF_GC, LL_INFO10000, "New commit: %Ix",
10890 (size_t)heap_segment_committed (seg));
10892 assert (heap_segment_committed (seg) <= heap_segment_reserved (seg));
10894 assert (high_address <= heap_segment_committed (seg));
10900 int gc_heap::grow_heap_segment (heap_segment* seg, uint8_t* allocated, uint8_t* old_loc, size_t size, BOOL pad_front_p REQD_ALIGN_AND_OFFSET_DCL)
10903 if ((old_loc != 0) && pad_front_p)
10905 allocated = allocated + Align (min_obj_size);
10907 #endif //SHORT_PLUGS
10909 if (!((old_loc == 0) || same_large_alignment_p (old_loc, allocated)))
10910 size = size + switch_alignment_size (FALSE);
10911 #ifdef FEATURE_STRUCTALIGN
10912 size_t pad = ComputeStructAlignPad(allocated, requiredAlignment, alignmentOffset);
10913 return grow_heap_segment (seg, allocated + pad + size);
10914 #else // FEATURE_STRUCTALIGN
10915 return grow_heap_segment (seg, allocated + size);
10916 #endif // FEATURE_STRUCTALIGN
10919 //used only in older generation allocation (i.e during gc).
10920 void gc_heap::adjust_limit (uint8_t* start, size_t limit_size, generation* gen,
10923 UNREFERENCED_PARAMETER(gennum);
10924 dprintf (3, ("gc Expanding segment allocation"));
10925 heap_segment* seg = generation_allocation_segment (gen);
10926 if ((generation_allocation_limit (gen) != start) || (start != heap_segment_plan_allocated (seg)))
10928 if (generation_allocation_limit (gen) == heap_segment_plan_allocated (seg))
10930 assert (generation_allocation_pointer (gen) >= heap_segment_mem (seg));
10931 assert (generation_allocation_pointer (gen) <= heap_segment_committed (seg));
10932 heap_segment_plan_allocated (generation_allocation_segment (gen)) = generation_allocation_pointer (gen);
10936 uint8_t* hole = generation_allocation_pointer (gen);
10937 size_t size = (generation_allocation_limit (gen) - generation_allocation_pointer (gen));
10941 dprintf (3, ("filling up hole: %Ix, size %Ix", hole, size));
10942 size_t allocated_size = generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen);
10943 if (size >= Align (min_free_list))
10945 if (allocated_size < min_free_list)
10947 if (size >= (Align (min_free_list) + Align (min_obj_size)))
10949 //split hole into min obj + threadable free item
10950 make_unused_array (hole, min_obj_size);
10951 generation_free_obj_space (gen) += Align (min_obj_size);
10952 make_unused_array (hole + Align (min_obj_size), size - Align (min_obj_size));
10953 generation_free_list_space (gen) += size - Align (min_obj_size);
10954 generation_allocator(gen)->thread_item_front (hole + Align (min_obj_size),
10955 size - Align (min_obj_size));
10956 add_gen_free (gen->gen_num, (size - Align (min_obj_size)));
10960 dprintf (3, ("allocated size too small, can't put back rest on free list %Ix", allocated_size));
10961 make_unused_array (hole, size);
10962 generation_free_obj_space (gen) += size;
10967 dprintf (3, ("threading hole in front of free list"));
10968 make_unused_array (hole, size);
10969 generation_free_list_space (gen) += size;
10970 generation_allocator(gen)->thread_item_front (hole, size);
10971 add_gen_free (gen->gen_num, size);
10976 make_unused_array (hole, size);
10977 generation_free_obj_space (gen) += size;
10981 generation_allocation_pointer (gen) = start;
10982 generation_allocation_context_start_region (gen) = start;
10984 generation_allocation_limit (gen) = (start + limit_size);
10987 void verify_mem_cleared (uint8_t* start, size_t size)
10989 if (!Aligned (size))
10994 PTR_PTR curr_ptr = (PTR_PTR) start;
10995 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
10997 if (*(curr_ptr++) != 0)
11004 #if defined (VERIFY_HEAP) && defined (BACKGROUND_GC)
11005 void gc_heap::set_batch_mark_array_bits (uint8_t* start, uint8_t* end)
11007 size_t start_mark_bit = mark_bit_of (start);
11008 size_t end_mark_bit = mark_bit_of (end);
11009 unsigned int startbit = mark_bit_bit (start_mark_bit);
11010 unsigned int endbit = mark_bit_bit (end_mark_bit);
11011 size_t startwrd = mark_bit_word (start_mark_bit);
11012 size_t endwrd = mark_bit_word (end_mark_bit);
11014 dprintf (3, ("Setting all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
11015 (size_t)start, (size_t)start_mark_bit,
11016 (size_t)end, (size_t)end_mark_bit));
11018 unsigned int firstwrd = ~(lowbits (~0, startbit));
11019 unsigned int lastwrd = ~(highbits (~0, endbit));
11021 if (startwrd == endwrd)
11023 unsigned int wrd = firstwrd & lastwrd;
11024 mark_array[startwrd] |= wrd;
11028 // set the first mark word.
11031 mark_array[startwrd] |= firstwrd;
11035 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
11037 mark_array[wrdtmp] = ~(unsigned int)0;
11040 // set the last mark word.
11043 mark_array[endwrd] |= lastwrd;
11047 // makes sure that the mark array bits between start and end are 0.
11048 void gc_heap::check_batch_mark_array_bits (uint8_t* start, uint8_t* end)
11050 size_t start_mark_bit = mark_bit_of (start);
11051 size_t end_mark_bit = mark_bit_of (end);
11052 unsigned int startbit = mark_bit_bit (start_mark_bit);
11053 unsigned int endbit = mark_bit_bit (end_mark_bit);
11054 size_t startwrd = mark_bit_word (start_mark_bit);
11055 size_t endwrd = mark_bit_word (end_mark_bit);
11057 //dprintf (3, ("Setting all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
11058 // (size_t)start, (size_t)start_mark_bit,
11059 // (size_t)end, (size_t)end_mark_bit));
11061 unsigned int firstwrd = ~(lowbits (~0, startbit));
11062 unsigned int lastwrd = ~(highbits (~0, endbit));
11064 if (startwrd == endwrd)
11066 unsigned int wrd = firstwrd & lastwrd;
11067 if (mark_array[startwrd] & wrd)
11069 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11071 mark_array [startwrd], mark_word_address (startwrd)));
11077 // set the first mark word.
11080 if (mark_array[startwrd] & firstwrd)
11082 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11083 firstwrd, startwrd,
11084 mark_array [startwrd], mark_word_address (startwrd)));
11091 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
11093 if (mark_array[wrdtmp])
11095 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11097 mark_array [wrdtmp], mark_word_address (wrdtmp)));
11102 // set the last mark word.
11105 if (mark_array[endwrd] & lastwrd)
11107 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11109 mark_array [lastwrd], mark_word_address (lastwrd)));
11114 #endif //VERIFY_HEAP && BACKGROUND_GC
11116 allocator::allocator (unsigned int num_b, size_t fbs, alloc_list* b)
11118 assert (num_b < MAX_BUCKET_COUNT);
11119 num_buckets = num_b;
11120 frst_bucket_size = fbs;
11124 alloc_list& allocator::alloc_list_of (unsigned int bn)
11126 assert (bn < num_buckets);
11128 return first_bucket;
11130 return buckets [bn-1];
11133 size_t& allocator::alloc_list_damage_count_of (unsigned int bn)
11135 assert (bn < num_buckets);
11137 return first_bucket.alloc_list_damage_count();
11139 return buckets [bn-1].alloc_list_damage_count();
11142 void allocator::unlink_item (unsigned int bn, uint8_t* item, uint8_t* prev_item, BOOL use_undo_p)
11144 //unlink the free_item
11145 alloc_list* al = &alloc_list_of (bn);
11148 if (use_undo_p && (free_list_undo (prev_item) == UNDO_EMPTY))
11150 assert (item == free_list_slot (prev_item));
11151 free_list_undo (prev_item) = item;
11152 alloc_list_damage_count_of (bn)++;
11154 free_list_slot (prev_item) = free_list_slot(item);
11158 al->alloc_list_head() = (uint8_t*)free_list_slot(item);
11160 if (al->alloc_list_tail() == item)
11162 al->alloc_list_tail() = prev_item;
11166 void allocator::clear()
11168 for (unsigned int i = 0; i < num_buckets; i++)
11170 alloc_list_head_of (i) = 0;
11171 alloc_list_tail_of (i) = 0;
11175 //always thread to the end.
11176 void allocator::thread_free_item (uint8_t* item, uint8_t*& head, uint8_t*& tail)
11178 free_list_slot (item) = 0;
11179 free_list_undo (item) = UNDO_EMPTY;
11180 assert (item != head);
11186 //TODO: This shouldn't happen anymore - verify that's the case.
11187 //the following is necessary because the last free element
11188 //may have been truncated, and tail isn't updated.
11189 else if (free_list_slot (head) == 0)
11191 free_list_slot (head) = item;
11195 assert (item != tail);
11196 assert (free_list_slot(tail) == 0);
11197 free_list_slot (tail) = item;
11202 void allocator::thread_item (uint8_t* item, size_t size)
11204 size_t sz = frst_bucket_size;
11205 unsigned int a_l_number = 0;
11207 for (; a_l_number < (num_buckets-1); a_l_number++)
11215 alloc_list* al = &alloc_list_of (a_l_number);
11216 thread_free_item (item,
11217 al->alloc_list_head(),
11218 al->alloc_list_tail());
11221 void allocator::thread_item_front (uint8_t* item, size_t size)
11223 //find right free list
11224 size_t sz = frst_bucket_size;
11225 unsigned int a_l_number = 0;
11226 for (; a_l_number < (num_buckets-1); a_l_number++)
11234 alloc_list* al = &alloc_list_of (a_l_number);
11235 free_list_slot (item) = al->alloc_list_head();
11236 free_list_undo (item) = UNDO_EMPTY;
11238 if (al->alloc_list_tail() == 0)
11240 al->alloc_list_tail() = al->alloc_list_head();
11242 al->alloc_list_head() = item;
11243 if (al->alloc_list_tail() == 0)
11245 al->alloc_list_tail() = item;
11249 void allocator::copy_to_alloc_list (alloc_list* toalist)
11251 for (unsigned int i = 0; i < num_buckets; i++)
11253 toalist [i] = alloc_list_of (i);
11254 #ifdef FL_VERIFICATION
11255 uint8_t* free_item = alloc_list_head_of (i);
11260 free_item = free_list_slot (free_item);
11263 toalist[i].item_count = count;
11264 #endif //FL_VERIFICATION
11268 void allocator::copy_from_alloc_list (alloc_list* fromalist)
11270 BOOL repair_list = !discard_if_no_fit_p ();
11271 for (unsigned int i = 0; i < num_buckets; i++)
11273 size_t count = alloc_list_damage_count_of (i);
11274 alloc_list_of (i) = fromalist [i];
11275 assert (alloc_list_damage_count_of (i) == 0);
11279 //repair the the list
11280 //new items may have been added during the plan phase
11281 //items may have been unlinked.
11282 uint8_t* free_item = alloc_list_head_of (i);
11283 while (free_item && count)
11285 assert (((CObjectHeader*)free_item)->IsFree());
11286 if ((free_list_undo (free_item) != UNDO_EMPTY))
11289 free_list_slot (free_item) = free_list_undo (free_item);
11290 free_list_undo (free_item) = UNDO_EMPTY;
11293 free_item = free_list_slot (free_item);
11296 #ifdef FL_VERIFICATION
11297 free_item = alloc_list_head_of (i);
11298 size_t item_count = 0;
11302 free_item = free_list_slot (free_item);
11305 assert (item_count == alloc_list_of (i).item_count);
11306 #endif //FL_VERIFICATION
11309 uint8_t* tail_item = alloc_list_tail_of (i);
11310 assert ((tail_item == 0) || (free_list_slot (tail_item) == 0));
11315 void allocator::commit_alloc_list_changes()
11317 BOOL repair_list = !discard_if_no_fit_p ();
11320 for (unsigned int i = 0; i < num_buckets; i++)
11322 //remove the undo info from list.
11323 uint8_t* free_item = alloc_list_head_of (i);
11324 size_t count = alloc_list_damage_count_of (i);
11325 while (free_item && count)
11327 assert (((CObjectHeader*)free_item)->IsFree());
11329 if (free_list_undo (free_item) != UNDO_EMPTY)
11331 free_list_undo (free_item) = UNDO_EMPTY;
11335 free_item = free_list_slot (free_item);
11338 alloc_list_damage_count_of (i) = 0;
11343 void gc_heap::adjust_limit_clr (uint8_t* start, size_t limit_size,
11344 alloc_context* acontext, heap_segment* seg,
11345 int align_const, int gen_number)
11347 size_t aligned_min_obj_size = Align(min_obj_size, align_const);
11349 //probably should pass seg==0 for free lists.
11352 assert (heap_segment_used (seg) <= heap_segment_committed (seg));
11355 dprintf (3, ("Expanding segment allocation [%Ix, %Ix[", (size_t)start,
11356 (size_t)start + limit_size - aligned_min_obj_size));
11358 if ((acontext->alloc_limit != start) &&
11359 (acontext->alloc_limit + aligned_min_obj_size)!= start)
11361 uint8_t* hole = acontext->alloc_ptr;
11364 size_t size = (acontext->alloc_limit - acontext->alloc_ptr);
11365 dprintf (3, ("filling up hole [%Ix, %Ix[", (size_t)hole, (size_t)hole + size + Align (min_obj_size, align_const)));
11366 // when we are finishing an allocation from a free list
11367 // we know that the free area was Align(min_obj_size) larger
11368 acontext->alloc_bytes -= size;
11369 size_t free_obj_size = size + aligned_min_obj_size;
11370 make_unused_array (hole, free_obj_size);
11371 generation_free_obj_space (generation_of (gen_number)) += free_obj_size;
11373 acontext->alloc_ptr = start;
11377 // If the next alloc context is right up against the current one it means we are absorbing the min
11378 // object, so need to account for that.
11379 acontext->alloc_bytes += (start - acontext->alloc_limit);
11382 acontext->alloc_limit = (start + limit_size - aligned_min_obj_size);
11383 acontext->alloc_bytes += limit_size - ((gen_number < max_generation + 1) ? aligned_min_obj_size : 0);
11385 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
11388 AppDomain* alloc_appdomain = GetAppDomain();
11389 alloc_appdomain->RecordAllocBytes (limit_size, heap_number);
11391 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
11393 uint8_t* saved_used = 0;
11397 saved_used = heap_segment_used (seg);
11400 if (seg == ephemeral_heap_segment)
11402 //Sometimes the allocated size is advanced without clearing the
11403 //memory. Let's catch up here
11404 if (heap_segment_used (seg) < (alloc_allocated - plug_skew))
11407 #ifndef BACKGROUND_GC
11408 clear_mark_array (heap_segment_used (seg) + plug_skew, alloc_allocated);
11409 #endif //BACKGROUND_GC
11410 #endif //MARK_ARRAY
11411 heap_segment_used (seg) = alloc_allocated - plug_skew;
11414 #ifdef BACKGROUND_GC
11417 uint8_t* old_allocated = heap_segment_allocated (seg) - plug_skew - limit_size;
11418 #ifdef FEATURE_LOH_COMPACTION
11419 old_allocated -= Align (loh_padding_obj_size, align_const);
11420 #endif //FEATURE_LOH_COMPACTION
11422 assert (heap_segment_used (seg) >= old_allocated);
11424 #endif //BACKGROUND_GC
11426 (start - plug_skew + limit_size) <= heap_segment_used (seg))
11428 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to clear memory(1)", heap_number));
11429 add_saved_spinlock_info (me_release, mt_clr_mem);
11430 leave_spin_lock (&more_space_lock);
11431 dprintf (3, ("clearing memory at %Ix for %d bytes", (start - plug_skew), limit_size));
11432 memclr (start - plug_skew, limit_size);
11436 uint8_t* used = heap_segment_used (seg);
11437 heap_segment_used (seg) = start + limit_size - plug_skew;
11439 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to clear memory", heap_number));
11440 add_saved_spinlock_info (me_release, mt_clr_mem);
11441 leave_spin_lock (&more_space_lock);
11442 if ((start - plug_skew) < used)
11444 if (used != saved_used)
11449 dprintf (2, ("clearing memory before used at %Ix for %Id bytes",
11450 (start - plug_skew), (plug_skew + used - start)));
11451 memclr (start - plug_skew, used - (start - plug_skew));
11455 //this portion can be done after we release the lock
11456 if (seg == ephemeral_heap_segment)
11458 #ifdef FFIND_OBJECT
11459 if (gen0_must_clear_bricks > 0)
11461 //set the brick table to speed up find_object
11462 size_t b = brick_of (acontext->alloc_ptr);
11463 set_brick (b, acontext->alloc_ptr - brick_address (b));
11465 dprintf (3, ("Allocation Clearing bricks [%Ix, %Ix[",
11466 b, brick_of (align_on_brick (start + limit_size))));
11467 volatile short* x = &brick_table [b];
11468 short* end_x = &brick_table [brick_of (align_on_brick (start + limit_size))];
11470 for (;x < end_x;x++)
11474 #endif //FFIND_OBJECT
11476 gen0_bricks_cleared = FALSE;
11480 // verifying the memory is completely cleared.
11481 //verify_mem_cleared (start - plug_skew, limit_size);
11484 /* in order to make the allocator faster, allocate returns a
11485 * 0 filled object. Care must be taken to set the allocation limit to the
11486 * allocation pointer after gc
11489 size_t gc_heap::limit_from_size (size_t size, size_t room, int gen_number,
11492 size_t new_limit = new_allocation_limit ((size + Align (min_obj_size, align_const)),
11493 min (room,max (size + Align (min_obj_size, align_const),
11494 ((gen_number < max_generation+1) ?
11495 allocation_quantum :
11498 assert (new_limit >= (size + Align (min_obj_size, align_const)));
11499 dprintf (100, ("requested to allocate %Id bytes, actual size is %Id", size, new_limit));
11503 void gc_heap::handle_oom (int heap_num, oom_reason reason, size_t alloc_size,
11504 uint8_t* allocated, uint8_t* reserved)
11506 dprintf (1, ("total committed on the heap is %Id", get_total_committed_size()));
11508 UNREFERENCED_PARAMETER(heap_num);
11510 if (reason == oom_budget)
11512 alloc_size = dd_min_size (dynamic_data_of (0)) / 2;
11515 if ((reason == oom_budget) && ((!fgm_result.loh_p) && (fgm_result.fgm != fgm_no_failure)))
11517 // This means during the last GC we needed to reserve and/or commit more memory
11518 // but we couldn't. We proceeded with the GC and ended up not having enough
11519 // memory at the end. This is a legitimate OOM situtation. Otherwise we
11520 // probably made a mistake and didn't expand the heap when we should have.
11521 reason = oom_low_mem;
11524 oom_info.reason = reason;
11525 oom_info.allocated = allocated;
11526 oom_info.reserved = reserved;
11527 oom_info.alloc_size = alloc_size;
11528 oom_info.gc_index = settings.gc_index;
11529 oom_info.fgm = fgm_result.fgm;
11530 oom_info.size = fgm_result.size;
11531 oom_info.available_pagefile_mb = fgm_result.available_pagefile_mb;
11532 oom_info.loh_p = fgm_result.loh_p;
11534 fgm_result.fgm = fgm_no_failure;
11536 // Break early - before the more_space_lock is release so no other threads
11537 // could have allocated on the same heap when OOM happened.
11538 if (GCConfig::GetBreakOnOOM())
11540 GCToOSInterface::DebugBreak();
11544 #ifdef BACKGROUND_GC
11545 BOOL gc_heap::background_allowed_p()
11547 return ( gc_can_use_concurrent && ((settings.pause_mode == pause_interactive) || (settings.pause_mode == pause_sustained_low_latency)) );
11549 #endif //BACKGROUND_GC
11551 void gc_heap::check_for_full_gc (int gen_num, size_t size)
11553 BOOL should_notify = FALSE;
11554 // if we detect full gc because of the allocation budget specified this is TRUE;
11555 // it's FALSE if it's due to other factors.
11556 BOOL alloc_factor = TRUE;
11559 int n_initial = gen_num;
11560 BOOL local_blocking_collection = FALSE;
11561 BOOL local_elevation_requested = FALSE;
11562 int new_alloc_remain_percent = 0;
11564 if (full_gc_approach_event_set)
11569 if (gen_num != (max_generation + 1))
11571 gen_num = max_generation;
11574 dynamic_data* dd_full = dynamic_data_of (gen_num);
11575 ptrdiff_t new_alloc_remain = 0;
11576 uint32_t pct = ((gen_num == (max_generation + 1)) ? fgn_loh_percent : fgn_maxgen_percent);
11578 for (int gen_index = 0; gen_index <= (max_generation + 1); gen_index++)
11580 dprintf (2, ("FGN: h#%d: gen%d: %Id(%Id)",
11581 heap_number, gen_index,
11582 dd_new_allocation (dynamic_data_of (gen_index)),
11583 dd_desired_allocation (dynamic_data_of (gen_index))));
11586 // For small object allocations we only check every fgn_check_quantum bytes.
11587 if (n_initial == 0)
11589 dprintf (2, ("FGN: gen0 last recorded alloc: %Id", fgn_last_alloc));
11590 dynamic_data* dd_0 = dynamic_data_of (n_initial);
11591 if (((fgn_last_alloc - dd_new_allocation (dd_0)) < fgn_check_quantum) &&
11592 (dd_new_allocation (dd_0) >= 0))
11598 fgn_last_alloc = dd_new_allocation (dd_0);
11599 dprintf (2, ("FGN: gen0 last recorded alloc is now: %Id", fgn_last_alloc));
11602 // We don't consider the size that came from soh 'cause it doesn't contribute to the
11607 for (i = n+1; i <= max_generation; i++)
11609 if (get_new_allocation (i) <= 0)
11611 n = min (i, max_generation);
11617 dprintf (2, ("FGN: h#%d: gen%d budget exceeded", heap_number, n));
11618 if (gen_num == max_generation)
11620 // If it's small object heap we should first see if we will even be looking at gen2 budget
11621 // in the next GC or not. If not we should go directly to checking other factors.
11622 if (n < (max_generation - 1))
11624 goto check_other_factors;
11628 new_alloc_remain = dd_new_allocation (dd_full) - size;
11630 new_alloc_remain_percent = (int)(((float)(new_alloc_remain) / (float)dd_desired_allocation (dd_full)) * 100);
11632 dprintf (2, ("FGN: alloc threshold for gen%d is %d%%, current threshold is %d%%",
11633 gen_num, pct, new_alloc_remain_percent));
11635 if (new_alloc_remain_percent <= (int)pct)
11637 #ifdef BACKGROUND_GC
11638 // If background GC is enabled, we still want to check whether this will
11639 // be a blocking GC or not because we only want to notify when it's a
11640 // blocking full GC.
11641 if (background_allowed_p())
11643 goto check_other_factors;
11645 #endif //BACKGROUND_GC
11647 should_notify = TRUE;
11651 check_other_factors:
11653 dprintf (2, ("FGC: checking other factors"));
11654 n = generation_to_condemn (n,
11655 &local_blocking_collection,
11656 &local_elevation_requested,
11659 if (local_elevation_requested && (n == max_generation))
11661 if (settings.should_lock_elevation)
11663 int local_elevation_locked_count = settings.elevation_locked_count + 1;
11664 if (local_elevation_locked_count != 6)
11666 dprintf (2, ("FGN: lock count is %d - Condemning max_generation-1",
11667 local_elevation_locked_count));
11668 n = max_generation - 1;
11673 dprintf (2, ("FGN: we estimate gen%d will be collected", n));
11675 #ifdef BACKGROUND_GC
11676 // When background GC is enabled it decreases the accuracy of our predictability -
11677 // by the time the GC happens, we may not be under BGC anymore. If we try to
11678 // predict often enough it should be ok.
11679 if ((n == max_generation) &&
11680 (recursive_gc_sync::background_running_p()))
11682 n = max_generation - 1;
11683 dprintf (2, ("FGN: bgc - 1 instead of 2"));
11686 if ((n == max_generation) && !local_blocking_collection)
11688 if (!background_allowed_p())
11690 local_blocking_collection = TRUE;
11693 #endif //BACKGROUND_GC
11695 dprintf (2, ("FGN: we estimate gen%d will be collected: %s",
11697 (local_blocking_collection ? "blocking" : "background")));
11699 if ((n == max_generation) && local_blocking_collection)
11701 alloc_factor = FALSE;
11702 should_notify = TRUE;
11710 dprintf (2, ("FGN: gen%d detecting full GC approaching(%s) (GC#%d) (%Id%% left in gen%d)",
11712 (alloc_factor ? "alloc" : "other"),
11713 dd_collection_count (dynamic_data_of (0)),
11714 new_alloc_remain_percent,
11717 send_full_gc_notification (n_initial, alloc_factor);
11721 void gc_heap::send_full_gc_notification (int gen_num, BOOL due_to_alloc_p)
11723 if (!full_gc_approach_event_set)
11725 assert (full_gc_approach_event.IsValid());
11726 FIRE_EVENT(GCFullNotify_V1, gen_num, due_to_alloc_p);
11728 full_gc_end_event.Reset();
11729 full_gc_approach_event.Set();
11730 full_gc_approach_event_set = true;
11734 wait_full_gc_status gc_heap::full_gc_wait (GCEvent *event, int time_out_ms)
11736 if (fgn_maxgen_percent == 0)
11738 return wait_full_gc_na;
11741 uint32_t wait_result = user_thread_wait(event, FALSE, time_out_ms);
11743 if ((wait_result == WAIT_OBJECT_0) || (wait_result == WAIT_TIMEOUT))
11745 if (fgn_maxgen_percent == 0)
11747 return wait_full_gc_cancelled;
11750 if (wait_result == WAIT_OBJECT_0)
11752 #ifdef BACKGROUND_GC
11753 if (fgn_last_gc_was_concurrent)
11755 fgn_last_gc_was_concurrent = FALSE;
11756 return wait_full_gc_na;
11759 #endif //BACKGROUND_GC
11761 return wait_full_gc_success;
11766 return wait_full_gc_timeout;
11771 return wait_full_gc_failed;
11775 size_t gc_heap::get_full_compact_gc_count()
11777 return full_gc_counts[gc_type_compacting];
11780 // DTREVIEW - we should check this in dt_low_ephemeral_space_p
11783 BOOL gc_heap::short_on_end_of_seg (int gen_number,
11787 UNREFERENCED_PARAMETER(gen_number);
11788 uint8_t* allocated = heap_segment_allocated(seg);
11790 return (!a_size_fit_p (end_space_after_gc(),
11792 heap_segment_reserved (seg),
11797 #pragma warning(disable:4706) // "assignment within conditional expression" is intentional in this function.
11801 BOOL gc_heap::a_fit_free_list_p (int gen_number,
11803 alloc_context* acontext,
11806 BOOL can_fit = FALSE;
11807 generation* gen = generation_of (gen_number);
11808 allocator* gen_allocator = generation_allocator (gen);
11809 size_t sz_list = gen_allocator->first_bucket_size();
11810 for (unsigned int a_l_idx = 0; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
11812 if ((size < sz_list) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
11814 uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
11815 uint8_t* prev_free_item = 0;
11817 while (free_list != 0)
11819 dprintf (3, ("considering free list %Ix", (size_t)free_list));
11820 size_t free_list_size = unused_array_size (free_list);
11821 if ((size + Align (min_obj_size, align_const)) <= free_list_size)
11823 dprintf (3, ("Found adequate unused area: [%Ix, size: %Id",
11824 (size_t)free_list, free_list_size));
11826 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
11827 // We ask for more Align (min_obj_size)
11828 // to make sure that we can insert a free object
11829 // in adjust_limit will set the limit lower
11830 size_t limit = limit_from_size (size, free_list_size, gen_number, align_const);
11832 uint8_t* remain = (free_list + limit);
11833 size_t remain_size = (free_list_size - limit);
11834 if (remain_size >= Align(min_free_list, align_const))
11836 make_unused_array (remain, remain_size);
11837 gen_allocator->thread_item_front (remain, remain_size);
11838 assert (remain_size >= Align (min_obj_size, align_const));
11842 //absorb the entire free list
11843 limit += remain_size;
11845 generation_free_list_space (gen) -= limit;
11847 adjust_limit_clr (free_list, limit, acontext, 0, align_const, gen_number);
11852 else if (gen_allocator->discard_if_no_fit_p())
11854 assert (prev_free_item == 0);
11855 dprintf (3, ("couldn't use this free area, discarding"));
11856 generation_free_obj_space (gen) += free_list_size;
11858 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
11859 generation_free_list_space (gen) -= free_list_size;
11863 prev_free_item = free_list;
11865 free_list = free_list_slot (free_list);
11868 sz_list = sz_list * 2;
11875 #ifdef BACKGROUND_GC
11876 void gc_heap::bgc_loh_alloc_clr (uint8_t* alloc_start,
11878 alloc_context* acontext,
11884 make_unused_array (alloc_start, size);
11886 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
11889 AppDomain* alloc_appdomain = GetAppDomain();
11890 alloc_appdomain->RecordAllocBytes (size, heap_number);
11892 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
11894 size_t size_of_array_base = sizeof(ArrayBase);
11896 bgc_alloc_lock->loh_alloc_done_with_index (lock_index);
11898 // clear memory while not holding the lock.
11899 size_t size_to_skip = size_of_array_base;
11900 size_t size_to_clear = size - size_to_skip - plug_skew;
11901 size_t saved_size_to_clear = size_to_clear;
11904 uint8_t* end = alloc_start + size - plug_skew;
11905 uint8_t* used = heap_segment_used (seg);
11908 if ((alloc_start + size_to_skip) < used)
11910 size_to_clear = used - (alloc_start + size_to_skip);
11916 dprintf (2, ("bgc loh: setting used to %Ix", end));
11917 heap_segment_used (seg) = end;
11920 dprintf (2, ("bgc loh: used: %Ix, alloc: %Ix, end of alloc: %Ix, clear %Id bytes",
11921 used, alloc_start, end, size_to_clear));
11925 dprintf (2, ("bgc loh: [%Ix-[%Ix(%Id)", alloc_start, alloc_start+size, size));
11929 // since we filled in 0xcc for free object when we verify heap,
11930 // we need to make sure we clear those bytes.
11931 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
11933 if (size_to_clear < saved_size_to_clear)
11935 size_to_clear = saved_size_to_clear;
11938 #endif //VERIFY_HEAP
11940 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to clear large obj", heap_number));
11941 add_saved_spinlock_info (me_release, mt_clr_large_mem);
11942 leave_spin_lock (&more_space_lock);
11943 memclr (alloc_start + size_to_skip, size_to_clear);
11945 bgc_alloc_lock->loh_alloc_set (alloc_start);
11947 acontext->alloc_ptr = alloc_start;
11948 acontext->alloc_limit = (alloc_start + size - Align (min_obj_size, align_const));
11950 // need to clear the rest of the object before we hand it out.
11951 clear_unused_array(alloc_start, size);
11953 #endif //BACKGROUND_GC
11955 BOOL gc_heap::a_fit_free_list_large_p (size_t size,
11956 alloc_context* acontext,
11959 #ifdef BACKGROUND_GC
11960 wait_for_background_planning (awr_loh_alloc_during_plan);
11961 #endif //BACKGROUND_GC
11963 BOOL can_fit = FALSE;
11964 int gen_number = max_generation + 1;
11965 generation* gen = generation_of (gen_number);
11966 allocator* loh_allocator = generation_allocator (gen);
11968 #ifdef FEATURE_LOH_COMPACTION
11969 size_t loh_pad = Align (loh_padding_obj_size, align_const);
11970 #endif //FEATURE_LOH_COMPACTION
11972 #ifdef BACKGROUND_GC
11974 #endif //BACKGROUND_GC
11975 size_t sz_list = loh_allocator->first_bucket_size();
11976 for (unsigned int a_l_idx = 0; a_l_idx < loh_allocator->number_of_buckets(); a_l_idx++)
11978 if ((size < sz_list) || (a_l_idx == (loh_allocator->number_of_buckets()-1)))
11980 uint8_t* free_list = loh_allocator->alloc_list_head_of (a_l_idx);
11981 uint8_t* prev_free_item = 0;
11982 while (free_list != 0)
11984 dprintf (3, ("considering free list %Ix", (size_t)free_list));
11986 size_t free_list_size = unused_array_size(free_list);
11988 #ifdef FEATURE_LOH_COMPACTION
11989 if ((size + loh_pad) <= free_list_size)
11991 if (((size + Align (min_obj_size, align_const)) <= free_list_size)||
11992 (size == free_list_size))
11993 #endif //FEATURE_LOH_COMPACTION
11995 #ifdef BACKGROUND_GC
11996 cookie = bgc_alloc_lock->loh_alloc_set (free_list);
11997 #endif //BACKGROUND_GC
11999 //unlink the free_item
12000 loh_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
12002 // Substract min obj size because limit_from_size adds it. Not needed for LOH
12003 size_t limit = limit_from_size (size - Align(min_obj_size, align_const), free_list_size,
12004 gen_number, align_const);
12006 #ifdef FEATURE_LOH_COMPACTION
12007 make_unused_array (free_list, loh_pad);
12009 free_list += loh_pad;
12010 free_list_size -= loh_pad;
12011 #endif //FEATURE_LOH_COMPACTION
12013 uint8_t* remain = (free_list + limit);
12014 size_t remain_size = (free_list_size - limit);
12015 if (remain_size != 0)
12017 assert (remain_size >= Align (min_obj_size, align_const));
12018 make_unused_array (remain, remain_size);
12020 if (remain_size >= Align(min_free_list, align_const))
12022 loh_thread_gap_front (remain, remain_size, gen);
12023 assert (remain_size >= Align (min_obj_size, align_const));
12027 generation_free_obj_space (gen) += remain_size;
12029 generation_free_list_space (gen) -= free_list_size;
12030 dprintf (3, ("found fit on loh at %Ix", free_list));
12031 #ifdef BACKGROUND_GC
12034 bgc_loh_alloc_clr (free_list, limit, acontext, align_const, cookie, FALSE, 0);
12037 #endif //BACKGROUND_GC
12039 adjust_limit_clr (free_list, limit, acontext, 0, align_const, gen_number);
12042 //fix the limit to compensate for adjust_limit_clr making it too short
12043 acontext->alloc_limit += Align (min_obj_size, align_const);
12047 prev_free_item = free_list;
12048 free_list = free_list_slot (free_list);
12051 sz_list = sz_list * 2;
12058 #pragma warning(default:4706)
12061 BOOL gc_heap::a_fit_segment_end_p (int gen_number,
12064 alloc_context* acontext,
12066 BOOL* commit_failed_p)
12068 *commit_failed_p = FALSE;
12070 #ifdef BACKGROUND_GC
12072 #endif //BACKGROUND_GC
12074 uint8_t*& allocated = ((gen_number == 0) ?
12076 heap_segment_allocated(seg));
12078 size_t pad = Align (min_obj_size, align_const);
12080 #ifdef FEATURE_LOH_COMPACTION
12081 if (gen_number == (max_generation + 1))
12083 pad += Align (loh_padding_obj_size, align_const);
12085 #endif //FEATURE_LOH_COMPACTION
12087 uint8_t* end = heap_segment_committed (seg) - pad;
12089 if (a_size_fit_p (size, allocated, end, align_const))
12091 limit = limit_from_size (size,
12093 gen_number, align_const);
12097 end = heap_segment_reserved (seg) - pad;
12099 if (a_size_fit_p (size, allocated, end, align_const))
12101 limit = limit_from_size (size,
12103 gen_number, align_const);
12104 if (grow_heap_segment (seg, allocated + limit))
12110 dprintf (2, ("can't grow segment, doing a full gc"));
12111 *commit_failed_p = TRUE;
12118 #ifdef BACKGROUND_GC
12119 if (gen_number != 0)
12121 cookie = bgc_alloc_lock->loh_alloc_set (allocated);
12123 #endif //BACKGROUND_GC
12125 uint8_t* old_alloc;
12126 old_alloc = allocated;
12127 #ifdef FEATURE_LOH_COMPACTION
12128 if (gen_number == (max_generation + 1))
12130 size_t loh_pad = Align (loh_padding_obj_size, align_const);
12131 make_unused_array (old_alloc, loh_pad);
12132 old_alloc += loh_pad;
12133 allocated += loh_pad;
12136 #endif //FEATURE_LOH_COMPACTION
12138 #if defined (VERIFY_HEAP) && defined (_DEBUG)
12139 ((void**) allocated)[-1] = 0; //clear the sync block
12140 #endif //VERIFY_HEAP && _DEBUG
12141 allocated += limit;
12143 dprintf (3, ("found fit at end of seg: %Ix", old_alloc));
12145 #ifdef BACKGROUND_GC
12148 bgc_loh_alloc_clr (old_alloc, limit, acontext, align_const, cookie, TRUE, seg);
12151 #endif //BACKGROUND_GC
12153 adjust_limit_clr (old_alloc, limit, acontext, seg, align_const, gen_number);
12163 BOOL gc_heap::loh_a_fit_segment_end_p (int gen_number,
12165 alloc_context* acontext,
12167 BOOL* commit_failed_p,
12170 *commit_failed_p = FALSE;
12171 heap_segment* seg = generation_allocation_segment (generation_of (gen_number));
12172 BOOL can_allocate_p = FALSE;
12176 if (a_fit_segment_end_p (gen_number, seg, (size - Align (min_obj_size, align_const)),
12177 acontext, align_const, commit_failed_p))
12179 acontext->alloc_limit += Align (min_obj_size, align_const);
12180 can_allocate_p = TRUE;
12185 if (*commit_failed_p)
12187 *oom_r = oom_cant_commit;
12192 seg = heap_segment_next_rw (seg);
12197 return can_allocate_p;
12200 #ifdef BACKGROUND_GC
12202 void gc_heap::wait_for_background (alloc_wait_reason awr)
12204 dprintf (2, ("BGC is already in progress, waiting for it to finish"));
12205 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to wait for bgc done", heap_number));
12206 add_saved_spinlock_info (me_release, mt_wait_bgc);
12207 leave_spin_lock (&more_space_lock);
12208 background_gc_wait (awr);
12209 enter_spin_lock (&more_space_lock);
12210 add_saved_spinlock_info (me_acquire, mt_wait_bgc);
12211 dprintf (SPINLOCK_LOG, ("[%d]Emsl after waiting for bgc done", heap_number));
12214 void gc_heap::wait_for_bgc_high_memory (alloc_wait_reason awr)
12216 if (recursive_gc_sync::background_running_p())
12218 uint32_t memory_load;
12219 get_memory_info (&memory_load);
12220 if (memory_load >= 95)
12222 dprintf (GTC_LOG, ("high mem - wait for BGC to finish, wait reason: %d", awr));
12223 wait_for_background (awr);
12228 #endif //BACKGROUND_GC
12230 // We request to trigger an ephemeral GC but we may get a full compacting GC.
12231 // return TRUE if that's the case.
12232 BOOL gc_heap::trigger_ephemeral_gc (gc_reason gr)
12234 #ifdef BACKGROUND_GC
12235 wait_for_bgc_high_memory (awr_loh_oos_bgc);
12236 #endif //BACKGROUND_GC
12238 BOOL did_full_compact_gc = FALSE;
12240 dprintf (2, ("triggering a gen1 GC"));
12241 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12242 vm_heap->GarbageCollectGeneration(max_generation - 1, gr);
12244 #ifdef MULTIPLE_HEAPS
12245 enter_spin_lock (&more_space_lock);
12246 add_saved_spinlock_info (me_acquire, mt_t_eph_gc);
12247 dprintf (SPINLOCK_LOG, ("[%d]Emsl after a GC", heap_number));
12248 #endif //MULTIPLE_HEAPS
12250 size_t current_full_compact_gc_count = get_full_compact_gc_count();
12252 if (current_full_compact_gc_count > last_full_compact_gc_count)
12254 dprintf (2, ("attempted to trigger an ephemeral GC and got a full compacting GC"));
12255 did_full_compact_gc = TRUE;
12258 return did_full_compact_gc;
12261 BOOL gc_heap::soh_try_fit (int gen_number,
12263 alloc_context* acontext,
12265 BOOL* commit_failed_p,
12266 BOOL* short_seg_end_p)
12268 BOOL can_allocate = TRUE;
12269 if (short_seg_end_p)
12271 *short_seg_end_p = FALSE;
12274 can_allocate = a_fit_free_list_p (gen_number, size, acontext, align_const);
12277 if (short_seg_end_p)
12279 *short_seg_end_p = short_on_end_of_seg (gen_number, ephemeral_heap_segment, align_const);
12281 // If the caller doesn't care, we always try to fit at the end of seg;
12282 // otherwise we would only try if we are actually not short at end of seg.
12283 if (!short_seg_end_p || !(*short_seg_end_p))
12285 can_allocate = a_fit_segment_end_p (gen_number, ephemeral_heap_segment, size,
12286 acontext, align_const, commit_failed_p);
12290 return can_allocate;
12293 BOOL gc_heap::allocate_small (int gen_number,
12295 alloc_context* acontext,
12298 #if defined (BACKGROUND_GC) && !defined (MULTIPLE_HEAPS)
12299 if (recursive_gc_sync::background_running_p())
12301 background_soh_alloc_count++;
12302 if ((background_soh_alloc_count % bgc_alloc_spin_count) == 0)
12304 add_saved_spinlock_info (me_release, mt_alloc_small);
12305 dprintf (SPINLOCK_LOG, ("[%d]spin Lmsl", heap_number));
12306 leave_spin_lock (&more_space_lock);
12307 bool cooperative_mode = enable_preemptive ();
12308 GCToOSInterface::Sleep (bgc_alloc_spin);
12309 disable_preemptive (cooperative_mode);
12310 enter_spin_lock (&more_space_lock);
12311 add_saved_spinlock_info (me_acquire, mt_alloc_small);
12312 dprintf (SPINLOCK_LOG, ("[%d]spin Emsl", heap_number));
12316 //GCToOSInterface::YieldThread (0);
12319 #endif //BACKGROUND_GC && !MULTIPLE_HEAPS
12321 gc_reason gr = reason_oos_soh;
12322 oom_reason oom_r = oom_no_failure;
12324 // No variable values should be "carried over" from one state to the other.
12325 // That's why there are local variable for each state
12327 allocation_state soh_alloc_state = a_state_start;
12329 // If we can get a new seg it means allocation will succeed.
12332 dprintf (3, ("[h%d]soh state is %s", heap_number, allocation_state_str[soh_alloc_state]));
12333 switch (soh_alloc_state)
12335 case a_state_can_allocate:
12336 case a_state_cant_allocate:
12340 case a_state_start:
12342 soh_alloc_state = a_state_try_fit;
12345 case a_state_try_fit:
12347 BOOL commit_failed_p = FALSE;
12348 BOOL can_use_existing_p = FALSE;
12350 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12351 align_const, &commit_failed_p,
12353 soh_alloc_state = (can_use_existing_p ?
12354 a_state_can_allocate :
12356 a_state_trigger_full_compact_gc :
12357 a_state_trigger_ephemeral_gc));
12360 case a_state_try_fit_after_bgc:
12362 BOOL commit_failed_p = FALSE;
12363 BOOL can_use_existing_p = FALSE;
12364 BOOL short_seg_end_p = FALSE;
12366 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12367 align_const, &commit_failed_p,
12369 soh_alloc_state = (can_use_existing_p ?
12370 a_state_can_allocate :
12372 a_state_trigger_2nd_ephemeral_gc :
12373 a_state_trigger_full_compact_gc));
12376 case a_state_try_fit_after_cg:
12378 BOOL commit_failed_p = FALSE;
12379 BOOL can_use_existing_p = FALSE;
12380 BOOL short_seg_end_p = FALSE;
12382 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12383 align_const, &commit_failed_p,
12385 if (short_seg_end_p)
12387 soh_alloc_state = a_state_cant_allocate;
12388 oom_r = oom_budget;
12392 if (can_use_existing_p)
12394 soh_alloc_state = a_state_can_allocate;
12398 #ifdef MULTIPLE_HEAPS
12399 if (!commit_failed_p)
12401 // some other threads already grabbed the more space lock and allocated
12402 // so we should attempt an ephemeral GC again.
12403 assert (heap_segment_allocated (ephemeral_heap_segment) < alloc_allocated);
12404 soh_alloc_state = a_state_trigger_ephemeral_gc;
12407 #endif //MULTIPLE_HEAPS
12409 assert (commit_failed_p);
12410 soh_alloc_state = a_state_cant_allocate;
12411 oom_r = oom_cant_commit;
12417 case a_state_check_and_wait_for_bgc:
12419 BOOL bgc_in_progress_p = FALSE;
12420 BOOL did_full_compacting_gc = FALSE;
12422 bgc_in_progress_p = check_and_wait_for_bgc (awr_gen0_oos_bgc, &did_full_compacting_gc);
12423 soh_alloc_state = (did_full_compacting_gc ?
12424 a_state_try_fit_after_cg :
12425 a_state_try_fit_after_bgc);
12428 case a_state_trigger_ephemeral_gc:
12430 BOOL commit_failed_p = FALSE;
12431 BOOL can_use_existing_p = FALSE;
12432 BOOL short_seg_end_p = FALSE;
12433 BOOL bgc_in_progress_p = FALSE;
12434 BOOL did_full_compacting_gc = FALSE;
12436 did_full_compacting_gc = trigger_ephemeral_gc (gr);
12437 if (did_full_compacting_gc)
12439 soh_alloc_state = a_state_try_fit_after_cg;
12443 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12444 align_const, &commit_failed_p,
12446 #ifdef BACKGROUND_GC
12447 bgc_in_progress_p = recursive_gc_sync::background_running_p();
12448 #endif //BACKGROUND_GC
12450 if (short_seg_end_p)
12452 soh_alloc_state = (bgc_in_progress_p ?
12453 a_state_check_and_wait_for_bgc :
12454 a_state_trigger_full_compact_gc);
12456 if (fgn_maxgen_percent)
12458 dprintf (2, ("FGN: doing last GC before we throw OOM"));
12459 send_full_gc_notification (max_generation, FALSE);
12464 if (can_use_existing_p)
12466 soh_alloc_state = a_state_can_allocate;
12470 #ifdef MULTIPLE_HEAPS
12471 if (!commit_failed_p)
12473 // some other threads already grabbed the more space lock and allocated
12474 // so we should attempt an ephemeral GC again.
12475 assert (heap_segment_allocated (ephemeral_heap_segment) < alloc_allocated);
12476 soh_alloc_state = a_state_trigger_ephemeral_gc;
12479 #endif //MULTIPLE_HEAPS
12481 soh_alloc_state = a_state_trigger_full_compact_gc;
12482 if (fgn_maxgen_percent)
12484 dprintf (2, ("FGN: failed to commit, doing full compacting GC"));
12485 send_full_gc_notification (max_generation, FALSE);
12493 case a_state_trigger_2nd_ephemeral_gc:
12495 BOOL commit_failed_p = FALSE;
12496 BOOL can_use_existing_p = FALSE;
12497 BOOL short_seg_end_p = FALSE;
12498 BOOL did_full_compacting_gc = FALSE;
12501 did_full_compacting_gc = trigger_ephemeral_gc (gr);
12503 if (did_full_compacting_gc)
12505 soh_alloc_state = a_state_try_fit_after_cg;
12509 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12510 align_const, &commit_failed_p,
12512 if (short_seg_end_p || commit_failed_p)
12514 soh_alloc_state = a_state_trigger_full_compact_gc;
12518 assert (can_use_existing_p);
12519 soh_alloc_state = a_state_can_allocate;
12524 case a_state_trigger_full_compact_gc:
12526 BOOL got_full_compacting_gc = FALSE;
12528 got_full_compacting_gc = trigger_full_compact_gc (gr, &oom_r);
12529 soh_alloc_state = (got_full_compacting_gc ? a_state_try_fit_after_cg : a_state_cant_allocate);
12534 assert (!"Invalid state!");
12541 if (soh_alloc_state == a_state_cant_allocate)
12543 assert (oom_r != oom_no_failure);
12544 handle_oom (heap_number,
12547 heap_segment_allocated (ephemeral_heap_segment),
12548 heap_segment_reserved (ephemeral_heap_segment));
12550 dprintf (SPINLOCK_LOG, ("[%d]Lmsl for oom", heap_number));
12551 add_saved_spinlock_info (me_release, mt_alloc_small_cant);
12552 leave_spin_lock (&more_space_lock);
12555 return (soh_alloc_state == a_state_can_allocate);
12558 #ifdef BACKGROUND_GC
12560 void gc_heap::wait_for_background_planning (alloc_wait_reason awr)
12562 while (current_c_gc_state == c_gc_state_planning)
12564 dprintf (3, ("lh state planning, cannot allocate"));
12566 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to wait for bgc plan", heap_number));
12567 add_saved_spinlock_info (me_release, mt_wait_bgc_plan);
12568 leave_spin_lock (&more_space_lock);
12569 background_gc_wait_lh (awr);
12570 enter_spin_lock (&more_space_lock);
12571 add_saved_spinlock_info (me_acquire, mt_wait_bgc_plan);
12572 dprintf (SPINLOCK_LOG, ("[%d]Emsl after waiting for bgc plan", heap_number));
12574 assert ((current_c_gc_state == c_gc_state_free) ||
12575 (current_c_gc_state == c_gc_state_marking));
12578 BOOL gc_heap::bgc_loh_should_allocate()
12580 size_t min_gc_size = dd_min_size(dynamic_data_of (max_generation + 1));
12582 if ((bgc_begin_loh_size + bgc_loh_size_increased) < (min_gc_size * 10))
12587 if (((bgc_begin_loh_size / end_loh_size) >= 2) || (bgc_loh_size_increased >= bgc_begin_loh_size))
12589 if ((bgc_begin_loh_size / end_loh_size) > 2)
12591 dprintf (3, ("alloc-ed too much before bgc started"));
12595 dprintf (3, ("alloc-ed too much after bgc started"));
12601 bgc_alloc_spin_loh = (uint32_t)(((float)bgc_loh_size_increased / (float)bgc_begin_loh_size) * 10);
12605 #endif //BACKGROUND_GC
12607 size_t gc_heap::get_large_seg_size (size_t size)
12609 size_t default_seg_size = min_loh_segment_size;
12610 #ifdef SEG_MAPPING_TABLE
12611 size_t align_size = default_seg_size;
12612 #else //SEG_MAPPING_TABLE
12613 size_t align_size = default_seg_size / 2;
12614 #endif //SEG_MAPPING_TABLE
12615 int align_const = get_alignment_constant (FALSE);
12616 size_t large_seg_size = align_on_page (
12617 max (default_seg_size,
12618 ((size + 2 * Align(min_obj_size, align_const) + OS_PAGE_SIZE +
12619 align_size) / align_size * align_size)));
12620 return large_seg_size;
12623 BOOL gc_heap::loh_get_new_seg (generation* gen,
12626 BOOL* did_full_compact_gc,
12629 UNREFERENCED_PARAMETER(gen);
12630 UNREFERENCED_PARAMETER(align_const);
12632 *did_full_compact_gc = FALSE;
12634 size_t seg_size = get_large_seg_size (size);
12636 heap_segment* new_seg = get_large_segment (seg_size, did_full_compact_gc);
12640 loh_alloc_since_cg += seg_size;
12647 return (new_seg != 0);
12650 BOOL gc_heap::retry_full_compact_gc (size_t size)
12652 size_t seg_size = get_large_seg_size (size);
12654 if (loh_alloc_since_cg >= (2 * (uint64_t)seg_size))
12659 #ifdef MULTIPLE_HEAPS
12660 uint64_t total_alloc_size = 0;
12661 for (int i = 0; i < n_heaps; i++)
12663 total_alloc_size += g_heaps[i]->loh_alloc_since_cg;
12666 if (total_alloc_size >= (2 * (uint64_t)seg_size))
12670 #endif //MULTIPLE_HEAPS
12675 BOOL gc_heap::check_and_wait_for_bgc (alloc_wait_reason awr,
12676 BOOL* did_full_compact_gc)
12678 BOOL bgc_in_progress = FALSE;
12679 *did_full_compact_gc = FALSE;
12680 #ifdef BACKGROUND_GC
12681 if (recursive_gc_sync::background_running_p())
12683 bgc_in_progress = TRUE;
12684 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12685 wait_for_background (awr);
12686 size_t current_full_compact_gc_count = get_full_compact_gc_count();
12687 if (current_full_compact_gc_count > last_full_compact_gc_count)
12689 *did_full_compact_gc = TRUE;
12692 #endif //BACKGROUND_GC
12694 return bgc_in_progress;
12697 BOOL gc_heap::loh_try_fit (int gen_number,
12699 alloc_context* acontext,
12701 BOOL* commit_failed_p,
12704 BOOL can_allocate = TRUE;
12706 if (!a_fit_free_list_large_p (size, acontext, align_const))
12708 can_allocate = loh_a_fit_segment_end_p (gen_number, size,
12709 acontext, align_const,
12710 commit_failed_p, oom_r);
12712 #ifdef BACKGROUND_GC
12713 if (can_allocate && recursive_gc_sync::background_running_p())
12715 bgc_loh_size_increased += size;
12717 #endif //BACKGROUND_GC
12719 #ifdef BACKGROUND_GC
12722 if (recursive_gc_sync::background_running_p())
12724 bgc_loh_allocated_in_free += size;
12727 #endif //BACKGROUND_GC
12729 return can_allocate;
12732 BOOL gc_heap::trigger_full_compact_gc (gc_reason gr,
12735 BOOL did_full_compact_gc = FALSE;
12737 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12739 // Set this so the next GC will be a full compacting GC.
12740 if (!last_gc_before_oom)
12742 last_gc_before_oom = TRUE;
12745 #ifdef BACKGROUND_GC
12746 if (recursive_gc_sync::background_running_p())
12748 wait_for_background ((gr == reason_oos_soh) ? awr_gen0_oos_bgc : awr_loh_oos_bgc);
12749 dprintf (2, ("waited for BGC - done"));
12751 #endif //BACKGROUND_GC
12753 size_t current_full_compact_gc_count = get_full_compact_gc_count();
12754 if (current_full_compact_gc_count > last_full_compact_gc_count)
12756 dprintf (3, ("a full compacting GC triggered while waiting for BGC (%d->%d)", last_full_compact_gc_count, current_full_compact_gc_count));
12757 assert (current_full_compact_gc_count > last_full_compact_gc_count);
12758 did_full_compact_gc = TRUE;
12762 dprintf (3, ("h%d full GC", heap_number));
12763 vm_heap->GarbageCollectGeneration(max_generation, gr);
12765 #ifdef MULTIPLE_HEAPS
12766 enter_spin_lock (&more_space_lock);
12767 dprintf (SPINLOCK_LOG, ("[%d]Emsl after full gc", heap_number));
12768 add_saved_spinlock_info (me_acquire, mt_t_full_gc);
12769 #endif //MULTIPLE_HEAPS
12771 current_full_compact_gc_count = get_full_compact_gc_count();
12773 if (current_full_compact_gc_count == last_full_compact_gc_count)
12775 dprintf (2, ("attempted to trigger a full compacting GC but didn't get it"));
12776 // We requested a full GC but didn't get because of the elevation logic
12777 // which means we should fail.
12778 *oom_r = oom_unproductive_full_gc;
12782 dprintf (3, ("h%d: T full compacting GC (%d->%d)",
12784 last_full_compact_gc_count,
12785 current_full_compact_gc_count));
12787 assert (current_full_compact_gc_count > last_full_compact_gc_count);
12788 did_full_compact_gc = TRUE;
12792 return did_full_compact_gc;
12795 #ifdef RECORD_LOH_STATE
12796 void gc_heap::add_saved_loh_state (allocation_state loh_state_to_save, EEThreadId thread_id)
12798 // When the state is can_allocate we already have released the more
12799 // space lock. So we are not logging states here since this code
12800 // is not thread safe.
12801 if (loh_state_to_save != a_state_can_allocate)
12803 last_loh_states[loh_state_index].alloc_state = loh_state_to_save;
12804 last_loh_states[loh_state_index].thread_id = thread_id;
12807 if (loh_state_index == max_saved_loh_states)
12809 loh_state_index = 0;
12812 assert (loh_state_index < max_saved_loh_states);
12815 #endif //RECORD_LOH_STATE
12817 BOOL gc_heap::allocate_large (int gen_number,
12819 alloc_context* acontext,
12822 #ifdef BACKGROUND_GC
12823 if (recursive_gc_sync::background_running_p() && (current_c_gc_state != c_gc_state_planning))
12825 background_loh_alloc_count++;
12826 //if ((background_loh_alloc_count % bgc_alloc_spin_count_loh) == 0)
12828 if (bgc_loh_should_allocate())
12830 if (!bgc_alloc_spin_loh)
12832 add_saved_spinlock_info (me_release, mt_alloc_large);
12833 dprintf (SPINLOCK_LOG, ("[%d]spin Lmsl loh", heap_number));
12834 leave_spin_lock (&more_space_lock);
12835 bool cooperative_mode = enable_preemptive ();
12836 GCToOSInterface::YieldThread (bgc_alloc_spin_loh);
12837 disable_preemptive (cooperative_mode);
12838 enter_spin_lock (&more_space_lock);
12839 add_saved_spinlock_info (me_acquire, mt_alloc_large);
12840 dprintf (SPINLOCK_LOG, ("[%d]spin Emsl loh", heap_number));
12845 wait_for_background (awr_loh_alloc_during_bgc);
12849 #endif //BACKGROUND_GC
12851 gc_reason gr = reason_oos_loh;
12852 generation* gen = generation_of (gen_number);
12853 oom_reason oom_r = oom_no_failure;
12854 size_t current_full_compact_gc_count = 0;
12856 // No variable values should be "carried over" from one state to the other.
12857 // That's why there are local variable for each state
12858 allocation_state loh_alloc_state = a_state_start;
12859 #ifdef RECORD_LOH_STATE
12860 EEThreadId current_thread_id;
12861 current_thread_id.SetToCurrentThread();
12862 #endif //RECORD_LOH_STATE
12864 // If we can get a new seg it means allocation will succeed.
12867 dprintf (3, ("[h%d]loh state is %s", heap_number, allocation_state_str[loh_alloc_state]));
12869 #ifdef RECORD_LOH_STATE
12870 add_saved_loh_state (loh_alloc_state, current_thread_id);
12871 #endif //RECORD_LOH_STATE
12872 switch (loh_alloc_state)
12874 case a_state_can_allocate:
12875 case a_state_cant_allocate:
12879 case a_state_start:
12881 loh_alloc_state = a_state_try_fit;
12884 case a_state_try_fit:
12886 BOOL commit_failed_p = FALSE;
12887 BOOL can_use_existing_p = FALSE;
12889 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12890 align_const, &commit_failed_p, &oom_r);
12891 loh_alloc_state = (can_use_existing_p ?
12892 a_state_can_allocate :
12894 a_state_trigger_full_compact_gc :
12895 a_state_acquire_seg));
12896 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12899 case a_state_try_fit_new_seg:
12901 BOOL commit_failed_p = FALSE;
12902 BOOL can_use_existing_p = FALSE;
12904 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12905 align_const, &commit_failed_p, &oom_r);
12906 // Even after we got a new seg it doesn't necessarily mean we can allocate,
12907 // another LOH allocating thread could have beat us to acquire the msl so
12908 // we need to try again.
12909 loh_alloc_state = (can_use_existing_p ? a_state_can_allocate : a_state_try_fit);
12910 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12913 case a_state_try_fit_new_seg_after_cg:
12915 BOOL commit_failed_p = FALSE;
12916 BOOL can_use_existing_p = FALSE;
12918 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12919 align_const, &commit_failed_p, &oom_r);
12920 // Even after we got a new seg it doesn't necessarily mean we can allocate,
12921 // another LOH allocating thread could have beat us to acquire the msl so
12922 // we need to try again. However, if we failed to commit, which means we
12923 // did have space on the seg, we bail right away 'cause we already did a
12924 // full compacting GC.
12925 loh_alloc_state = (can_use_existing_p ?
12926 a_state_can_allocate :
12928 a_state_cant_allocate :
12929 a_state_acquire_seg_after_cg));
12930 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12933 case a_state_try_fit_no_seg:
12935 BOOL commit_failed_p = FALSE;
12936 BOOL can_use_existing_p = FALSE;
12938 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12939 align_const, &commit_failed_p, &oom_r);
12940 loh_alloc_state = (can_use_existing_p ? a_state_can_allocate : a_state_cant_allocate);
12941 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12942 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
12945 case a_state_try_fit_after_cg:
12947 BOOL commit_failed_p = FALSE;
12948 BOOL can_use_existing_p = FALSE;
12950 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12951 align_const, &commit_failed_p, &oom_r);
12952 loh_alloc_state = (can_use_existing_p ?
12953 a_state_can_allocate :
12955 a_state_cant_allocate :
12956 a_state_acquire_seg_after_cg));
12957 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12960 case a_state_try_fit_after_bgc:
12962 BOOL commit_failed_p = FALSE;
12963 BOOL can_use_existing_p = FALSE;
12965 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12966 align_const, &commit_failed_p, &oom_r);
12967 loh_alloc_state = (can_use_existing_p ?
12968 a_state_can_allocate :
12970 a_state_trigger_full_compact_gc :
12971 a_state_acquire_seg_after_bgc));
12972 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12975 case a_state_acquire_seg:
12977 BOOL can_get_new_seg_p = FALSE;
12978 BOOL did_full_compacting_gc = FALSE;
12980 current_full_compact_gc_count = get_full_compact_gc_count();
12982 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
12983 loh_alloc_state = (can_get_new_seg_p ?
12984 a_state_try_fit_new_seg :
12985 (did_full_compacting_gc ?
12986 a_state_check_retry_seg :
12987 a_state_check_and_wait_for_bgc));
12990 case a_state_acquire_seg_after_cg:
12992 BOOL can_get_new_seg_p = FALSE;
12993 BOOL did_full_compacting_gc = FALSE;
12995 current_full_compact_gc_count = get_full_compact_gc_count();
12997 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
12998 // Since we release the msl before we try to allocate a seg, other
12999 // threads could have allocated a bunch of segments before us so
13000 // we might need to retry.
13001 loh_alloc_state = (can_get_new_seg_p ?
13002 a_state_try_fit_new_seg_after_cg :
13003 a_state_check_retry_seg);
13006 case a_state_acquire_seg_after_bgc:
13008 BOOL can_get_new_seg_p = FALSE;
13009 BOOL did_full_compacting_gc = FALSE;
13011 current_full_compact_gc_count = get_full_compact_gc_count();
13013 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
13014 loh_alloc_state = (can_get_new_seg_p ?
13015 a_state_try_fit_new_seg :
13016 (did_full_compacting_gc ?
13017 a_state_check_retry_seg :
13018 a_state_trigger_full_compact_gc));
13019 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13022 case a_state_check_and_wait_for_bgc:
13024 BOOL bgc_in_progress_p = FALSE;
13025 BOOL did_full_compacting_gc = FALSE;
13027 if (fgn_maxgen_percent)
13029 dprintf (2, ("FGN: failed to acquire seg, may need to do a full blocking GC"));
13030 send_full_gc_notification (max_generation, FALSE);
13033 bgc_in_progress_p = check_and_wait_for_bgc (awr_loh_oos_bgc, &did_full_compacting_gc);
13034 loh_alloc_state = (!bgc_in_progress_p ?
13035 a_state_trigger_full_compact_gc :
13036 (did_full_compacting_gc ?
13037 a_state_try_fit_after_cg :
13038 a_state_try_fit_after_bgc));
13041 case a_state_trigger_full_compact_gc:
13043 BOOL got_full_compacting_gc = FALSE;
13045 got_full_compacting_gc = trigger_full_compact_gc (gr, &oom_r);
13046 loh_alloc_state = (got_full_compacting_gc ? a_state_try_fit_after_cg : a_state_cant_allocate);
13047 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13050 case a_state_check_retry_seg:
13052 BOOL should_retry_gc = retry_full_compact_gc (size);
13053 BOOL should_retry_get_seg = FALSE;
13054 if (!should_retry_gc)
13056 size_t last_full_compact_gc_count = current_full_compact_gc_count;
13057 current_full_compact_gc_count = get_full_compact_gc_count();
13059 if (current_full_compact_gc_count > (last_full_compact_gc_count + 1))
13061 should_retry_get_seg = TRUE;
13065 loh_alloc_state = (should_retry_gc ?
13066 a_state_trigger_full_compact_gc :
13067 (should_retry_get_seg ?
13068 a_state_acquire_seg_after_cg :
13069 a_state_cant_allocate));
13070 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13075 assert (!"Invalid state!");
13082 if (loh_alloc_state == a_state_cant_allocate)
13084 assert (oom_r != oom_no_failure);
13085 handle_oom (heap_number,
13091 add_saved_spinlock_info (me_release, mt_alloc_large_cant);
13092 dprintf (SPINLOCK_LOG, ("[%d]Lmsl for loh oom", heap_number));
13093 leave_spin_lock (&more_space_lock);
13096 return (loh_alloc_state == a_state_can_allocate);
13099 int gc_heap::try_allocate_more_space (alloc_context* acontext, size_t size,
13102 if (gc_heap::gc_started)
13104 wait_for_gc_done();
13108 #ifdef SYNCHRONIZATION_STATS
13109 int64_t msl_acquire_start = GCToOSInterface::QueryPerformanceCounter();
13110 #endif //SYNCHRONIZATION_STATS
13111 enter_spin_lock (&more_space_lock);
13112 add_saved_spinlock_info (me_acquire, mt_try_alloc);
13113 dprintf (SPINLOCK_LOG, ("[%d]Emsl for alloc", heap_number));
13114 #ifdef SYNCHRONIZATION_STATS
13115 int64_t msl_acquire = GCToOSInterface::QueryPerformanceCounter() - msl_acquire_start;
13116 total_msl_acquire += msl_acquire;
13117 num_msl_acquired++;
13118 if (msl_acquire > 200)
13120 num_high_msl_acquire++;
13124 num_low_msl_acquire++;
13126 #endif //SYNCHRONIZATION_STATS
13129 // We are commenting this out 'cause we don't see the point - we already
13130 // have checked gc_started when we were acquiring the msl - no need to check
13131 // again. This complicates the logic in bgc_suspend_EE 'cause that one would
13132 // need to release msl which causes all sorts of trouble.
13133 if (gc_heap::gc_started)
13135 #ifdef SYNCHRONIZATION_STATS
13137 #endif //SYNCHRONIZATION_STATS
13138 BOOL fStress = (g_pConfig->GetGCStressLevel() & GCConfig::GCSTRESS_TRANSITION) != 0;
13141 //Rendez vous early (MP scaling issue)
13142 //dprintf (1, ("[%d]waiting for gc", heap_number));
13143 wait_for_gc_done();
13144 #ifdef MULTIPLE_HEAPS
13146 #endif //MULTIPLE_HEAPS
13151 dprintf (3, ("requested to allocate %d bytes on gen%d", size, gen_number));
13153 int align_const = get_alignment_constant (gen_number != (max_generation+1));
13155 if (fgn_maxgen_percent)
13157 check_for_full_gc (gen_number, size);
13160 if (!(new_allocation_allowed (gen_number)))
13162 if (fgn_maxgen_percent && (gen_number == 0))
13164 // We only check gen0 every so often, so take this opportunity to check again.
13165 check_for_full_gc (gen_number, size);
13168 #ifdef BACKGROUND_GC
13169 wait_for_bgc_high_memory (awr_gen0_alloc);
13170 #endif //BACKGROUND_GC
13172 #ifdef SYNCHRONIZATION_STATS
13174 #endif //SYNCHRONIZATION_STATS
13175 dprintf (/*100*/ 2, ("running out of budget on gen%d, gc", gen_number));
13177 if (!settings.concurrent || (gen_number == 0))
13179 vm_heap->GarbageCollectGeneration (0, ((gen_number == 0) ? reason_alloc_soh : reason_alloc_loh));
13180 #ifdef MULTIPLE_HEAPS
13181 enter_spin_lock (&more_space_lock);
13182 add_saved_spinlock_info (me_acquire, mt_try_budget);
13183 dprintf (SPINLOCK_LOG, ("[%d]Emsl out budget", heap_number));
13184 #endif //MULTIPLE_HEAPS
13188 BOOL can_allocate = ((gen_number == 0) ?
13189 allocate_small (gen_number, size, acontext, align_const) :
13190 allocate_large (gen_number, size, acontext, align_const));
13194 size_t alloc_context_bytes = acontext->alloc_limit + Align (min_obj_size, align_const) - acontext->alloc_ptr;
13195 int etw_allocation_index = ((gen_number == 0) ? 0 : 1);
13197 etw_allocation_running_amount[etw_allocation_index] += alloc_context_bytes;
13200 if (etw_allocation_running_amount[etw_allocation_index] > etw_allocation_tick)
13202 #ifdef FEATURE_REDHAWK
13203 FIRE_EVENT(GCAllocationTick_V1, (uint32_t)etw_allocation_running_amount[etw_allocation_index],
13204 (gen_number == 0) ? gc_etw_alloc_soh : gc_etw_alloc_loh);
13206 // Unfortunately some of the ETW macros do not check whether the ETW feature is enabled.
13207 // The ones that do are much less efficient.
13208 #if defined(FEATURE_EVENT_TRACE)
13209 if (EVENT_ENABLED(GCAllocationTick_V3))
13211 fire_etw_allocation_event (etw_allocation_running_amount[etw_allocation_index], gen_number, acontext->alloc_ptr);
13213 #endif //FEATURE_EVENT_TRACE
13214 #endif //FEATURE_REDHAWK
13215 etw_allocation_running_amount[etw_allocation_index] = 0;
13219 return (int)can_allocate;
13222 #ifdef MULTIPLE_HEAPS
13223 void gc_heap::balance_heaps (alloc_context* acontext)
13226 if (acontext->alloc_count < 4)
13228 if (acontext->alloc_count == 0)
13230 acontext->set_home_heap(GCHeap::GetHeap( heap_select::select_heap(acontext, 0) ));
13231 gc_heap* hp = acontext->get_home_heap()->pGenGCHeap;
13232 dprintf (3, ("First allocation for context %Ix on heap %d\n", (size_t)acontext, (size_t)hp->heap_number));
13233 acontext->set_alloc_heap(acontext->get_home_heap());
13234 hp->alloc_context_count++;
13239 BOOL set_home_heap = FALSE;
13242 if (heap_select::can_find_heap_fast())
13244 if (acontext->get_home_heap() != NULL)
13245 hint = acontext->get_home_heap()->pGenGCHeap->heap_number;
13246 if (acontext->get_home_heap() != GCHeap::GetHeap(hint = heap_select::select_heap(acontext, hint)) || ((acontext->alloc_count & 15) == 0))
13248 set_home_heap = TRUE;
13254 if ((acontext->alloc_count & 3) == 0)
13255 set_home_heap = TRUE;
13261 // Since we are balancing up to MAX_SUPPORTED_CPUS, no need for this.
13262 if (n_heaps > MAX_SUPPORTED_CPUS)
13264 // on machines with many processors cache affinity is really king, so don't even try
13265 // to balance on these.
13266 acontext->home_heap = GCHeap::GetHeap( heap_select::select_heap(acontext, hint) );
13267 acontext->alloc_heap = acontext->home_heap;
13272 gc_heap* org_hp = acontext->get_alloc_heap()->pGenGCHeap;
13274 dynamic_data* dd = org_hp->dynamic_data_of (0);
13275 ptrdiff_t org_size = dd_new_allocation (dd);
13276 int org_alloc_context_count;
13277 int max_alloc_context_count;
13279 ptrdiff_t max_size;
13280 size_t delta = dd_min_size (dd)/4;
13282 int start, end, finish;
13283 heap_select::get_heap_range_for_heap(org_hp->heap_number, &start, &end);
13284 finish = start + n_heaps;
13290 max_size = org_size + delta;
13291 acontext->set_home_heap(GCHeap::GetHeap( heap_select::select_heap(acontext, hint) ));
13293 if (org_hp == acontext->get_home_heap()->pGenGCHeap)
13294 max_size = max_size + delta;
13296 org_alloc_context_count = org_hp->alloc_context_count;
13297 max_alloc_context_count = org_alloc_context_count;
13298 if (max_alloc_context_count > 1)
13299 max_size /= max_alloc_context_count;
13301 for (int i = start; i < end; i++)
13303 gc_heap* hp = GCHeap::GetHeap(i%n_heaps)->pGenGCHeap;
13304 dd = hp->dynamic_data_of (0);
13305 ptrdiff_t size = dd_new_allocation (dd);
13306 if (hp == acontext->get_home_heap()->pGenGCHeap)
13307 size = size + delta;
13308 int hp_alloc_context_count = hp->alloc_context_count;
13309 if (hp_alloc_context_count > 0)
13310 size /= (hp_alloc_context_count + 1);
13311 if (size > max_size)
13315 max_alloc_context_count = hp_alloc_context_count;
13319 while (org_alloc_context_count != org_hp->alloc_context_count ||
13320 max_alloc_context_count != max_hp->alloc_context_count);
13322 if ((max_hp == org_hp) && (end < finish))
13324 start = end; end = finish;
13325 delta = dd_min_size(dd)/2; // Make it twice as hard to balance to remote nodes on NUMA.
13329 if (max_hp != org_hp)
13331 org_hp->alloc_context_count--;
13332 max_hp->alloc_context_count++;
13333 acontext->set_alloc_heap(GCHeap::GetHeap(max_hp->heap_number));
13334 if (GCToOSInterface::CanEnableGCCPUGroups())
13335 { //only set ideal processor when max_hp and org_hp are in the same cpu
13336 //group. DO NOT MOVE THREADS ACROSS CPU GROUPS
13337 uint16_t org_gn = heap_select::find_cpu_group_from_heap_no(org_hp->heap_number);
13338 uint16_t max_gn = heap_select::find_cpu_group_from_heap_no(max_hp->heap_number);
13339 if (org_gn == max_gn) //only set within CPU group, so SetThreadIdealProcessor is enough
13341 uint16_t group_proc_no = heap_select::find_group_proc_from_heap_no(max_hp->heap_number);
13343 GCThreadAffinity affinity;
13344 affinity.Processor = group_proc_no;
13345 affinity.Group = org_gn;
13346 if (!GCToOSInterface::SetCurrentThreadIdealAffinity(&affinity))
13348 dprintf (3, ("Failed to set the ideal processor and group for heap %d.",
13349 org_hp->heap_number));
13355 uint16_t proc_no = heap_select::find_proc_no_from_heap_no(max_hp->heap_number);
13357 GCThreadAffinity affinity;
13358 affinity.Processor = proc_no;
13359 affinity.Group = GCThreadAffinity::None;
13361 if (!GCToOSInterface::SetCurrentThreadIdealAffinity(&affinity))
13363 dprintf (3, ("Failed to set the ideal processor for heap %d.",
13364 org_hp->heap_number));
13367 dprintf (3, ("Switching context %p (home heap %d) ",
13369 acontext->get_home_heap()->pGenGCHeap->heap_number));
13370 dprintf (3, (" from heap %d (%Id free bytes, %d contexts) ",
13371 org_hp->heap_number,
13373 org_alloc_context_count));
13374 dprintf (3, (" to heap %d (%Id free bytes, %d contexts)\n",
13375 max_hp->heap_number,
13376 dd_new_allocation(max_hp->dynamic_data_of(0)),
13377 max_alloc_context_count));
13382 acontext->alloc_count++;
13385 gc_heap* gc_heap::balance_heaps_loh (alloc_context* acontext, size_t /*size*/)
13387 gc_heap* org_hp = acontext->get_alloc_heap()->pGenGCHeap;
13388 //dprintf (1, ("LA: %Id", size));
13390 //if (size > 128*1024)
13393 dynamic_data* dd = org_hp->dynamic_data_of (max_generation + 1);
13395 ptrdiff_t org_size = dd_new_allocation (dd);
13397 ptrdiff_t max_size;
13398 size_t delta = dd_min_size (dd) * 4;
13400 int start, end, finish;
13401 heap_select::get_heap_range_for_heap(org_hp->heap_number, &start, &end);
13402 finish = start + n_heaps;
13407 max_size = org_size + delta;
13408 dprintf (3, ("orig hp: %d, max size: %d",
13409 org_hp->heap_number,
13412 for (int i = start; i < end; i++)
13414 gc_heap* hp = GCHeap::GetHeap(i%n_heaps)->pGenGCHeap;
13415 dd = hp->dynamic_data_of (max_generation + 1);
13416 ptrdiff_t size = dd_new_allocation (dd);
13417 dprintf (3, ("hp: %d, size: %d",
13420 if (size > max_size)
13424 dprintf (3, ("max hp: %d, max size: %d",
13425 max_hp->heap_number,
13431 if ((max_hp == org_hp) && (end < finish))
13433 start = end; end = finish;
13434 delta = dd_min_size(dd) * 4; // Need to tuning delta
13438 if (max_hp != org_hp)
13440 dprintf (3, ("loh: %d(%Id)->%d(%Id)",
13441 org_hp->heap_number, dd_new_allocation (org_hp->dynamic_data_of (max_generation + 1)),
13442 max_hp->heap_number, dd_new_allocation (max_hp->dynamic_data_of (max_generation + 1))));
13452 #endif //MULTIPLE_HEAPS
13454 BOOL gc_heap::allocate_more_space(alloc_context* acontext, size_t size,
13455 int alloc_generation_number)
13460 #ifdef MULTIPLE_HEAPS
13461 if (alloc_generation_number == 0)
13463 balance_heaps (acontext);
13464 status = acontext->get_alloc_heap()->pGenGCHeap->try_allocate_more_space (acontext, size, alloc_generation_number);
13468 gc_heap* alloc_heap = balance_heaps_loh (acontext, size);
13469 status = alloc_heap->try_allocate_more_space (acontext, size, alloc_generation_number);
13472 status = try_allocate_more_space (acontext, size, alloc_generation_number);
13473 #endif //MULTIPLE_HEAPS
13475 while (status == -1);
13477 return (status != 0);
13481 CObjectHeader* gc_heap::allocate (size_t jsize, alloc_context* acontext)
13483 size_t size = Align (jsize);
13484 assert (size >= Align (min_obj_size));
13487 uint8_t* result = acontext->alloc_ptr;
13488 acontext->alloc_ptr+=size;
13489 if (acontext->alloc_ptr <= acontext->alloc_limit)
13491 CObjectHeader* obj = (CObjectHeader*)result;
13497 acontext->alloc_ptr -= size;
13500 #pragma inline_depth(0)
13503 if (! allocate_more_space (acontext, size, 0))
13507 #pragma inline_depth(20)
13516 CObjectHeader* gc_heap::try_fast_alloc (size_t jsize)
13518 size_t size = Align (jsize);
13519 assert (size >= Align (min_obj_size));
13520 generation* gen = generation_of (0);
13521 uint8_t* result = generation_allocation_pointer (gen);
13522 generation_allocation_pointer (gen) += size;
13523 if (generation_allocation_pointer (gen) <=
13524 generation_allocation_limit (gen))
13526 return (CObjectHeader*)result;
13530 generation_allocation_pointer (gen) -= size;
13534 void gc_heap::leave_allocation_segment (generation* gen)
13536 adjust_limit (0, 0, gen, max_generation);
13539 void gc_heap::init_free_and_plug()
13541 #ifdef FREE_USAGE_STATS
13542 for (int i = 0; i <= settings.condemned_generation; i++)
13544 generation* gen = generation_of (i);
13545 memset (gen->gen_free_spaces, 0, sizeof (gen->gen_free_spaces));
13546 memset (gen->gen_plugs, 0, sizeof (gen->gen_plugs));
13547 memset (gen->gen_current_pinned_free_spaces, 0, sizeof (gen->gen_current_pinned_free_spaces));
13550 if (settings.condemned_generation != max_generation)
13552 for (int i = (settings.condemned_generation + 1); i <= max_generation; i++)
13554 generation* gen = generation_of (i);
13555 memset (gen->gen_plugs, 0, sizeof (gen->gen_plugs));
13558 #endif //FREE_USAGE_STATS
13561 void gc_heap::print_free_and_plug (const char* msg)
13563 #if defined(FREE_USAGE_STATS) && defined(SIMPLE_DPRINTF)
13564 int older_gen = ((settings.condemned_generation == max_generation) ? max_generation : (settings.condemned_generation + 1));
13565 for (int i = 0; i <= older_gen; i++)
13567 generation* gen = generation_of (i);
13568 for (int j = 0; j < NUM_GEN_POWER2; j++)
13570 if ((gen->gen_free_spaces[j] != 0) || (gen->gen_plugs[j] != 0))
13572 dprintf (2, ("[%s][h%d][%s#%d]gen%d: 2^%d: F: %Id, P: %Id",
13575 (settings.concurrent ? "BGC" : "GC"),
13578 (j + 9), gen->gen_free_spaces[j], gen->gen_plugs[j]));
13583 UNREFERENCED_PARAMETER(msg);
13584 #endif //FREE_USAGE_STATS && SIMPLE_DPRINTF
13587 void gc_heap::add_gen_plug (int gen_number, size_t plug_size)
13589 #ifdef FREE_USAGE_STATS
13590 dprintf (3, ("adding plug size %Id to gen%d", plug_size, gen_number));
13591 generation* gen = generation_of (gen_number);
13592 size_t sz = BASE_GEN_SIZE;
13595 for (; i < NUM_GEN_POWER2; i++)
13597 if (plug_size < sz)
13604 (gen->gen_plugs[i])++;
13606 UNREFERENCED_PARAMETER(gen_number);
13607 UNREFERENCED_PARAMETER(plug_size);
13608 #endif //FREE_USAGE_STATS
13611 void gc_heap::add_item_to_current_pinned_free (int gen_number, size_t free_size)
13613 #ifdef FREE_USAGE_STATS
13614 generation* gen = generation_of (gen_number);
13615 size_t sz = BASE_GEN_SIZE;
13618 for (; i < NUM_GEN_POWER2; i++)
13620 if (free_size < sz)
13627 (gen->gen_current_pinned_free_spaces[i])++;
13628 generation_pinned_free_obj_space (gen) += free_size;
13629 dprintf (3, ("left pin free %Id(2^%d) to gen%d, total %Id bytes (%Id)",
13630 free_size, (i + 10), gen_number,
13631 generation_pinned_free_obj_space (gen),
13632 gen->gen_current_pinned_free_spaces[i]));
13634 UNREFERENCED_PARAMETER(gen_number);
13635 UNREFERENCED_PARAMETER(free_size);
13636 #endif //FREE_USAGE_STATS
13639 void gc_heap::add_gen_free (int gen_number, size_t free_size)
13641 #ifdef FREE_USAGE_STATS
13642 dprintf (3, ("adding free size %Id to gen%d", free_size, gen_number));
13643 generation* gen = generation_of (gen_number);
13644 size_t sz = BASE_GEN_SIZE;
13647 for (; i < NUM_GEN_POWER2; i++)
13649 if (free_size < sz)
13656 (gen->gen_free_spaces[i])++;
13658 UNREFERENCED_PARAMETER(gen_number);
13659 UNREFERENCED_PARAMETER(free_size);
13660 #endif //FREE_USAGE_STATS
13663 void gc_heap::remove_gen_free (int gen_number, size_t free_size)
13665 #ifdef FREE_USAGE_STATS
13666 dprintf (3, ("removing free %Id from gen%d", free_size, gen_number));
13667 generation* gen = generation_of (gen_number);
13668 size_t sz = BASE_GEN_SIZE;
13671 for (; i < NUM_GEN_POWER2; i++)
13673 if (free_size < sz)
13680 (gen->gen_free_spaces[i])--;
13682 UNREFERENCED_PARAMETER(gen_number);
13683 UNREFERENCED_PARAMETER(free_size);
13684 #endif //FREE_USAGE_STATS
13687 uint8_t* gc_heap::allocate_in_older_generation (generation* gen, size_t size,
13688 int from_gen_number,
13689 uint8_t* old_loc REQD_ALIGN_AND_OFFSET_DCL)
13691 size = Align (size);
13692 assert (size >= Align (min_obj_size));
13693 assert (from_gen_number < max_generation);
13694 assert (from_gen_number >= 0);
13695 assert (generation_of (from_gen_number + 1) == gen);
13697 allocator* gen_allocator = generation_allocator (gen);
13698 BOOL discard_p = gen_allocator->discard_if_no_fit_p ();
13699 int pad_in_front = (old_loc != 0)? USE_PADDING_FRONT : 0;
13701 size_t real_size = size + Align (min_obj_size);
13703 real_size += Align (min_obj_size);
13705 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
13706 generation_allocation_limit (gen), old_loc, USE_PADDING_TAIL | pad_in_front)))
13708 size_t sz_list = gen_allocator->first_bucket_size();
13709 for (unsigned int a_l_idx = 0; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
13711 if ((real_size < (sz_list / 2)) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
13713 uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
13714 uint8_t* prev_free_item = 0;
13715 while (free_list != 0)
13717 dprintf (3, ("considering free list %Ix", (size_t)free_list));
13719 size_t free_list_size = unused_array_size (free_list);
13721 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, free_list, (free_list + free_list_size),
13722 old_loc, USE_PADDING_TAIL | pad_in_front))
13724 dprintf (4, ("F:%Ix-%Id",
13725 (size_t)free_list, free_list_size));
13727 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, !discard_p);
13728 generation_free_list_space (gen) -= free_list_size;
13729 remove_gen_free (gen->gen_num, free_list_size);
13731 adjust_limit (free_list, free_list_size, gen, from_gen_number+1);
13734 // We do first fit on bucket 0 because we are not guaranteed to find a fit there.
13735 else if (discard_p || (a_l_idx == 0))
13737 dprintf (3, ("couldn't use this free area, discarding"));
13738 generation_free_obj_space (gen) += free_list_size;
13740 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
13741 generation_free_list_space (gen) -= free_list_size;
13742 remove_gen_free (gen->gen_num, free_list_size);
13746 prev_free_item = free_list;
13748 free_list = free_list_slot (free_list);
13751 sz_list = sz_list * 2;
13753 //go back to the beginning of the segment list
13754 generation_allocate_end_seg_p (gen) = TRUE;
13755 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
13756 if (seg != generation_allocation_segment (gen))
13758 leave_allocation_segment (gen);
13759 generation_allocation_segment (gen) = seg;
13761 while (seg != ephemeral_heap_segment)
13763 if (size_fit_p(size REQD_ALIGN_AND_OFFSET_ARG, heap_segment_plan_allocated (seg),
13764 heap_segment_committed (seg), old_loc, USE_PADDING_TAIL | pad_in_front))
13766 dprintf (3, ("using what's left in committed"));
13767 adjust_limit (heap_segment_plan_allocated (seg),
13768 heap_segment_committed (seg) -
13769 heap_segment_plan_allocated (seg),
13770 gen, from_gen_number+1);
13771 // dformat (t, 3, "Expanding segment allocation");
13772 heap_segment_plan_allocated (seg) =
13773 heap_segment_committed (seg);
13778 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, heap_segment_plan_allocated (seg),
13779 heap_segment_reserved (seg), old_loc, USE_PADDING_TAIL | pad_in_front) &&
13780 grow_heap_segment (seg, heap_segment_plan_allocated (seg), old_loc, size, pad_in_front REQD_ALIGN_AND_OFFSET_ARG))
13782 dprintf (3, ("using what's left in reserved"));
13783 adjust_limit (heap_segment_plan_allocated (seg),
13784 heap_segment_committed (seg) -
13785 heap_segment_plan_allocated (seg),
13786 gen, from_gen_number+1);
13787 heap_segment_plan_allocated (seg) =
13788 heap_segment_committed (seg);
13794 leave_allocation_segment (gen);
13795 heap_segment* next_seg = heap_segment_next_rw (seg);
13798 dprintf (3, ("getting next segment"));
13799 generation_allocation_segment (gen) = next_seg;
13800 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
13801 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
13810 seg = generation_allocation_segment (gen);
13812 //No need to fix the last region. Will be done later
13823 uint8_t* result = generation_allocation_pointer (gen);
13827 if ((pad_in_front & USE_PADDING_FRONT) &&
13828 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
13829 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
13831 pad = Align (min_obj_size);
13832 set_plug_padded (old_loc);
13834 #endif //SHORT_PLUGS
13836 #ifdef FEATURE_STRUCTALIGN
13837 _ASSERTE(!old_loc || alignmentOffset != 0);
13838 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
13841 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
13842 set_node_aligninfo (old_loc, requiredAlignment, pad1);
13845 #else // FEATURE_STRUCTALIGN
13846 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
13848 pad += switch_alignment_size (is_plug_padded (old_loc));
13849 set_node_realigned (old_loc);
13850 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
13851 (size_t)old_loc, (size_t)(result+pad)));
13852 assert (same_large_alignment_p (result + pad, old_loc));
13854 #endif // FEATURE_STRUCTALIGN
13855 dprintf (3, ("Allocate %Id bytes", size));
13857 if ((old_loc == 0) || (pad != 0))
13859 //allocating a non plug or a gap, so reset the start region
13860 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
13863 generation_allocation_pointer (gen) += size + pad;
13864 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
13865 if (generation_allocate_end_seg_p (gen))
13867 generation_end_seg_allocated (gen) += size;
13871 generation_free_list_allocated (gen) += size;
13873 generation_allocation_size (gen) += size;
13875 dprintf (3, ("aio: ptr: %Ix, limit: %Ix, sr: %Ix",
13876 generation_allocation_pointer (gen), generation_allocation_limit (gen),
13877 generation_allocation_context_start_region (gen)));
13879 return result + pad;;
13883 void gc_heap::repair_allocation_in_expanded_heap (generation* consing_gen)
13885 //make sure that every generation has a planned allocation start
13886 int gen_number = max_generation - 1;
13887 while (gen_number>= 0)
13889 generation* gen = generation_of (gen_number);
13890 if (0 == generation_plan_allocation_start (gen))
13892 realloc_plan_generation_start (gen, consing_gen);
13894 assert (generation_plan_allocation_start (gen));
13899 // now we know the planned allocation size
13900 size_t size = (generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
13901 heap_segment* seg = generation_allocation_segment (consing_gen);
13902 if (generation_allocation_limit (consing_gen) == heap_segment_plan_allocated (seg))
13906 heap_segment_plan_allocated (seg) = generation_allocation_pointer (consing_gen);
13911 assert (settings.condemned_generation == max_generation);
13912 uint8_t* first_address = generation_allocation_limit (consing_gen);
13913 //look through the pinned plugs for relevant ones.
13914 //Look for the right pinned plug to start from.
13917 while (mi != mark_stack_tos)
13919 m = pinned_plug_of (mi);
13920 if ((pinned_plug (m) == first_address))
13925 assert (mi != mark_stack_tos);
13926 pinned_len (m) = size;
13930 //tododefrag optimize for new segment (plan_allocated == mem)
13931 uint8_t* gc_heap::allocate_in_expanded_heap (generation* gen,
13936 BOOL set_padding_on_saved_p,
13937 mark* pinned_plug_entry,
13938 #endif //SHORT_PLUGS
13939 BOOL consider_bestfit,
13940 int active_new_gen_number
13941 REQD_ALIGN_AND_OFFSET_DCL)
13943 UNREFERENCED_PARAMETER(active_new_gen_number);
13944 dprintf (3, ("aie: P: %Ix, size: %Ix", old_loc, size));
13946 size = Align (size);
13947 assert (size >= Align (min_obj_size));
13948 int pad_in_front = (old_loc != 0) ? USE_PADDING_FRONT : 0;
13950 if (consider_bestfit && use_bestfit)
13952 assert (bestfit_seg);
13953 dprintf (SEG_REUSE_LOG_1, ("reallocating 0x%Ix in expanded heap, size: %Id",
13955 return bestfit_seg->fit (old_loc,
13957 set_padding_on_saved_p,
13959 #endif //SHORT_PLUGS
13960 size REQD_ALIGN_AND_OFFSET_ARG);
13963 heap_segment* seg = generation_allocation_segment (gen);
13965 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
13966 generation_allocation_limit (gen), old_loc,
13967 ((generation_allocation_limit (gen) !=
13968 heap_segment_plan_allocated (seg))? USE_PADDING_TAIL : 0) | pad_in_front)))
13970 dprintf (3, ("aie: can't fit: ptr: %Ix, limit: %Ix", generation_allocation_pointer (gen),
13971 generation_allocation_limit (gen)));
13974 uint8_t* first_address = (generation_allocation_limit (gen) ?
13975 generation_allocation_limit (gen) :
13976 heap_segment_mem (seg));
13977 assert (in_range_for_segment (first_address, seg));
13979 uint8_t* end_address = heap_segment_reserved (seg);
13981 dprintf (3, ("aie: first_addr: %Ix, gen alloc limit: %Ix, end_address: %Ix",
13982 first_address, generation_allocation_limit (gen), end_address));
13987 if (heap_segment_allocated (seg) != heap_segment_mem (seg))
13989 assert (settings.condemned_generation == max_generation);
13990 //look through the pinned plugs for relevant ones.
13991 //Look for the right pinned plug to start from.
13992 while (mi != mark_stack_tos)
13994 m = pinned_plug_of (mi);
13995 if ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address))
13997 dprintf (3, ("aie: found pin: %Ix", pinned_plug (m)));
14003 if (mi != mark_stack_tos)
14005 //fix old free list.
14006 size_t hsize = (generation_allocation_limit (gen) - generation_allocation_pointer (gen));
14008 dprintf(3,("gc filling up hole"));
14009 ptrdiff_t mi1 = (ptrdiff_t)mi;
14010 while ((mi1 >= 0) &&
14011 (pinned_plug (pinned_plug_of(mi1)) != generation_allocation_limit (gen)))
14013 dprintf (3, ("aie: checking pin %Ix", pinned_plug (pinned_plug_of(mi1))));
14018 size_t saved_pinned_len = pinned_len (pinned_plug_of(mi1));
14019 pinned_len (pinned_plug_of(mi1)) = hsize;
14020 dprintf (3, ("changing %Ix len %Ix->%Ix",
14021 pinned_plug (pinned_plug_of(mi1)),
14022 saved_pinned_len, pinned_len (pinned_plug_of(mi1))));
14029 assert (generation_allocation_limit (gen) ==
14030 generation_allocation_pointer (gen));
14031 mi = mark_stack_tos;
14034 while ((mi != mark_stack_tos) && in_range_for_segment (pinned_plug (m), seg))
14036 size_t len = pinned_len (m);
14037 uint8_t* free_list = (pinned_plug (m) - len);
14038 dprintf (3, ("aie: testing free item: %Ix->%Ix(%Ix)",
14039 free_list, (free_list + len), len));
14040 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, free_list, (free_list + len), old_loc, USE_PADDING_TAIL | pad_in_front))
14042 dprintf (3, ("aie: Found adequate unused area: %Ix, size: %Id",
14043 (size_t)free_list, len));
14045 generation_allocation_pointer (gen) = free_list;
14046 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14047 generation_allocation_limit (gen) = (free_list + len);
14049 goto allocate_in_free;
14052 m = pinned_plug_of (mi);
14055 //switch to the end of the segment.
14056 generation_allocation_pointer (gen) = heap_segment_plan_allocated (seg);
14057 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14058 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14059 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14060 dprintf (3, ("aie: switching to end of seg: %Ix->%Ix(%Ix)",
14061 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14062 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
14064 if (!size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14065 generation_allocation_limit (gen), old_loc, USE_PADDING_TAIL | pad_in_front))
14067 dprintf (3, ("aie: ptr: %Ix, limit: %Ix, can't alloc", generation_allocation_pointer (gen),
14068 generation_allocation_limit (gen)));
14069 assert (!"Can't allocate if no free space");
14080 uint8_t* result = generation_allocation_pointer (gen);
14084 if ((pad_in_front & USE_PADDING_FRONT) &&
14085 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
14086 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
14089 pad = Align (min_obj_size);
14090 set_padding_in_expand (old_loc, set_padding_on_saved_p, pinned_plug_entry);
14092 #endif //SHORT_PLUGS
14094 #ifdef FEATURE_STRUCTALIGN
14095 _ASSERTE(!old_loc || alignmentOffset != 0);
14096 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
14099 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
14100 set_node_aligninfo (old_loc, requiredAlignment, pad1);
14104 #else // FEATURE_STRUCTALIGN
14105 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
14107 pad += switch_alignment_size (is_plug_padded (old_loc));
14108 set_node_realigned (old_loc);
14109 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
14110 (size_t)old_loc, (size_t)(result+pad)));
14111 assert (same_large_alignment_p (result + pad, old_loc));
14114 #endif // FEATURE_STRUCTALIGN
14116 if ((old_loc == 0) || (pad != 0))
14118 //allocating a non plug or a gap, so reset the start region
14119 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14122 generation_allocation_pointer (gen) += size + pad;
14123 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
14124 dprintf (3, ("Allocated in expanded heap %Ix:%Id", (size_t)(result+pad), size));
14126 dprintf (3, ("aie: ptr: %Ix, limit: %Ix, sr: %Ix",
14127 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14128 generation_allocation_context_start_region (gen)));
14130 return result + pad;
14134 generation* gc_heap::ensure_ephemeral_heap_segment (generation* consing_gen)
14136 heap_segment* seg = generation_allocation_segment (consing_gen);
14137 if (seg != ephemeral_heap_segment)
14139 assert (generation_allocation_pointer (consing_gen)>= heap_segment_mem (seg));
14140 assert (generation_allocation_pointer (consing_gen)<= heap_segment_committed (seg));
14142 //fix the allocated size of the segment.
14143 heap_segment_plan_allocated (seg) = generation_allocation_pointer (consing_gen);
14145 generation* new_consing_gen = generation_of (max_generation - 1);
14146 generation_allocation_pointer (new_consing_gen) =
14147 heap_segment_mem (ephemeral_heap_segment);
14148 generation_allocation_limit (new_consing_gen) =
14149 generation_allocation_pointer (new_consing_gen);
14150 generation_allocation_context_start_region (new_consing_gen) =
14151 generation_allocation_pointer (new_consing_gen);
14152 generation_allocation_segment (new_consing_gen) = ephemeral_heap_segment;
14154 return new_consing_gen;
14157 return consing_gen;
14160 uint8_t* gc_heap::allocate_in_condemned_generations (generation* gen,
14162 int from_gen_number,
14164 BOOL* convert_to_pinned_p,
14165 uint8_t* next_pinned_plug,
14166 heap_segment* current_seg,
14167 #endif //SHORT_PLUGS
14169 REQD_ALIGN_AND_OFFSET_DCL)
14171 // Make sure that the youngest generation gap hasn't been allocated
14172 if (settings.promotion)
14174 assert (generation_plan_allocation_start (youngest_generation) == 0);
14177 size = Align (size);
14178 assert (size >= Align (min_obj_size));
14179 int to_gen_number = from_gen_number;
14180 if (from_gen_number != (int)max_generation)
14182 to_gen_number = from_gen_number + (settings.promotion ? 1 : 0);
14185 dprintf (3, ("aic gen%d: s: %Id, %d->%d, %Ix->%Ix", gen->gen_num, size, from_gen_number,
14186 to_gen_number, generation_allocation_pointer(gen), generation_allocation_limit(gen)));
14188 int pad_in_front = (old_loc != 0) ? USE_PADDING_FRONT : 0;
14190 if ((from_gen_number != -1) && (from_gen_number != (int)max_generation) && settings.promotion)
14192 generation_condemned_allocated (generation_of (from_gen_number + (settings.promotion ? 1 : 0))) += size;
14193 generation_allocation_size (generation_of (from_gen_number + (settings.promotion ? 1 : 0))) += size;
14197 heap_segment* seg = generation_allocation_segment (gen);
14198 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14199 generation_allocation_limit (gen), old_loc,
14200 ((generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))?USE_PADDING_TAIL:0)|pad_in_front)))
14202 if ((! (pinned_plug_que_empty_p()) &&
14203 (generation_allocation_limit (gen) ==
14204 pinned_plug (oldest_pin()))))
14206 size_t entry = deque_pinned_plug();
14207 mark* pinned_plug_entry = pinned_plug_of (entry);
14208 size_t len = pinned_len (pinned_plug_entry);
14209 uint8_t* plug = pinned_plug (pinned_plug_entry);
14210 set_new_pin_info (pinned_plug_entry, generation_allocation_pointer (gen));
14212 #ifdef FREE_USAGE_STATS
14213 generation_allocated_in_pinned_free (gen) += generation_allocated_since_last_pin (gen);
14214 dprintf (3, ("allocated %Id so far within pin %Ix, total->%Id",
14215 generation_allocated_since_last_pin (gen),
14217 generation_allocated_in_pinned_free (gen)));
14218 generation_allocated_since_last_pin (gen) = 0;
14220 add_item_to_current_pinned_free (gen->gen_num, pinned_len (pinned_plug_of (entry)));
14221 #endif //FREE_USAGE_STATS
14223 dprintf (3, ("mark stack bos: %Id, tos: %Id, aic: p %Ix len: %Ix->%Ix",
14224 mark_stack_bos, mark_stack_tos, plug, len, pinned_len (pinned_plug_of (entry))));
14226 assert(mark_stack_array[entry].len == 0 ||
14227 mark_stack_array[entry].len >= Align(min_obj_size));
14228 generation_allocation_pointer (gen) = plug + len;
14229 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14230 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14231 set_allocator_next_pin (gen);
14233 //Add the size of the pinned plug to the right pinned allocations
14234 //find out which gen this pinned plug came from
14235 int frgn = object_gennum (plug);
14236 if ((frgn != (int)max_generation) && settings.promotion)
14238 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
14239 int togn = object_gennum_plan (plug);
14242 generation_pinned_allocation_compact_size (generation_of (togn)) += len;
14248 if (generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))
14250 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14251 dprintf (3, ("changed limit to plan alloc: %Ix", generation_allocation_limit (gen)));
14255 if (heap_segment_plan_allocated (seg) != heap_segment_committed (seg))
14257 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14258 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14259 dprintf (3, ("changed limit to commit: %Ix", generation_allocation_limit (gen)));
14263 #ifndef RESPECT_LARGE_ALIGNMENT
14264 assert (gen != youngest_generation);
14265 #endif //RESPECT_LARGE_ALIGNMENT
14267 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14268 heap_segment_reserved (seg), old_loc, USE_PADDING_TAIL | pad_in_front) &&
14269 (grow_heap_segment (seg, generation_allocation_pointer (gen), old_loc,
14270 size, pad_in_front REQD_ALIGN_AND_OFFSET_ARG)))
14272 dprintf (3, ("Expanded segment allocation by committing more memory"));
14273 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14274 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14278 heap_segment* next_seg = heap_segment_next (seg);
14279 assert (generation_allocation_pointer (gen)>=
14280 heap_segment_mem (seg));
14281 // Verify that all pinned plugs for this segment are consumed
14282 if (!pinned_plug_que_empty_p() &&
14283 ((pinned_plug (oldest_pin()) <
14284 heap_segment_allocated (seg)) &&
14285 (pinned_plug (oldest_pin()) >=
14286 generation_allocation_pointer (gen))))
14288 LOG((LF_GC, LL_INFO10, "remaining pinned plug %Ix while leaving segment on allocation",
14289 pinned_plug (oldest_pin())));
14292 assert (generation_allocation_pointer (gen)>=
14293 heap_segment_mem (seg));
14294 assert (generation_allocation_pointer (gen)<=
14295 heap_segment_committed (seg));
14296 heap_segment_plan_allocated (seg) = generation_allocation_pointer (gen);
14300 generation_allocation_segment (gen) = next_seg;
14301 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
14302 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
14303 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14307 return 0; //should only happen during allocation of generation 0 gap
14308 // in that case we are going to grow the heap anyway
14313 set_allocator_next_pin (gen);
14320 assert (generation_allocation_pointer (gen)>=
14321 heap_segment_mem (generation_allocation_segment (gen)));
14322 uint8_t* result = generation_allocation_pointer (gen);
14325 if ((pad_in_front & USE_PADDING_FRONT) &&
14326 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
14327 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
14329 ptrdiff_t dist = old_loc - result;
14332 dprintf (3, ("old alloc: %Ix, same as new alloc, not padding", old_loc));
14337 if ((dist > 0) && (dist < (ptrdiff_t)Align (min_obj_size)))
14339 dprintf (3, ("old alloc: %Ix, only %d bytes > new alloc! Shouldn't happen", old_loc, dist));
14343 pad = Align (min_obj_size);
14344 set_plug_padded (old_loc);
14347 #endif //SHORT_PLUGS
14348 #ifdef FEATURE_STRUCTALIGN
14349 _ASSERTE(!old_loc || alignmentOffset != 0);
14350 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
14351 if ((old_loc != 0))
14353 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
14354 set_node_aligninfo (old_loc, requiredAlignment, pad1);
14357 #else // FEATURE_STRUCTALIGN
14358 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
14360 pad += switch_alignment_size (is_plug_padded (old_loc));
14361 set_node_realigned(old_loc);
14362 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
14363 (size_t)old_loc, (size_t)(result+pad)));
14364 assert (same_large_alignment_p (result + pad, old_loc));
14366 #endif // FEATURE_STRUCTALIGN
14369 if ((next_pinned_plug != 0) && (pad != 0) && (generation_allocation_segment (gen) == current_seg))
14371 assert (old_loc != 0);
14372 ptrdiff_t dist_to_next_pin = (ptrdiff_t)(next_pinned_plug - (generation_allocation_pointer (gen) + size + pad));
14373 assert (dist_to_next_pin >= 0);
14375 if ((dist_to_next_pin >= 0) && (dist_to_next_pin < (ptrdiff_t)Align (min_obj_size)))
14377 dprintf (3, ("%Ix->(%Ix,%Ix),%Ix(%Ix)(%Ix),NP->PP",
14379 generation_allocation_pointer (gen),
14380 generation_allocation_limit (gen),
14383 dist_to_next_pin));
14384 clear_plug_padded (old_loc);
14386 *convert_to_pinned_p = TRUE;
14387 record_interesting_data_point (idp_converted_pin);
14392 #endif //SHORT_PLUGS
14394 if ((old_loc == 0) || (pad != 0))
14396 //allocating a non plug or a gap, so reset the start region
14397 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14400 generation_allocation_pointer (gen) += size + pad;
14401 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
14403 #ifdef FREE_USAGE_STATS
14404 generation_allocated_since_last_pin (gen) += size;
14405 #endif //FREE_USAGE_STATS
14407 dprintf (3, ("aic: ptr: %Ix, limit: %Ix, sr: %Ix",
14408 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14409 generation_allocation_context_start_region (gen)));
14411 assert (result + pad);
14412 return result + pad;
14416 inline int power (int x, int y)
14419 for (int i = 0; i < y; i++)
14426 int gc_heap::joined_generation_to_condemn (BOOL should_evaluate_elevation,
14428 BOOL* blocking_collection_p
14429 STRESS_HEAP_ARG(int n_original))
14432 #ifdef MULTIPLE_HEAPS
14433 BOOL blocking_p = *blocking_collection_p;
14436 for (int i = 0; i < n_heaps; i++)
14438 if (g_heaps[i]->last_gc_before_oom)
14440 dprintf (GTC_LOG, ("h%d is setting blocking to TRUE", i));
14441 *blocking_collection_p = TRUE;
14446 #endif //MULTIPLE_HEAPS
14448 if (should_evaluate_elevation && (n == max_generation))
14450 dprintf (GTC_LOG, ("lock: %d(%d)",
14451 (settings.should_lock_elevation ? 1 : 0),
14452 settings.elevation_locked_count));
14454 if (settings.should_lock_elevation)
14456 settings.elevation_locked_count++;
14457 if (settings.elevation_locked_count == 6)
14459 settings.elevation_locked_count = 0;
14463 n = max_generation - 1;
14464 settings.elevation_reduced = TRUE;
14469 settings.elevation_locked_count = 0;
14474 settings.should_lock_elevation = FALSE;
14475 settings.elevation_locked_count = 0;
14479 #ifdef BACKGROUND_GC
14480 // We can only do Concurrent GC Stress if the caller did not explicitly ask for all
14481 // generations to be collected,
14483 // [LOCALGC TODO] STRESS_HEAP is not defined for a standalone GC so there are multiple
14484 // things that need to be fixed in this code block.
14485 if (n_original != max_generation &&
14486 g_pConfig->GetGCStressLevel() && gc_can_use_concurrent)
14488 #ifndef FEATURE_REDHAWK
14489 // for the GC stress mix mode throttle down gen2 collections
14490 if (g_pConfig->IsGCStressMix())
14492 size_t current_gc_count = 0;
14494 #ifdef MULTIPLE_HEAPS
14495 current_gc_count = (size_t)dd_collection_count (g_heaps[0]->dynamic_data_of (0));
14497 current_gc_count = (size_t)dd_collection_count (dynamic_data_of (0));
14498 #endif //MULTIPLE_HEAPS
14499 // in gc stress, only escalate every 10th non-gen2 collection to a gen2...
14500 if ((current_gc_count % 10) == 0)
14502 n = max_generation;
14505 // for traditional GC stress
14507 #endif // !FEATURE_REDHAWK
14508 if (*blocking_collection_p)
14510 // We call StressHeap() a lot for Concurrent GC Stress. However,
14511 // if we can not do a concurrent collection, no need to stress anymore.
14512 // @TODO: Enable stress when the memory pressure goes down again
14513 GCStressPolicy::GlobalDisable();
14517 n = max_generation;
14520 #endif //BACKGROUND_GC
14521 #endif //STRESS_HEAP
14527 size_t get_survived_size (gc_history_per_heap* hist)
14529 size_t surv_size = 0;
14530 gc_generation_data* gen_data;
14532 for (int gen_number = 0; gen_number <= (max_generation + 1); gen_number++)
14534 gen_data = &(hist->gen_data[gen_number]);
14535 surv_size += (gen_data->size_after -
14536 gen_data->free_list_space_after -
14537 gen_data->free_obj_space_after);
14543 size_t gc_heap::get_total_survived_size()
14545 size_t total_surv_size = 0;
14546 #ifdef MULTIPLE_HEAPS
14547 for (int i = 0; i < gc_heap::n_heaps; i++)
14549 gc_heap* hp = gc_heap::g_heaps[i];
14550 gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap();
14551 total_surv_size += get_survived_size (current_gc_data_per_heap);
14554 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
14555 total_surv_size = get_survived_size (current_gc_data_per_heap);
14556 #endif //MULTIPLE_HEAPS
14557 return total_surv_size;
14560 // Gets what's allocated on both SOH and LOH that hasn't been collected.
14561 size_t gc_heap::get_current_allocated()
14563 dynamic_data* dd = dynamic_data_of (0);
14564 size_t current_alloc = dd_desired_allocation (dd) - dd_new_allocation (dd);
14565 dd = dynamic_data_of (max_generation + 1);
14566 current_alloc += dd_desired_allocation (dd) - dd_new_allocation (dd);
14568 return current_alloc;
14571 size_t gc_heap::get_total_allocated()
14573 size_t total_current_allocated = 0;
14574 #ifdef MULTIPLE_HEAPS
14575 for (int i = 0; i < gc_heap::n_heaps; i++)
14577 gc_heap* hp = gc_heap::g_heaps[i];
14578 total_current_allocated += hp->get_current_allocated();
14581 total_current_allocated = get_current_allocated();
14582 #endif //MULTIPLE_HEAPS
14583 return total_current_allocated;
14586 size_t gc_heap::current_generation_size (int gen_number)
14588 dynamic_data* dd = dynamic_data_of (gen_number);
14589 size_t gen_size = (dd_current_size (dd) + dd_desired_allocation (dd)
14590 - dd_new_allocation (dd));
14596 #pragma warning(push)
14597 #pragma warning(disable:6326) // "Potential comparison of a constant with another constant" is intentional in this function.
14601 This is called by when we are actually doing a GC, or when we are just checking whether
14602 we would do a full blocking GC, in which case check_only_p is TRUE.
14604 The difference between calling this with check_only_p TRUE and FALSE is that when it's
14606 settings.reason is ignored
14607 budgets are not checked (since they are checked before this is called)
14608 it doesn't change anything non local like generation_skip_ratio
14610 int gc_heap::generation_to_condemn (int n_initial,
14611 BOOL* blocking_collection_p,
14612 BOOL* elevation_requested_p,
14615 gc_mechanisms temp_settings = settings;
14616 gen_to_condemn_tuning temp_condemn_reasons;
14617 gc_mechanisms* local_settings = (check_only_p ? &temp_settings : &settings);
14618 gen_to_condemn_tuning* local_condemn_reasons = (check_only_p ? &temp_condemn_reasons : &gen_to_condemn_reasons);
14621 if ((local_settings->reason == reason_oos_soh) || (local_settings->reason == reason_oos_loh))
14623 assert (n_initial >= 1);
14626 assert (settings.reason != reason_empty);
14629 local_condemn_reasons->init();
14633 if (heap_number == 0)
14635 dprintf (GTC_LOG, ("init: %d(%d)", n_initial, settings.reason));
14639 BOOL low_memory_detected = g_low_memory_status;
14640 uint32_t memory_load = 0;
14641 uint64_t available_physical = 0;
14642 uint64_t available_page_file = 0;
14643 BOOL check_memory = FALSE;
14644 BOOL high_fragmentation = FALSE;
14645 BOOL v_high_memory_load = FALSE;
14646 BOOL high_memory_load = FALSE;
14647 BOOL low_ephemeral_space = FALSE;
14648 BOOL evaluate_elevation = TRUE;
14649 *elevation_requested_p = FALSE;
14650 *blocking_collection_p = FALSE;
14652 BOOL check_max_gen_alloc = TRUE;
14656 #endif //STRESS_HEAP
14660 dd_fragmentation (dynamic_data_of (0)) =
14661 generation_free_list_space (youngest_generation) +
14662 generation_free_obj_space (youngest_generation);
14664 dd_fragmentation (dynamic_data_of (max_generation + 1)) =
14665 generation_free_list_space (large_object_generation) +
14666 generation_free_obj_space (large_object_generation);
14668 //save new_allocation
14669 for (i = 0; i <= max_generation+1; i++)
14671 dynamic_data* dd = dynamic_data_of (i);
14672 dprintf (GTC_LOG, ("h%d: g%d: l: %Id (%Id)",
14674 dd_new_allocation (dd),
14675 dd_desired_allocation (dd)));
14676 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
14679 local_condemn_reasons->set_gen (gen_initial, n);
14682 #ifdef BACKGROUND_GC
14683 if (recursive_gc_sync::background_running_p())
14685 dprintf (GTC_LOG, ("bgc in prog, 1"));
14686 check_max_gen_alloc = FALSE;
14688 #endif //BACKGROUND_GC
14690 if (check_max_gen_alloc)
14692 //figure out if large objects need to be collected.
14693 if (get_new_allocation (max_generation+1) <= 0)
14695 n = max_generation;
14696 local_condemn_reasons->set_gen (gen_alloc_budget, n);
14700 //figure out which generation ran out of allocation
14701 for (i = n+1; i <= (check_max_gen_alloc ? max_generation : (max_generation - 1)); i++)
14703 if (get_new_allocation (i) <= 0)
14714 local_condemn_reasons->set_gen (gen_alloc_budget, n);
14717 dprintf (GTC_LOG, ("h%d: g%d budget", heap_number, ((get_new_allocation (max_generation+1) <= 0) ? 3 : n)));
14721 #if defined(BACKGROUND_GC) && !defined(MULTIPLE_HEAPS)
14722 //time based tuning
14723 // if enough time has elapsed since the last gc
14724 // and the number of gc is too low (1/10 of lower gen) then collect
14725 // This should also be enabled if we have memory concerns
14726 int n_time_max = max_generation;
14730 if (recursive_gc_sync::background_running_p())
14732 n_time_max = max_generation - 1;
14736 if ((local_settings->pause_mode == pause_interactive) ||
14737 (local_settings->pause_mode == pause_sustained_low_latency))
14739 dynamic_data* dd0 = dynamic_data_of (0);
14740 size_t now = GetHighPrecisionTimeStamp();
14742 for (i = (temp_gen+1); i <= n_time_max; i++)
14744 dynamic_data* dd = dynamic_data_of (i);
14745 if ((now > dd_time_clock(dd) + dd_time_clock_interval(dd)) &&
14746 (dd_gc_clock (dd0) > (dd_gc_clock (dd) + dd_gc_clock_interval(dd))) &&
14747 ((n < max_generation) || ((dd_current_size (dd) < dd_max_size (dd0)))))
14749 n = min (i, n_time_max);
14750 dprintf (GTC_LOG, ("time %d", n));
14755 local_condemn_reasons->set_gen (gen_time_tuning, n);
14761 dprintf (GTC_LOG, ("Condemning %d based on time tuning and fragmentation", n));
14763 #endif //BACKGROUND_GC && !MULTIPLE_HEAPS
14765 if (n < (max_generation - 1))
14767 if (dt_low_card_table_efficiency_p (tuning_deciding_condemned_gen))
14769 n = max (n, max_generation - 1);
14770 local_settings->promotion = TRUE;
14771 dprintf (GTC_LOG, ("h%d: skip %d, c %d",
14772 heap_number, generation_skip_ratio, n));
14773 local_condemn_reasons->set_condition (gen_low_card_p);
14779 generation_skip_ratio = 100;
14782 if (dt_low_ephemeral_space_p (check_only_p ?
14783 tuning_deciding_full_gc :
14784 tuning_deciding_condemned_gen))
14786 low_ephemeral_space = TRUE;
14788 n = max (n, max_generation - 1);
14789 local_condemn_reasons->set_condition (gen_low_ephemeral_p);
14790 dprintf (GTC_LOG, ("h%d: low eph", heap_number));
14792 #ifdef BACKGROUND_GC
14793 if (!gc_can_use_concurrent || (generation_free_list_space (generation_of (max_generation)) == 0))
14794 #endif //BACKGROUND_GC
14796 //It is better to defragment first if we are running out of space for
14797 //the ephemeral generation but we have enough fragmentation to make up for it
14798 //in the non ephemeral generation. Essentially we are trading a gen2 for
14799 // having to expand heap in ephemeral collections.
14800 if (dt_high_frag_p (tuning_deciding_condemned_gen,
14801 max_generation - 1,
14804 high_fragmentation = TRUE;
14805 local_condemn_reasons->set_condition (gen_max_high_frag_e_p);
14806 dprintf (GTC_LOG, ("heap%d: gen1 frag", heap_number));
14811 //figure out which ephemeral generation is too fragramented
14813 for (i = n+1; i < max_generation; i++)
14815 if (dt_high_frag_p (tuning_deciding_condemned_gen, i))
14817 dprintf (GTC_LOG, ("h%d g%d too frag", heap_number, i));
14824 if (low_ephemeral_space)
14827 local_settings->promotion = TRUE;
14832 local_condemn_reasons->set_condition (gen_eph_high_frag_p);
14837 if (settings.pause_mode == pause_low_latency)
14839 if (!is_induced (settings.reason))
14841 n = min (n, max_generation - 1);
14842 dprintf (GTC_LOG, ("low latency mode is enabled, condemning %d", n));
14843 evaluate_elevation = FALSE;
14849 // It's hard to catch when we get to the point that the memory load is so high
14850 // we get an induced GC from the finalizer thread so we are checking the memory load
14851 // for every gen0 GC.
14852 check_memory = (check_only_p ?
14854 ((n >= 1) || low_memory_detected));
14858 //find out if we are short on memory
14859 get_memory_info (&memory_load, &available_physical, &available_page_file);
14860 if (heap_number == 0)
14862 dprintf (GTC_LOG, ("ml: %d", memory_load));
14865 // Need to get it early enough for all heaps to use.
14866 entry_available_physical_mem = available_physical;
14867 local_settings->entry_memory_load = memory_load;
14869 // @TODO: Force compaction more often under GCSTRESS
14870 if (memory_load >= high_memory_load_th || low_memory_detected)
14872 #ifdef SIMPLE_DPRINTF
14873 // stress log can't handle any parameter that's bigger than a void*.
14874 if (heap_number == 0)
14876 dprintf (GTC_LOG, ("tp: %I64d, ap: %I64d", total_physical_mem, available_physical));
14878 #endif //SIMPLE_DPRINTF
14880 high_memory_load = TRUE;
14882 if (memory_load >= v_high_memory_load_th || low_memory_detected)
14884 // TODO: Perhaps in 64-bit we should be estimating gen1's fragmentation as well since
14885 // gen1/gen0 may take a lot more memory than gen2.
14886 if (!high_fragmentation)
14888 high_fragmentation = dt_estimate_reclaim_space_p (tuning_deciding_condemned_gen, max_generation);
14890 v_high_memory_load = TRUE;
14894 if (!high_fragmentation)
14896 high_fragmentation = dt_estimate_high_frag_p (tuning_deciding_condemned_gen, max_generation, available_physical);
14900 if (high_fragmentation)
14902 if (high_memory_load)
14904 local_condemn_reasons->set_condition (gen_max_high_frag_m_p);
14906 else if (v_high_memory_load)
14908 local_condemn_reasons->set_condition (gen_max_high_frag_vm_p);
14914 dprintf (GTC_LOG, ("h%d: le: %d, hm: %d, vm: %d, f: %d",
14915 heap_number, low_ephemeral_space, high_memory_load, v_high_memory_load,
14916 high_fragmentation));
14918 if (should_expand_in_full_gc)
14920 dprintf (GTC_LOG, ("h%d: expand_in_full - BLOCK", heap_number));
14921 *blocking_collection_p = TRUE;
14924 should_expand_in_full_gc = FALSE;
14926 evaluate_elevation = FALSE;
14927 n = max_generation;
14928 local_condemn_reasons->set_condition (gen_expand_fullgc_p);
14931 if (last_gc_before_oom)
14933 dprintf (GTC_LOG, ("h%d: alloc full - BLOCK", heap_number));
14934 n = max_generation;
14935 *blocking_collection_p = TRUE;
14936 if ((local_settings->reason == reason_oos_loh) ||
14937 (local_settings->reason == reason_alloc_loh))
14938 evaluate_elevation = FALSE;
14940 local_condemn_reasons->set_condition (gen_before_oom);
14945 if (is_induced_blocking (settings.reason) &&
14946 n_initial == max_generation
14947 IN_STRESS_HEAP( && !settings.stress_induced ))
14949 if (heap_number == 0)
14951 dprintf (GTC_LOG, ("induced - BLOCK"));
14954 *blocking_collection_p = TRUE;
14955 local_condemn_reasons->set_condition (gen_induced_fullgc_p);
14956 evaluate_elevation = FALSE;
14959 if (settings.reason == reason_induced_noforce)
14961 local_condemn_reasons->set_condition (gen_induced_noforce_p);
14962 evaluate_elevation = FALSE;
14966 if (evaluate_elevation && (low_ephemeral_space || high_memory_load || v_high_memory_load))
14968 *elevation_requested_p = TRUE;
14970 // if we are in high memory load and have consumed 10% of the gen2 budget, do a gen2 now.
14971 if (high_memory_load || v_high_memory_load)
14973 dynamic_data* dd_max = dynamic_data_of (max_generation);
14974 if (((float)dd_new_allocation (dd_max) / (float)dd_desired_allocation (dd_max)) < 0.9)
14976 dprintf (GTC_LOG, ("%Id left in gen2 alloc (%Id)",
14977 dd_new_allocation (dd_max), dd_desired_allocation (dd_max)));
14978 n = max_generation;
14979 local_condemn_reasons->set_condition (gen_almost_max_alloc);
14983 if (n <= max_generation)
14986 if (high_fragmentation)
14988 //elevate to max_generation
14989 n = max_generation;
14990 dprintf (GTC_LOG, ("h%d: f full", heap_number));
14992 #ifdef BACKGROUND_GC
14993 if (high_memory_load || v_high_memory_load)
14995 // For background GC we want to do blocking collections more eagerly because we don't
14996 // want to get into the situation where the memory load becomes high while we are in
14997 // a background GC and we'd have to wait for the background GC to finish to start
14998 // a blocking collection (right now the implemenation doesn't handle converting
14999 // a background GC to a blocking collection midway.
15000 dprintf (GTC_LOG, ("h%d: bgc - BLOCK", heap_number));
15001 *blocking_collection_p = TRUE;
15004 if (v_high_memory_load)
15006 dprintf (GTC_LOG, ("h%d: - BLOCK", heap_number));
15007 *blocking_collection_p = TRUE;
15009 #endif //BACKGROUND_GC
15013 n = max (n, max_generation - 1);
15014 dprintf (GTC_LOG, ("h%d: nf c %d", heap_number, n));
15021 if ((n == (max_generation - 1)) && (n_alloc < (max_generation -1)))
15023 dprintf (GTC_LOG, ("h%d: budget %d, check 2",
15024 heap_number, n_alloc));
15025 if (get_new_allocation (max_generation) <= 0)
15027 dprintf (GTC_LOG, ("h%d: budget alloc", heap_number));
15028 n = max_generation;
15029 local_condemn_reasons->set_condition (gen_max_gen1);
15033 //figure out if max_generation is too fragmented -> blocking collection
15034 if (n == max_generation)
15036 if (dt_high_frag_p (tuning_deciding_condemned_gen, n))
15038 dprintf (GTC_LOG, ("h%d: g%d too frag", heap_number, n));
15039 local_condemn_reasons->set_condition (gen_max_high_frag_p);
15040 if (local_settings->pause_mode != pause_sustained_low_latency)
15042 *blocking_collection_p = TRUE;
15047 #ifdef BACKGROUND_GC
15048 if (n == max_generation)
15050 if (heap_number == 0)
15052 BOOL bgc_heap_too_small = TRUE;
15053 size_t gen2size = 0;
15054 size_t gen3size = 0;
15055 #ifdef MULTIPLE_HEAPS
15056 for (int i = 0; i < n_heaps; i++)
15058 if (((g_heaps[i]->current_generation_size (max_generation)) > bgc_min_per_heap) ||
15059 ((g_heaps[i]->current_generation_size (max_generation + 1)) > bgc_min_per_heap))
15061 bgc_heap_too_small = FALSE;
15065 #else //MULTIPLE_HEAPS
15066 if ((current_generation_size (max_generation) > bgc_min_per_heap) ||
15067 (current_generation_size (max_generation + 1) > bgc_min_per_heap))
15069 bgc_heap_too_small = FALSE;
15071 #endif //MULTIPLE_HEAPS
15073 if (bgc_heap_too_small)
15075 dprintf (GTC_LOG, ("gen2 and gen3 too small"));
15078 // do not turn stress-induced collections into blocking GCs
15079 if (!settings.stress_induced)
15080 #endif //STRESS_HEAP
15082 *blocking_collection_p = TRUE;
15085 local_condemn_reasons->set_condition (gen_gen2_too_small);
15089 #endif //BACKGROUND_GC
15095 #ifdef BACKGROUND_GC
15096 // We can only do Concurrent GC Stress if the caller did not explicitly ask for all
15097 // generations to be collected,
15099 if (orig_gen != max_generation &&
15100 g_pConfig->GetGCStressLevel() && gc_can_use_concurrent)
15102 *elevation_requested_p = FALSE;
15104 #endif //BACKGROUND_GC
15105 #endif //STRESS_HEAP
15109 fgm_result.available_pagefile_mb = (size_t)(available_page_file / (1024 * 1024));
15112 local_condemn_reasons->set_gen (gen_final_per_heap, n);
15113 get_gc_data_per_heap()->gen_to_condemn_reasons.init (local_condemn_reasons);
15116 local_condemn_reasons->print (heap_number);
15119 if ((local_settings->reason == reason_oos_soh) ||
15120 (local_settings->reason == reason_oos_loh))
15130 #pragma warning(pop)
15134 size_t gc_heap::min_reclaim_fragmentation_threshold (uint32_t num_heaps)
15136 // if the memory load is higher, the threshold we'd want to collect gets lower.
15137 size_t min_mem_based_on_available =
15138 (500 - (settings.entry_memory_load - high_memory_load_th) * 40) * 1024 * 1024 / num_heaps;
15139 size_t ten_percent_size = (size_t)((float)generation_size (max_generation) * 0.10);
15140 uint64_t three_percent_mem = mem_one_percent * 3 / num_heaps;
15142 #ifdef SIMPLE_DPRINTF
15143 dprintf (GTC_LOG, ("min av: %Id, 10%% gen2: %Id, 3%% mem: %I64d",
15144 min_mem_based_on_available, ten_percent_size, three_percent_mem));
15145 #endif //SIMPLE_DPRINTF
15146 return (size_t)(min (min_mem_based_on_available, min (ten_percent_size, three_percent_mem)));
15150 uint64_t gc_heap::min_high_fragmentation_threshold(uint64_t available_mem, uint32_t num_heaps)
15152 return min (available_mem, (256*1024*1024)) / num_heaps;
15156 CORINFO_EXCEPTION_GC = 0xE0004743 // 'GC'
15160 #ifdef BACKGROUND_GC
15161 void gc_heap::init_background_gc ()
15163 //reset the allocation so foreground gc can allocate into older (max_generation) generation
15164 generation* gen = generation_of (max_generation);
15165 generation_allocation_pointer (gen)= 0;
15166 generation_allocation_limit (gen) = 0;
15167 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
15169 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
15171 //reset the plan allocation for each segment
15172 for (heap_segment* seg = generation_allocation_segment (gen); seg != ephemeral_heap_segment;
15173 seg = heap_segment_next_rw (seg))
15175 heap_segment_plan_allocated (seg) = heap_segment_allocated (seg);
15178 if (heap_number == 0)
15180 dprintf (2, ("heap%d: bgc lowest: %Ix, highest: %Ix",
15182 background_saved_lowest_address,
15183 background_saved_highest_address));
15186 gc_lh_block_event.Reset();
15189 #endif //BACKGROUND_GC
15192 void fire_drain_mark_list_event (size_t mark_list_objects)
15194 FIRE_EVENT(BGCDrainMark, mark_list_objects);
15198 void fire_revisit_event (size_t dirtied_pages,
15199 size_t marked_objects,
15200 BOOL large_objects_p)
15202 FIRE_EVENT(BGCRevisit, dirtied_pages, marked_objects, large_objects_p);
15206 void fire_overflow_event (uint8_t* overflow_min,
15207 uint8_t* overflow_max,
15208 size_t marked_objects,
15209 int large_objects_p)
15211 FIRE_EVENT(BGCOverflow, (uint64_t)overflow_min, (uint64_t)overflow_max, marked_objects, large_objects_p);
15214 void gc_heap::concurrent_print_time_delta (const char* msg)
15217 size_t current_time = GetHighPrecisionTimeStamp();
15218 size_t elapsed_time = current_time - time_bgc_last;
15219 time_bgc_last = current_time;
15221 dprintf (2, ("h%d: %s T %Id ms", heap_number, msg, elapsed_time));
15223 UNREFERENCED_PARAMETER(msg);
15227 void gc_heap::free_list_info (int gen_num, const char* msg)
15229 UNREFERENCED_PARAMETER(gen_num);
15230 #if defined (BACKGROUND_GC) && defined (TRACE_GC)
15231 dprintf (3, ("h%d: %s", heap_number, msg));
15232 for (int i = 0; i <= (max_generation + 1); i++)
15234 generation* gen = generation_of (i);
15235 if ((generation_allocation_size (gen) == 0) &&
15236 (generation_free_list_space (gen) == 0) &&
15237 (generation_free_obj_space (gen) == 0))
15239 // don't print if everything is 0.
15243 dprintf (3, ("h%d: g%d: a-%Id, fl-%Id, fo-%Id",
15245 generation_allocation_size (gen),
15246 generation_free_list_space (gen),
15247 generation_free_obj_space (gen)));
15251 UNREFERENCED_PARAMETER(msg);
15252 #endif // BACKGROUND_GC && TRACE_GC
15255 void gc_heap::update_collection_counts_for_no_gc()
15257 assert (settings.pause_mode == pause_no_gc);
15259 settings.condemned_generation = max_generation;
15260 #ifdef MULTIPLE_HEAPS
15261 for (int i = 0; i < n_heaps; i++)
15262 g_heaps[i]->update_collection_counts();
15263 #else //MULTIPLE_HEAPS
15264 update_collection_counts();
15265 #endif //MULTIPLE_HEAPS
15267 full_gc_counts[gc_type_blocking]++;
15270 BOOL gc_heap::should_proceed_with_gc()
15272 if (gc_heap::settings.pause_mode == pause_no_gc)
15274 if (current_no_gc_region_info.started)
15276 // The no_gc mode was already in progress yet we triggered another GC,
15277 // this effectively exits the no_gc mode.
15278 restore_data_for_no_gc();
15281 return should_proceed_for_no_gc();
15287 //internal part of gc used by the serial and concurrent version
15288 void gc_heap::gc1()
15290 #ifdef BACKGROUND_GC
15291 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15292 #endif //BACKGROUND_GC
15295 mark_time = plan_time = reloc_time = compact_time = sweep_time = 0;
15298 verify_soh_segment_list();
15300 int n = settings.condemned_generation;
15302 update_collection_counts ();
15304 #ifdef BACKGROUND_GC
15305 bgc_alloc_lock->check();
15306 #endif //BACKGROUND_GC
15308 free_list_info (max_generation, "beginning");
15310 vm_heap->GcCondemnedGeneration = settings.condemned_generation;
15312 assert (g_gc_card_table == card_table);
15314 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
15315 assert (g_gc_card_bundle_table == card_bundle_table);
15319 if (n == max_generation)
15321 gc_low = lowest_address;
15322 gc_high = highest_address;
15326 gc_low = generation_allocation_start (generation_of (n));
15327 gc_high = heap_segment_reserved (ephemeral_heap_segment);
15329 #ifdef BACKGROUND_GC
15330 if (settings.concurrent)
15333 time_bgc_last = GetHighPrecisionTimeStamp();
15336 FIRE_EVENT(BGCBegin);
15338 concurrent_print_time_delta ("BGC");
15340 //#ifdef WRITE_WATCH
15341 //reset_write_watch (FALSE);
15342 //#endif //WRITE_WATCH
15344 concurrent_print_time_delta ("RW");
15345 background_mark_phase();
15346 free_list_info (max_generation, "after mark phase");
15348 background_sweep();
15349 free_list_info (max_generation, "after sweep phase");
15352 #endif //BACKGROUND_GC
15354 mark_phase (n, FALSE);
15356 GCScan::GcRuntimeStructuresValid (FALSE);
15358 GCScan::GcRuntimeStructuresValid (TRUE);
15362 size_t end_gc_time = GetHighPrecisionTimeStamp();
15363 // printf ("generation: %d, elapsed time: %Id\n", n, end_gc_time - dd_time_clock (dynamic_data_of (0)));
15365 //adjust the allocation size from the pinned quantities.
15366 for (int gen_number = 0; gen_number <= min (max_generation,n+1); gen_number++)
15368 generation* gn = generation_of (gen_number);
15369 if (settings.compaction)
15371 generation_pinned_allocated (gn) += generation_pinned_allocation_compact_size (gn);
15372 generation_allocation_size (generation_of (gen_number)) += generation_pinned_allocation_compact_size (gn);
15376 generation_pinned_allocated (gn) += generation_pinned_allocation_sweep_size (gn);
15377 generation_allocation_size (generation_of (gen_number)) += generation_pinned_allocation_sweep_size (gn);
15379 generation_pinned_allocation_sweep_size (gn) = 0;
15380 generation_pinned_allocation_compact_size (gn) = 0;
15383 #ifdef BACKGROUND_GC
15384 if (settings.concurrent)
15386 dynamic_data* dd = dynamic_data_of (n);
15387 dd_gc_elapsed_time (dd) = end_gc_time - dd_time_clock (dd);
15389 free_list_info (max_generation, "after computing new dynamic data");
15391 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
15393 for (int gen_number = 0; gen_number < max_generation; gen_number++)
15395 dprintf (2, ("end of BGC: gen%d new_alloc: %Id",
15396 gen_number, dd_desired_allocation (dynamic_data_of (gen_number))));
15397 current_gc_data_per_heap->gen_data[gen_number].size_after = generation_size (gen_number);
15398 current_gc_data_per_heap->gen_data[gen_number].free_list_space_after = generation_free_list_space (generation_of (gen_number));
15399 current_gc_data_per_heap->gen_data[gen_number].free_obj_space_after = generation_free_obj_space (generation_of (gen_number));
15403 #endif //BACKGROUND_GC
15405 free_list_info (max_generation, "end");
15406 for (int gen_number = 0; gen_number <= n; gen_number++)
15408 dynamic_data* dd = dynamic_data_of (gen_number);
15409 dd_gc_elapsed_time (dd) = end_gc_time - dd_time_clock (dd);
15410 compute_new_dynamic_data (gen_number);
15413 if (n != max_generation)
15415 int gen_num_for_data = ((n < (max_generation - 1)) ? (n + 1) : (max_generation + 1));
15416 for (int gen_number = (n + 1); gen_number <= gen_num_for_data; gen_number++)
15418 get_gc_data_per_heap()->gen_data[gen_number].size_after = generation_size (gen_number);
15419 get_gc_data_per_heap()->gen_data[gen_number].free_list_space_after = generation_free_list_space (generation_of (gen_number));
15420 get_gc_data_per_heap()->gen_data[gen_number].free_obj_space_after = generation_free_obj_space (generation_of (gen_number));
15424 get_gc_data_per_heap()->maxgen_size_info.running_free_list_efficiency = (uint32_t)(generation_allocator_efficiency (generation_of (max_generation)) * 100);
15426 free_list_info (max_generation, "after computing new dynamic data");
15428 if (heap_number == 0)
15430 dprintf (GTC_LOG, ("GC#%d(gen%d) took %Idms",
15431 dd_collection_count (dynamic_data_of (0)),
15432 settings.condemned_generation,
15433 dd_gc_elapsed_time (dynamic_data_of (0))));
15436 for (int gen_number = 0; gen_number <= (max_generation + 1); gen_number++)
15438 dprintf (2, ("end of FGC/NGC: gen%d new_alloc: %Id",
15439 gen_number, dd_desired_allocation (dynamic_data_of (gen_number))));
15443 if (n < max_generation)
15445 compute_promoted_allocation (1 + n);
15447 dynamic_data* dd = dynamic_data_of (1 + n);
15448 size_t new_fragmentation = generation_free_list_space (generation_of (1 + n)) +
15449 generation_free_obj_space (generation_of (1 + n));
15451 #ifdef BACKGROUND_GC
15452 if (current_c_gc_state != c_gc_state_planning)
15453 #endif //BACKGROUND_GC
15455 if (settings.promotion)
15457 dd_fragmentation (dd) = new_fragmentation;
15461 //assert (dd_fragmentation (dd) == new_fragmentation);
15466 #ifdef BACKGROUND_GC
15467 if (!settings.concurrent)
15468 #endif //BACKGROUND_GC
15470 #ifndef FEATURE_REDHAWK
15471 // GCToEEInterface::IsGCThread() always returns false on CoreRT, but this assert is useful in CoreCLR.
15472 assert(GCToEEInterface::IsGCThread());
15473 #endif // FEATURE_REDHAWK
15474 adjust_ephemeral_limits();
15477 #ifdef BACKGROUND_GC
15478 assert (ephemeral_low == generation_allocation_start (generation_of ( max_generation -1)));
15479 assert (ephemeral_high == heap_segment_reserved (ephemeral_heap_segment));
15480 #endif //BACKGROUND_GC
15482 if (fgn_maxgen_percent)
15484 if (settings.condemned_generation == (max_generation - 1))
15486 check_for_full_gc (max_generation - 1, 0);
15488 else if (settings.condemned_generation == max_generation)
15490 if (full_gc_approach_event_set
15491 #ifdef MULTIPLE_HEAPS
15492 && (heap_number == 0)
15493 #endif //MULTIPLE_HEAPS
15496 dprintf (2, ("FGN-GC: setting gen2 end event"));
15498 full_gc_approach_event.Reset();
15499 #ifdef BACKGROUND_GC
15500 // By definition WaitForFullGCComplete only succeeds if it's full, *blocking* GC, otherwise need to return N/A
15501 fgn_last_gc_was_concurrent = settings.concurrent ? TRUE : FALSE;
15502 #endif //BACKGROUND_GC
15503 full_gc_end_event.Set();
15504 full_gc_approach_event_set = false;
15509 #ifdef BACKGROUND_GC
15510 if (!settings.concurrent)
15511 #endif //BACKGROUND_GC
15513 //decide on the next allocation quantum
15514 if (alloc_contexts_used >= 1)
15516 allocation_quantum = Align (min ((size_t)CLR_SIZE,
15517 (size_t)max (1024, get_new_allocation (0) / (2 * alloc_contexts_used))),
15518 get_alignment_constant(FALSE));
15519 dprintf (3, ("New allocation quantum: %d(0x%Ix)", allocation_quantum, allocation_quantum));
15523 descr_generations (FALSE);
15525 verify_soh_segment_list();
15527 #ifdef BACKGROUND_GC
15528 add_to_history_per_heap();
15529 if (heap_number == 0)
15533 #endif // BACKGROUND_GC
15536 if (GCStatistics::Enabled() && heap_number == 0)
15537 g_GCStatistics.AddGCStats(settings,
15538 dd_gc_elapsed_time(dynamic_data_of(settings.condemned_generation)));
15542 fprintf (stdout, "%d,%d,%d,%d,%d,%d\n",
15543 n, mark_time, plan_time, reloc_time, compact_time, sweep_time);
15546 #ifdef BACKGROUND_GC
15547 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15548 #endif //BACKGROUND_GC
15550 #if defined(VERIFY_HEAP) || (defined (FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC))
15553 // Note that right now g_pConfig->GetHeapVerifyLevel always returns the same
15554 // value. If we ever allow randomly adjusting this as the process runs,
15555 // we cannot call it this way as joins need to match - we must have the same
15556 // value for all heaps like we do with bgc_heap_walk_for_etw_p.
15557 || (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
15559 #if defined(FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC)
15560 || (bgc_heap_walk_for_etw_p && settings.concurrent)
15564 #ifdef BACKGROUND_GC
15565 bool cooperative_mode = true;
15567 if (settings.concurrent)
15569 cooperative_mode = enable_preemptive ();
15571 #ifdef MULTIPLE_HEAPS
15572 bgc_t_join.join(this, gc_join_suspend_ee_verify);
15573 if (bgc_t_join.joined())
15575 bgc_threads_sync_event.Reset();
15577 dprintf(2, ("Joining BGC threads to suspend EE for verify heap"));
15578 bgc_t_join.restart();
15580 if (heap_number == 0)
15583 bgc_threads_sync_event.Set();
15587 bgc_threads_sync_event.Wait(INFINITE, FALSE);
15588 dprintf (2, ("bgc_threads_sync_event is signalled"));
15592 #endif //MULTIPLE_HEAPS
15594 //fix the allocation area so verify_heap can proceed.
15595 fix_allocation_contexts (FALSE);
15597 #endif //BACKGROUND_GC
15599 #ifdef BACKGROUND_GC
15600 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15601 #ifdef FEATURE_EVENT_TRACE
15602 if (bgc_heap_walk_for_etw_p && settings.concurrent)
15604 GCToEEInterface::DiagWalkBGCSurvivors(__this);
15606 #ifdef MULTIPLE_HEAPS
15607 bgc_t_join.join(this, gc_join_after_profiler_heap_walk);
15608 if (bgc_t_join.joined())
15610 bgc_t_join.restart();
15612 #endif // MULTIPLE_HEAPS
15614 #endif // FEATURE_EVENT_TRACE
15615 #endif //BACKGROUND_GC
15618 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
15619 verify_heap (FALSE);
15620 #endif // VERIFY_HEAP
15622 #ifdef BACKGROUND_GC
15623 if (settings.concurrent)
15625 repair_allocation_contexts (TRUE);
15627 #ifdef MULTIPLE_HEAPS
15628 bgc_t_join.join(this, gc_join_restart_ee_verify);
15629 if (bgc_t_join.joined())
15631 bgc_threads_sync_event.Reset();
15633 dprintf(2, ("Joining BGC threads to restart EE after verify heap"));
15634 bgc_t_join.restart();
15636 if (heap_number == 0)
15639 bgc_threads_sync_event.Set();
15643 bgc_threads_sync_event.Wait(INFINITE, FALSE);
15644 dprintf (2, ("bgc_threads_sync_event is signalled"));
15648 #endif //MULTIPLE_HEAPS
15650 disable_preemptive (cooperative_mode);
15652 #endif //BACKGROUND_GC
15654 #endif // defined(VERIFY_HEAP) || (defined(FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC))
15656 #ifdef MULTIPLE_HEAPS
15657 if (!settings.concurrent)
15659 gc_t_join.join(this, gc_join_done);
15660 if (gc_t_join.joined ())
15662 gc_heap::internal_gc_done = false;
15664 //equalize the new desired size of the generations
15665 int limit = settings.condemned_generation;
15666 if (limit == max_generation)
15668 limit = max_generation+1;
15670 for (int gen = 0; gen <= limit; gen++)
15672 size_t total_desired = 0;
15674 for (int i = 0; i < gc_heap::n_heaps; i++)
15676 gc_heap* hp = gc_heap::g_heaps[i];
15677 dynamic_data* dd = hp->dynamic_data_of (gen);
15678 size_t temp_total_desired = total_desired + dd_desired_allocation (dd);
15679 if (temp_total_desired < total_desired)
15682 total_desired = (size_t)MAX_PTR;
15685 total_desired = temp_total_desired;
15688 size_t desired_per_heap = Align (total_desired/gc_heap::n_heaps,
15689 get_alignment_constant ((gen != (max_generation+1))));
15693 #if 1 //subsumed by the linear allocation model
15694 // to avoid spikes in mem usage due to short terms fluctuations in survivorship,
15695 // apply some smoothing.
15696 static size_t smoothed_desired_per_heap = 0;
15697 size_t smoothing = 3; // exponential smoothing factor
15698 if (smoothing > VolatileLoad(&settings.gc_index))
15699 smoothing = VolatileLoad(&settings.gc_index);
15700 smoothed_desired_per_heap = desired_per_heap / smoothing + ((smoothed_desired_per_heap / smoothing) * (smoothing-1));
15701 dprintf (1, ("sn = %Id n = %Id", smoothed_desired_per_heap, desired_per_heap));
15702 desired_per_heap = Align(smoothed_desired_per_heap, get_alignment_constant (true));
15705 // if desired_per_heap is close to min_gc_size, trim it
15706 // down to min_gc_size to stay in the cache
15707 gc_heap* hp = gc_heap::g_heaps[0];
15708 dynamic_data* dd = hp->dynamic_data_of (gen);
15709 size_t min_gc_size = dd_min_size(dd);
15710 // if min GC size larger than true on die cache, then don't bother
15711 // limiting the desired size
15712 if ((min_gc_size <= GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE)) &&
15713 desired_per_heap <= 2*min_gc_size)
15715 desired_per_heap = min_gc_size;
15718 desired_per_heap = joined_youngest_desired (desired_per_heap);
15719 dprintf (2, ("final gen0 new_alloc: %Id", desired_per_heap));
15722 gc_data_global.final_youngest_desired = desired_per_heap;
15724 #if 1 //subsumed by the linear allocation model
15725 if (gen == (max_generation + 1))
15727 // to avoid spikes in mem usage due to short terms fluctuations in survivorship,
15728 // apply some smoothing.
15729 static size_t smoothed_desired_per_heap_loh = 0;
15730 size_t smoothing = 3; // exponential smoothing factor
15731 size_t loh_count = dd_collection_count (dynamic_data_of (max_generation));
15732 if (smoothing > loh_count)
15733 smoothing = loh_count;
15734 smoothed_desired_per_heap_loh = desired_per_heap / smoothing + ((smoothed_desired_per_heap_loh / smoothing) * (smoothing-1));
15735 dprintf( 2, ("smoothed_desired_per_heap_loh = %Id desired_per_heap = %Id", smoothed_desired_per_heap_loh, desired_per_heap));
15736 desired_per_heap = Align(smoothed_desired_per_heap_loh, get_alignment_constant (false));
15739 for (int i = 0; i < gc_heap::n_heaps; i++)
15741 gc_heap* hp = gc_heap::g_heaps[i];
15742 dynamic_data* dd = hp->dynamic_data_of (gen);
15743 dd_desired_allocation (dd) = desired_per_heap;
15744 dd_gc_new_allocation (dd) = desired_per_heap;
15745 dd_new_allocation (dd) = desired_per_heap;
15749 hp->fgn_last_alloc = desired_per_heap;
15754 #ifdef FEATURE_LOH_COMPACTION
15755 BOOL all_heaps_compacted_p = TRUE;
15756 #endif //FEATURE_LOH_COMPACTION
15757 for (int i = 0; i < gc_heap::n_heaps; i++)
15759 gc_heap* hp = gc_heap::g_heaps[i];
15760 hp->decommit_ephemeral_segment_pages();
15761 hp->rearrange_large_heap_segments();
15762 #ifdef FEATURE_LOH_COMPACTION
15763 all_heaps_compacted_p &= hp->loh_compacted_p;
15764 #endif //FEATURE_LOH_COMPACTION
15767 #ifdef FEATURE_LOH_COMPACTION
15768 check_loh_compact_mode (all_heaps_compacted_p);
15769 #endif //FEATURE_LOH_COMPACTION
15773 gc_t_join.restart();
15775 alloc_context_count = 0;
15776 heap_select::mark_heap (heap_number);
15780 gc_data_global.final_youngest_desired =
15781 dd_desired_allocation (dynamic_data_of (0));
15783 check_loh_compact_mode (loh_compacted_p);
15785 decommit_ephemeral_segment_pages();
15788 if (!(settings.concurrent))
15790 rearrange_large_heap_segments();
15794 #ifdef BACKGROUND_GC
15795 recover_bgc_settings();
15796 #endif //BACKGROUND_GC
15797 #endif //MULTIPLE_HEAPS
15800 void gc_heap::save_data_for_no_gc()
15802 current_no_gc_region_info.saved_pause_mode = settings.pause_mode;
15803 #ifdef MULTIPLE_HEAPS
15804 // This is to affect heap balancing.
15805 for (int i = 0; i < n_heaps; i++)
15807 current_no_gc_region_info.saved_gen0_min_size = dd_min_size (g_heaps[i]->dynamic_data_of (0));
15808 dd_min_size (g_heaps[i]->dynamic_data_of (0)) = min_balance_threshold;
15809 current_no_gc_region_info.saved_gen3_min_size = dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1));
15810 dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1)) = 0;
15812 #endif //MULTIPLE_HEAPS
15815 void gc_heap::restore_data_for_no_gc()
15817 gc_heap::settings.pause_mode = current_no_gc_region_info.saved_pause_mode;
15818 #ifdef MULTIPLE_HEAPS
15819 for (int i = 0; i < n_heaps; i++)
15821 dd_min_size (g_heaps[i]->dynamic_data_of (0)) = current_no_gc_region_info.saved_gen0_min_size;
15822 dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1)) = current_no_gc_region_info.saved_gen3_min_size;
15824 #endif //MULTIPLE_HEAPS
15827 start_no_gc_region_status gc_heap::prepare_for_no_gc_region (uint64_t total_size,
15828 BOOL loh_size_known,
15830 BOOL disallow_full_blocking)
15832 if (current_no_gc_region_info.started)
15834 return start_no_gc_in_progress;
15837 start_no_gc_region_status status = start_no_gc_success;
15839 save_data_for_no_gc();
15840 settings.pause_mode = pause_no_gc;
15841 current_no_gc_region_info.start_status = start_no_gc_success;
15843 uint64_t allocation_no_gc_loh = 0;
15844 uint64_t allocation_no_gc_soh = 0;
15845 assert(total_size != 0);
15846 if (loh_size_known)
15848 assert(loh_size != 0);
15849 assert(loh_size <= total_size);
15850 allocation_no_gc_loh = loh_size;
15851 allocation_no_gc_soh = total_size - loh_size;
15855 allocation_no_gc_soh = total_size;
15856 allocation_no_gc_loh = total_size;
15859 int soh_align_const = get_alignment_constant (TRUE);
15860 size_t max_soh_allocated = soh_segment_size - segment_info_size - eph_gen_starts_size;
15861 size_t size_per_heap = 0;
15862 const double scale_factor = 1.05;
15865 #ifdef MULTIPLE_HEAPS
15866 num_heaps = n_heaps;
15867 #endif // MULTIPLE_HEAPS
15869 uint64_t total_allowed_soh_allocation = max_soh_allocated * num_heaps;
15871 // In theory, the upper limit here is the physical memory of the machine, not
15872 // SIZE_T_MAX. This is not true today because total_physical_mem can be
15873 // larger than SIZE_T_MAX if running in wow64 on a machine with more than
15874 // 4GB of RAM. Once Local GC code divergence is resolved and code is flowing
15875 // more freely between branches, it would be good to clean this up to use
15876 // total_physical_mem instead of SIZE_T_MAX.
15877 assert(total_allowed_soh_allocation <= SIZE_T_MAX);
15878 uint64_t total_allowed_loh_allocation = SIZE_T_MAX;
15879 uint64_t total_allowed_soh_alloc_scaled = allocation_no_gc_soh > 0 ? static_cast<uint64_t>(total_allowed_soh_allocation / scale_factor) : 0;
15880 uint64_t total_allowed_loh_alloc_scaled = allocation_no_gc_loh > 0 ? static_cast<uint64_t>(total_allowed_loh_allocation / scale_factor) : 0;
15882 if (allocation_no_gc_soh > total_allowed_soh_alloc_scaled ||
15883 allocation_no_gc_loh > total_allowed_loh_alloc_scaled)
15885 status = start_no_gc_too_large;
15889 if (allocation_no_gc_soh > 0)
15891 allocation_no_gc_soh = static_cast<uint64_t>(allocation_no_gc_soh * scale_factor);
15892 allocation_no_gc_soh = min (allocation_no_gc_soh, total_allowed_soh_alloc_scaled);
15895 if (allocation_no_gc_loh > 0)
15897 allocation_no_gc_loh = static_cast<uint64_t>(allocation_no_gc_loh * scale_factor);
15898 allocation_no_gc_loh = min (allocation_no_gc_loh, total_allowed_loh_alloc_scaled);
15901 if (disallow_full_blocking)
15902 current_no_gc_region_info.minimal_gc_p = TRUE;
15904 if (allocation_no_gc_soh != 0)
15906 current_no_gc_region_info.soh_allocation_size = static_cast<size_t>(allocation_no_gc_soh);
15907 size_per_heap = current_no_gc_region_info.soh_allocation_size;
15908 #ifdef MULTIPLE_HEAPS
15909 size_per_heap /= n_heaps;
15910 for (int i = 0; i < n_heaps; i++)
15912 // due to heap balancing we need to allow some room before we even look to balance to another heap.
15913 g_heaps[i]->soh_allocation_no_gc = min (Align ((size_per_heap + min_balance_threshold), soh_align_const), max_soh_allocated);
15915 #else //MULTIPLE_HEAPS
15916 soh_allocation_no_gc = min (Align (size_per_heap, soh_align_const), max_soh_allocated);
15917 #endif //MULTIPLE_HEAPS
15920 if (allocation_no_gc_loh != 0)
15922 current_no_gc_region_info.loh_allocation_size = static_cast<size_t>(allocation_no_gc_loh);
15923 size_per_heap = current_no_gc_region_info.loh_allocation_size;
15924 #ifdef MULTIPLE_HEAPS
15925 size_per_heap /= n_heaps;
15926 for (int i = 0; i < n_heaps; i++)
15927 g_heaps[i]->loh_allocation_no_gc = Align (size_per_heap, get_alignment_constant (FALSE));
15928 #else //MULTIPLE_HEAPS
15929 loh_allocation_no_gc = Align (size_per_heap, get_alignment_constant (FALSE));
15930 #endif //MULTIPLE_HEAPS
15934 if (status != start_no_gc_success)
15935 restore_data_for_no_gc();
15939 void gc_heap::handle_failure_for_no_gc()
15941 gc_heap::restore_data_for_no_gc();
15942 // sets current_no_gc_region_info.started to FALSE here.
15943 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
15946 start_no_gc_region_status gc_heap::get_start_no_gc_region_status()
15948 return current_no_gc_region_info.start_status;
15951 void gc_heap::record_gcs_during_no_gc()
15953 if (current_no_gc_region_info.started)
15955 current_no_gc_region_info.num_gcs++;
15956 if (is_induced (settings.reason))
15957 current_no_gc_region_info.num_gcs_induced++;
15961 BOOL gc_heap::find_loh_free_for_no_gc()
15963 allocator* loh_allocator = generation_allocator (generation_of (max_generation + 1));
15964 size_t sz_list = loh_allocator->first_bucket_size();
15965 size_t size = loh_allocation_no_gc;
15966 for (unsigned int a_l_idx = 0; a_l_idx < loh_allocator->number_of_buckets(); a_l_idx++)
15968 if ((size < sz_list) || (a_l_idx == (loh_allocator->number_of_buckets()-1)))
15970 uint8_t* free_list = loh_allocator->alloc_list_head_of (a_l_idx);
15973 size_t free_list_size = unused_array_size(free_list);
15975 if (free_list_size > loh_allocation_no_gc)
15977 dprintf (3, ("free item %Ix(%Id) for no gc", (size_t)free_list, free_list_size));
15981 free_list = free_list_slot (free_list);
15984 sz_list = sz_list * 2;
15990 BOOL gc_heap::find_loh_space_for_no_gc()
15992 saved_loh_segment_no_gc = 0;
15994 if (find_loh_free_for_no_gc())
15997 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16001 size_t remaining = heap_segment_reserved (seg) - heap_segment_allocated (seg);
16002 if (remaining >= loh_allocation_no_gc)
16004 saved_loh_segment_no_gc = seg;
16007 seg = heap_segment_next (seg);
16010 if (!saved_loh_segment_no_gc && current_no_gc_region_info.minimal_gc_p)
16012 // If no full GC is allowed, we try to get a new seg right away.
16013 saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc)
16014 #ifdef MULTIPLE_HEAPS
16016 #endif //MULTIPLE_HEAPS
16020 return (saved_loh_segment_no_gc != 0);
16023 BOOL gc_heap::loh_allocated_for_no_gc()
16025 if (!saved_loh_segment_no_gc)
16028 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16031 if (seg == saved_loh_segment_no_gc)
16035 seg = heap_segment_next (seg);
16041 BOOL gc_heap::commit_loh_for_no_gc (heap_segment* seg)
16043 uint8_t* end_committed = heap_segment_allocated (seg) + loh_allocation_no_gc;
16044 assert (end_committed <= heap_segment_reserved (seg));
16045 return (grow_heap_segment (seg, end_committed));
16048 void gc_heap::thread_no_gc_loh_segments()
16050 #ifdef MULTIPLE_HEAPS
16051 for (int i = 0; i < n_heaps; i++)
16053 gc_heap* hp = g_heaps[i];
16054 if (hp->loh_allocated_for_no_gc())
16056 hp->thread_loh_segment (hp->saved_loh_segment_no_gc);
16057 hp->saved_loh_segment_no_gc = 0;
16060 #else //MULTIPLE_HEAPS
16061 if (loh_allocated_for_no_gc())
16063 thread_loh_segment (saved_loh_segment_no_gc);
16064 saved_loh_segment_no_gc = 0;
16066 #endif //MULTIPLE_HEAPS
16069 void gc_heap::set_loh_allocations_for_no_gc()
16071 if (current_no_gc_region_info.loh_allocation_size != 0)
16073 dynamic_data* dd = dynamic_data_of (max_generation + 1);
16074 dd_new_allocation (dd) = loh_allocation_no_gc;
16075 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
16079 void gc_heap::set_soh_allocations_for_no_gc()
16081 if (current_no_gc_region_info.soh_allocation_size != 0)
16083 dynamic_data* dd = dynamic_data_of (0);
16084 dd_new_allocation (dd) = soh_allocation_no_gc;
16085 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
16086 #ifdef MULTIPLE_HEAPS
16087 alloc_context_count = 0;
16088 #endif //MULTIPLE_HEAPS
16092 void gc_heap::set_allocations_for_no_gc()
16094 #ifdef MULTIPLE_HEAPS
16095 for (int i = 0; i < n_heaps; i++)
16097 gc_heap* hp = g_heaps[i];
16098 hp->set_loh_allocations_for_no_gc();
16099 hp->set_soh_allocations_for_no_gc();
16101 #else //MULTIPLE_HEAPS
16102 set_loh_allocations_for_no_gc();
16103 set_soh_allocations_for_no_gc();
16104 #endif //MULTIPLE_HEAPS
16107 BOOL gc_heap::should_proceed_for_no_gc()
16109 BOOL gc_requested = FALSE;
16110 BOOL loh_full_gc_requested = FALSE;
16111 BOOL soh_full_gc_requested = FALSE;
16112 BOOL no_gc_requested = FALSE;
16113 BOOL get_new_loh_segments = FALSE;
16115 if (current_no_gc_region_info.soh_allocation_size)
16117 #ifdef MULTIPLE_HEAPS
16118 for (int i = 0; i < n_heaps; i++)
16120 gc_heap* hp = g_heaps[i];
16121 if ((size_t)(heap_segment_reserved (hp->ephemeral_heap_segment) - hp->alloc_allocated) < hp->soh_allocation_no_gc)
16123 gc_requested = TRUE;
16127 #else //MULTIPLE_HEAPS
16128 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated) < soh_allocation_no_gc)
16129 gc_requested = TRUE;
16130 #endif //MULTIPLE_HEAPS
16134 #ifdef MULTIPLE_HEAPS
16135 for (int i = 0; i < n_heaps; i++)
16137 gc_heap* hp = g_heaps[i];
16138 if (!(hp->grow_heap_segment (hp->ephemeral_heap_segment, (hp->alloc_allocated + hp->soh_allocation_no_gc))))
16140 soh_full_gc_requested = TRUE;
16144 #else //MULTIPLE_HEAPS
16145 if (!grow_heap_segment (ephemeral_heap_segment, (alloc_allocated + soh_allocation_no_gc)))
16146 soh_full_gc_requested = TRUE;
16147 #endif //MULTIPLE_HEAPS
16151 if (!current_no_gc_region_info.minimal_gc_p && gc_requested)
16153 soh_full_gc_requested = TRUE;
16156 no_gc_requested = !(soh_full_gc_requested || gc_requested);
16158 if (soh_full_gc_requested && current_no_gc_region_info.minimal_gc_p)
16160 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16164 if (!soh_full_gc_requested && current_no_gc_region_info.loh_allocation_size)
16166 // Check to see if we have enough reserved space.
16167 #ifdef MULTIPLE_HEAPS
16168 for (int i = 0; i < n_heaps; i++)
16170 gc_heap* hp = g_heaps[i];
16171 if (!hp->find_loh_space_for_no_gc())
16173 loh_full_gc_requested = TRUE;
16177 #else //MULTIPLE_HEAPS
16178 if (!find_loh_space_for_no_gc())
16179 loh_full_gc_requested = TRUE;
16180 #endif //MULTIPLE_HEAPS
16182 // Check to see if we have committed space.
16183 if (!loh_full_gc_requested)
16185 #ifdef MULTIPLE_HEAPS
16186 for (int i = 0; i < n_heaps; i++)
16188 gc_heap* hp = g_heaps[i];
16189 if (hp->saved_loh_segment_no_gc &&!hp->commit_loh_for_no_gc (hp->saved_loh_segment_no_gc))
16191 loh_full_gc_requested = TRUE;
16195 #else //MULTIPLE_HEAPS
16196 if (saved_loh_segment_no_gc && !commit_loh_for_no_gc (saved_loh_segment_no_gc))
16197 loh_full_gc_requested = TRUE;
16198 #endif //MULTIPLE_HEAPS
16202 if (loh_full_gc_requested || soh_full_gc_requested)
16204 if (current_no_gc_region_info.minimal_gc_p)
16205 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16208 no_gc_requested = !(loh_full_gc_requested || soh_full_gc_requested || gc_requested);
16210 if (current_no_gc_region_info.start_status == start_no_gc_success)
16212 if (no_gc_requested)
16213 set_allocations_for_no_gc();
16218 if ((current_no_gc_region_info.start_status == start_no_gc_success) && !no_gc_requested)
16222 // We are done with starting the no_gc_region.
16223 current_no_gc_region_info.started = TRUE;
16228 end_no_gc_region_status gc_heap::end_no_gc_region()
16230 dprintf (1, ("end no gc called"));
16232 end_no_gc_region_status status = end_no_gc_success;
16234 if (!(current_no_gc_region_info.started))
16235 status = end_no_gc_not_in_progress;
16236 if (current_no_gc_region_info.num_gcs_induced)
16237 status = end_no_gc_induced;
16238 else if (current_no_gc_region_info.num_gcs)
16239 status = end_no_gc_alloc_exceeded;
16241 if (settings.pause_mode == pause_no_gc)
16242 restore_data_for_no_gc();
16244 // sets current_no_gc_region_info.started to FALSE here.
16245 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
16251 void gc_heap::update_collection_counts ()
16253 dynamic_data* dd0 = dynamic_data_of (0);
16254 dd_gc_clock (dd0) += 1;
16256 size_t now = GetHighPrecisionTimeStamp();
16258 for (int i = 0; i <= settings.condemned_generation;i++)
16260 dynamic_data* dd = dynamic_data_of (i);
16261 dd_collection_count (dd)++;
16262 //this is needed by the linear allocation model
16263 if (i == max_generation)
16264 dd_collection_count (dynamic_data_of (max_generation+1))++;
16265 dd_gc_clock (dd) = dd_gc_clock (dd0);
16266 dd_time_clock (dd) = now;
16270 BOOL gc_heap::expand_soh_with_minimal_gc()
16272 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment)) >= soh_allocation_no_gc)
16275 heap_segment* new_seg = soh_get_segment_to_expand();
16278 if (g_gc_card_table != card_table)
16279 copy_brick_card_table();
16281 settings.promotion = TRUE;
16282 settings.demotion = FALSE;
16283 ephemeral_promotion = TRUE;
16284 int condemned_gen_number = max_generation - 1;
16286 generation* gen = 0;
16287 int align_const = get_alignment_constant (TRUE);
16289 for (int i = 0; i <= condemned_gen_number; i++)
16291 gen = generation_of (i);
16292 saved_ephemeral_plan_start[i] = generation_allocation_start (gen);
16293 saved_ephemeral_plan_start_size[i] = Align (size (generation_allocation_start (gen)), align_const);
16296 // We do need to clear the bricks here as we are converting a bunch of ephemeral objects to gen2
16297 // and need to make sure that there are no left over bricks from the previous GCs for the space
16298 // we just used for gen0 allocation. We will need to go through the bricks for these objects for
16299 // ephemeral GCs later.
16300 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
16301 b < brick_of (align_on_brick (heap_segment_allocated (ephemeral_heap_segment)));
16307 size_t ephemeral_size = (heap_segment_allocated (ephemeral_heap_segment) -
16308 generation_allocation_start (generation_of (max_generation - 1)));
16309 heap_segment_next (ephemeral_heap_segment) = new_seg;
16310 ephemeral_heap_segment = new_seg;
16311 uint8_t* start = heap_segment_mem (ephemeral_heap_segment);
16313 for (int i = condemned_gen_number; i >= 0; i--)
16315 gen = generation_of (i);
16316 size_t gen_start_size = Align (min_obj_size);
16317 make_generation (generation_table[i], ephemeral_heap_segment, start, 0);
16318 generation_plan_allocation_start (gen) = start;
16319 generation_plan_allocation_start_size (gen) = gen_start_size;
16320 start += gen_start_size;
16322 heap_segment_used (ephemeral_heap_segment) = start - plug_skew;
16323 heap_segment_plan_allocated (ephemeral_heap_segment) = start;
16325 fix_generation_bounds (condemned_gen_number, generation_of (0));
16327 dd_gc_new_allocation (dynamic_data_of (max_generation)) -= ephemeral_size;
16328 dd_new_allocation (dynamic_data_of (max_generation)) = dd_gc_new_allocation (dynamic_data_of (max_generation));
16330 adjust_ephemeral_limits();
16337 // Only to be done on the thread that calls restart in a join for server GC
16338 // and reset the oom status per heap.
16339 void gc_heap::check_and_set_no_gc_oom()
16341 #ifdef MULTIPLE_HEAPS
16342 for (int i = 0; i < n_heaps; i++)
16344 gc_heap* hp = g_heaps[i];
16345 if (hp->no_gc_oom_p)
16347 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16348 hp->no_gc_oom_p = false;
16354 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16355 no_gc_oom_p = false;
16357 #endif //MULTIPLE_HEAPS
16360 void gc_heap::allocate_for_no_gc_after_gc()
16362 if (current_no_gc_region_info.minimal_gc_p)
16363 repair_allocation_contexts (TRUE);
16365 no_gc_oom_p = false;
16367 if (current_no_gc_region_info.start_status != start_no_gc_no_memory)
16369 if (current_no_gc_region_info.soh_allocation_size != 0)
16371 if (((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment)) < soh_allocation_no_gc) ||
16372 (!grow_heap_segment (ephemeral_heap_segment, (heap_segment_allocated (ephemeral_heap_segment) + soh_allocation_no_gc))))
16374 no_gc_oom_p = true;
16377 #ifdef MULTIPLE_HEAPS
16378 gc_t_join.join(this, gc_join_after_commit_soh_no_gc);
16379 if (gc_t_join.joined())
16381 #endif //MULTIPLE_HEAPS
16383 check_and_set_no_gc_oom();
16385 #ifdef MULTIPLE_HEAPS
16386 gc_t_join.restart();
16388 #endif //MULTIPLE_HEAPS
16391 if ((current_no_gc_region_info.start_status == start_no_gc_success) &&
16392 !(current_no_gc_region_info.minimal_gc_p) &&
16393 (current_no_gc_region_info.loh_allocation_size != 0))
16395 gc_policy = policy_compact;
16396 saved_loh_segment_no_gc = 0;
16398 if (!find_loh_free_for_no_gc())
16400 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16401 BOOL found_seg_p = FALSE;
16404 if ((size_t)(heap_segment_reserved (seg) - heap_segment_allocated (seg)) >= loh_allocation_no_gc)
16406 found_seg_p = TRUE;
16407 if (!commit_loh_for_no_gc (seg))
16409 no_gc_oom_p = true;
16413 seg = heap_segment_next (seg);
16417 gc_policy = policy_expand;
16420 #ifdef MULTIPLE_HEAPS
16421 gc_t_join.join(this, gc_join_expand_loh_no_gc);
16422 if (gc_t_join.joined())
16424 check_and_set_no_gc_oom();
16426 if (current_no_gc_region_info.start_status == start_no_gc_success)
16428 for (int i = 0; i < n_heaps; i++)
16430 gc_heap* hp = g_heaps[i];
16431 if (hp->gc_policy == policy_expand)
16433 hp->saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc), hp);
16434 if (!(hp->saved_loh_segment_no_gc))
16436 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16443 gc_t_join.restart();
16445 #else //MULTIPLE_HEAPS
16446 check_and_set_no_gc_oom();
16448 if ((current_no_gc_region_info.start_status == start_no_gc_success) && (gc_policy == policy_expand))
16450 saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc));
16451 if (!saved_loh_segment_no_gc)
16452 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16454 #endif //MULTIPLE_HEAPS
16456 if ((current_no_gc_region_info.start_status == start_no_gc_success) && saved_loh_segment_no_gc)
16458 if (!commit_loh_for_no_gc (saved_loh_segment_no_gc))
16460 no_gc_oom_p = true;
16466 #ifdef MULTIPLE_HEAPS
16467 gc_t_join.join(this, gc_join_final_no_gc);
16468 if (gc_t_join.joined())
16470 #endif //MULTIPLE_HEAPS
16472 check_and_set_no_gc_oom();
16474 if (current_no_gc_region_info.start_status == start_no_gc_success)
16476 set_allocations_for_no_gc();
16477 current_no_gc_region_info.started = TRUE;
16480 #ifdef MULTIPLE_HEAPS
16481 gc_t_join.restart();
16483 #endif //MULTIPLE_HEAPS
16486 void gc_heap::init_records()
16488 memset (&gc_data_per_heap, 0, sizeof (gc_data_per_heap));
16489 gc_data_per_heap.heap_index = heap_number;
16490 if (heap_number == 0)
16491 memset (&gc_data_global, 0, sizeof (gc_data_global));
16493 #ifdef GC_CONFIG_DRIVEN
16494 memset (interesting_data_per_gc, 0, sizeof (interesting_data_per_gc));
16495 #endif //GC_CONFIG_DRIVEN
16498 int gc_heap::garbage_collect (int n)
16500 //reset the number of alloc contexts
16501 alloc_contexts_used = 0;
16503 fix_allocation_contexts (TRUE);
16504 #ifdef MULTIPLE_HEAPS
16506 gc_t_join.start_ts(this);
16507 #endif //JOIN_STATS
16508 clear_gen0_bricks();
16509 #endif //MULTIPLE_HEAPS
16511 if ((settings.pause_mode == pause_no_gc) && current_no_gc_region_info.minimal_gc_p)
16513 #ifdef MULTIPLE_HEAPS
16514 gc_t_join.join(this, gc_join_minimal_gc);
16515 if (gc_t_join.joined())
16517 #endif //MULTIPLE_HEAPS
16519 #ifdef MULTIPLE_HEAPS
16520 // this is serialized because we need to get a segment
16521 for (int i = 0; i < n_heaps; i++)
16523 if (!(g_heaps[i]->expand_soh_with_minimal_gc()))
16524 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16527 if (!expand_soh_with_minimal_gc())
16528 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16529 #endif //MULTIPLE_HEAPS
16531 update_collection_counts_for_no_gc();
16533 #ifdef MULTIPLE_HEAPS
16534 gc_t_join.restart();
16536 #endif //MULTIPLE_HEAPS
16542 memset (&fgm_result, 0, sizeof (fgm_result));
16544 settings.reason = gc_trigger_reason;
16545 verify_pinned_queue_p = FALSE;
16547 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
16548 num_pinned_objects = 0;
16549 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
16552 if (settings.reason == reason_gcstress)
16554 settings.reason = reason_induced;
16555 settings.stress_induced = TRUE;
16557 #endif // STRESS_HEAP
16559 #ifdef MULTIPLE_HEAPS
16560 //align all heaps on the max generation to condemn
16561 dprintf (3, ("Joining for max generation to condemn"));
16562 condemned_generation_num = generation_to_condemn (n,
16563 &blocking_collection,
16564 &elevation_requested,
16566 gc_t_join.join(this, gc_join_generation_determined);
16567 if (gc_t_join.joined())
16568 #endif //MULTIPLE_HEAPS
16570 #ifdef MULTIPLE_HEAPS
16571 #if !defined(SEG_MAPPING_TABLE) && !defined(FEATURE_BASICFREEZE)
16572 //delete old slots from the segment table
16573 seg_table->delete_old_slots();
16574 #endif //!SEG_MAPPING_TABLE && !FEATURE_BASICFREEZE
16575 for (int i = 0; i < n_heaps; i++)
16577 //copy the card and brick tables
16578 if (g_gc_card_table != g_heaps[i]->card_table)
16580 g_heaps[i]->copy_brick_card_table();
16583 g_heaps[i]->rearrange_large_heap_segments();
16584 if (!recursive_gc_sync::background_running_p())
16586 g_heaps[i]->rearrange_small_heap_segments();
16589 #else //MULTIPLE_HEAPS
16590 #ifdef BACKGROUND_GC
16591 //delete old slots from the segment table
16592 #if !defined(SEG_MAPPING_TABLE) && !defined(FEATURE_BASICFREEZE)
16593 seg_table->delete_old_slots();
16594 #endif //!SEG_MAPPING_TABLE && !FEATURE_BASICFREEZE
16595 rearrange_large_heap_segments();
16596 if (!recursive_gc_sync::background_running_p())
16598 rearrange_small_heap_segments();
16600 #endif //BACKGROUND_GC
16601 // check for card table growth
16602 if (g_gc_card_table != card_table)
16603 copy_brick_card_table();
16605 #endif //MULTIPLE_HEAPS
16607 BOOL should_evaluate_elevation = FALSE;
16608 BOOL should_do_blocking_collection = FALSE;
16610 #ifdef MULTIPLE_HEAPS
16611 int gen_max = condemned_generation_num;
16612 for (int i = 0; i < n_heaps; i++)
16614 if (gen_max < g_heaps[i]->condemned_generation_num)
16615 gen_max = g_heaps[i]->condemned_generation_num;
16616 if ((!should_evaluate_elevation) && (g_heaps[i]->elevation_requested))
16617 should_evaluate_elevation = TRUE;
16618 if ((!should_do_blocking_collection) && (g_heaps[i]->blocking_collection))
16619 should_do_blocking_collection = TRUE;
16622 settings.condemned_generation = gen_max;
16623 #else //MULTIPLE_HEAPS
16624 settings.condemned_generation = generation_to_condemn (n,
16625 &blocking_collection,
16626 &elevation_requested,
16628 should_evaluate_elevation = elevation_requested;
16629 should_do_blocking_collection = blocking_collection;
16630 #endif //MULTIPLE_HEAPS
16632 settings.condemned_generation = joined_generation_to_condemn (
16633 should_evaluate_elevation,
16634 settings.condemned_generation,
16635 &should_do_blocking_collection
16639 STRESS_LOG1(LF_GCROOTS|LF_GC|LF_GCALLOC, LL_INFO10,
16640 "condemned generation num: %d\n", settings.condemned_generation);
16642 record_gcs_during_no_gc();
16644 if (settings.condemned_generation > 1)
16645 settings.promotion = TRUE;
16647 #ifdef HEAP_ANALYZE
16648 // At this point we've decided what generation is condemned
16649 // See if we've been requested to analyze survivors after the mark phase
16650 if (GCToEEInterface::AnalyzeSurvivorsRequested(settings.condemned_generation))
16652 heap_analyze_enabled = TRUE;
16654 #endif // HEAP_ANALYZE
16656 GCToEEInterface::DiagGCStart(settings.condemned_generation, settings.reason == reason_induced);
16658 #ifdef BACKGROUND_GC
16659 if ((settings.condemned_generation == max_generation) &&
16660 (recursive_gc_sync::background_running_p()))
16662 //TODO BACKGROUND_GC If we just wait for the end of gc, it won't work
16663 // because we have to collect 0 and 1 properly
16664 // in particular, the allocation contexts are gone.
16665 // For now, it is simpler to collect max_generation-1
16666 settings.condemned_generation = max_generation - 1;
16667 dprintf (GTC_LOG, ("bgc - 1 instead of 2"));
16670 if ((settings.condemned_generation == max_generation) &&
16671 (should_do_blocking_collection == FALSE) &&
16672 gc_can_use_concurrent &&
16673 !temp_disable_concurrent_p &&
16674 ((settings.pause_mode == pause_interactive) || (settings.pause_mode == pause_sustained_low_latency)))
16676 keep_bgc_threads_p = TRUE;
16677 c_write (settings.concurrent, TRUE);
16679 #endif //BACKGROUND_GC
16681 settings.gc_index = (uint32_t)dd_collection_count (dynamic_data_of (0)) + 1;
16683 // Call the EE for start of GC work
16684 // just one thread for MP GC
16685 GCToEEInterface::GcStartWork (settings.condemned_generation,
16688 // TODO: we could fire an ETW event to say this GC as a concurrent GC but later on due to not being able to
16689 // create threads or whatever, this could be a non concurrent GC. Maybe for concurrent GC we should fire
16690 // it in do_background_gc and if it failed to be a CGC we fire it in gc1... in other words, this should be
16694 #ifdef MULTIPLE_HEAPS
16695 gc_start_event.Reset();
16696 //start all threads on the roots.
16697 dprintf(3, ("Starting all gc threads for gc"));
16698 gc_t_join.restart();
16699 #endif //MULTIPLE_HEAPS
16703 int gen_num_for_data = max_generation + 1;
16704 for (int i = 0; i <= gen_num_for_data; i++)
16706 gc_data_per_heap.gen_data[i].size_before = generation_size (i);
16707 generation* gen = generation_of (i);
16708 gc_data_per_heap.gen_data[i].free_list_space_before = generation_free_list_space (gen);
16709 gc_data_per_heap.gen_data[i].free_obj_space_before = generation_free_obj_space (gen);
16712 descr_generations (TRUE);
16713 // descr_card_table();
16716 if ((GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC) &&
16717 !(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_POST_GC_ONLY))
16719 verify_heap (TRUE);
16721 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_BARRIERCHECK)
16722 checkGCWriteBarrier();
16724 #endif // VERIFY_HEAP
16726 #ifdef BACKGROUND_GC
16727 if (settings.concurrent)
16729 // We need to save the settings because we'll need to restore it after each FGC.
16730 assert (settings.condemned_generation == max_generation);
16731 settings.compaction = FALSE;
16732 saved_bgc_settings = settings;
16734 #ifdef MULTIPLE_HEAPS
16735 if (heap_number == 0)
16737 for (int i = 0; i < n_heaps; i++)
16739 prepare_bgc_thread (g_heaps[i]);
16741 dprintf (2, ("setting bgc_threads_sync_event"));
16742 bgc_threads_sync_event.Set();
16746 bgc_threads_sync_event.Wait(INFINITE, FALSE);
16747 dprintf (2, ("bgc_threads_sync_event is signalled"));
16750 prepare_bgc_thread(0);
16751 #endif //MULTIPLE_HEAPS
16753 #ifdef MULTIPLE_HEAPS
16754 gc_t_join.join(this, gc_join_start_bgc);
16755 if (gc_t_join.joined())
16756 #endif //MULTIPLE_HEAPS
16758 do_concurrent_p = TRUE;
16759 do_ephemeral_gc_p = FALSE;
16760 #ifdef MULTIPLE_HEAPS
16761 dprintf(2, ("Joined to perform a background GC"));
16763 for (int i = 0; i < n_heaps; i++)
16765 gc_heap* hp = g_heaps[i];
16766 if (!(hp->bgc_thread) || !hp->commit_mark_array_bgc_init (hp->mark_array))
16768 do_concurrent_p = FALSE;
16773 hp->background_saved_lowest_address = hp->lowest_address;
16774 hp->background_saved_highest_address = hp->highest_address;
16778 do_concurrent_p = (!!bgc_thread && commit_mark_array_bgc_init (mark_array));
16779 if (do_concurrent_p)
16781 background_saved_lowest_address = lowest_address;
16782 background_saved_highest_address = highest_address;
16784 #endif //MULTIPLE_HEAPS
16786 if (do_concurrent_p)
16788 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
16789 SoftwareWriteWatch::EnableForGCHeap();
16790 #endif //FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
16792 #ifdef MULTIPLE_HEAPS
16793 for (int i = 0; i < n_heaps; i++)
16794 g_heaps[i]->current_bgc_state = bgc_initialized;
16796 current_bgc_state = bgc_initialized;
16797 #endif //MULTIPLE_HEAPS
16799 int gen = check_for_ephemeral_alloc();
16800 // always do a gen1 GC before we start BGC.
16801 // This is temporary for testing purpose.
16802 //int gen = max_generation - 1;
16803 dont_restart_ee_p = TRUE;
16806 // If we decide to not do a GC before the BGC we need to
16807 // restore the gen0 alloc context.
16808 #ifdef MULTIPLE_HEAPS
16809 for (int i = 0; i < n_heaps; i++)
16811 generation_allocation_pointer (g_heaps[i]->generation_of (0)) = 0;
16812 generation_allocation_limit (g_heaps[i]->generation_of (0)) = 0;
16815 generation_allocation_pointer (youngest_generation) = 0;
16816 generation_allocation_limit (youngest_generation) = 0;
16817 #endif //MULTIPLE_HEAPS
16821 do_ephemeral_gc_p = TRUE;
16823 settings.init_mechanisms();
16824 settings.condemned_generation = gen;
16825 settings.gc_index = (size_t)dd_collection_count (dynamic_data_of (0)) + 2;
16828 // TODO BACKGROUND_GC need to add the profiling stuff here.
16829 dprintf (GTC_LOG, ("doing gen%d before doing a bgc", gen));
16832 //clear the cards so they don't bleed in gen 1 during collection
16833 // shouldn't this always be done at the beginning of any GC?
16834 //clear_card_for_addresses (
16835 // generation_allocation_start (generation_of (0)),
16836 // heap_segment_allocated (ephemeral_heap_segment));
16838 if (!do_ephemeral_gc_p)
16840 do_background_gc();
16845 settings.compaction = TRUE;
16846 c_write (settings.concurrent, FALSE);
16849 #ifdef MULTIPLE_HEAPS
16850 gc_t_join.restart();
16851 #endif //MULTIPLE_HEAPS
16854 if (do_concurrent_p)
16856 // At this point we are sure we'll be starting a BGC, so save its per heap data here.
16857 // global data is only calculated at the end of the GC so we don't need to worry about
16858 // FGCs overwriting it.
16859 memset (&bgc_data_per_heap, 0, sizeof (bgc_data_per_heap));
16860 memcpy (&bgc_data_per_heap, &gc_data_per_heap, sizeof(gc_data_per_heap));
16862 if (do_ephemeral_gc_p)
16864 dprintf (2, ("GC threads running, doing gen%d GC", settings.condemned_generation));
16866 gen_to_condemn_reasons.init();
16867 gen_to_condemn_reasons.set_condition (gen_before_bgc);
16868 gc_data_per_heap.gen_to_condemn_reasons.init (&gen_to_condemn_reasons);
16870 #ifdef MULTIPLE_HEAPS
16871 gc_t_join.join(this, gc_join_bgc_after_ephemeral);
16872 if (gc_t_join.joined())
16873 #endif //MULTIPLE_HEAPS
16875 #ifdef MULTIPLE_HEAPS
16877 #endif //MULTIPLE_HEAPS
16878 settings = saved_bgc_settings;
16879 assert (settings.concurrent);
16881 do_background_gc();
16883 #ifdef MULTIPLE_HEAPS
16884 gc_t_join.restart();
16885 #endif //MULTIPLE_HEAPS
16891 dprintf (2, ("couldn't create BGC threads, reverting to doing a blocking GC"));
16896 #endif //BACKGROUND_GC
16900 #ifndef MULTIPLE_HEAPS
16901 allocation_running_time = (size_t)GCToOSInterface::GetLowPrecisionTimeStamp();
16902 allocation_running_amount = dd_new_allocation (dynamic_data_of (0));
16903 fgn_last_alloc = dd_new_allocation (dynamic_data_of (0));
16904 #endif //MULTIPLE_HEAPS
16907 if (settings.pause_mode == pause_no_gc)
16908 allocate_for_no_gc_after_gc();
16910 int gn = settings.condemned_generation;
16914 #define mark_stack_empty_p() (mark_stack_base == mark_stack_tos)
16917 size_t& gc_heap::promoted_bytes(int thread)
16919 #ifdef MULTIPLE_HEAPS
16920 return g_promoted [thread*16];
16921 #else //MULTIPLE_HEAPS
16922 UNREFERENCED_PARAMETER(thread);
16924 #endif //MULTIPLE_HEAPS
16927 #ifdef INTERIOR_POINTERS
16928 heap_segment* gc_heap::find_segment (uint8_t* interior, BOOL small_segment_only_p)
16930 #ifdef SEG_MAPPING_TABLE
16931 heap_segment* seg = seg_mapping_table_segment_of (interior);
16934 if (small_segment_only_p && heap_segment_loh_p (seg))
16938 #else //SEG_MAPPING_TABLE
16939 #ifdef MULTIPLE_HEAPS
16940 for (int i = 0; i < gc_heap::n_heaps; i++)
16942 gc_heap* h = gc_heap::g_heaps [i];
16943 hs = h->find_segment_per_heap (o, small_segment_only_p);
16951 gc_heap* h = pGenGCHeap;
16952 hs = h->find_segment_per_heap (o, small_segment_only_p);
16954 #endif //MULTIPLE_HEAPS
16955 #endif //SEG_MAPPING_TABLE
16958 heap_segment* gc_heap::find_segment_per_heap (uint8_t* interior, BOOL small_segment_only_p)
16960 #ifdef SEG_MAPPING_TABLE
16961 return find_segment (interior, small_segment_only_p);
16962 #else //SEG_MAPPING_TABLE
16963 if (in_range_for_segment (interior, ephemeral_heap_segment))
16965 return ephemeral_heap_segment;
16969 heap_segment* found_seg = 0;
16972 heap_segment* seg = generation_start_segment (generation_of (max_generation));
16975 if (in_range_for_segment (interior, seg))
16978 goto end_find_segment;
16981 } while ((seg = heap_segment_next (seg)) != 0);
16983 if (!small_segment_only_p)
16985 #ifdef BACKGROUND_GC
16987 ptrdiff_t delta = 0;
16988 heap_segment* seg = segment_of (interior, delta);
16989 if (seg && in_range_for_segment (interior, seg))
16993 goto end_find_segment;
16995 #else //BACKGROUND_GC
16996 heap_segment* seg = generation_start_segment (generation_of (max_generation+1));
16999 if (in_range_for_segment(interior, seg))
17002 goto end_find_segment;
17005 } while ((seg = heap_segment_next (seg)) != 0);
17006 #endif //BACKGROUND_GC
17012 #endif //SEG_MAPPING_TABLE
17014 #endif //INTERIOR_POINTERS
17016 #if !defined(_DEBUG) && !defined(__GNUC__)
17017 inline // This causes link errors if global optimization is off
17018 #endif //!_DEBUG && !__GNUC__
17019 gc_heap* gc_heap::heap_of (uint8_t* o)
17021 #ifdef MULTIPLE_HEAPS
17023 return g_heaps [0];
17024 #ifdef SEG_MAPPING_TABLE
17025 gc_heap* hp = seg_mapping_table_heap_of (o);
17026 return (hp ? hp : g_heaps[0]);
17027 #else //SEG_MAPPING_TABLE
17028 ptrdiff_t delta = 0;
17029 heap_segment* seg = segment_of (o, delta);
17030 return (seg ? heap_segment_heap (seg) : g_heaps [0]);
17031 #endif //SEG_MAPPING_TABLE
17032 #else //MULTIPLE_HEAPS
17033 UNREFERENCED_PARAMETER(o);
17035 #endif //MULTIPLE_HEAPS
17039 gc_heap* gc_heap::heap_of_gc (uint8_t* o)
17041 #ifdef MULTIPLE_HEAPS
17043 return g_heaps [0];
17044 #ifdef SEG_MAPPING_TABLE
17045 gc_heap* hp = seg_mapping_table_heap_of_gc (o);
17046 return (hp ? hp : g_heaps[0]);
17047 #else //SEG_MAPPING_TABLE
17048 ptrdiff_t delta = 0;
17049 heap_segment* seg = segment_of (o, delta);
17050 return (seg ? heap_segment_heap (seg) : g_heaps [0]);
17051 #endif //SEG_MAPPING_TABLE
17052 #else //MULTIPLE_HEAPS
17053 UNREFERENCED_PARAMETER(o);
17055 #endif //MULTIPLE_HEAPS
17058 #ifdef INTERIOR_POINTERS
17059 // will find all heap objects (large and small)
17060 uint8_t* gc_heap::find_object (uint8_t* interior, uint8_t* low)
17062 if (!gen0_bricks_cleared)
17064 #ifdef MULTIPLE_HEAPS
17065 assert (!"Should have already been done in server GC");
17066 #endif //MULTIPLE_HEAPS
17067 gen0_bricks_cleared = TRUE;
17068 //initialize brick table for gen 0
17069 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
17070 b < brick_of (align_on_brick
17071 (heap_segment_allocated (ephemeral_heap_segment)));
17077 #ifdef FFIND_OBJECT
17078 //indicate that in the future this needs to be done during allocation
17079 #ifdef MULTIPLE_HEAPS
17080 gen0_must_clear_bricks = FFIND_DECAY*gc_heap::n_heaps;
17082 gen0_must_clear_bricks = FFIND_DECAY;
17083 #endif //MULTIPLE_HEAPS
17084 #endif //FFIND_OBJECT
17086 int brick_entry = get_brick_entry(brick_of (interior));
17087 if (brick_entry == 0)
17089 // this is a pointer to a large object
17090 heap_segment* seg = find_segment_per_heap (interior, FALSE);
17092 #ifdef FEATURE_CONSERVATIVE_GC
17093 && (GCConfig::GetConservativeGC() || interior <= heap_segment_allocated(seg))
17097 // If interior falls within the first free object at the beginning of a generation,
17098 // we don't have brick entry for it, and we may incorrectly treat it as on large object heap.
17099 int align_const = get_alignment_constant (heap_segment_read_only_p (seg)
17100 #ifdef FEATURE_CONSERVATIVE_GC
17101 || (GCConfig::GetConservativeGC() && !heap_segment_loh_p (seg))
17104 //int align_const = get_alignment_constant (heap_segment_read_only_p (seg));
17105 assert (interior < heap_segment_allocated (seg));
17107 uint8_t* o = heap_segment_mem (seg);
17108 while (o < heap_segment_allocated (seg))
17110 uint8_t* next_o = o + Align (size (o), align_const);
17111 assert (next_o > o);
17112 if ((o <= interior) && (interior < next_o))
17123 else if (interior >= low)
17125 heap_segment* seg = find_segment_per_heap (interior, TRUE);
17128 #ifdef FEATURE_CONSERVATIVE_GC
17129 if (interior >= heap_segment_allocated (seg))
17132 assert (interior < heap_segment_allocated (seg));
17134 uint8_t* o = find_first_object (interior, heap_segment_mem (seg));
17145 gc_heap::find_object_for_relocation (uint8_t* interior, uint8_t* low, uint8_t* high)
17147 uint8_t* old_address = interior;
17148 if (!((old_address >= low) && (old_address < high)))
17151 size_t brick = brick_of (old_address);
17152 int brick_entry = brick_table [ brick ];
17153 if (brick_entry != 0)
17157 while (brick_entry < 0)
17159 brick = (brick + brick_entry);
17160 brick_entry = brick_table [ brick ];
17162 uint8_t* old_loc = old_address;
17163 uint8_t* node = tree_search ((brick_address (brick) + brick_entry-1),
17165 if (node <= old_loc)
17170 brick_entry = brick_table [ brick ];
17176 //find the object by going along the plug
17178 while (o <= interior)
17180 uint8_t* next_o = o + Align (size (o));
17181 assert (next_o > o);
17182 if (next_o > interior)
17188 assert ((o <= interior) && ((o + Align (size (o))) > interior));
17193 // this is a pointer to a large object
17194 heap_segment* seg = find_segment_per_heap (interior, FALSE);
17197 assert (interior < heap_segment_allocated (seg));
17199 uint8_t* o = heap_segment_mem (seg);
17200 while (o < heap_segment_allocated (seg))
17202 uint8_t* next_o = o + Align (size (o));
17203 assert (next_o > o);
17204 if ((o < interior) && (interior < next_o))
17216 #else //INTERIOR_POINTERS
17218 uint8_t* gc_heap::find_object (uint8_t* o, uint8_t* low)
17222 #endif //INTERIOR_POINTERS
17225 #ifdef GC_CONFIG_DRIVEN
17226 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;} else {mark_list_index++;} if (slow > o) slow = o; if (shigh < o) shigh = o;}
17228 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;}if (slow > o) slow = o; if (shigh < o) shigh = o;}
17229 #endif //GC_CONFIG_DRIVEN
17231 #define m_boundary(o) {if (slow > o) slow = o; if (shigh < o) shigh = o;}
17234 #define m_boundary_fullgc(o) {if (slow > o) slow = o; if (shigh < o) shigh = o;}
17236 #define method_table(o) ((CObjectHeader*)(o))->GetMethodTable()
17239 BOOL gc_heap::gc_mark1 (uint8_t* o)
17241 BOOL marked = !marked (o);
17243 dprintf (3, ("*%Ix*, newly marked: %d", (size_t)o, marked));
17248 BOOL gc_heap::gc_mark (uint8_t* o, uint8_t* low, uint8_t* high)
17250 BOOL marked = FALSE;
17251 if ((o >= low) && (o < high))
17252 marked = gc_mark1 (o);
17253 #ifdef MULTIPLE_HEAPS
17257 gc_heap* hp = heap_of_gc (o);
17259 if ((o >= hp->gc_low) && (o < hp->gc_high))
17260 marked = gc_mark1 (o);
17263 snoop_stat.objects_checked_count++;
17267 snoop_stat.objects_marked_count++;
17271 snoop_stat.zero_ref_count++;
17274 #endif //SNOOP_STATS
17275 #endif //MULTIPLE_HEAPS
17279 #ifdef BACKGROUND_GC
17282 BOOL gc_heap::background_marked (uint8_t* o)
17284 return mark_array_marked (o);
17287 BOOL gc_heap::background_mark1 (uint8_t* o)
17289 BOOL to_mark = !mark_array_marked (o);
17291 dprintf (3, ("b*%Ix*b(%d)", (size_t)o, (to_mark ? 1 : 0)));
17294 mark_array_set_marked (o);
17295 dprintf (4, ("n*%Ix*n", (size_t)o));
17302 // TODO: we could consider filtering out NULL's here instead of going to
17303 // look for it on other heaps
17305 BOOL gc_heap::background_mark (uint8_t* o, uint8_t* low, uint8_t* high)
17307 BOOL marked = FALSE;
17308 if ((o >= low) && (o < high))
17309 marked = background_mark1 (o);
17310 #ifdef MULTIPLE_HEAPS
17314 gc_heap* hp = heap_of (o);
17316 if ((o >= hp->background_saved_lowest_address) && (o < hp->background_saved_highest_address))
17317 marked = background_mark1 (o);
17319 #endif //MULTIPLE_HEAPS
17323 #endif //BACKGROUND_GC
17326 uint8_t* gc_heap::next_end (heap_segment* seg, uint8_t* f)
17328 if (seg == ephemeral_heap_segment)
17331 return heap_segment_allocated (seg);
17334 #define new_start() {if (ppstop <= start) {break;} else {parm = start}}
17335 #define ignore_start 0
17336 #define use_start 1
17338 #define go_through_object(mt,o,size,parm,start,start_useful,limit,exp) \
17340 CGCDesc* map = CGCDesc::GetCGCDescFromMT((MethodTable*)(mt)); \
17341 CGCDescSeries* cur = map->GetHighestSeries(); \
17342 ptrdiff_t cnt = (ptrdiff_t) map->GetNumSeries(); \
17346 CGCDescSeries* last = map->GetLowestSeries(); \
17347 uint8_t** parm = 0; \
17350 assert (parm <= (uint8_t**)((o) + cur->GetSeriesOffset())); \
17351 parm = (uint8_t**)((o) + cur->GetSeriesOffset()); \
17352 uint8_t** ppstop = \
17353 (uint8_t**)((uint8_t*)parm + cur->GetSeriesSize() + (size));\
17354 if (!start_useful || (uint8_t*)ppstop > (start)) \
17356 if (start_useful && (uint8_t*)parm < (start)) parm = (uint8_t**)(start);\
17357 while (parm < ppstop) \
17365 } while (cur >= last); \
17369 /* Handle the repeating case - array of valuetypes */ \
17370 uint8_t** parm = (uint8_t**)((o) + cur->startoffset); \
17371 if (start_useful && start > (uint8_t*)parm) \
17373 ptrdiff_t cs = mt->RawGetComponentSize(); \
17374 parm = (uint8_t**)((uint8_t*)parm + (((start) - (uint8_t*)parm)/cs)*cs); \
17376 while ((uint8_t*)parm < ((o)+(size)-plug_skew)) \
17378 for (ptrdiff_t __i = 0; __i > cnt; __i--) \
17380 HALF_SIZE_T skip = cur->val_serie[__i].skip; \
17381 HALF_SIZE_T nptrs = cur->val_serie[__i].nptrs; \
17382 uint8_t** ppstop = parm + nptrs; \
17383 if (!start_useful || (uint8_t*)ppstop > (start)) \
17385 if (start_useful && (uint8_t*)parm < (start)) parm = (uint8_t**)(start); \
17390 } while (parm < ppstop); \
17392 parm = (uint8_t**)((uint8_t*)ppstop + skip); \
17398 #define go_through_object_nostart(mt,o,size,parm,exp) {go_through_object(mt,o,size,parm,o,ignore_start,(o + size),exp); }
17400 // 1 thing to note about this macro:
17401 // 1) you can use *parm safely but in general you don't want to use parm
17402 // because for the collectible types it's not an address on the managed heap.
17403 #ifndef COLLECTIBLE_CLASS
17404 #define go_through_object_cl(mt,o,size,parm,exp) \
17406 if (header(o)->ContainsPointers()) \
17408 go_through_object_nostart(mt,o,size,parm,exp); \
17411 #else //COLLECTIBLE_CLASS
17412 #define go_through_object_cl(mt,o,size,parm,exp) \
17414 if (header(o)->Collectible()) \
17416 uint8_t* class_obj = get_class_object (o); \
17417 uint8_t** parm = &class_obj; \
17418 do {exp} while (false); \
17420 if (header(o)->ContainsPointers()) \
17422 go_through_object_nostart(mt,o,size,parm,exp); \
17425 #endif //COLLECTIBLE_CLASS
17427 // This starts a plug. But mark_stack_tos isn't increased until set_pinned_info is called.
17428 void gc_heap::enque_pinned_plug (uint8_t* plug,
17429 BOOL save_pre_plug_info_p,
17430 uint8_t* last_object_in_last_plug)
17432 if (mark_stack_array_length <= mark_stack_tos)
17434 if (!grow_mark_stack (mark_stack_array, mark_stack_array_length, MARK_STACK_INITIAL_LENGTH))
17436 // we don't want to continue here due to security
17437 // risks. This happens very rarely and fixing it in the
17438 // way so that we can continue is a bit involved and will
17439 // not be done in Dev10.
17440 GCToEEInterface::HandleFatalError(CORINFO_EXCEPTION_GC);
17444 dprintf (3, ("enqueuing P #%Id(%Ix): %Ix. oldest: %Id, LO: %Ix, pre: %d",
17445 mark_stack_tos, &mark_stack_array[mark_stack_tos], plug, mark_stack_bos, last_object_in_last_plug, (save_pre_plug_info_p ? 1 : 0)));
17446 mark& m = mark_stack_array[mark_stack_tos];
17448 // Must be set now because if we have a short object we'll need the value of saved_pre_p.
17449 m.saved_pre_p = save_pre_plug_info_p;
17451 if (save_pre_plug_info_p)
17454 BOOL is_padded = is_plug_padded (last_object_in_last_plug);
17456 clear_plug_padded (last_object_in_last_plug);
17457 #endif //SHORT_PLUGS
17458 memcpy (&(m.saved_pre_plug), &(((plug_and_gap*)plug)[-1]), sizeof (gap_reloc_pair));
17461 set_plug_padded (last_object_in_last_plug);
17462 #endif //SHORT_PLUGS
17464 memcpy (&(m.saved_pre_plug_reloc), &(((plug_and_gap*)plug)[-1]), sizeof (gap_reloc_pair));
17466 // If the last object in the last plug is too short, it requires special handling.
17467 size_t last_obj_size = plug - last_object_in_last_plug;
17468 if (last_obj_size < min_pre_pin_obj_size)
17470 record_interesting_data_point (idp_pre_short);
17473 record_interesting_data_point (idp_pre_short_padded);
17474 #endif //SHORT_PLUGS
17475 dprintf (3, ("encountered a short object %Ix right before pinned plug %Ix!",
17476 last_object_in_last_plug, plug));
17477 // Need to set the short bit regardless of having refs or not because we need to
17478 // indicate that this object is not walkable.
17481 #ifdef COLLECTIBLE_CLASS
17482 if (is_collectible (last_object_in_last_plug))
17484 m.set_pre_short_collectible();
17486 #endif //COLLECTIBLE_CLASS
17488 if (contain_pointers (last_object_in_last_plug))
17490 dprintf (3, ("short object: %Ix(%Ix)", last_object_in_last_plug, last_obj_size));
17492 go_through_object_nostart (method_table(last_object_in_last_plug), last_object_in_last_plug, last_obj_size, pval,
17494 size_t gap_offset = (((size_t)pval - (size_t)(plug - sizeof (gap_reloc_pair) - plug_skew))) / sizeof (uint8_t*);
17495 dprintf (3, ("member: %Ix->%Ix, %Id ptrs from beginning of gap", (uint8_t*)pval, *pval, gap_offset));
17496 m.set_pre_short_bit (gap_offset);
17503 m.saved_post_p = FALSE;
17506 void gc_heap::save_post_plug_info (uint8_t* last_pinned_plug, uint8_t* last_object_in_last_plug, uint8_t* post_plug)
17508 UNREFERENCED_PARAMETER(last_pinned_plug);
17510 mark& m = mark_stack_array[mark_stack_tos - 1];
17511 assert (last_pinned_plug == m.first);
17512 m.saved_post_plug_info_start = (uint8_t*)&(((plug_and_gap*)post_plug)[-1]);
17515 BOOL is_padded = is_plug_padded (last_object_in_last_plug);
17517 clear_plug_padded (last_object_in_last_plug);
17518 #endif //SHORT_PLUGS
17519 memcpy (&(m.saved_post_plug), m.saved_post_plug_info_start, sizeof (gap_reloc_pair));
17522 set_plug_padded (last_object_in_last_plug);
17523 #endif //SHORT_PLUGS
17525 memcpy (&(m.saved_post_plug_reloc), m.saved_post_plug_info_start, sizeof (gap_reloc_pair));
17527 // This is important - we need to clear all bits here except the last one.
17528 m.saved_post_p = TRUE;
17531 m.saved_post_plug_debug.gap = 1;
17534 dprintf (3, ("PP %Ix has NP %Ix right after", last_pinned_plug, post_plug));
17536 size_t last_obj_size = post_plug - last_object_in_last_plug;
17537 if (last_obj_size < min_pre_pin_obj_size)
17539 dprintf (3, ("PP %Ix last obj %Ix is too short", last_pinned_plug, last_object_in_last_plug));
17540 record_interesting_data_point (idp_post_short);
17543 record_interesting_data_point (idp_post_short_padded);
17544 #endif //SHORT_PLUGS
17545 m.set_post_short();
17546 verify_pinned_queue_p = TRUE;
17548 #ifdef COLLECTIBLE_CLASS
17549 if (is_collectible (last_object_in_last_plug))
17551 m.set_post_short_collectible();
17553 #endif //COLLECTIBLE_CLASS
17555 if (contain_pointers (last_object_in_last_plug))
17557 dprintf (3, ("short object: %Ix(%Ix)", last_object_in_last_plug, last_obj_size));
17559 // TODO: since we won't be able to walk this object in relocation, we still need to
17560 // take care of collectible assemblies here.
17561 go_through_object_nostart (method_table(last_object_in_last_plug), last_object_in_last_plug, last_obj_size, pval,
17563 size_t gap_offset = (((size_t)pval - (size_t)(post_plug - sizeof (gap_reloc_pair) - plug_skew))) / sizeof (uint8_t*);
17564 dprintf (3, ("member: %Ix->%Ix, %Id ptrs from beginning of gap", (uint8_t*)pval, *pval, gap_offset));
17565 m.set_post_short_bit (gap_offset);
17574 __declspec(naked) void __fastcall Prefetch(void* addr)
17582 inline void Prefetch (void* addr)
17584 UNREFERENCED_PARAMETER(addr);
17589 VOLATILE(uint8_t*)& gc_heap::ref_mark_stack (gc_heap* hp, int index)
17591 return ((VOLATILE(uint8_t*)*)(hp->mark_stack_array))[index];
17594 #endif //MH_SC_MARK
17598 #define partial_object 3
17600 uint8_t* ref_from_slot (uint8_t* r)
17602 return (uint8_t*)((size_t)r & ~(stolen | partial));
17605 BOOL stolen_p (uint8_t* r)
17607 return (((size_t)r&2) && !((size_t)r&1));
17610 BOOL ready_p (uint8_t* r)
17612 return ((size_t)r != 1);
17615 BOOL partial_p (uint8_t* r)
17617 return (((size_t)r&1) && !((size_t)r&2));
17620 BOOL straight_ref_p (uint8_t* r)
17622 return (!stolen_p (r) && !partial_p (r));
17625 BOOL partial_object_p (uint8_t* r)
17627 return (((size_t)r & partial_object) == partial_object);
17630 BOOL ref_p (uint8_t* r)
17632 return (straight_ref_p (r) || partial_object_p (r));
17635 void gc_heap::mark_object_simple1 (uint8_t* oo, uint8_t* start THREAD_NUMBER_DCL)
17637 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_tos = (SERVER_SC_MARK_VOLATILE(uint8_t*)*)mark_stack_array;
17638 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_limit = (SERVER_SC_MARK_VOLATILE(uint8_t*)*)&mark_stack_array[mark_stack_array_length];
17639 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_base = mark_stack_tos;
17640 #ifdef SORT_MARK_STACK
17641 SERVER_SC_MARK_VOLATILE(uint8_t*)* sorted_tos = mark_stack_base;
17642 #endif //SORT_MARK_STACK
17644 // If we are doing a full GC we don't use mark list anyway so use m_boundary_fullgc that doesn't
17645 // update mark list.
17646 BOOL full_p = (settings.condemned_generation == max_generation);
17648 assert ((start >= oo) && (start < oo+size(oo)));
17651 *mark_stack_tos = oo;
17652 #endif //!MH_SC_MARK
17656 #ifdef MULTIPLE_HEAPS
17657 #else //MULTIPLE_HEAPS
17658 const int thread = 0;
17659 #endif //MULTIPLE_HEAPS
17661 if (oo && ((size_t)oo != 4))
17669 else if (!partial_p (oo) && ((s = size (oo)) < (partial_size_th*sizeof (uint8_t*))))
17671 BOOL overflow_p = FALSE;
17673 if (mark_stack_tos + (s) /sizeof (uint8_t*) >= (mark_stack_limit - 1))
17675 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
17676 if (mark_stack_tos + CGCDesc::GetNumPointers(method_table(oo), s, num_components) >= (mark_stack_limit - 1))
17682 if (overflow_p == FALSE)
17684 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
17686 go_through_object_cl (method_table(oo), oo, s, ppslot,
17688 uint8_t* o = *ppslot;
17690 if (gc_mark (o, gc_low, gc_high))
17694 m_boundary_fullgc (o);
17700 size_t obj_size = size (o);
17701 promoted_bytes (thread) += obj_size;
17702 if (contain_pointers_or_collectible (o))
17704 *(mark_stack_tos++) = o;
17712 dprintf(3,("mark stack overflow for object %Ix ", (size_t)oo));
17713 min_overflow_address = min (min_overflow_address, oo);
17714 max_overflow_address = max (max_overflow_address, oo);
17719 if (partial_p (oo))
17721 start = ref_from_slot (oo);
17722 oo = ref_from_slot (*(--mark_stack_tos));
17723 dprintf (4, ("oo: %Ix, start: %Ix\n", (size_t)oo, (size_t)start));
17724 assert ((oo < start) && (start < (oo + size (oo))));
17726 #ifdef COLLECTIBLE_CLASS
17729 // If there's a class object, push it now. We are guaranteed to have the slot since
17730 // we just popped one object off.
17731 if (is_collectible (oo))
17733 uint8_t* class_obj = get_class_object (oo);
17734 if (gc_mark (class_obj, gc_low, gc_high))
17738 m_boundary_fullgc (class_obj);
17742 m_boundary (class_obj);
17745 size_t obj_size = size (class_obj);
17746 promoted_bytes (thread) += obj_size;
17747 *(mark_stack_tos++) = class_obj;
17748 // The code below expects that the oo is still stored in the stack slot that was
17749 // just popped and it "pushes" it back just by incrementing the mark_stack_tos.
17750 // But the class_obj has just overwritten that stack slot and so the oo needs to
17751 // be stored to the new slot that's pointed to by the mark_stack_tos.
17752 *mark_stack_tos = oo;
17756 #endif //COLLECTIBLE_CLASS
17760 BOOL overflow_p = FALSE;
17762 if (mark_stack_tos + (num_partial_refs + 2) >= mark_stack_limit)
17766 if (overflow_p == FALSE)
17768 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
17770 //push the object and its current
17771 SERVER_SC_MARK_VOLATILE(uint8_t*)* place = ++mark_stack_tos;
17775 *(place) = (uint8_t*)partial;
17776 #endif //MH_SC_MARK
17777 int i = num_partial_refs;
17778 uint8_t* ref_to_continue = 0;
17780 go_through_object (method_table(oo), oo, s, ppslot,
17781 start, use_start, (oo + s),
17783 uint8_t* o = *ppslot;
17785 if (gc_mark (o, gc_low, gc_high))
17789 m_boundary_fullgc (o);
17795 size_t obj_size = size (o);
17796 promoted_bytes (thread) += obj_size;
17797 if (contain_pointers_or_collectible (o))
17799 *(mark_stack_tos++) = o;
17802 ref_to_continue = (uint8_t*)((size_t)(ppslot+1) | partial);
17811 //we are finished with this object
17812 assert (ref_to_continue == 0);
17814 assert ((*(place-1)) == (uint8_t*)0);
17817 #endif //MH_SC_MARK
17819 // shouldn't we decrease tos by 2 here??
17822 if (ref_to_continue)
17826 assert ((*(place-1)) == (uint8_t*)0);
17827 *(place-1) = (uint8_t*)((size_t)oo | partial_object);
17828 assert (((*place) == (uint8_t*)1) || ((*place) == (uint8_t*)2));
17829 #endif //MH_SC_MARK
17830 *place = ref_to_continue;
17835 dprintf(3,("mark stack overflow for object %Ix ", (size_t)oo));
17836 min_overflow_address = min (min_overflow_address, oo);
17837 max_overflow_address = max (max_overflow_address, oo);
17840 #ifdef SORT_MARK_STACK
17841 if (mark_stack_tos > sorted_tos + mark_stack_array_length/8)
17843 rqsort1 (sorted_tos, mark_stack_tos-1);
17844 sorted_tos = mark_stack_tos-1;
17846 #endif //SORT_MARK_STACK
17849 if (!(mark_stack_empty_p()))
17851 oo = *(--mark_stack_tos);
17854 #ifdef SORT_MARK_STACK
17855 sorted_tos = min ((size_t)sorted_tos, (size_t)mark_stack_tos);
17856 #endif //SORT_MARK_STACK
17864 BOOL same_numa_node_p (int hn1, int hn2)
17866 return (heap_select::find_numa_node_from_heap_no (hn1) == heap_select::find_numa_node_from_heap_no (hn2));
17869 int find_next_buddy_heap (int this_heap_number, int current_buddy, int n_heaps)
17871 int hn = (current_buddy+1)%n_heaps;
17872 while (hn != current_buddy)
17874 if ((this_heap_number != hn) && (same_numa_node_p (this_heap_number, hn)))
17876 hn = (hn+1)%n_heaps;
17878 return current_buddy;
17882 gc_heap::mark_steal()
17884 mark_stack_busy() = 0;
17885 //clear the mark stack in the snooping range
17886 for (int i = 0; i < max_snoop_level; i++)
17888 ((VOLATILE(uint8_t*)*)(mark_stack_array))[i] = 0;
17891 //pick the next heap as our buddy
17892 int thpn = find_next_buddy_heap (heap_number, heap_number, n_heaps);
17895 dprintf (SNOOP_LOG, ("(GC%d)heap%d: start snooping %d", settings.gc_index, heap_number, (heap_number+1)%n_heaps));
17896 uint32_t begin_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
17897 #endif //SNOOP_STATS
17899 int idle_loop_count = 0;
17900 int first_not_ready_level = 0;
17904 gc_heap* hp = g_heaps [thpn];
17905 int level = first_not_ready_level;
17906 first_not_ready_level = 0;
17908 while (check_next_mark_stack (hp) && (level < (max_snoop_level-1)))
17910 idle_loop_count = 0;
17912 snoop_stat.busy_count++;
17913 dprintf (SNOOP_LOG, ("heap%d: looking at next heap level %d stack contents: %Ix",
17914 heap_number, level, (int)((uint8_t**)(hp->mark_stack_array))[level]));
17915 #endif //SNOOP_STATS
17917 uint8_t* o = ref_mark_stack (hp, level);
17919 uint8_t* start = o;
17922 mark_stack_busy() = 1;
17924 BOOL success = TRUE;
17925 uint8_t* next = (ref_mark_stack (hp, level+1));
17928 if (((size_t)o > 4) && !partial_object_p (o))
17930 //this is a normal object, not a partial mark tuple
17931 //success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level), 0, o)==o);
17932 success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level), (uint8_t*)4, o)==o);
17934 snoop_stat.interlocked_count++;
17936 snoop_stat.normal_count++;
17937 #endif //SNOOP_STATS
17941 //it is a stolen entry, or beginning/ending of a partial mark
17944 snoop_stat.stolen_or_pm_count++;
17945 #endif //SNOOP_STATS
17949 else if (stolen_p (next))
17951 //ignore the stolen guy and go to the next level
17955 snoop_stat.stolen_entry_count++;
17956 #endif //SNOOP_STATS
17960 assert (partial_p (next));
17961 start = ref_from_slot (next);
17962 //re-read the object
17963 o = ref_from_slot (ref_mark_stack (hp, level));
17967 success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level+1), (uint8_t*)stolen, next)==next);
17969 snoop_stat.interlocked_count++;
17972 snoop_stat.partial_mark_parent_count++;
17974 #endif //SNOOP_STATS
17978 // stack is not ready, or o is completely different from the last time we read from this stack level.
17979 // go up 2 levels to steal children or totally unrelated objects.
17981 if (first_not_ready_level == 0)
17983 first_not_ready_level = level;
17987 snoop_stat.pm_not_ready_count++;
17988 #endif //SNOOP_STATS
17995 dprintf (SNOOP_LOG, ("heap%d: marking %Ix from %d [%d] tl:%dms",
17996 heap_number, (size_t)o, (heap_number+1)%n_heaps, level,
17997 (GCToOSInterface::GetLowPrecisionTimeStamp()-begin_tick)));
17998 uint32_t start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
17999 #endif //SNOOP_STATS
18001 mark_object_simple1 (o, start, heap_number);
18004 dprintf (SNOOP_LOG, ("heap%d: done marking %Ix from %d [%d] %dms tl:%dms",
18005 heap_number, (size_t)o, (heap_number+1)%n_heaps, level,
18006 (GCToOSInterface::GetLowPrecisionTimeStamp()-start_tick),(GCToOSInterface::GetLowPrecisionTimeStamp()-begin_tick)));
18007 #endif //SNOOP_STATS
18009 mark_stack_busy() = 0;
18011 //clear the mark stack in snooping range
18012 for (int i = 0; i < max_snoop_level; i++)
18014 if (((uint8_t**)mark_stack_array)[i] != 0)
18016 ((VOLATILE(uint8_t*)*)(mark_stack_array))[i] = 0;
18018 snoop_stat.stack_bottom_clear_count++;
18019 #endif //SNOOP_STATS
18025 mark_stack_busy() = 0;
18029 //slot is either partial or stolen
18033 if ((first_not_ready_level != 0) && hp->mark_stack_busy())
18037 if (!hp->mark_stack_busy())
18039 first_not_ready_level = 0;
18042 if ((idle_loop_count % (6) )==1)
18045 snoop_stat.switch_to_thread_count++;
18046 #endif //SNOOP_STATS
18047 GCToOSInterface::Sleep(1);
18049 int free_count = 1;
18051 snoop_stat.stack_idle_count++;
18052 //dprintf (SNOOP_LOG, ("heap%d: counting idle threads", heap_number));
18053 #endif //SNOOP_STATS
18054 for (int hpn = (heap_number+1)%n_heaps; hpn != heap_number;)
18056 if (!((g_heaps [hpn])->mark_stack_busy()))
18060 dprintf (SNOOP_LOG, ("heap%d: %d idle", heap_number, free_count));
18061 #endif //SNOOP_STATS
18063 else if (same_numa_node_p (hpn, heap_number) || ((idle_loop_count%1000))==999)
18068 hpn = (hpn+1)%n_heaps;
18071 if (free_count == n_heaps)
18080 BOOL gc_heap::check_next_mark_stack (gc_heap* next_heap)
18083 snoop_stat.check_level_count++;
18084 #endif //SNOOP_STATS
18085 return (next_heap->mark_stack_busy()>=1);
18087 #endif //MH_SC_MARK
18090 void gc_heap::print_snoop_stat()
18092 dprintf (1234, ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s",
18093 "heap", "check", "zero", "mark", "stole", "pstack", "nstack", "nonsk"));
18094 dprintf (1234, ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d",
18095 snoop_stat.heap_index,
18096 snoop_stat.objects_checked_count,
18097 snoop_stat.zero_ref_count,
18098 snoop_stat.objects_marked_count,
18099 snoop_stat.stolen_stack_count,
18100 snoop_stat.partial_stack_count,
18101 snoop_stat.normal_stack_count,
18102 snoop_stat.non_stack_count));
18103 dprintf (1234, ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s",
18104 "heap", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "clear"));
18105 dprintf (1234, ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
18106 snoop_stat.heap_index,
18107 snoop_stat.check_level_count,
18108 snoop_stat.busy_count,
18109 snoop_stat.interlocked_count,
18110 snoop_stat.partial_mark_parent_count,
18111 snoop_stat.stolen_or_pm_count,
18112 snoop_stat.stolen_entry_count,
18113 snoop_stat.pm_not_ready_count,
18114 snoop_stat.normal_count,
18115 snoop_stat.stack_bottom_clear_count));
18117 printf ("\n%4s | %8s | %8s | %8s | %8s | %8s\n",
18118 "heap", "check", "zero", "mark", "idle", "switch");
18119 printf ("%4d | %8d | %8d | %8d | %8d | %8d\n",
18120 snoop_stat.heap_index,
18121 snoop_stat.objects_checked_count,
18122 snoop_stat.zero_ref_count,
18123 snoop_stat.objects_marked_count,
18124 snoop_stat.stack_idle_count,
18125 snoop_stat.switch_to_thread_count);
18126 printf ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s\n",
18127 "heap", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "normal", "clear");
18128 printf ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
18129 snoop_stat.heap_index,
18130 snoop_stat.check_level_count,
18131 snoop_stat.busy_count,
18132 snoop_stat.interlocked_count,
18133 snoop_stat.partial_mark_parent_count,
18134 snoop_stat.stolen_or_pm_count,
18135 snoop_stat.stolen_entry_count,
18136 snoop_stat.pm_not_ready_count,
18137 snoop_stat.normal_count,
18138 snoop_stat.stack_bottom_clear_count);
18140 #endif //SNOOP_STATS
18142 #ifdef HEAP_ANALYZE
18144 gc_heap::ha_mark_object_simple (uint8_t** po THREAD_NUMBER_DCL)
18146 if (!internal_root_array)
18148 internal_root_array = new (nothrow) uint8_t* [internal_root_array_length];
18149 if (!internal_root_array)
18151 heap_analyze_success = FALSE;
18155 if (heap_analyze_success && (internal_root_array_length <= internal_root_array_index))
18157 size_t new_size = 2*internal_root_array_length;
18159 uint64_t available_physical = 0;
18160 get_memory_info (NULL, &available_physical);
18161 if (new_size > (size_t)(available_physical / 10))
18163 heap_analyze_success = FALSE;
18167 uint8_t** tmp = new (nothrow) uint8_t* [new_size];
18170 memcpy (tmp, internal_root_array,
18171 internal_root_array_length*sizeof (uint8_t*));
18172 delete[] internal_root_array;
18173 internal_root_array = tmp;
18174 internal_root_array_length = new_size;
18178 heap_analyze_success = FALSE;
18183 if (heap_analyze_success)
18185 PREFIX_ASSUME(internal_root_array_index < internal_root_array_length);
18187 uint8_t* ref = (uint8_t*)po;
18188 if (!current_obj ||
18189 !((ref >= current_obj) && (ref < (current_obj + current_obj_size))))
18191 gc_heap* hp = gc_heap::heap_of (ref);
18192 current_obj = hp->find_object (ref, hp->lowest_address);
18193 current_obj_size = size (current_obj);
18195 internal_root_array[internal_root_array_index] = current_obj;
18196 internal_root_array_index++;
18200 mark_object_simple (po THREAD_NUMBER_ARG);
18202 #endif //HEAP_ANALYZE
18204 //this method assumes that *po is in the [low. high[ range
18206 gc_heap::mark_object_simple (uint8_t** po THREAD_NUMBER_DCL)
18209 #ifdef MULTIPLE_HEAPS
18210 #else //MULTIPLE_HEAPS
18211 const int thread = 0;
18212 #endif //MULTIPLE_HEAPS
18215 snoop_stat.objects_checked_count++;
18216 #endif //SNOOP_STATS
18221 size_t s = size (o);
18222 promoted_bytes (thread) += s;
18224 go_through_object_cl (method_table(o), o, s, poo,
18226 uint8_t* oo = *poo;
18227 if (gc_mark (oo, gc_low, gc_high))
18230 size_t obj_size = size (oo);
18231 promoted_bytes (thread) += obj_size;
18233 if (contain_pointers_or_collectible (oo))
18234 mark_object_simple1 (oo, oo THREAD_NUMBER_ARG);
18244 uint8_t* gc_heap::mark_object (uint8_t* o THREAD_NUMBER_DCL)
18246 if ((o >= gc_low) && (o < gc_high))
18247 mark_object_simple (&o THREAD_NUMBER_ARG);
18248 #ifdef MULTIPLE_HEAPS
18252 gc_heap* hp = heap_of (o);
18254 if ((o >= hp->gc_low) && (o < hp->gc_high))
18255 mark_object_simple (&o THREAD_NUMBER_ARG);
18257 #endif //MULTIPLE_HEAPS
18262 #ifdef BACKGROUND_GC
18264 void gc_heap::background_mark_simple1 (uint8_t* oo THREAD_NUMBER_DCL)
18266 uint8_t** mark_stack_limit = &background_mark_stack_array[background_mark_stack_array_length];
18268 #ifdef SORT_MARK_STACK
18269 uint8_t** sorted_tos = background_mark_stack_array;
18270 #endif //SORT_MARK_STACK
18272 background_mark_stack_tos = background_mark_stack_array;
18276 #ifdef MULTIPLE_HEAPS
18277 #else //MULTIPLE_HEAPS
18278 const int thread = 0;
18279 #endif //MULTIPLE_HEAPS
18283 if ((((size_t)oo & 1) == 0) && ((s = size (oo)) < (partial_size_th*sizeof (uint8_t*))))
18285 BOOL overflow_p = FALSE;
18287 if (background_mark_stack_tos + (s) /sizeof (uint8_t*) >= (mark_stack_limit - 1))
18289 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
18290 size_t num_pointers = CGCDesc::GetNumPointers(method_table(oo), s, num_components);
18291 if (background_mark_stack_tos + num_pointers >= (mark_stack_limit - 1))
18293 dprintf (2, ("h%d: %Id left, obj (mt: %Ix) %Id ptrs",
18295 (size_t)(mark_stack_limit - 1 - background_mark_stack_tos),
18299 bgc_overflow_count++;
18304 if (overflow_p == FALSE)
18306 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18308 go_through_object_cl (method_table(oo), oo, s, ppslot,
18310 uint8_t* o = *ppslot;
18312 if (background_mark (o,
18313 background_saved_lowest_address,
18314 background_saved_highest_address))
18317 size_t obj_size = size (o);
18318 bpromoted_bytes (thread) += obj_size;
18319 if (contain_pointers_or_collectible (o))
18321 *(background_mark_stack_tos++) = o;
18330 dprintf (3,("mark stack overflow for object %Ix ", (size_t)oo));
18331 background_min_overflow_address = min (background_min_overflow_address, oo);
18332 background_max_overflow_address = max (background_max_overflow_address, oo);
18337 uint8_t* start = oo;
18338 if ((size_t)oo & 1)
18340 oo = (uint8_t*)((size_t)oo & ~1);
18341 start = *(--background_mark_stack_tos);
18342 dprintf (4, ("oo: %Ix, start: %Ix\n", (size_t)oo, (size_t)start));
18344 #ifdef COLLECTIBLE_CLASS
18347 // If there's a class object, push it now. We are guaranteed to have the slot since
18348 // we just popped one object off.
18349 if (is_collectible (oo))
18351 uint8_t* class_obj = get_class_object (oo);
18352 if (background_mark (class_obj,
18353 background_saved_lowest_address,
18354 background_saved_highest_address))
18356 size_t obj_size = size (class_obj);
18357 bpromoted_bytes (thread) += obj_size;
18359 *(background_mark_stack_tos++) = class_obj;
18363 #endif //COLLECTIBLE_CLASS
18367 BOOL overflow_p = FALSE;
18369 if (background_mark_stack_tos + (num_partial_refs + 2) >= mark_stack_limit)
18371 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
18372 size_t num_pointers = CGCDesc::GetNumPointers(method_table(oo), s, num_components);
18374 dprintf (2, ("h%d: PM: %Id left, obj %Ix (mt: %Ix) start: %Ix, total: %Id",
18376 (size_t)(mark_stack_limit - background_mark_stack_tos),
18382 bgc_overflow_count++;
18385 if (overflow_p == FALSE)
18387 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18389 //push the object and its current
18390 uint8_t** place = background_mark_stack_tos++;
18392 *(background_mark_stack_tos++) = (uint8_t*)((size_t)oo | 1);
18394 int i = num_partial_refs;
18396 go_through_object (method_table(oo), oo, s, ppslot,
18397 start, use_start, (oo + s),
18399 uint8_t* o = *ppslot;
18402 if (background_mark (o,
18403 background_saved_lowest_address,
18404 background_saved_highest_address))
18407 size_t obj_size = size (o);
18408 bpromoted_bytes (thread) += obj_size;
18409 if (contain_pointers_or_collectible (o))
18411 *(background_mark_stack_tos++) = o;
18415 *place = (uint8_t*)(ppslot+1);
18424 //we are finished with this object
18432 dprintf (3,("mark stack overflow for object %Ix ", (size_t)oo));
18433 background_min_overflow_address = min (background_min_overflow_address, oo);
18434 background_max_overflow_address = max (background_max_overflow_address, oo);
18438 #ifdef SORT_MARK_STACK
18439 if (background_mark_stack_tos > sorted_tos + mark_stack_array_length/8)
18441 rqsort1 (sorted_tos, background_mark_stack_tos-1);
18442 sorted_tos = background_mark_stack_tos-1;
18444 #endif //SORT_MARK_STACK
18448 if (!(background_mark_stack_tos == background_mark_stack_array))
18450 oo = *(--background_mark_stack_tos);
18452 #ifdef SORT_MARK_STACK
18453 sorted_tos = (uint8_t**)min ((size_t)sorted_tos, (size_t)background_mark_stack_tos);
18454 #endif //SORT_MARK_STACK
18460 assert (background_mark_stack_tos == background_mark_stack_array);
18465 //this version is different than the foreground GC because
18466 //it can't keep pointers to the inside of an object
18467 //while calling background_mark_simple1. The object could be moved
18468 //by an intervening foreground gc.
18469 //this method assumes that *po is in the [low. high[ range
18471 gc_heap::background_mark_simple (uint8_t* o THREAD_NUMBER_DCL)
18473 #ifdef MULTIPLE_HEAPS
18474 #else //MULTIPLE_HEAPS
18475 const int thread = 0;
18476 #endif //MULTIPLE_HEAPS
18478 dprintf (3, ("bmarking %Ix", o));
18480 if (background_mark1 (o))
18483 size_t s = size (o);
18484 bpromoted_bytes (thread) += s;
18486 if (contain_pointers_or_collectible (o))
18488 background_mark_simple1 (o THREAD_NUMBER_ARG);
18495 uint8_t* gc_heap::background_mark_object (uint8_t* o THREAD_NUMBER_DCL)
18497 if ((o >= background_saved_lowest_address) && (o < background_saved_highest_address))
18499 background_mark_simple (o THREAD_NUMBER_ARG);
18505 dprintf (3, ("or-%Ix", o));
18511 void gc_heap::background_verify_mark (Object*& object, ScanContext* sc, uint32_t flags)
18513 UNREFERENCED_PARAMETER(sc);
18515 assert (settings.concurrent);
18516 uint8_t* o = (uint8_t*)object;
18518 gc_heap* hp = gc_heap::heap_of (o);
18519 #ifdef INTERIOR_POINTERS
18520 if (flags & GC_CALL_INTERIOR)
18522 o = hp->find_object (o, background_saved_lowest_address);
18524 #endif //INTERIOR_POINTERS
18526 if (!background_object_marked (o, FALSE))
18532 void gc_heap::background_promote (Object** ppObject, ScanContext* sc, uint32_t flags)
18534 UNREFERENCED_PARAMETER(sc);
18535 //in order to save space on the array, mark the object,
18536 //knowing that it will be visited later
18537 assert (settings.concurrent);
18539 THREAD_NUMBER_FROM_CONTEXT;
18540 #ifndef MULTIPLE_HEAPS
18541 const int thread = 0;
18542 #endif //!MULTIPLE_HEAPS
18544 uint8_t* o = (uint8_t*)*ppObject;
18549 #ifdef DEBUG_DestroyedHandleValue
18550 // we can race with destroy handle during concurrent scan
18551 if (o == (uint8_t*)DEBUG_DestroyedHandleValue)
18553 #endif //DEBUG_DestroyedHandleValue
18557 gc_heap* hp = gc_heap::heap_of (o);
18559 if ((o < hp->background_saved_lowest_address) || (o >= hp->background_saved_highest_address))
18564 #ifdef INTERIOR_POINTERS
18565 if (flags & GC_CALL_INTERIOR)
18567 o = hp->find_object (o, hp->background_saved_lowest_address);
18571 #endif //INTERIOR_POINTERS
18573 #ifdef FEATURE_CONSERVATIVE_GC
18574 // For conservative GC, a value on stack may point to middle of a free object.
18575 // In this case, we don't need to promote the pointer.
18576 if (GCConfig::GetConservativeGC() && ((CObjectHeader*)o)->IsFree())
18580 #endif //FEATURE_CONSERVATIVE_GC
18583 ((CObjectHeader*)o)->Validate();
18586 dprintf (BGC_LOG, ("Background Promote %Ix", (size_t)o));
18588 //needs to be called before the marking because it is possible for a foreground
18589 //gc to take place during the mark and move the object
18590 STRESS_LOG3(LF_GC|LF_GCROOTS, LL_INFO1000000, " GCHeap::Promote: Promote GC Root *%p = %p MT = %pT", ppObject, o, o ? ((Object*) o)->GetGCSafeMethodTable() : NULL);
18592 hpt->background_mark_simple (o THREAD_NUMBER_ARG);
18595 //used by the ephemeral collection to scan the local background structures
18596 //containing references.
18598 gc_heap::scan_background_roots (promote_func* fn, int hn, ScanContext *pSC)
18604 pSC->thread_number = hn;
18606 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
18607 pSC->pCurrentDomain = 0;
18610 BOOL relocate_p = (fn == &GCHeap::Relocate);
18612 dprintf (3, ("Scanning background mark list"));
18615 size_t mark_list_finger = 0;
18616 while (mark_list_finger < c_mark_list_index)
18618 uint8_t** o = &c_mark_list [mark_list_finger];
18621 // We may not be able to calculate the size during relocate as POPO
18622 // may have written over the object.
18623 size_t s = size (*o);
18624 assert (Align (s) >= Align (min_obj_size));
18625 dprintf(3,("background root %Ix", (size_t)*o));
18627 (*fn) ((Object**)o, pSC, 0);
18628 mark_list_finger++;
18631 //scan the mark stack
18632 dprintf (3, ("Scanning background mark stack"));
18634 uint8_t** finger = background_mark_stack_array;
18635 while (finger < background_mark_stack_tos)
18637 if ((finger + 1) < background_mark_stack_tos)
18639 // We need to check for the partial mark case here.
18640 uint8_t* parent_obj = *(finger + 1);
18641 if ((size_t)parent_obj & 1)
18643 uint8_t* place = *finger;
18644 size_t place_offset = 0;
18645 uint8_t* real_parent_obj = (uint8_t*)((size_t)parent_obj & ~1);
18649 *(finger + 1) = real_parent_obj;
18650 place_offset = place - real_parent_obj;
18651 dprintf(3,("relocating background root %Ix", (size_t)real_parent_obj));
18652 (*fn) ((Object**)(finger + 1), pSC, 0);
18653 real_parent_obj = *(finger + 1);
18654 *finger = real_parent_obj + place_offset;
18655 *(finger + 1) = (uint8_t*)((size_t)real_parent_obj | 1);
18656 dprintf(3,("roots changed to %Ix, %Ix", *finger, *(finger + 1)));
18660 uint8_t** temp = &real_parent_obj;
18661 dprintf(3,("marking background root %Ix", (size_t)real_parent_obj));
18662 (*fn) ((Object**)temp, pSC, 0);
18669 dprintf(3,("background root %Ix", (size_t)*finger));
18670 (*fn) ((Object**)finger, pSC, 0);
18676 void gc_heap::background_mark_through_object (uint8_t* oo THREAD_NUMBER_DCL)
18678 if (contain_pointers (oo))
18680 size_t total_refs = 0;
18681 size_t s = size (oo);
18682 go_through_object_nostart (method_table(oo), oo, s, po,
18686 background_mark_object (o THREAD_NUMBER_ARG);
18690 dprintf (3,("Background marking through %Ix went through %Id refs",
18696 uint8_t* gc_heap::background_seg_end (heap_segment* seg, BOOL concurrent_p)
18698 if (concurrent_p && (seg == saved_overflow_ephemeral_seg))
18700 // for now we stop at where gen1 started when we started processing
18701 return background_min_soh_overflow_address;
18705 return heap_segment_allocated (seg);
18709 uint8_t* gc_heap::background_first_overflow (uint8_t* min_add,
18712 BOOL small_object_p)
18716 if (small_object_p)
18718 if (in_range_for_segment (min_add, seg))
18720 // min_add was the beginning of gen1 when we did the concurrent
18721 // overflow. Now we could be in a situation where min_add is
18722 // actually the same as allocated for that segment (because
18723 // we expanded heap), in which case we can not call
18724 // find first on this address or we will AV.
18725 if (min_add >= heap_segment_allocated (seg))
18731 if (concurrent_p &&
18732 ((seg == saved_overflow_ephemeral_seg) && (min_add >= background_min_soh_overflow_address)))
18734 return background_min_soh_overflow_address;
18738 o = find_first_object (min_add, heap_segment_mem (seg));
18745 o = max (heap_segment_mem (seg), min_add);
18749 void gc_heap::background_process_mark_overflow_internal (int condemned_gen_number,
18750 uint8_t* min_add, uint8_t* max_add,
18755 current_bgc_state = bgc_overflow_soh;
18758 size_t total_marked_objects = 0;
18760 #ifdef MULTIPLE_HEAPS
18761 int thread = heap_number;
18762 #endif //MULTIPLE_HEAPS
18764 exclusive_sync* loh_alloc_lock = 0;
18766 dprintf (2,("Processing Mark overflow [%Ix %Ix]", (size_t)min_add, (size_t)max_add));
18767 #ifdef MULTIPLE_HEAPS
18768 // We don't have each heap scan all heaps concurrently because we are worried about
18769 // multiple threads calling things like find_first_object.
18770 int h_start = (concurrent_p ? heap_number : 0);
18771 int h_end = (concurrent_p ? (heap_number + 1) : n_heaps);
18772 for (int hi = h_start; hi < h_end; hi++)
18774 gc_heap* hp = (concurrent_p ? this : g_heaps [(heap_number + hi) % n_heaps]);
18780 #endif //MULTIPLE_HEAPS
18781 BOOL small_object_segments = TRUE;
18782 int align_const = get_alignment_constant (small_object_segments);
18783 generation* gen = hp->generation_of (condemned_gen_number);
18784 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
18785 PREFIX_ASSUME(seg != NULL);
18786 loh_alloc_lock = hp->bgc_alloc_lock;
18788 uint8_t* o = hp->background_first_overflow (min_add,
18791 small_object_segments);
18795 while ((o < hp->background_seg_end (seg, concurrent_p)) && (o <= max_add))
18797 dprintf (3, ("considering %Ix", (size_t)o));
18801 if (concurrent_p && !small_object_segments)
18803 loh_alloc_lock->bgc_mark_set (o);
18805 if (((CObjectHeader*)o)->IsFree())
18807 s = unused_array_size (o);
18819 if (background_object_marked (o, FALSE) && contain_pointers_or_collectible (o))
18821 total_marked_objects++;
18822 go_through_object_cl (method_table(o), o, s, poo,
18823 uint8_t* oo = *poo;
18824 background_mark_object (oo THREAD_NUMBER_ARG);
18828 if (concurrent_p && !small_object_segments)
18830 loh_alloc_lock->bgc_mark_done ();
18833 o = o + Align (s, align_const);
18841 dprintf (2, ("went through overflow objects in segment %Ix (%d) (so far %Id marked)",
18842 heap_segment_mem (seg), (small_object_segments ? 0 : 1), total_marked_objects));
18844 if ((concurrent_p && (seg == hp->saved_overflow_ephemeral_seg)) ||
18845 (seg = heap_segment_next_in_range (seg)) == 0)
18847 if (small_object_segments)
18851 current_bgc_state = bgc_overflow_loh;
18854 dprintf (2, ("h%d: SOH: ov-mo: %Id", heap_number, total_marked_objects));
18855 fire_overflow_event (min_add, max_add, total_marked_objects, !small_object_segments);
18856 concurrent_print_time_delta (concurrent_p ? "Cov SOH" : "Nov SOH");
18857 total_marked_objects = 0;
18858 small_object_segments = FALSE;
18859 align_const = get_alignment_constant (small_object_segments);
18860 seg = heap_segment_in_range (generation_start_segment (hp->generation_of (max_generation+1)));
18862 PREFIX_ASSUME(seg != NULL);
18864 o = max (heap_segment_mem (seg), min_add);
18869 dprintf (GTC_LOG, ("h%d: LOH: ov-mo: %Id", heap_number, total_marked_objects));
18870 fire_overflow_event (min_add, max_add, total_marked_objects, !small_object_segments);
18876 o = hp->background_first_overflow (min_add,
18879 small_object_segments);
18886 BOOL gc_heap::background_process_mark_overflow (BOOL concurrent_p)
18888 BOOL grow_mark_array_p = TRUE;
18892 assert (!processed_soh_overflow_p);
18894 if ((background_max_overflow_address != 0) &&
18895 (background_min_overflow_address != MAX_PTR))
18897 // We have overflow to process but we know we can't process the ephemeral generations
18898 // now (we actually could process till the current gen1 start but since we are going to
18899 // make overflow per segment, for now I'll just stop at the saved gen1 start.
18900 saved_overflow_ephemeral_seg = ephemeral_heap_segment;
18901 background_max_soh_overflow_address = heap_segment_reserved (saved_overflow_ephemeral_seg);
18902 background_min_soh_overflow_address = generation_allocation_start (generation_of (max_generation-1));
18907 assert ((saved_overflow_ephemeral_seg == 0) ||
18908 ((background_max_soh_overflow_address != 0) &&
18909 (background_min_soh_overflow_address != MAX_PTR)));
18911 if (!processed_soh_overflow_p)
18913 // if there was no more overflow we just need to process what we didn't process
18914 // on the saved ephemeral segment.
18915 if ((background_max_overflow_address == 0) && (background_min_overflow_address == MAX_PTR))
18917 dprintf (2, ("final processing mark overflow - no more overflow since last time"));
18918 grow_mark_array_p = FALSE;
18921 background_min_overflow_address = min (background_min_overflow_address,
18922 background_min_soh_overflow_address);
18923 background_max_overflow_address = max (background_max_overflow_address,
18924 background_max_soh_overflow_address);
18925 processed_soh_overflow_p = TRUE;
18929 BOOL overflow_p = FALSE;
18931 if ((! ((background_max_overflow_address == 0)) ||
18932 ! ((background_min_overflow_address == MAX_PTR))))
18936 if (grow_mark_array_p)
18938 // Try to grow the array.
18939 size_t new_size = max (MARK_STACK_INITIAL_LENGTH, 2*background_mark_stack_array_length);
18941 if ((new_size * sizeof(mark)) > 100*1024)
18943 size_t new_max_size = (get_total_heap_size() / 10) / sizeof(mark);
18945 new_size = min(new_max_size, new_size);
18948 if ((background_mark_stack_array_length < new_size) &&
18949 ((new_size - background_mark_stack_array_length) > (background_mark_stack_array_length / 2)))
18951 dprintf (2, ("h%d: ov grow to %Id", heap_number, new_size));
18953 uint8_t** tmp = new (nothrow) uint8_t* [new_size];
18956 delete background_mark_stack_array;
18957 background_mark_stack_array = tmp;
18958 background_mark_stack_array_length = new_size;
18959 background_mark_stack_tos = background_mark_stack_array;
18965 grow_mark_array_p = TRUE;
18968 uint8_t* min_add = background_min_overflow_address;
18969 uint8_t* max_add = background_max_overflow_address;
18971 background_max_overflow_address = 0;
18972 background_min_overflow_address = MAX_PTR;
18974 background_process_mark_overflow_internal (max_generation, min_add, max_add, concurrent_p);
18984 #endif //BACKGROUND_GC
18987 void gc_heap::mark_through_object (uint8_t* oo, BOOL mark_class_object_p THREAD_NUMBER_DCL)
18989 #ifndef COLLECTIBLE_CLASS
18990 UNREFERENCED_PARAMETER(mark_class_object_p);
18991 BOOL to_mark_class_object = FALSE;
18992 #else //COLLECTIBLE_CLASS
18993 BOOL to_mark_class_object = (mark_class_object_p && (is_collectible(oo)));
18994 #endif //COLLECTIBLE_CLASS
18995 if (contain_pointers (oo) || to_mark_class_object)
18997 dprintf(3,( "Marking through %Ix", (size_t)oo));
18998 size_t s = size (oo);
19000 #ifdef COLLECTIBLE_CLASS
19001 if (to_mark_class_object)
19003 uint8_t* class_obj = get_class_object (oo);
19004 mark_object (class_obj THREAD_NUMBER_ARG);
19006 #endif //COLLECTIBLE_CLASS
19008 if (contain_pointers (oo))
19010 go_through_object_nostart (method_table(oo), oo, s, po,
19012 mark_object (o THREAD_NUMBER_ARG);
19018 size_t gc_heap::get_total_heap_size()
19020 size_t total_heap_size = 0;
19022 #ifdef MULTIPLE_HEAPS
19025 for (hn = 0; hn < gc_heap::n_heaps; hn++)
19027 gc_heap* hp2 = gc_heap::g_heaps [hn];
19028 total_heap_size += hp2->generation_size (max_generation + 1) + hp2->generation_sizes (hp2->generation_of (max_generation));
19031 total_heap_size = generation_size (max_generation + 1) + generation_sizes (generation_of (max_generation));
19032 #endif //MULTIPLE_HEAPS
19034 return total_heap_size;
19037 size_t gc_heap::get_total_fragmentation()
19039 size_t total_fragmentation = 0;
19041 #ifdef MULTIPLE_HEAPS
19042 for (int i = 0; i < gc_heap::n_heaps; i++)
19044 gc_heap* hp = gc_heap::g_heaps[i];
19045 #else //MULTIPLE_HEAPS
19047 gc_heap* hp = pGenGCHeap;
19048 #endif //MULTIPLE_HEAPS
19049 for (int i = 0; i <= (max_generation + 1); i++)
19051 generation* gen = hp->generation_of (i);
19052 total_fragmentation += (generation_free_list_space (gen) + generation_free_obj_space (gen));
19056 return total_fragmentation;
19059 size_t gc_heap::committed_size()
19061 generation* gen = generation_of (max_generation);
19062 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
19063 size_t total_committed = 0;
19067 total_committed += heap_segment_committed (seg) - (uint8_t*)seg;
19069 seg = heap_segment_next (seg);
19072 if (gen != large_object_generation)
19074 gen = generation_of (max_generation + 1);
19075 seg = generation_start_segment (gen);
19082 return total_committed;
19085 size_t gc_heap::get_total_committed_size()
19087 size_t total_committed = 0;
19089 #ifdef MULTIPLE_HEAPS
19092 for (hn = 0; hn < gc_heap::n_heaps; hn++)
19094 gc_heap* hp = gc_heap::g_heaps [hn];
19095 total_committed += hp->committed_size();
19098 total_committed = committed_size();
19099 #endif //MULTIPLE_HEAPS
19101 return total_committed;
19104 void gc_heap::get_memory_info (uint32_t* memory_load,
19105 uint64_t* available_physical,
19106 uint64_t* available_page_file)
19108 GCToOSInterface::GetMemoryStatus(memory_load, available_physical, available_page_file);
19111 void fire_mark_event (int heap_num, int root_type, size_t bytes_marked)
19113 dprintf (DT_LOG_0, ("-----------[%d]mark %d: %Id", heap_num, root_type, bytes_marked));
19114 FIRE_EVENT(GCMarkWithType, heap_num, root_type, bytes_marked);
19117 //returns TRUE is an overflow happened.
19118 BOOL gc_heap::process_mark_overflow(int condemned_gen_number)
19120 size_t last_promoted_bytes = promoted_bytes (heap_number);
19121 BOOL overflow_p = FALSE;
19123 if ((! (max_overflow_address == 0) ||
19124 ! (min_overflow_address == MAX_PTR)))
19127 // Try to grow the array.
19129 max (MARK_STACK_INITIAL_LENGTH, 2*mark_stack_array_length);
19131 if ((new_size * sizeof(mark)) > 100*1024)
19133 size_t new_max_size = (get_total_heap_size() / 10) / sizeof(mark);
19135 new_size = min(new_max_size, new_size);
19138 if ((mark_stack_array_length < new_size) &&
19139 ((new_size - mark_stack_array_length) > (mark_stack_array_length / 2)))
19141 mark* tmp = new (nothrow) mark [new_size];
19144 delete mark_stack_array;
19145 mark_stack_array = tmp;
19146 mark_stack_array_length = new_size;
19150 uint8_t* min_add = min_overflow_address;
19151 uint8_t* max_add = max_overflow_address;
19152 max_overflow_address = 0;
19153 min_overflow_address = MAX_PTR;
19154 process_mark_overflow_internal (condemned_gen_number, min_add, max_add);
19158 size_t current_promoted_bytes = promoted_bytes (heap_number);
19160 if (current_promoted_bytes != last_promoted_bytes)
19161 fire_mark_event (heap_number, ETW::GC_ROOT_OVERFLOW, (current_promoted_bytes - last_promoted_bytes));
19165 void gc_heap::process_mark_overflow_internal (int condemned_gen_number,
19166 uint8_t* min_add, uint8_t* max_add)
19168 #ifdef MULTIPLE_HEAPS
19169 int thread = heap_number;
19170 #endif //MULTIPLE_HEAPS
19171 BOOL full_p = (condemned_gen_number == max_generation);
19173 dprintf(3,("Processing Mark overflow [%Ix %Ix]", (size_t)min_add, (size_t)max_add));
19174 #ifdef MULTIPLE_HEAPS
19175 for (int hi = 0; hi < n_heaps; hi++)
19177 gc_heap* hp = g_heaps [(heap_number + hi) % n_heaps];
19183 #endif //MULTIPLE_HEAPS
19184 BOOL small_object_segments = TRUE;
19185 int align_const = get_alignment_constant (small_object_segments);
19186 generation* gen = hp->generation_of (condemned_gen_number);
19187 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
19189 PREFIX_ASSUME(seg != NULL);
19190 uint8_t* o = max (heap_segment_mem (seg), min_add);
19193 uint8_t* end = heap_segment_allocated (seg);
19195 while ((o < end) && (o <= max_add))
19197 assert ((min_add <= o) && (max_add >= o));
19198 dprintf (3, ("considering %Ix", (size_t)o));
19201 mark_through_object (o, TRUE THREAD_NUMBER_ARG);
19204 o = o + Align (size (o), align_const);
19207 if (( seg = heap_segment_next_in_range (seg)) == 0)
19209 if (small_object_segments && full_p)
19211 small_object_segments = FALSE;
19212 align_const = get_alignment_constant (small_object_segments);
19213 seg = heap_segment_in_range (generation_start_segment (hp->generation_of (max_generation+1)));
19215 PREFIX_ASSUME(seg != NULL);
19217 o = max (heap_segment_mem (seg), min_add);
19227 o = max (heap_segment_mem (seg), min_add);
19234 // Scanning for promotion for dependent handles need special handling. Because the primary holds a strong
19235 // reference to the secondary (when the primary itself is reachable) and this can cause a cascading series of
19236 // promotions (the secondary of one handle is or promotes the primary of another) we might need to perform the
19237 // promotion scan multiple times.
19238 // This helper encapsulates the logic to complete all dependent handle promotions when running a server GC. It
19239 // also has the effect of processing any mark stack overflow.
19241 #ifdef MULTIPLE_HEAPS
19242 // When multiple heaps are enabled we have must utilize a more complex algorithm in order to keep all the GC
19243 // worker threads synchronized. The algorithms are sufficiently divergent that we have different
19244 // implementations based on whether MULTIPLE_HEAPS is defined or not.
19246 // Define some static variables used for synchronization in the method below. These should really be defined
19247 // locally but MSVC complains when the VOLATILE macro is expanded into an instantiation of the Volatile class.
19249 // A note about the synchronization used within this method. Communication between the worker threads is
19250 // achieved via two shared booleans (defined below). These both act as latches that are transitioned only from
19251 // false -> true by unsynchronized code. They are only read or reset to false by a single thread under the
19252 // protection of a join.
19253 static VOLATILE(BOOL) s_fUnpromotedHandles = FALSE;
19254 static VOLATILE(BOOL) s_fUnscannedPromotions = FALSE;
19255 static VOLATILE(BOOL) s_fScanRequired;
19256 void gc_heap::scan_dependent_handles (int condemned_gen_number, ScanContext *sc, BOOL initial_scan_p)
19258 // Whenever we call this method there may have been preceding object promotions. So set
19259 // s_fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
19260 // based on the how the scanning proceeded).
19261 s_fUnscannedPromotions = TRUE;
19263 // We don't know how many times we need to loop yet. In particular we can't base the loop condition on
19264 // the state of this thread's portion of the dependent handle table. That's because promotions on other
19265 // threads could cause handle promotions to become necessary here. Even if there are definitely no more
19266 // promotions possible in this thread's handles, we still have to stay in lock-step with those worker
19267 // threads that haven't finished yet (each GC worker thread has to join exactly the same number of times
19268 // as all the others or they'll get out of step).
19271 // The various worker threads are all currently racing in this code. We need to work out if at least
19272 // one of them think they have work to do this cycle. Each thread needs to rescan its portion of the
19273 // dependent handle table when both of the following conditions apply:
19274 // 1) At least one (arbitrary) object might have been promoted since the last scan (because if this
19275 // object happens to correspond to a primary in one of our handles we might potentially have to
19276 // promote the associated secondary).
19277 // 2) The table for this thread has at least one handle with a secondary that isn't promoted yet.
19279 // The first condition is represented by s_fUnscannedPromotions. This is always non-zero for the first
19280 // iteration of this loop (see comment above) and in subsequent cycles each thread updates this
19281 // whenever a mark stack overflow occurs or scanning their dependent handles results in a secondary
19282 // being promoted. This value is cleared back to zero in a synchronized fashion in the join that
19283 // follows below. Note that we can't read this outside of the join since on any iteration apart from
19284 // the first threads will be racing between reading this value and completing their previous
19285 // iteration's table scan.
19287 // The second condition is tracked by the dependent handle code itself on a per worker thread basis
19288 // (and updated by the GcDhReScan() method). We call GcDhUnpromotedHandlesExist() on each thread to
19289 // determine the local value and collect the results into the s_fUnpromotedHandles variable in what is
19290 // effectively an OR operation. As per s_fUnscannedPromotions we can't read the final result until
19291 // we're safely joined.
19292 if (GCScan::GcDhUnpromotedHandlesExist(sc))
19293 s_fUnpromotedHandles = TRUE;
19295 // Synchronize all the threads so we can read our state variables safely. The shared variable
19296 // s_fScanRequired, indicating whether we should scan the tables or terminate the loop, will be set by
19297 // a single thread inside the join.
19298 gc_t_join.join(this, gc_join_scan_dependent_handles);
19299 if (gc_t_join.joined())
19301 // We're synchronized so it's safe to read our shared state variables. We update another shared
19302 // variable to indicate to all threads whether we'll be scanning for another cycle or terminating
19303 // the loop. We scan if there has been at least one object promotion since last time and at least
19304 // one thread has a dependent handle table with a potential handle promotion possible.
19305 s_fScanRequired = s_fUnscannedPromotions && s_fUnpromotedHandles;
19307 // Reset our shared state variables (ready to be set again on this scan or with a good initial
19308 // value for the next call if we're terminating the loop).
19309 s_fUnscannedPromotions = FALSE;
19310 s_fUnpromotedHandles = FALSE;
19312 if (!s_fScanRequired)
19314 // We're terminating the loop. Perform any last operations that require single threaded access.
19315 if (!initial_scan_p)
19317 // On the second invocation we reconcile all mark overflow ranges across the heaps. This can help
19318 // load balance if some of the heaps have an abnormally large workload.
19319 uint8_t* all_heaps_max = 0;
19320 uint8_t* all_heaps_min = MAX_PTR;
19322 for (i = 0; i < n_heaps; i++)
19324 if (all_heaps_max < g_heaps[i]->max_overflow_address)
19325 all_heaps_max = g_heaps[i]->max_overflow_address;
19326 if (all_heaps_min > g_heaps[i]->min_overflow_address)
19327 all_heaps_min = g_heaps[i]->min_overflow_address;
19329 for (i = 0; i < n_heaps; i++)
19331 g_heaps[i]->max_overflow_address = all_heaps_max;
19332 g_heaps[i]->min_overflow_address = all_heaps_min;
19337 // Restart all the workers.
19338 dprintf(3, ("Starting all gc thread mark stack overflow processing"));
19339 gc_t_join.restart();
19342 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
19343 // being visible. If there really was an overflow (process_mark_overflow returns true) then set the
19344 // global flag indicating that at least one object promotion may have occurred (the usual comment
19345 // about races applies). (Note it's OK to set this flag even if we're about to terminate the loop and
19346 // exit the method since we unconditionally set this variable on method entry anyway).
19347 if (process_mark_overflow(condemned_gen_number))
19348 s_fUnscannedPromotions = TRUE;
19350 // If we decided that no scan was required we can terminate the loop now.
19351 if (!s_fScanRequired)
19354 // Otherwise we must join with the other workers to ensure that all mark stack overflows have been
19355 // processed before we start scanning dependent handle tables (if overflows remain while we scan we
19356 // could miss noting the promotion of some primary objects).
19357 gc_t_join.join(this, gc_join_rescan_dependent_handles);
19358 if (gc_t_join.joined())
19360 // Restart all the workers.
19361 dprintf(3, ("Starting all gc thread for dependent handle promotion"));
19362 gc_t_join.restart();
19365 // If the portion of the dependent handle table managed by this worker has handles that could still be
19366 // promoted perform a rescan. If the rescan resulted in at least one promotion note this fact since it
19367 // could require a rescan of handles on this or other workers.
19368 if (GCScan::GcDhUnpromotedHandlesExist(sc))
19369 if (GCScan::GcDhReScan(sc))
19370 s_fUnscannedPromotions = TRUE;
19373 #else //MULTIPLE_HEAPS
19374 // Non-multiple heap version of scan_dependent_handles: much simpler without the need to keep multiple worker
19375 // threads synchronized.
19376 void gc_heap::scan_dependent_handles (int condemned_gen_number, ScanContext *sc, BOOL initial_scan_p)
19378 UNREFERENCED_PARAMETER(initial_scan_p);
19380 // Whenever we call this method there may have been preceding object promotions. So set
19381 // fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
19382 // based on the how the scanning proceeded).
19383 bool fUnscannedPromotions = true;
19385 // Loop until there are either no more dependent handles that can have their secondary promoted or we've
19386 // managed to perform a scan without promoting anything new.
19387 while (GCScan::GcDhUnpromotedHandlesExist(sc) && fUnscannedPromotions)
19389 // On each iteration of the loop start with the assumption that no further objects have been promoted.
19390 fUnscannedPromotions = false;
19392 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
19393 // being visible. If there was an overflow (process_mark_overflow returned true) then additional
19394 // objects now appear to be promoted and we should set the flag.
19395 if (process_mark_overflow(condemned_gen_number))
19396 fUnscannedPromotions = true;
19398 // Perform the scan and set the flag if any promotions resulted.
19399 if (GCScan::GcDhReScan(sc))
19400 fUnscannedPromotions = true;
19403 // Process any mark stack overflow that may have resulted from scanning handles (or if we didn't need to
19404 // scan any handles at all this is the processing of overflows that may have occurred prior to this method
19406 process_mark_overflow(condemned_gen_number);
19408 #endif //MULTIPLE_HEAPS
19410 void gc_heap::mark_phase (int condemned_gen_number, BOOL mark_only_p)
19412 assert (settings.concurrent == FALSE);
19415 sc.thread_number = heap_number;
19416 sc.promotion = TRUE;
19417 sc.concurrent = FALSE;
19419 dprintf(2,("---- Mark Phase condemning %d ----", condemned_gen_number));
19420 BOOL full_p = (condemned_gen_number == max_generation);
19425 start = GetCycleCount32();
19428 int gen_to_init = condemned_gen_number;
19429 if (condemned_gen_number == max_generation)
19431 gen_to_init = max_generation + 1;
19433 for (int gen_idx = 0; gen_idx <= gen_to_init; gen_idx++)
19435 dynamic_data* dd = dynamic_data_of (gen_idx);
19436 dd_begin_data_size (dd) = generation_size (gen_idx) -
19437 dd_fragmentation (dd) -
19438 Align (size (generation_allocation_start (generation_of (gen_idx))));
19439 dprintf (2, ("begin data size for gen%d is %Id", gen_idx, dd_begin_data_size (dd)));
19440 dd_survived_size (dd) = 0;
19441 dd_pinned_survived_size (dd) = 0;
19442 dd_artificial_pinned_survived_size (dd) = 0;
19443 dd_added_pinned_size (dd) = 0;
19445 dd_padding_size (dd) = 0;
19446 #endif //SHORT_PLUGS
19447 #if defined (RESPECT_LARGE_ALIGNMENT) || defined (FEATURE_STRUCTALIGN)
19448 dd_num_npinned_plugs (dd) = 0;
19449 #endif //RESPECT_LARGE_ALIGNMENT || FEATURE_STRUCTALIGN
19452 #ifdef FFIND_OBJECT
19453 if (gen0_must_clear_bricks > 0)
19454 gen0_must_clear_bricks--;
19455 #endif //FFIND_OBJECT
19457 size_t last_promoted_bytes = 0;
19459 promoted_bytes (heap_number) = 0;
19460 reset_mark_stack();
19463 memset (&snoop_stat, 0, sizeof(snoop_stat));
19464 snoop_stat.heap_index = heap_number;
19465 #endif //SNOOP_STATS
19470 //initialize the mark stack
19471 for (int i = 0; i < max_snoop_level; i++)
19473 ((uint8_t**)(mark_stack_array))[i] = 0;
19476 mark_stack_busy() = 1;
19478 #endif //MH_SC_MARK
19480 static uint32_t num_sizedrefs = 0;
19483 static BOOL do_mark_steal_p = FALSE;
19484 #endif //MH_SC_MARK
19486 #ifdef MULTIPLE_HEAPS
19487 gc_t_join.join(this, gc_join_begin_mark_phase);
19488 if (gc_t_join.joined())
19490 #endif //MULTIPLE_HEAPS
19492 num_sizedrefs = GCToEEInterface::GetTotalNumSizedRefHandles();
19494 #ifdef MULTIPLE_HEAPS
19499 size_t total_heap_size = get_total_heap_size();
19501 if (total_heap_size > (100 * 1024 * 1024))
19503 do_mark_steal_p = TRUE;
19507 do_mark_steal_p = FALSE;
19512 do_mark_steal_p = FALSE;
19514 #endif //MH_SC_MARK
19516 gc_t_join.restart();
19518 #endif //MULTIPLE_HEAPS
19523 //set up the mark lists from g_mark_list
19524 assert (g_mark_list);
19525 #ifdef MULTIPLE_HEAPS
19526 mark_list = &g_mark_list [heap_number*mark_list_size];
19528 mark_list = g_mark_list;
19529 #endif //MULTIPLE_HEAPS
19530 //dont use the mark list for full gc
19531 //because multiple segments are more complex to handle and the list
19532 //is likely to overflow
19533 if (condemned_gen_number != max_generation)
19534 mark_list_end = &mark_list [mark_list_size-1];
19536 mark_list_end = &mark_list [0];
19537 mark_list_index = &mark_list [0];
19540 shigh = (uint8_t*) 0;
19543 //%type% category = quote (mark);
19545 if ((condemned_gen_number == max_generation) && (num_sizedrefs > 0))
19547 GCScan::GcScanSizedRefs(GCHeap::Promote, condemned_gen_number, max_generation, &sc);
19548 fire_mark_event (heap_number, ETW::GC_ROOT_SIZEDREF, (promoted_bytes (heap_number) - last_promoted_bytes));
19549 last_promoted_bytes = promoted_bytes (heap_number);
19551 #ifdef MULTIPLE_HEAPS
19552 gc_t_join.join(this, gc_join_scan_sizedref_done);
19553 if (gc_t_join.joined())
19555 dprintf(3, ("Done with marking all sized refs. Starting all gc thread for marking other strong roots"));
19556 gc_t_join.restart();
19558 #endif //MULTIPLE_HEAPS
19561 dprintf(3,("Marking Roots"));
19563 GCScan::GcScanRoots(GCHeap::Promote,
19564 condemned_gen_number, max_generation,
19567 fire_mark_event (heap_number, ETW::GC_ROOT_STACK, (promoted_bytes (heap_number) - last_promoted_bytes));
19568 last_promoted_bytes = promoted_bytes (heap_number);
19570 #ifdef BACKGROUND_GC
19571 if (recursive_gc_sync::background_running_p())
19573 scan_background_roots (GCHeap::Promote, heap_number, &sc);
19575 #endif //BACKGROUND_GC
19577 #ifdef FEATURE_PREMORTEM_FINALIZATION
19578 dprintf(3, ("Marking finalization data"));
19579 finalize_queue->GcScanRoots(GCHeap::Promote, heap_number, 0);
19580 #endif // FEATURE_PREMORTEM_FINALIZATION
19582 fire_mark_event (heap_number, ETW::GC_ROOT_FQ, (promoted_bytes (heap_number) - last_promoted_bytes));
19583 last_promoted_bytes = promoted_bytes (heap_number);
19588 dprintf(3,("Marking handle table"));
19589 GCScan::GcScanHandles(GCHeap::Promote,
19590 condemned_gen_number, max_generation,
19592 fire_mark_event (heap_number, ETW::GC_ROOT_HANDLES, (promoted_bytes (heap_number) - last_promoted_bytes));
19593 last_promoted_bytes = promoted_bytes (heap_number);
19597 size_t promoted_before_cards = promoted_bytes (heap_number);
19600 dprintf (3, ("before cards: %Id", promoted_before_cards));
19604 #ifdef MULTIPLE_HEAPS
19605 if (gc_t_join.r_join(this, gc_r_join_update_card_bundle))
19607 #endif //MULTIPLE_HEAPS
19609 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
19610 // If we are manually managing card bundles, every write to the card table should already be
19611 // accounted for in the card bundle table so there's nothing to update here.
19612 update_card_table_bundle();
19614 if (card_bundles_enabled())
19616 verify_card_bundles();
19619 #ifdef MULTIPLE_HEAPS
19620 gc_t_join.r_restart();
19622 #endif //MULTIPLE_HEAPS
19623 #endif //CARD_BUNDLE
19625 card_fn mark_object_fn = &gc_heap::mark_object_simple;
19626 #ifdef HEAP_ANALYZE
19627 heap_analyze_success = TRUE;
19628 if (heap_analyze_enabled)
19630 internal_root_array_index = 0;
19632 current_obj_size = 0;
19633 mark_object_fn = &gc_heap::ha_mark_object_simple;
19635 #endif //HEAP_ANALYZE
19637 dprintf(3,("Marking cross generation pointers"));
19638 mark_through_cards_for_segments (mark_object_fn, FALSE);
19640 dprintf(3,("Marking cross generation pointers for large objects"));
19641 mark_through_cards_for_large_objects (mark_object_fn, FALSE);
19643 dprintf (3, ("marked by cards: %Id",
19644 (promoted_bytes (heap_number) - promoted_before_cards)));
19645 fire_mark_event (heap_number, ETW::GC_ROOT_OLDER, (promoted_bytes (heap_number) - last_promoted_bytes));
19646 last_promoted_bytes = promoted_bytes (heap_number);
19651 if (do_mark_steal_p)
19655 #endif //MH_SC_MARK
19657 // Dependent handles need to be scanned with a special algorithm (see the header comment on
19658 // scan_dependent_handles for more detail). We perform an initial scan without synchronizing with other
19659 // worker threads or processing any mark stack overflow. This is not guaranteed to complete the operation
19660 // but in a common case (where there are no dependent handles that are due to be collected) it allows us
19661 // to optimize away further scans. The call to scan_dependent_handles is what will cycle through more
19662 // iterations if required and will also perform processing of any mark stack overflow once the dependent
19663 // handle table has been fully promoted.
19664 GCScan::GcDhInitialScan(GCHeap::Promote, condemned_gen_number, max_generation, &sc);
19665 scan_dependent_handles(condemned_gen_number, &sc, true);
19667 #ifdef MULTIPLE_HEAPS
19668 dprintf(3, ("Joining for short weak handle scan"));
19669 gc_t_join.join(this, gc_join_null_dead_short_weak);
19670 if (gc_t_join.joined())
19671 #endif //MULTIPLE_HEAPS
19673 #ifdef HEAP_ANALYZE
19674 heap_analyze_enabled = FALSE;
19675 GCToEEInterface::AnalyzeSurvivorsFinished(condemned_gen_number);
19676 #endif // HEAP_ANALYZE
19677 GCToEEInterface::AfterGcScanRoots (condemned_gen_number, max_generation, &sc);
19679 #ifdef MULTIPLE_HEAPS
19682 // we used r_join and need to reinitialize states for it here.
19683 gc_t_join.r_init();
19686 //start all threads on the roots.
19687 dprintf(3, ("Starting all gc thread for short weak handle scan"));
19688 gc_t_join.restart();
19689 #endif //MULTIPLE_HEAPS
19693 // null out the target of short weakref that were not promoted.
19694 GCScan::GcShortWeakPtrScan(GCHeap::Promote, condemned_gen_number, max_generation,&sc);
19696 // MTHTS: keep by single thread
19697 #ifdef MULTIPLE_HEAPS
19698 dprintf(3, ("Joining for finalization"));
19699 gc_t_join.join(this, gc_join_scan_finalization);
19700 if (gc_t_join.joined())
19701 #endif //MULTIPLE_HEAPS
19704 #ifdef MULTIPLE_HEAPS
19705 //start all threads on the roots.
19706 dprintf(3, ("Starting all gc thread for Finalization"));
19707 gc_t_join.restart();
19708 #endif //MULTIPLE_HEAPS
19711 //Handle finalization.
19712 size_t promoted_bytes_live = promoted_bytes (heap_number);
19714 #ifdef FEATURE_PREMORTEM_FINALIZATION
19715 dprintf (3, ("Finalize marking"));
19716 finalize_queue->ScanForFinalization (GCHeap::Promote, condemned_gen_number, mark_only_p, __this);
19718 GCToEEInterface::DiagWalkFReachableObjects(__this);
19719 #endif // FEATURE_PREMORTEM_FINALIZATION
19721 // Scan dependent handles again to promote any secondaries associated with primaries that were promoted
19722 // for finalization. As before scan_dependent_handles will also process any mark stack overflow.
19723 scan_dependent_handles(condemned_gen_number, &sc, false);
19725 #ifdef MULTIPLE_HEAPS
19726 dprintf(3, ("Joining for weak pointer deletion"));
19727 gc_t_join.join(this, gc_join_null_dead_long_weak);
19728 if (gc_t_join.joined())
19730 //start all threads on the roots.
19731 dprintf(3, ("Starting all gc thread for weak pointer deletion"));
19732 gc_t_join.restart();
19734 #endif //MULTIPLE_HEAPS
19736 // null out the target of long weakref that were not promoted.
19737 GCScan::GcWeakPtrScan (GCHeap::Promote, condemned_gen_number, max_generation, &sc);
19739 // MTHTS: keep by single thread
19740 #ifdef MULTIPLE_HEAPS
19742 #ifdef PARALLEL_MARK_LIST_SORT
19743 // unsigned long start = GetCycleCount32();
19745 // printf("sort_mark_list took %u cycles\n", GetCycleCount32() - start);
19746 #endif //PARALLEL_MARK_LIST_SORT
19749 dprintf (3, ("Joining for sync block cache entry scanning"));
19750 gc_t_join.join(this, gc_join_null_dead_syncblk);
19751 if (gc_t_join.joined())
19752 #endif //MULTIPLE_HEAPS
19754 // scan for deleted entries in the syncblk cache
19755 GCScan::GcWeakPtrScanBySingleThread (condemned_gen_number, max_generation, &sc);
19757 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
19760 size_t promoted_all_heaps = 0;
19761 #ifdef MULTIPLE_HEAPS
19762 for (int i = 0; i < n_heaps; i++)
19764 promoted_all_heaps += promoted_bytes (i);
19767 promoted_all_heaps = promoted_bytes (heap_number);
19768 #endif //MULTIPLE_HEAPS
19769 SystemDomain::RecordTotalSurvivedBytes (promoted_all_heaps);
19771 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
19773 #ifdef MULTIPLE_HEAPS
19776 #ifndef PARALLEL_MARK_LIST_SORT
19777 //compact g_mark_list and sort it.
19778 combine_mark_lists();
19779 #endif //PARALLEL_MARK_LIST_SORT
19782 //decide on promotion
19783 if (!settings.promotion)
19786 for (int n = 0; n <= condemned_gen_number;n++)
19788 m += (size_t)(dd_min_size (dynamic_data_of (n))*(n+1)*0.1);
19791 for (int i = 0; i < n_heaps; i++)
19793 dynamic_data* dd = g_heaps[i]->dynamic_data_of (min (condemned_gen_number +1,
19795 size_t older_gen_size = (dd_current_size (dd) +
19796 (dd_desired_allocation (dd) -
19797 dd_new_allocation (dd)));
19799 if ((m > (older_gen_size)) ||
19800 (promoted_bytes (i) > m))
19802 settings.promotion = TRUE;
19808 if (do_mark_steal_p)
19810 size_t objects_checked_count = 0;
19811 size_t zero_ref_count = 0;
19812 size_t objects_marked_count = 0;
19813 size_t check_level_count = 0;
19814 size_t busy_count = 0;
19815 size_t interlocked_count = 0;
19816 size_t partial_mark_parent_count = 0;
19817 size_t stolen_or_pm_count = 0;
19818 size_t stolen_entry_count = 0;
19819 size_t pm_not_ready_count = 0;
19820 size_t normal_count = 0;
19821 size_t stack_bottom_clear_count = 0;
19823 for (int i = 0; i < n_heaps; i++)
19825 gc_heap* hp = g_heaps[i];
19826 hp->print_snoop_stat();
19827 objects_checked_count += hp->snoop_stat.objects_checked_count;
19828 zero_ref_count += hp->snoop_stat.zero_ref_count;
19829 objects_marked_count += hp->snoop_stat.objects_marked_count;
19830 check_level_count += hp->snoop_stat.check_level_count;
19831 busy_count += hp->snoop_stat.busy_count;
19832 interlocked_count += hp->snoop_stat.interlocked_count;
19833 partial_mark_parent_count += hp->snoop_stat.partial_mark_parent_count;
19834 stolen_or_pm_count += hp->snoop_stat.stolen_or_pm_count;
19835 stolen_entry_count += hp->snoop_stat.stolen_entry_count;
19836 pm_not_ready_count += hp->snoop_stat.pm_not_ready_count;
19837 normal_count += hp->snoop_stat.normal_count;
19838 stack_bottom_clear_count += hp->snoop_stat.stack_bottom_clear_count;
19843 printf ("-------total stats-------\n");
19844 printf ("%8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s\n",
19845 "checked", "zero", "marked", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "normal", "clear");
19846 printf ("%8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
19847 objects_checked_count,
19849 objects_marked_count,
19853 partial_mark_parent_count,
19854 stolen_or_pm_count,
19855 stolen_entry_count,
19856 pm_not_ready_count,
19858 stack_bottom_clear_count);
19860 #endif //SNOOP_STATS
19862 //start all threads.
19863 dprintf(3, ("Starting all threads for end of mark phase"));
19864 gc_t_join.restart();
19865 #else //MULTIPLE_HEAPS
19867 //decide on promotion
19868 if (!settings.promotion)
19871 for (int n = 0; n <= condemned_gen_number;n++)
19873 m += (size_t)(dd_min_size (dynamic_data_of (n))*(n+1)*0.06);
19875 dynamic_data* dd = dynamic_data_of (min (condemned_gen_number +1,
19877 size_t older_gen_size = (dd_current_size (dd) +
19878 (dd_desired_allocation (dd) -
19879 dd_new_allocation (dd)));
19881 dprintf (2, ("promotion threshold: %Id, promoted bytes: %Id size n+1: %Id",
19882 m, promoted_bytes (heap_number), older_gen_size));
19884 if ((m > older_gen_size) ||
19885 (promoted_bytes (heap_number) > m))
19887 settings.promotion = TRUE;
19891 #endif //MULTIPLE_HEAPS
19894 #ifdef MULTIPLE_HEAPS
19896 #ifdef PARALLEL_MARK_LIST_SORT
19897 // start = GetCycleCount32();
19898 merge_mark_lists();
19899 // printf("merge_mark_lists took %u cycles\n", GetCycleCount32() - start);
19900 #endif //PARALLEL_MARK_LIST_SORT
19902 #endif //MULTIPLE_HEAPS
19904 #ifdef BACKGROUND_GC
19905 total_promoted_bytes = promoted_bytes (heap_number);
19906 #endif //BACKGROUND_GC
19908 promoted_bytes (heap_number) -= promoted_bytes_live;
19911 finish = GetCycleCount32();
19912 mark_time = finish - start;
19915 dprintf(2,("---- End of mark phase ----"));
19919 void gc_heap::pin_object (uint8_t* o, uint8_t** ppObject, uint8_t* low, uint8_t* high)
19921 dprintf (3, ("Pinning %Ix", (size_t)o));
19922 if ((o >= low) && (o < high))
19924 dprintf(3,("^%Ix^", (size_t)o));
19927 #ifdef FEATURE_EVENT_TRACE
19928 if(EVENT_ENABLED(PinObjectAtGCTime))
19930 fire_etw_pin_object_event(o, ppObject);
19932 #endif // FEATURE_EVENT_TRACE
19934 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
19935 num_pinned_objects++;
19936 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
19940 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
19941 size_t gc_heap::get_total_pinned_objects()
19943 #ifdef MULTIPLE_HEAPS
19944 size_t total_num_pinned_objects = 0;
19945 for (int i = 0; i < gc_heap::n_heaps; i++)
19947 gc_heap* hp = gc_heap::g_heaps[i];
19948 total_num_pinned_objects += hp->num_pinned_objects;
19950 return total_num_pinned_objects;
19951 #else //MULTIPLE_HEAPS
19952 return num_pinned_objects;
19953 #endif //MULTIPLE_HEAPS
19955 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
19957 void gc_heap::reset_mark_stack ()
19959 reset_pinned_queue();
19960 max_overflow_address = 0;
19961 min_overflow_address = MAX_PTR;
19964 #ifdef FEATURE_STRUCTALIGN
19966 // The word with left child, right child, and align info is laid out as follows:
19968 // | upper short word | lower short word |
19969 // |<------------> <----->|<------------> <----->|
19970 // | left child info hi| right child info lo|
19971 // x86: | 10 bits 6 bits| 10 bits 6 bits|
19973 // where left/right child are signed values and concat(info hi, info lo) is unsigned.
19975 // The "align info" encodes two numbers: the required alignment (a power of two)
19976 // and the misalignment (the number of machine words the destination address needs
19977 // to be adjusted by to provide alignment - so this number is always smaller than
19978 // the required alignment). Thus, the two can be represented as the "logical or"
19979 // of the two numbers. Note that the actual pad is computed from the misalignment
19980 // by adding the alignment iff the misalignment is non-zero and less than min_obj_size.
19983 // The number of bits in a brick.
19984 #if defined (_TARGET_AMD64_)
19985 #define brick_bits (12)
19987 #define brick_bits (11)
19988 #endif //_TARGET_AMD64_
19989 C_ASSERT(brick_size == (1 << brick_bits));
19991 // The number of bits needed to represent the offset to a child node.
19992 // "brick_bits + 1" allows us to represent a signed offset within a brick.
19993 #define child_bits (brick_bits + 1 - LOG2_PTRSIZE)
19995 // The number of bits in each of the pad hi, pad lo fields.
19996 #define pad_bits (sizeof(short) * 8 - child_bits)
19998 #define child_from_short(w) (((signed short)(w) / (1 << (pad_bits - LOG2_PTRSIZE))) & ~((1 << LOG2_PTRSIZE) - 1))
19999 #define pad_mask ((1 << pad_bits) - 1)
20000 #define pad_from_short(w) ((size_t)(w) & pad_mask)
20001 #else // FEATURE_STRUCTALIGN
20002 #define child_from_short(w) (w)
20003 #endif // FEATURE_STRUCTALIGN
20006 short node_left_child(uint8_t* node)
20008 return child_from_short(((plug_and_pair*)node)[-1].m_pair.left);
20012 void set_node_left_child(uint8_t* node, ptrdiff_t val)
20014 assert (val > -(ptrdiff_t)brick_size);
20015 assert (val < (ptrdiff_t)brick_size);
20016 assert (Aligned (val));
20017 #ifdef FEATURE_STRUCTALIGN
20018 size_t pad = pad_from_short(((plug_and_pair*)node)[-1].m_pair.left);
20019 ((plug_and_pair*)node)[-1].m_pair.left = ((short)val << (pad_bits - LOG2_PTRSIZE)) | (short)pad;
20020 #else // FEATURE_STRUCTALIGN
20021 ((plug_and_pair*)node)[-1].m_pair.left = (short)val;
20022 #endif // FEATURE_STRUCTALIGN
20023 assert (node_left_child (node) == val);
20027 short node_right_child(uint8_t* node)
20029 return child_from_short(((plug_and_pair*)node)[-1].m_pair.right);
20033 void set_node_right_child(uint8_t* node, ptrdiff_t val)
20035 assert (val > -(ptrdiff_t)brick_size);
20036 assert (val < (ptrdiff_t)brick_size);
20037 assert (Aligned (val));
20038 #ifdef FEATURE_STRUCTALIGN
20039 size_t pad = pad_from_short(((plug_and_pair*)node)[-1].m_pair.right);
20040 ((plug_and_pair*)node)[-1].m_pair.right = ((short)val << (pad_bits - LOG2_PTRSIZE)) | (short)pad;
20041 #else // FEATURE_STRUCTALIGN
20042 ((plug_and_pair*)node)[-1].m_pair.right = (short)val;
20043 #endif // FEATURE_STRUCTALIGN
20044 assert (node_right_child (node) == val);
20047 #ifdef FEATURE_STRUCTALIGN
20048 void node_aligninfo (uint8_t* node, int& requiredAlignment, ptrdiff_t& pad)
20050 // Extract the single-number aligninfo from the fields.
20051 short left = ((plug_and_pair*)node)[-1].m_pair.left;
20052 short right = ((plug_and_pair*)node)[-1].m_pair.right;
20053 ptrdiff_t pad_shifted = (pad_from_short(left) << pad_bits) | pad_from_short(right);
20054 ptrdiff_t aligninfo = pad_shifted * DATA_ALIGNMENT;
20056 // Replicate the topmost bit into all lower bits.
20057 ptrdiff_t x = aligninfo;
20063 // Clear all bits but the highest.
20064 requiredAlignment = (int)(x ^ (x >> 1));
20065 pad = aligninfo - requiredAlignment;
20066 pad += AdjustmentForMinPadSize(pad, requiredAlignment);
20070 ptrdiff_t node_alignpad (uint8_t* node)
20072 int requiredAlignment;
20073 ptrdiff_t alignpad;
20074 node_aligninfo (node, requiredAlignment, alignpad);
20078 void clear_node_aligninfo (uint8_t* node)
20080 ((plug_and_pair*)node)[-1].m_pair.left &= ~0 << pad_bits;
20081 ((plug_and_pair*)node)[-1].m_pair.right &= ~0 << pad_bits;
20084 void set_node_aligninfo (uint8_t* node, int requiredAlignment, ptrdiff_t pad)
20086 // Encode the alignment requirement and alignment offset as a single number
20087 // as described above.
20088 ptrdiff_t aligninfo = (size_t)requiredAlignment + (pad & (requiredAlignment-1));
20089 assert (Aligned (aligninfo));
20090 ptrdiff_t aligninfo_shifted = aligninfo / DATA_ALIGNMENT;
20091 assert (aligninfo_shifted < (1 << (pad_bits + pad_bits)));
20093 ptrdiff_t hi = aligninfo_shifted >> pad_bits;
20094 assert (pad_from_short(((plug_and_gap*)node)[-1].m_pair.left) == 0);
20095 ((plug_and_pair*)node)[-1].m_pair.left |= hi;
20097 ptrdiff_t lo = aligninfo_shifted & pad_mask;
20098 assert (pad_from_short(((plug_and_gap*)node)[-1].m_pair.right) == 0);
20099 ((plug_and_pair*)node)[-1].m_pair.right |= lo;
20102 int requiredAlignment2;
20104 node_aligninfo (node, requiredAlignment2, pad2);
20105 assert (requiredAlignment == requiredAlignment2);
20106 assert (pad == pad2);
20109 #endif // FEATURE_STRUCTALIGN
20112 void loh_set_node_relocation_distance(uint8_t* node, ptrdiff_t val)
20114 ptrdiff_t* place = &(((loh_obj_and_pad*)node)[-1].reloc);
20119 ptrdiff_t loh_node_relocation_distance(uint8_t* node)
20121 return (((loh_obj_and_pad*)node)[-1].reloc);
20125 ptrdiff_t node_relocation_distance (uint8_t* node)
20127 return (((plug_and_reloc*)(node))[-1].reloc & ~3);
20131 void set_node_relocation_distance(uint8_t* node, ptrdiff_t val)
20133 assert (val == (val & ~3));
20134 ptrdiff_t* place = &(((plug_and_reloc*)node)[-1].reloc);
20135 //clear the left bit and the relocation field
20141 #define node_left_p(node) (((plug_and_reloc*)(node))[-1].reloc & 2)
20143 #define set_node_left(node) ((plug_and_reloc*)(node))[-1].reloc |= 2;
20145 #ifndef FEATURE_STRUCTALIGN
20146 void set_node_realigned(uint8_t* node)
20148 ((plug_and_reloc*)(node))[-1].reloc |= 1;
20151 void clear_node_realigned(uint8_t* node)
20153 #ifdef RESPECT_LARGE_ALIGNMENT
20154 ((plug_and_reloc*)(node))[-1].reloc &= ~1;
20156 UNREFERENCED_PARAMETER(node);
20157 #endif //RESPECT_LARGE_ALIGNMENT
20159 #endif // FEATURE_STRUCTALIGN
20162 size_t node_gap_size (uint8_t* node)
20164 return ((plug_and_gap *)node)[-1].gap;
20167 void set_gap_size (uint8_t* node, size_t size)
20169 assert (Aligned (size));
20171 // clear the 2 uint32_t used by the node.
20172 ((plug_and_gap *)node)[-1].reloc = 0;
20173 ((plug_and_gap *)node)[-1].lr =0;
20174 ((plug_and_gap *)node)[-1].gap = size;
20176 assert ((size == 0 )||(size >= sizeof(plug_and_reloc)));
20180 uint8_t* gc_heap::insert_node (uint8_t* new_node, size_t sequence_number,
20181 uint8_t* tree, uint8_t* last_node)
20183 dprintf (3, ("IN: %Ix(%Ix), T: %Ix(%Ix), L: %Ix(%Ix) [%Ix]",
20184 (size_t)new_node, brick_of(new_node),
20185 (size_t)tree, brick_of(tree),
20186 (size_t)last_node, brick_of(last_node),
20188 if (power_of_two_p (sequence_number))
20190 set_node_left_child (new_node, (tree - new_node));
20191 dprintf (3, ("NT: %Ix, LC->%Ix", (size_t)new_node, (tree - new_node)));
20196 if (oddp (sequence_number))
20198 set_node_right_child (last_node, (new_node - last_node));
20199 dprintf (3, ("%Ix RC->%Ix", last_node, (new_node - last_node)));
20203 uint8_t* earlier_node = tree;
20204 size_t imax = logcount(sequence_number) - 2;
20205 for (size_t i = 0; i != imax; i++)
20207 earlier_node = earlier_node + node_right_child (earlier_node);
20209 int tmp_offset = node_right_child (earlier_node);
20210 assert (tmp_offset); // should never be empty
20211 set_node_left_child (new_node, ((earlier_node + tmp_offset ) - new_node));
20212 set_node_right_child (earlier_node, (new_node - earlier_node));
20214 dprintf (3, ("%Ix LC->%Ix, %Ix RC->%Ix",
20215 new_node, ((earlier_node + tmp_offset ) - new_node),
20216 earlier_node, (new_node - earlier_node)));
20222 size_t gc_heap::update_brick_table (uint8_t* tree, size_t current_brick,
20223 uint8_t* x, uint8_t* plug_end)
20225 dprintf (3, ("tree: %Ix, current b: %Ix, x: %Ix, plug_end: %Ix",
20226 tree, current_brick, x, plug_end));
20230 dprintf (3, ("b- %Ix->%Ix pointing to tree %Ix",
20231 current_brick, (size_t)(tree - brick_address (current_brick)), tree));
20232 set_brick (current_brick, (tree - brick_address (current_brick)));
20236 dprintf (3, ("b- %Ix->-1", current_brick));
20237 set_brick (current_brick, -1);
20239 size_t b = 1 + current_brick;
20240 ptrdiff_t offset = 0;
20241 size_t last_br = brick_of (plug_end-1);
20242 current_brick = brick_of (x-1);
20243 dprintf (3, ("ubt: %Ix->%Ix]->%Ix]", b, last_br, current_brick));
20244 while (b <= current_brick)
20248 set_brick (b, --offset);
20256 return brick_of (x);
20259 void gc_heap::plan_generation_start (generation* gen, generation* consing_gen, uint8_t* next_plug_to_allocate)
20262 // We should never demote big plugs to gen0.
20263 if (gen == youngest_generation)
20265 heap_segment* seg = ephemeral_heap_segment;
20266 size_t mark_stack_large_bos = mark_stack_bos;
20267 size_t large_plug_pos = 0;
20268 while (mark_stack_large_bos < mark_stack_tos)
20270 if (mark_stack_array[mark_stack_large_bos].len > demotion_plug_len_th)
20272 while (mark_stack_bos <= mark_stack_large_bos)
20274 size_t entry = deque_pinned_plug();
20275 size_t len = pinned_len (pinned_plug_of (entry));
20276 uint8_t* plug = pinned_plug (pinned_plug_of(entry));
20277 if (len > demotion_plug_len_th)
20279 dprintf (2, ("ps(%d): S %Ix (%Id)(%Ix)", gen->gen_num, plug, len, (plug+len)));
20281 pinned_len (pinned_plug_of (entry)) = plug - generation_allocation_pointer (consing_gen);
20282 assert(mark_stack_array[entry].len == 0 ||
20283 mark_stack_array[entry].len >= Align(min_obj_size));
20284 generation_allocation_pointer (consing_gen) = plug + len;
20285 generation_allocation_limit (consing_gen) = heap_segment_plan_allocated (seg);
20286 set_allocator_next_pin (consing_gen);
20290 mark_stack_large_bos++;
20295 generation_plan_allocation_start (gen) =
20296 allocate_in_condemned_generations (consing_gen, Align (min_obj_size), -1);
20297 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
20298 size_t allocation_left = (size_t)(generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
20299 if (next_plug_to_allocate)
20301 size_t dist_to_next_plug = (size_t)(next_plug_to_allocate - generation_allocation_pointer (consing_gen));
20302 if (allocation_left > dist_to_next_plug)
20304 allocation_left = dist_to_next_plug;
20307 if (allocation_left < Align (min_obj_size))
20309 generation_plan_allocation_start_size (gen) += allocation_left;
20310 generation_allocation_pointer (consing_gen) += allocation_left;
20313 dprintf (1, ("plan alloc gen%d(%Ix) start at %Ix (ptr: %Ix, limit: %Ix, next: %Ix)", gen->gen_num,
20314 generation_plan_allocation_start (gen),
20315 generation_plan_allocation_start_size (gen),
20316 generation_allocation_pointer (consing_gen), generation_allocation_limit (consing_gen),
20317 next_plug_to_allocate));
20320 void gc_heap::realloc_plan_generation_start (generation* gen, generation* consing_gen)
20322 BOOL adjacentp = FALSE;
20324 generation_plan_allocation_start (gen) =
20325 allocate_in_expanded_heap (consing_gen, Align(min_obj_size), adjacentp, 0,
20328 #endif //SHORT_PLUGS
20329 FALSE, -1 REQD_ALIGN_AND_OFFSET_ARG);
20331 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
20332 size_t allocation_left = (size_t)(generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
20333 if ((allocation_left < Align (min_obj_size)) &&
20334 (generation_allocation_limit (consing_gen)!=heap_segment_plan_allocated (generation_allocation_segment (consing_gen))))
20336 generation_plan_allocation_start_size (gen) += allocation_left;
20337 generation_allocation_pointer (consing_gen) += allocation_left;
20340 dprintf (1, ("plan re-alloc gen%d start at %Ix (ptr: %Ix, limit: %Ix)", gen->gen_num,
20341 generation_plan_allocation_start (consing_gen),
20342 generation_allocation_pointer (consing_gen),
20343 generation_allocation_limit (consing_gen)));
20346 void gc_heap::plan_generation_starts (generation*& consing_gen)
20348 //make sure that every generation has a planned allocation start
20349 int gen_number = settings.condemned_generation;
20350 while (gen_number >= 0)
20352 if (gen_number < max_generation)
20354 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
20356 generation* gen = generation_of (gen_number);
20357 if (0 == generation_plan_allocation_start (gen))
20359 plan_generation_start (gen, consing_gen, 0);
20360 assert (generation_plan_allocation_start (gen));
20364 // now we know the planned allocation size
20365 heap_segment_plan_allocated (ephemeral_heap_segment) =
20366 generation_allocation_pointer (consing_gen);
20369 void gc_heap::advance_pins_for_demotion (generation* gen)
20371 uint8_t* original_youngest_start = generation_allocation_start (youngest_generation);
20372 heap_segment* seg = ephemeral_heap_segment;
20374 if ((!(pinned_plug_que_empty_p())))
20376 size_t gen1_pinned_promoted = generation_pinned_allocation_compact_size (generation_of (max_generation));
20377 size_t gen1_pins_left = dd_pinned_survived_size (dynamic_data_of (max_generation - 1)) - gen1_pinned_promoted;
20378 size_t total_space_to_skip = last_gen1_pin_end - generation_allocation_pointer (gen);
20379 float pin_frag_ratio = (float)gen1_pins_left / (float)total_space_to_skip;
20380 float pin_surv_ratio = (float)gen1_pins_left / (float)(dd_survived_size (dynamic_data_of (max_generation - 1)));
20381 if ((pin_frag_ratio > 0.15) && (pin_surv_ratio > 0.30))
20383 while (!pinned_plug_que_empty_p() &&
20384 (pinned_plug (oldest_pin()) < original_youngest_start))
20386 size_t entry = deque_pinned_plug();
20387 size_t len = pinned_len (pinned_plug_of (entry));
20388 uint8_t* plug = pinned_plug (pinned_plug_of(entry));
20389 pinned_len (pinned_plug_of (entry)) = plug - generation_allocation_pointer (gen);
20390 assert(mark_stack_array[entry].len == 0 ||
20391 mark_stack_array[entry].len >= Align(min_obj_size));
20392 generation_allocation_pointer (gen) = plug + len;
20393 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20394 set_allocator_next_pin (gen);
20396 //Add the size of the pinned plug to the right pinned allocations
20397 //find out which gen this pinned plug came from
20398 int frgn = object_gennum (plug);
20399 if ((frgn != (int)max_generation) && settings.promotion)
20401 int togn = object_gennum_plan (plug);
20402 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
20405 generation_pinned_allocation_compact_size (generation_of (togn)) += len;
20409 dprintf (2, ("skipping gap %d, pin %Ix (%Id)",
20410 pinned_len (pinned_plug_of (entry)), plug, len));
20413 dprintf (2, ("ad_p_d: PL: %Id, SL: %Id, pfr: %d, psr: %d",
20414 gen1_pins_left, total_space_to_skip, (int)(pin_frag_ratio*100), (int)(pin_surv_ratio*100)));
20418 void gc_heap::process_ephemeral_boundaries (uint8_t* x,
20419 int& active_new_gen_number,
20420 int& active_old_gen_number,
20421 generation*& consing_gen,
20422 BOOL& allocate_in_condemned)
20425 if ((active_old_gen_number > 0) &&
20426 (x >= generation_allocation_start (generation_of (active_old_gen_number - 1))))
20428 dprintf (1, ("crossing gen%d, x is %Ix", active_old_gen_number - 1, x));
20430 if (!pinned_plug_que_empty_p())
20432 dprintf (1, ("oldest pin: %Ix(%Id)",
20433 pinned_plug (oldest_pin()),
20434 (x - pinned_plug (oldest_pin()))));
20437 if (active_old_gen_number <= (settings.promotion ? (max_generation - 1) : max_generation))
20439 active_new_gen_number--;
20442 active_old_gen_number--;
20443 assert ((!settings.promotion) || (active_new_gen_number>0));
20445 if (active_new_gen_number == (max_generation - 1))
20447 #ifdef FREE_USAGE_STATS
20448 if (settings.condemned_generation == max_generation)
20450 // We need to do this before we skip the rest of the pinned plugs.
20451 generation* gen_2 = generation_of (max_generation);
20452 generation* gen_1 = generation_of (max_generation - 1);
20454 size_t total_num_pinned_free_spaces_left = 0;
20456 // We are about to allocate gen1, check to see how efficient fitting in gen2 pinned free spaces is.
20457 for (int j = 0; j < NUM_GEN_POWER2; j++)
20459 dprintf (1, ("[h%d][#%Id]2^%d: current: %Id, S: 2: %Id, 1: %Id(%Id)",
20463 gen_2->gen_current_pinned_free_spaces[j],
20464 gen_2->gen_plugs[j], gen_1->gen_plugs[j],
20465 (gen_2->gen_plugs[j] + gen_1->gen_plugs[j])));
20467 total_num_pinned_free_spaces_left += gen_2->gen_current_pinned_free_spaces[j];
20470 float pinned_free_list_efficiency = 0;
20471 size_t total_pinned_free_space = generation_allocated_in_pinned_free (gen_2) + generation_pinned_free_obj_space (gen_2);
20472 if (total_pinned_free_space != 0)
20474 pinned_free_list_efficiency = (float)(generation_allocated_in_pinned_free (gen_2)) / (float)total_pinned_free_space;
20477 dprintf (1, ("[h%d] gen2 allocated %Id bytes with %Id bytes pinned free spaces (effi: %d%%), %Id (%Id) left",
20479 generation_allocated_in_pinned_free (gen_2),
20480 total_pinned_free_space,
20481 (int)(pinned_free_list_efficiency * 100),
20482 generation_pinned_free_obj_space (gen_2),
20483 total_num_pinned_free_spaces_left));
20485 #endif //FREE_USAGE_STATS
20487 //Go past all of the pinned plugs for this generation.
20488 while (!pinned_plug_que_empty_p() &&
20489 (!in_range_for_segment ((pinned_plug (oldest_pin())), ephemeral_heap_segment)))
20491 size_t entry = deque_pinned_plug();
20492 mark* m = pinned_plug_of (entry);
20493 uint8_t* plug = pinned_plug (m);
20494 size_t len = pinned_len (m);
20495 // detect pinned block in different segment (later) than
20496 // allocation segment, skip those until the oldest pin is in the ephemeral seg.
20497 // adjust the allocation segment along the way (at the end it will
20498 // be the ephemeral segment.
20499 heap_segment* nseg = heap_segment_in_range (generation_allocation_segment (consing_gen));
20501 PREFIX_ASSUME(nseg != NULL);
20503 while (!((plug >= generation_allocation_pointer (consing_gen))&&
20504 (plug < heap_segment_allocated (nseg))))
20506 //adjust the end of the segment to be the end of the plug
20507 assert (generation_allocation_pointer (consing_gen)>=
20508 heap_segment_mem (nseg));
20509 assert (generation_allocation_pointer (consing_gen)<=
20510 heap_segment_committed (nseg));
20512 heap_segment_plan_allocated (nseg) =
20513 generation_allocation_pointer (consing_gen);
20514 //switch allocation segment
20515 nseg = heap_segment_next_rw (nseg);
20516 generation_allocation_segment (consing_gen) = nseg;
20517 //reset the allocation pointer and limits
20518 generation_allocation_pointer (consing_gen) =
20519 heap_segment_mem (nseg);
20521 set_new_pin_info (m, generation_allocation_pointer (consing_gen));
20522 assert(pinned_len(m) == 0 || pinned_len(m) >= Align(min_obj_size));
20523 generation_allocation_pointer (consing_gen) = plug + len;
20524 generation_allocation_limit (consing_gen) =
20525 generation_allocation_pointer (consing_gen);
20527 allocate_in_condemned = TRUE;
20528 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
20531 if (active_new_gen_number != max_generation)
20533 if (active_new_gen_number == (max_generation - 1))
20535 maxgen_pinned_compact_before_advance = generation_pinned_allocation_compact_size (generation_of (max_generation));
20536 if (!demote_gen1_p)
20537 advance_pins_for_demotion (consing_gen);
20540 plan_generation_start (generation_of (active_new_gen_number), consing_gen, x);
20542 dprintf (1, ("process eph: allocated gen%d start at %Ix",
20543 active_new_gen_number,
20544 generation_plan_allocation_start (generation_of (active_new_gen_number))));
20546 if ((demotion_low == MAX_PTR) && !pinned_plug_que_empty_p())
20548 uint8_t* pplug = pinned_plug (oldest_pin());
20549 if (object_gennum (pplug) > 0)
20551 demotion_low = pplug;
20552 dprintf (3, ("process eph: dlow->%Ix", demotion_low));
20556 assert (generation_plan_allocation_start (generation_of (active_new_gen_number)));
20564 void gc_heap::seg_clear_mark_bits (heap_segment* seg)
20566 uint8_t* o = heap_segment_mem (seg);
20567 while (o < heap_segment_allocated (seg))
20573 o = o + Align (size (o));
20577 #ifdef FEATURE_BASICFREEZE
20578 void gc_heap::sweep_ro_segments (heap_segment* start_seg)
20580 //go through all of the segment in range and reset the mark bit
20581 //TODO works only on small object segments
20583 heap_segment* seg = start_seg;
20587 if (heap_segment_read_only_p (seg) &&
20588 heap_segment_in_range_p (seg))
20590 #ifdef BACKGROUND_GC
20591 if (settings.concurrent)
20593 seg_clear_mark_array_bits_soh (seg);
20597 seg_clear_mark_bits (seg);
20599 #else //BACKGROUND_GC
20602 if(gc_can_use_concurrent)
20604 clear_mark_array (max (heap_segment_mem (seg), lowest_address),
20605 min (heap_segment_allocated (seg), highest_address),
20606 FALSE); // read_only segments need the mark clear
20609 seg_clear_mark_bits (seg);
20610 #endif //MARK_ARRAY
20612 #endif //BACKGROUND_GC
20614 seg = heap_segment_next (seg);
20617 #endif // FEATURE_BASICFREEZE
20619 #ifdef FEATURE_LOH_COMPACTION
20621 BOOL gc_heap::loh_pinned_plug_que_empty_p()
20623 return (loh_pinned_queue_bos == loh_pinned_queue_tos);
20626 void gc_heap::loh_set_allocator_next_pin()
20628 if (!(loh_pinned_plug_que_empty_p()))
20630 mark* oldest_entry = loh_oldest_pin();
20631 uint8_t* plug = pinned_plug (oldest_entry);
20632 generation* gen = large_object_generation;
20633 if ((plug >= generation_allocation_pointer (gen)) &&
20634 (plug < generation_allocation_limit (gen)))
20636 generation_allocation_limit (gen) = pinned_plug (oldest_entry);
20639 assert (!((plug < generation_allocation_pointer (gen)) &&
20640 (plug >= heap_segment_mem (generation_allocation_segment (gen)))));
20644 size_t gc_heap::loh_deque_pinned_plug ()
20646 size_t m = loh_pinned_queue_bos;
20647 loh_pinned_queue_bos++;
20652 mark* gc_heap::loh_pinned_plug_of (size_t bos)
20654 return &loh_pinned_queue[bos];
20658 mark* gc_heap::loh_oldest_pin()
20660 return loh_pinned_plug_of (loh_pinned_queue_bos);
20663 // If we can't grow the queue, then don't compact.
20664 BOOL gc_heap::loh_enque_pinned_plug (uint8_t* plug, size_t len)
20666 assert(len >= Align(min_obj_size, get_alignment_constant (FALSE)));
20668 if (loh_pinned_queue_length <= loh_pinned_queue_tos)
20670 if (!grow_mark_stack (loh_pinned_queue, loh_pinned_queue_length, LOH_PIN_QUEUE_LENGTH))
20675 dprintf (3, (" P: %Ix(%Id)", plug, len));
20676 mark& m = loh_pinned_queue[loh_pinned_queue_tos];
20679 loh_pinned_queue_tos++;
20680 loh_set_allocator_next_pin();
20685 BOOL gc_heap::loh_size_fit_p (size_t size, uint8_t* alloc_pointer, uint8_t* alloc_limit)
20687 dprintf (1235, ("trying to fit %Id(%Id) between %Ix and %Ix (%Id)",
20689 (2* AlignQword (loh_padding_obj_size) + size),
20692 (alloc_limit - alloc_pointer)));
20694 return ((alloc_pointer + 2* AlignQword (loh_padding_obj_size) + size) <= alloc_limit);
20697 uint8_t* gc_heap::loh_allocate_in_condemned (uint8_t* old_loc, size_t size)
20699 UNREFERENCED_PARAMETER(old_loc);
20701 generation* gen = large_object_generation;
20702 dprintf (1235, ("E: p:%Ix, l:%Ix, s: %Id",
20703 generation_allocation_pointer (gen),
20704 generation_allocation_limit (gen),
20709 heap_segment* seg = generation_allocation_segment (gen);
20710 if (!(loh_size_fit_p (size, generation_allocation_pointer (gen), generation_allocation_limit (gen))))
20712 if ((!(loh_pinned_plug_que_empty_p()) &&
20713 (generation_allocation_limit (gen) ==
20714 pinned_plug (loh_oldest_pin()))))
20716 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
20717 size_t len = pinned_len (m);
20718 uint8_t* plug = pinned_plug (m);
20719 dprintf (1235, ("AIC: %Ix->%Ix(%Id)", generation_allocation_pointer (gen), plug, plug - generation_allocation_pointer (gen)));
20720 pinned_len (m) = plug - generation_allocation_pointer (gen);
20721 generation_allocation_pointer (gen) = plug + len;
20723 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20724 loh_set_allocator_next_pin();
20725 dprintf (1235, ("s: p: %Ix, l: %Ix (%Id)",
20726 generation_allocation_pointer (gen),
20727 generation_allocation_limit (gen),
20728 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20733 if (generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))
20735 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20736 dprintf (1235, ("l->pa(%Ix)", generation_allocation_limit (gen)));
20740 if (heap_segment_plan_allocated (seg) != heap_segment_committed (seg))
20742 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
20743 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20744 dprintf (1235, ("l->c(%Ix)", generation_allocation_limit (gen)));
20748 if (loh_size_fit_p (size, generation_allocation_pointer (gen), heap_segment_reserved (seg)) &&
20749 (grow_heap_segment (seg, (generation_allocation_pointer (gen) + size + 2* AlignQword (loh_padding_obj_size)))))
20751 dprintf (1235, ("growing seg from %Ix to %Ix\n", heap_segment_committed (seg),
20752 (generation_allocation_pointer (gen) + size)));
20754 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
20755 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20757 dprintf (1235, ("g: p: %Ix, l: %Ix (%Id)",
20758 generation_allocation_pointer (gen),
20759 generation_allocation_limit (gen),
20760 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20764 heap_segment* next_seg = heap_segment_next (seg);
20765 assert (generation_allocation_pointer (gen)>=
20766 heap_segment_mem (seg));
20767 // Verify that all pinned plugs for this segment are consumed
20768 if (!loh_pinned_plug_que_empty_p() &&
20769 ((pinned_plug (loh_oldest_pin()) <
20770 heap_segment_allocated (seg)) &&
20771 (pinned_plug (loh_oldest_pin()) >=
20772 generation_allocation_pointer (gen))))
20774 LOG((LF_GC, LL_INFO10, "remaining pinned plug %Ix while leaving segment on allocation",
20775 pinned_plug (loh_oldest_pin())));
20776 dprintf (1236, ("queue empty: %d", loh_pinned_plug_que_empty_p()));
20779 assert (generation_allocation_pointer (gen)>=
20780 heap_segment_mem (seg));
20781 assert (generation_allocation_pointer (gen)<=
20782 heap_segment_committed (seg));
20783 heap_segment_plan_allocated (seg) = generation_allocation_pointer (gen);
20787 // for LOH do we want to try starting from the first LOH every time though?
20788 generation_allocation_segment (gen) = next_seg;
20789 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
20790 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
20792 dprintf (1235, ("n: p: %Ix, l: %Ix (%Id)",
20793 generation_allocation_pointer (gen),
20794 generation_allocation_limit (gen),
20795 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20799 dprintf (1, ("We ran out of space compacting, shouldn't happen"));
20805 loh_set_allocator_next_pin();
20807 dprintf (1235, ("r: p: %Ix, l: %Ix (%Id)",
20808 generation_allocation_pointer (gen),
20809 generation_allocation_limit (gen),
20810 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20817 assert (generation_allocation_pointer (gen)>=
20818 heap_segment_mem (generation_allocation_segment (gen)));
20819 uint8_t* result = generation_allocation_pointer (gen);
20820 size_t loh_pad = AlignQword (loh_padding_obj_size);
20822 generation_allocation_pointer (gen) += size + loh_pad;
20823 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
20825 dprintf (1235, ("p: %Ix, l: %Ix (%Id)",
20826 generation_allocation_pointer (gen),
20827 generation_allocation_limit (gen),
20828 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20830 assert (result + loh_pad);
20831 return result + loh_pad;
20835 BOOL gc_heap::should_compact_loh()
20837 return (loh_compaction_always_p || (loh_compaction_mode != loh_compaction_default));
20841 void gc_heap::check_loh_compact_mode (BOOL all_heaps_compacted_p)
20843 if (settings.loh_compaction && (loh_compaction_mode == loh_compaction_once))
20845 if (all_heaps_compacted_p)
20847 // If the compaction mode says to compact once and we are going to compact LOH,
20848 // we need to revert it back to no compaction.
20849 loh_compaction_mode = loh_compaction_default;
20854 BOOL gc_heap::plan_loh()
20856 if (!loh_pinned_queue)
20858 loh_pinned_queue = new (nothrow) (mark [LOH_PIN_QUEUE_LENGTH]);
20859 if (!loh_pinned_queue)
20861 dprintf (1, ("Cannot allocate the LOH pinned queue (%Id bytes), no compaction",
20862 LOH_PIN_QUEUE_LENGTH * sizeof (mark)));
20866 loh_pinned_queue_length = LOH_PIN_QUEUE_LENGTH;
20869 if (heap_number == 0)
20870 loh_pinned_queue_decay = LOH_PIN_DECAY;
20872 loh_pinned_queue_tos = 0;
20873 loh_pinned_queue_bos = 0;
20875 generation* gen = large_object_generation;
20876 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
20877 PREFIX_ASSUME(start_seg != NULL);
20878 heap_segment* seg = start_seg;
20879 uint8_t* o = generation_allocation_start (gen);
20881 dprintf (1235, ("before GC LOH size: %Id, free list: %Id, free obj: %Id\n",
20882 generation_size (max_generation + 1),
20883 generation_free_list_space (gen),
20884 generation_free_obj_space (gen)));
20888 heap_segment_plan_allocated (seg) = heap_segment_mem (seg);
20889 seg = heap_segment_next (seg);
20894 //Skip the generation gap object
20895 o = o + AlignQword (size (o));
20896 // We don't need to ever realloc gen3 start so don't touch it.
20897 heap_segment_plan_allocated (seg) = o;
20898 generation_allocation_pointer (gen) = o;
20899 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
20900 generation_allocation_segment (gen) = start_seg;
20902 uint8_t* free_space_start = o;
20903 uint8_t* free_space_end = o;
20904 uint8_t* new_address = 0;
20908 if (o >= heap_segment_allocated (seg))
20910 seg = heap_segment_next (seg);
20916 o = heap_segment_mem (seg);
20921 free_space_end = o;
20922 size_t size = AlignQword (size (o));
20923 dprintf (1235, ("%Ix(%Id) M", o, size));
20927 // We don't clear the pinned bit yet so we can check in
20928 // compact phase how big a free object we should allocate
20929 // in front of the pinned object. We use the reloc address
20930 // field to store this.
20931 if (!loh_enque_pinned_plug (o, size))
20939 new_address = loh_allocate_in_condemned (o, size);
20942 loh_set_node_relocation_distance (o, (new_address - o));
20943 dprintf (1235, ("lobj %Ix-%Ix -> %Ix-%Ix (%Id)", o, (o + size), new_address, (new_address + size), (new_address - o)));
20946 free_space_start = o;
20947 if (o < heap_segment_allocated (seg))
20949 assert (!marked (o));
20954 while (o < heap_segment_allocated (seg) && !marked (o))
20956 dprintf (1235, ("%Ix(%Id) F (%d)", o, AlignQword (size (o)), ((method_table (o) == g_gc_pFreeObjectMethodTable) ? 1 : 0)));
20957 o = o + AlignQword (size (o));
20962 while (!loh_pinned_plug_que_empty_p())
20964 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
20965 size_t len = pinned_len (m);
20966 uint8_t* plug = pinned_plug (m);
20968 // detect pinned block in different segment (later) than
20969 // allocation segment
20970 heap_segment* nseg = heap_segment_rw (generation_allocation_segment (gen));
20972 while ((plug < generation_allocation_pointer (gen)) ||
20973 (plug >= heap_segment_allocated (nseg)))
20975 assert ((plug < heap_segment_mem (nseg)) ||
20976 (plug > heap_segment_reserved (nseg)));
20977 //adjust the end of the segment to be the end of the plug
20978 assert (generation_allocation_pointer (gen)>=
20979 heap_segment_mem (nseg));
20980 assert (generation_allocation_pointer (gen)<=
20981 heap_segment_committed (nseg));
20983 heap_segment_plan_allocated (nseg) =
20984 generation_allocation_pointer (gen);
20985 //switch allocation segment
20986 nseg = heap_segment_next_rw (nseg);
20987 generation_allocation_segment (gen) = nseg;
20988 //reset the allocation pointer and limits
20989 generation_allocation_pointer (gen) =
20990 heap_segment_mem (nseg);
20993 dprintf (1235, ("SP: %Ix->%Ix(%Id)", generation_allocation_pointer (gen), plug, plug - generation_allocation_pointer (gen)));
20994 pinned_len (m) = plug - generation_allocation_pointer (gen);
20995 generation_allocation_pointer (gen) = plug + len;
20998 heap_segment_plan_allocated (generation_allocation_segment (gen)) = generation_allocation_pointer (gen);
20999 generation_allocation_pointer (gen) = 0;
21000 generation_allocation_limit (gen) = 0;
21005 void gc_heap::compact_loh()
21007 assert (should_compact_loh());
21009 generation* gen = large_object_generation;
21010 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
21011 PREFIX_ASSUME(start_seg != NULL);
21012 heap_segment* seg = start_seg;
21013 heap_segment* prev_seg = 0;
21014 uint8_t* o = generation_allocation_start (gen);
21016 //Skip the generation gap object
21017 o = o + AlignQword (size (o));
21018 // We don't need to ever realloc gen3 start so don't touch it.
21019 uint8_t* free_space_start = o;
21020 uint8_t* free_space_end = o;
21021 generation_allocator (gen)->clear();
21022 generation_free_list_space (gen) = 0;
21023 generation_free_obj_space (gen) = 0;
21025 loh_pinned_queue_bos = 0;
21029 if (o >= heap_segment_allocated (seg))
21031 heap_segment* next_seg = heap_segment_next (seg);
21033 if ((heap_segment_plan_allocated (seg) == heap_segment_mem (seg)) &&
21034 (seg != start_seg) && !heap_segment_read_only_p (seg))
21036 dprintf (3, ("Preparing empty large segment %Ix", (size_t)seg));
21038 heap_segment_next (prev_seg) = next_seg;
21039 heap_segment_next (seg) = freeable_large_heap_segment;
21040 freeable_large_heap_segment = seg;
21044 if (!heap_segment_read_only_p (seg))
21046 // We grew the segment to accommodate allocations.
21047 if (heap_segment_plan_allocated (seg) > heap_segment_allocated (seg))
21049 if ((heap_segment_plan_allocated (seg) - plug_skew) > heap_segment_used (seg))
21051 heap_segment_used (seg) = heap_segment_plan_allocated (seg) - plug_skew;
21055 heap_segment_allocated (seg) = heap_segment_plan_allocated (seg);
21056 dprintf (3, ("Trimming seg to %Ix[", heap_segment_allocated (seg)));
21057 decommit_heap_segment_pages (seg, 0);
21058 dprintf (1236, ("CLOH: seg: %Ix, alloc: %Ix, used: %Ix, committed: %Ix",
21060 heap_segment_allocated (seg),
21061 heap_segment_used (seg),
21062 heap_segment_committed (seg)));
21063 //heap_segment_used (seg) = heap_segment_allocated (seg) - plug_skew;
21064 dprintf (1236, ("CLOH: used is set to %Ix", heap_segment_used (seg)));
21074 o = heap_segment_mem (seg);
21080 free_space_end = o;
21081 size_t size = AlignQword (size (o));
21084 uint8_t* reloc = o;
21089 // We are relying on the fact the pinned objects are always looked at in the same order
21090 // in plan phase and in compact phase.
21091 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
21092 uint8_t* plug = pinned_plug (m);
21093 assert (plug == o);
21095 loh_pad = pinned_len (m);
21100 loh_pad = AlignQword (loh_padding_obj_size);
21102 reloc += loh_node_relocation_distance (o);
21103 gcmemcopy (reloc, o, size, TRUE);
21106 thread_gap ((reloc - loh_pad), loh_pad, gen);
21109 free_space_start = o;
21110 if (o < heap_segment_allocated (seg))
21112 assert (!marked (o));
21117 while (o < heap_segment_allocated (seg) && !marked (o))
21119 o = o + AlignQword (size (o));
21124 assert (loh_pinned_plug_que_empty_p());
21126 dprintf (1235, ("after GC LOH size: %Id, free list: %Id, free obj: %Id\n\n",
21127 generation_size (max_generation + 1),
21128 generation_free_list_space (gen),
21129 generation_free_obj_space (gen)));
21132 void gc_heap::relocate_in_loh_compact()
21134 generation* gen = large_object_generation;
21135 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
21136 uint8_t* o = generation_allocation_start (gen);
21138 //Skip the generation gap object
21139 o = o + AlignQword (size (o));
21141 relocate_args args;
21143 args.high = gc_high;
21144 args.last_plug = 0;
21148 if (o >= heap_segment_allocated (seg))
21150 seg = heap_segment_next (seg);
21156 o = heap_segment_mem (seg);
21161 size_t size = AlignQword (size (o));
21163 check_class_object_demotion (o);
21164 if (contain_pointers (o))
21166 go_through_object_nostart (method_table (o), o, size(o), pval,
21168 reloc_survivor_helper (pval);
21173 if (o < heap_segment_allocated (seg))
21175 assert (!marked (o));
21180 while (o < heap_segment_allocated (seg) && !marked (o))
21182 o = o + AlignQword (size (o));
21187 dprintf (1235, ("after GC LOH size: %Id, free list: %Id, free obj: %Id\n\n",
21188 generation_size (max_generation + 1),
21189 generation_free_list_space (gen),
21190 generation_free_obj_space (gen)));
21193 void gc_heap::walk_relocation_for_loh (void* profiling_context, record_surv_fn fn)
21195 generation* gen = large_object_generation;
21196 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
21197 uint8_t* o = generation_allocation_start (gen);
21199 //Skip the generation gap object
21200 o = o + AlignQword (size (o));
21204 if (o >= heap_segment_allocated (seg))
21206 seg = heap_segment_next (seg);
21212 o = heap_segment_mem (seg);
21217 size_t size = AlignQword (size (o));
21219 ptrdiff_t reloc = loh_node_relocation_distance (o);
21221 STRESS_LOG_PLUG_MOVE(o, (o + size), -reloc);
21223 fn (o, (o + size), reloc, profiling_context, !!settings.compaction, false);
21226 if (o < heap_segment_allocated (seg))
21228 assert (!marked (o));
21233 while (o < heap_segment_allocated (seg) && !marked (o))
21235 o = o + AlignQword (size (o));
21241 BOOL gc_heap::loh_object_p (uint8_t* o)
21243 #ifdef MULTIPLE_HEAPS
21244 gc_heap* hp = gc_heap::g_heaps [0];
21245 int brick_entry = hp->brick_table[hp->brick_of (o)];
21246 #else //MULTIPLE_HEAPS
21247 int brick_entry = brick_table[brick_of (o)];
21248 #endif //MULTIPLE_HEAPS
21250 return (brick_entry == 0);
21252 #endif //FEATURE_LOH_COMPACTION
21254 void gc_heap::convert_to_pinned_plug (BOOL& last_npinned_plug_p,
21255 BOOL& last_pinned_plug_p,
21256 BOOL& pinned_plug_p,
21258 size_t& artificial_pinned_size)
21260 last_npinned_plug_p = FALSE;
21261 last_pinned_plug_p = TRUE;
21262 pinned_plug_p = TRUE;
21263 artificial_pinned_size = ps;
21266 // Because we have the artificial pinning, we can't guarantee that pinned and npinned
21267 // plugs are always interleaved.
21268 void gc_heap::store_plug_gap_info (uint8_t* plug_start,
21270 BOOL& last_npinned_plug_p,
21271 BOOL& last_pinned_plug_p,
21272 uint8_t*& last_pinned_plug,
21273 BOOL& pinned_plug_p,
21274 uint8_t* last_object_in_last_plug,
21275 BOOL& merge_with_last_pin_p,
21276 // this is only for verification purpose
21277 size_t last_plug_len)
21279 UNREFERENCED_PARAMETER(last_plug_len);
21281 if (!last_npinned_plug_p && !last_pinned_plug_p)
21283 //dprintf (3, ("last full plug end: %Ix, full plug start: %Ix", plug_end, plug_start));
21284 dprintf (3, ("Free: %Ix", (plug_start - plug_end)));
21285 assert ((plug_start == plug_end) || ((size_t)(plug_start - plug_end) >= Align (min_obj_size)));
21286 set_gap_size (plug_start, plug_start - plug_end);
21289 if (pinned (plug_start))
21291 BOOL save_pre_plug_info_p = FALSE;
21293 if (last_npinned_plug_p || last_pinned_plug_p)
21295 //if (last_plug_len == Align (min_obj_size))
21297 // dprintf (3, ("debugging only - last npinned plug is min, check to see if it's correct"));
21298 // GCToOSInterface::DebugBreak();
21300 save_pre_plug_info_p = TRUE;
21303 pinned_plug_p = TRUE;
21304 last_npinned_plug_p = FALSE;
21306 if (last_pinned_plug_p)
21308 dprintf (3, ("last plug %Ix was also pinned, should merge", last_pinned_plug));
21309 merge_with_last_pin_p = TRUE;
21313 last_pinned_plug_p = TRUE;
21314 last_pinned_plug = plug_start;
21316 enque_pinned_plug (last_pinned_plug, save_pre_plug_info_p, last_object_in_last_plug);
21318 if (save_pre_plug_info_p)
21320 set_gap_size (plug_start, sizeof (gap_reloc_pair));
21326 if (last_pinned_plug_p)
21328 //if (Align (last_plug_len) < min_pre_pin_obj_size)
21330 // dprintf (3, ("debugging only - last pinned plug is min, check to see if it's correct"));
21331 // GCToOSInterface::DebugBreak();
21334 save_post_plug_info (last_pinned_plug, last_object_in_last_plug, plug_start);
21335 set_gap_size (plug_start, sizeof (gap_reloc_pair));
21337 verify_pins_with_post_plug_info("after saving post plug info");
21339 last_npinned_plug_p = TRUE;
21340 last_pinned_plug_p = FALSE;
21344 void gc_heap::record_interesting_data_point (interesting_data_point idp)
21346 #ifdef GC_CONFIG_DRIVEN
21347 (interesting_data_per_gc[idp])++;
21349 UNREFERENCED_PARAMETER(idp);
21350 #endif //GC_CONFIG_DRIVEN
21354 #pragma warning(push)
21355 #pragma warning(disable:21000) // Suppress PREFast warning about overly large function
21357 void gc_heap::plan_phase (int condemned_gen_number)
21359 size_t old_gen2_allocated = 0;
21360 size_t old_gen2_size = 0;
21362 if (condemned_gen_number == (max_generation - 1))
21364 old_gen2_allocated = generation_free_list_allocated (generation_of (max_generation));
21365 old_gen2_size = generation_size (max_generation);
21368 assert (settings.concurrent == FALSE);
21370 // %type% category = quote (plan);
21374 start = GetCycleCount32();
21377 dprintf (2,("---- Plan Phase ---- Condemned generation %d, promotion: %d",
21378 condemned_gen_number, settings.promotion ? 1 : 0));
21380 generation* condemned_gen1 = generation_of (condemned_gen_number);
21383 BOOL use_mark_list = FALSE;
21384 uint8_t** mark_list_next = &mark_list[0];
21385 #ifdef GC_CONFIG_DRIVEN
21386 dprintf (3, ("total number of marked objects: %Id (%Id)",
21387 (mark_list_index - &mark_list[0]), ((mark_list_end - &mark_list[0]))));
21389 dprintf (3, ("mark_list length: %Id",
21390 (mark_list_index - &mark_list[0])));
21391 #endif //GC_CONFIG_DRIVEN
21393 if ((condemned_gen_number < max_generation) &&
21394 (mark_list_index <= mark_list_end)
21395 #ifdef BACKGROUND_GC
21396 && (!recursive_gc_sync::background_running_p())
21397 #endif //BACKGROUND_GC
21400 #ifndef MULTIPLE_HEAPS
21401 _sort (&mark_list[0], mark_list_index-1, 0);
21402 //printf ("using mark list at GC #%d", dd_collection_count (dynamic_data_of (0)));
21403 //verify_qsort_array (&mark_list[0], mark_list_index-1);
21404 #endif //!MULTIPLE_HEAPS
21405 use_mark_list = TRUE;
21406 get_gc_data_per_heap()->set_mechanism_bit (gc_mark_list_bit);
21410 dprintf (3, ("mark_list not used"));
21415 #ifdef FEATURE_BASICFREEZE
21416 if ((generation_start_segment (condemned_gen1) != ephemeral_heap_segment) &&
21417 ro_segments_in_range)
21419 sweep_ro_segments (generation_start_segment (condemned_gen1));
21421 #endif // FEATURE_BASICFREEZE
21423 #ifndef MULTIPLE_HEAPS
21424 if (shigh != (uint8_t*)0)
21426 heap_segment* seg = heap_segment_rw (generation_start_segment (condemned_gen1));
21428 PREFIX_ASSUME(seg != NULL);
21430 heap_segment* fseg = seg;
21433 if (slow > heap_segment_mem (seg) &&
21434 slow < heap_segment_reserved (seg))
21438 uint8_t* o = generation_allocation_start (condemned_gen1) +
21439 Align (size (generation_allocation_start (condemned_gen1)));
21442 assert ((slow - o) >= (int)Align (min_obj_size));
21443 #ifdef BACKGROUND_GC
21444 if (current_c_gc_state == c_gc_state_marking)
21446 bgc_clear_batch_mark_array_bits (o, slow);
21448 #endif //BACKGROUND_GC
21449 make_unused_array (o, slow - o);
21454 assert (condemned_gen_number == max_generation);
21455 make_unused_array (heap_segment_mem (seg),
21456 slow - heap_segment_mem (seg));
21459 if (in_range_for_segment (shigh, seg))
21461 #ifdef BACKGROUND_GC
21462 if (current_c_gc_state == c_gc_state_marking)
21464 bgc_clear_batch_mark_array_bits ((shigh + Align (size (shigh))), heap_segment_allocated (seg));
21466 #endif //BACKGROUND_GC
21467 heap_segment_allocated (seg) = shigh + Align (size (shigh));
21469 // test if the segment is in the range of [slow, shigh]
21470 if (!((heap_segment_reserved (seg) >= slow) &&
21471 (heap_segment_mem (seg) <= shigh)))
21473 // shorten it to minimum
21474 heap_segment_allocated (seg) = heap_segment_mem (seg);
21476 seg = heap_segment_next_rw (seg);
21481 heap_segment* seg = heap_segment_rw (generation_start_segment (condemned_gen1));
21483 PREFIX_ASSUME(seg != NULL);
21485 heap_segment* sseg = seg;
21488 // shorten it to minimum
21491 // no survivors make all generations look empty
21492 uint8_t* o = generation_allocation_start (condemned_gen1) +
21493 Align (size (generation_allocation_start (condemned_gen1)));
21494 #ifdef BACKGROUND_GC
21495 if (current_c_gc_state == c_gc_state_marking)
21497 bgc_clear_batch_mark_array_bits (o, heap_segment_allocated (seg));
21499 #endif //BACKGROUND_GC
21500 heap_segment_allocated (seg) = o;
21504 assert (condemned_gen_number == max_generation);
21505 #ifdef BACKGROUND_GC
21506 if (current_c_gc_state == c_gc_state_marking)
21508 bgc_clear_batch_mark_array_bits (heap_segment_mem (seg), heap_segment_allocated (seg));
21510 #endif //BACKGROUND_GC
21511 heap_segment_allocated (seg) = heap_segment_mem (seg);
21513 seg = heap_segment_next_rw (seg);
21517 #endif //MULTIPLE_HEAPS
21519 heap_segment* seg1 = heap_segment_rw (generation_start_segment (condemned_gen1));
21521 PREFIX_ASSUME(seg1 != NULL);
21523 uint8_t* end = heap_segment_allocated (seg1);
21524 uint8_t* first_condemned_address = generation_allocation_start (condemned_gen1);
21525 uint8_t* x = first_condemned_address;
21527 assert (!marked (x));
21528 uint8_t* plug_end = x;
21530 size_t sequence_number = 0;
21531 uint8_t* last_node = 0;
21532 size_t current_brick = brick_of (x);
21533 BOOL allocate_in_condemned = ((condemned_gen_number == max_generation)||
21534 (settings.promotion == FALSE));
21535 int active_old_gen_number = condemned_gen_number;
21536 int active_new_gen_number = (allocate_in_condemned ? condemned_gen_number:
21537 (1 + condemned_gen_number));
21538 generation* older_gen = 0;
21539 generation* consing_gen = condemned_gen1;
21540 alloc_list r_free_list [MAX_BUCKET_COUNT];
21542 size_t r_free_list_space = 0;
21543 size_t r_free_obj_space = 0;
21544 size_t r_older_gen_free_list_allocated = 0;
21545 size_t r_older_gen_condemned_allocated = 0;
21546 size_t r_older_gen_end_seg_allocated = 0;
21547 uint8_t* r_allocation_pointer = 0;
21548 uint8_t* r_allocation_limit = 0;
21549 uint8_t* r_allocation_start_region = 0;
21550 heap_segment* r_allocation_segment = 0;
21551 #ifdef FREE_USAGE_STATS
21552 size_t r_older_gen_free_space[NUM_GEN_POWER2];
21553 #endif //FREE_USAGE_STATS
21555 if ((condemned_gen_number < max_generation))
21557 older_gen = generation_of (min (max_generation, 1 + condemned_gen_number));
21558 generation_allocator (older_gen)->copy_to_alloc_list (r_free_list);
21560 r_free_list_space = generation_free_list_space (older_gen);
21561 r_free_obj_space = generation_free_obj_space (older_gen);
21562 #ifdef FREE_USAGE_STATS
21563 memcpy (r_older_gen_free_space, older_gen->gen_free_spaces, sizeof (r_older_gen_free_space));
21564 #endif //FREE_USAGE_STATS
21565 generation_allocate_end_seg_p (older_gen) = FALSE;
21566 r_older_gen_free_list_allocated = generation_free_list_allocated (older_gen);
21567 r_older_gen_condemned_allocated = generation_condemned_allocated (older_gen);
21568 r_older_gen_end_seg_allocated = generation_end_seg_allocated (older_gen);
21569 r_allocation_limit = generation_allocation_limit (older_gen);
21570 r_allocation_pointer = generation_allocation_pointer (older_gen);
21571 r_allocation_start_region = generation_allocation_context_start_region (older_gen);
21572 r_allocation_segment = generation_allocation_segment (older_gen);
21573 heap_segment* start_seg = heap_segment_rw (generation_start_segment (older_gen));
21575 PREFIX_ASSUME(start_seg != NULL);
21577 if (start_seg != ephemeral_heap_segment)
21579 assert (condemned_gen_number == (max_generation - 1));
21580 while (start_seg && (start_seg != ephemeral_heap_segment))
21582 assert (heap_segment_allocated (start_seg) >=
21583 heap_segment_mem (start_seg));
21584 assert (heap_segment_allocated (start_seg) <=
21585 heap_segment_reserved (start_seg));
21586 heap_segment_plan_allocated (start_seg) =
21587 heap_segment_allocated (start_seg);
21588 start_seg = heap_segment_next_rw (start_seg);
21593 //reset all of the segment allocated sizes
21595 heap_segment* seg2 = heap_segment_rw (generation_start_segment (condemned_gen1));
21597 PREFIX_ASSUME(seg2 != NULL);
21601 heap_segment_plan_allocated (seg2) =
21602 heap_segment_mem (seg2);
21603 seg2 = heap_segment_next_rw (seg2);
21606 int condemned_gn = condemned_gen_number;
21608 int bottom_gen = 0;
21609 init_free_and_plug();
21611 while (condemned_gn >= bottom_gen)
21613 generation* condemned_gen2 = generation_of (condemned_gn);
21614 generation_allocator (condemned_gen2)->clear();
21615 generation_free_list_space (condemned_gen2) = 0;
21616 generation_free_obj_space (condemned_gen2) = 0;
21617 generation_allocation_size (condemned_gen2) = 0;
21618 generation_condemned_allocated (condemned_gen2) = 0;
21619 generation_pinned_allocated (condemned_gen2) = 0;
21620 generation_free_list_allocated(condemned_gen2) = 0;
21621 generation_end_seg_allocated (condemned_gen2) = 0;
21622 generation_pinned_allocation_sweep_size (condemned_gen2) = 0;
21623 generation_pinned_allocation_compact_size (condemned_gen2) = 0;
21624 #ifdef FREE_USAGE_STATS
21625 generation_pinned_free_obj_space (condemned_gen2) = 0;
21626 generation_allocated_in_pinned_free (condemned_gen2) = 0;
21627 generation_allocated_since_last_pin (condemned_gen2) = 0;
21628 #endif //FREE_USAGE_STATS
21629 generation_plan_allocation_start (condemned_gen2) = 0;
21630 generation_allocation_segment (condemned_gen2) =
21631 heap_segment_rw (generation_start_segment (condemned_gen2));
21633 PREFIX_ASSUME(generation_allocation_segment(condemned_gen2) != NULL);
21635 if (generation_start_segment (condemned_gen2) != ephemeral_heap_segment)
21637 generation_allocation_pointer (condemned_gen2) =
21638 heap_segment_mem (generation_allocation_segment (condemned_gen2));
21642 generation_allocation_pointer (condemned_gen2) = generation_allocation_start (condemned_gen2);
21645 generation_allocation_limit (condemned_gen2) = generation_allocation_pointer (condemned_gen2);
21646 generation_allocation_context_start_region (condemned_gen2) = generation_allocation_pointer (condemned_gen2);
21651 BOOL allocate_first_generation_start = FALSE;
21653 if (allocate_in_condemned)
21655 allocate_first_generation_start = TRUE;
21658 dprintf(3,( " From %Ix to %Ix", (size_t)x, (size_t)end));
21660 demotion_low = MAX_PTR;
21661 demotion_high = heap_segment_allocated (ephemeral_heap_segment);
21663 // If we are doing a gen1 only because of cards, it means we should not demote any pinned plugs
21664 // from gen1. They should get promoted to gen2.
21665 demote_gen1_p = !(settings.promotion &&
21666 (settings.condemned_generation == (max_generation - 1)) &&
21667 gen_to_condemn_reasons.is_only_condition (gen_low_card_p));
21669 total_ephemeral_size = 0;
21671 print_free_and_plug ("BP");
21673 for (int gen_idx = 0; gen_idx <= max_generation; gen_idx++)
21675 generation* temp_gen = generation_of (gen_idx);
21677 dprintf (2, ("gen%d start %Ix, plan start %Ix",
21679 generation_allocation_start (temp_gen),
21680 generation_plan_allocation_start (temp_gen)));
21683 BOOL fire_pinned_plug_events_p = ETW_EVENT_ENABLED(MICROSOFT_WINDOWS_DOTNETRUNTIME_PRIVATE_PROVIDER_Context, PinPlugAtGCTime);
21684 size_t last_plug_len = 0;
21691 assert (heap_segment_allocated (seg1) == end);
21692 heap_segment_allocated (seg1) = plug_end;
21694 current_brick = update_brick_table (tree, current_brick, x, plug_end);
21695 dprintf (3, ("end of seg: new tree, sequence# 0"));
21696 sequence_number = 0;
21699 if (heap_segment_next_rw (seg1))
21701 seg1 = heap_segment_next_rw (seg1);
21702 end = heap_segment_allocated (seg1);
21703 plug_end = x = heap_segment_mem (seg1);
21704 current_brick = brick_of (x);
21705 dprintf(3,( " From %Ix to %Ix", (size_t)x, (size_t)end));
21714 BOOL last_npinned_plug_p = FALSE;
21715 BOOL last_pinned_plug_p = FALSE;
21717 // last_pinned_plug is the beginning of the last pinned plug. If we merge a plug into a pinned
21718 // plug we do not change the value of last_pinned_plug. This happens with artificially pinned plugs -
21719 // it can be merged with a previous pinned plug and a pinned plug after it can be merged with it.
21720 uint8_t* last_pinned_plug = 0;
21721 size_t num_pinned_plugs_in_plug = 0;
21723 uint8_t* last_object_in_plug = 0;
21725 while ((x < end) && marked (x))
21727 uint8_t* plug_start = x;
21728 uint8_t* saved_plug_end = plug_end;
21729 BOOL pinned_plug_p = FALSE;
21730 BOOL npin_before_pin_p = FALSE;
21731 BOOL saved_last_npinned_plug_p = last_npinned_plug_p;
21732 uint8_t* saved_last_object_in_plug = last_object_in_plug;
21733 BOOL merge_with_last_pin_p = FALSE;
21735 size_t added_pinning_size = 0;
21736 size_t artificial_pinned_size = 0;
21738 store_plug_gap_info (plug_start, plug_end, last_npinned_plug_p, last_pinned_plug_p,
21739 last_pinned_plug, pinned_plug_p, last_object_in_plug,
21740 merge_with_last_pin_p, last_plug_len);
21742 #ifdef FEATURE_STRUCTALIGN
21743 int requiredAlignment = ((CObjectHeader*)plug_start)->GetRequiredAlignment();
21744 size_t alignmentOffset = OBJECT_ALIGNMENT_OFFSET;
21745 #endif // FEATURE_STRUCTALIGN
21749 while ((xl < end) && marked (xl) && (pinned (xl) == pinned_plug_p))
21756 #ifdef FEATURE_STRUCTALIGN
21759 int obj_requiredAlignment = ((CObjectHeader*)xl)->GetRequiredAlignment();
21760 if (obj_requiredAlignment > requiredAlignment)
21762 requiredAlignment = obj_requiredAlignment;
21763 alignmentOffset = xl - plug_start + OBJECT_ALIGNMENT_OFFSET;
21766 #endif // FEATURE_STRUCTALIGN
21770 dprintf(4, ("+%Ix+", (size_t)xl));
21771 assert ((size (xl) > 0));
21772 assert ((size (xl) <= LARGE_OBJECT_SIZE));
21774 last_object_in_plug = xl;
21776 xl = xl + Align (size (xl));
21780 BOOL next_object_marked_p = ((xl < end) && marked (xl));
21784 // If it is pinned we need to extend to the next marked object as we can't use part of
21785 // a pinned object to make the artificial gap (unless the last 3 ptr sized words are all
21786 // references but for now I am just using the next non pinned object for that).
21787 if (next_object_marked_p)
21790 last_object_in_plug = xl;
21791 size_t extra_size = Align (size (xl));
21792 xl = xl + extra_size;
21793 added_pinning_size = extra_size;
21798 if (next_object_marked_p)
21799 npin_before_pin_p = TRUE;
21802 assert (xl <= end);
21805 dprintf (3, ( "%Ix[", (size_t)x));
21807 size_t ps = plug_end - plug_start;
21808 last_plug_len = ps;
21809 dprintf (3, ( "%Ix[(%Ix)", (size_t)x, ps));
21810 uint8_t* new_address = 0;
21812 if (!pinned_plug_p)
21814 if (allocate_in_condemned &&
21815 (settings.condemned_generation == max_generation) &&
21816 (ps > OS_PAGE_SIZE))
21818 ptrdiff_t reloc = plug_start - generation_allocation_pointer (consing_gen);
21819 //reloc should >=0 except when we relocate
21820 //across segments and the dest seg is higher then the src
21822 if ((ps > (8*OS_PAGE_SIZE)) &&
21824 ((size_t)reloc < (ps/16)))
21826 dprintf (3, ("Pinning %Ix; reloc would have been: %Ix",
21827 (size_t)plug_start, reloc));
21828 // The last plug couldn't have been a npinned plug or it would have
21829 // included this plug.
21830 assert (!saved_last_npinned_plug_p);
21832 if (last_pinned_plug)
21834 dprintf (3, ("artificially pinned plug merged with last pinned plug"));
21835 merge_with_last_pin_p = TRUE;
21839 enque_pinned_plug (plug_start, FALSE, 0);
21840 last_pinned_plug = plug_start;
21843 convert_to_pinned_plug (last_npinned_plug_p, last_pinned_plug_p, pinned_plug_p,
21844 ps, artificial_pinned_size);
21849 if (allocate_first_generation_start)
21851 allocate_first_generation_start = FALSE;
21852 plan_generation_start (condemned_gen1, consing_gen, plug_start);
21853 assert (generation_plan_allocation_start (condemned_gen1));
21856 if (seg1 == ephemeral_heap_segment)
21858 process_ephemeral_boundaries (plug_start, active_new_gen_number,
21859 active_old_gen_number,
21861 allocate_in_condemned);
21864 dprintf (3, ("adding %Id to gen%d surv", ps, active_old_gen_number));
21866 dynamic_data* dd_active_old = dynamic_data_of (active_old_gen_number);
21867 dd_survived_size (dd_active_old) += ps;
21869 BOOL convert_to_pinned_p = FALSE;
21871 if (!pinned_plug_p)
21873 #if defined (RESPECT_LARGE_ALIGNMENT) || defined (FEATURE_STRUCTALIGN)
21874 dd_num_npinned_plugs (dd_active_old)++;
21875 #endif //RESPECT_LARGE_ALIGNMENT || FEATURE_STRUCTALIGN
21877 add_gen_plug (active_old_gen_number, ps);
21879 if (allocate_in_condemned)
21881 verify_pins_with_post_plug_info("before aic");
21884 allocate_in_condemned_generations (consing_gen,
21886 active_old_gen_number,
21888 &convert_to_pinned_p,
21889 (npin_before_pin_p ? plug_end : 0),
21891 #endif //SHORT_PLUGS
21892 plug_start REQD_ALIGN_AND_OFFSET_ARG);
21893 verify_pins_with_post_plug_info("after aic");
21897 new_address = allocate_in_older_generation (older_gen, ps, active_old_gen_number, plug_start REQD_ALIGN_AND_OFFSET_ARG);
21899 if (new_address != 0)
21901 if (settings.condemned_generation == (max_generation - 1))
21903 dprintf (3, (" NA: %Ix-%Ix -> %Ix, %Ix (%Ix)",
21904 plug_start, plug_end,
21905 (size_t)new_address, (size_t)new_address + (plug_end - plug_start),
21906 (size_t)(plug_end - plug_start)));
21911 allocate_in_condemned = TRUE;
21913 new_address = allocate_in_condemned_generations (consing_gen, ps, active_old_gen_number,
21915 &convert_to_pinned_p,
21916 (npin_before_pin_p ? plug_end : 0),
21918 #endif //SHORT_PLUGS
21919 plug_start REQD_ALIGN_AND_OFFSET_ARG);
21923 if (convert_to_pinned_p)
21925 assert (last_npinned_plug_p != FALSE);
21926 assert (last_pinned_plug_p == FALSE);
21927 convert_to_pinned_plug (last_npinned_plug_p, last_pinned_plug_p, pinned_plug_p,
21928 ps, artificial_pinned_size);
21929 enque_pinned_plug (plug_start, FALSE, 0);
21930 last_pinned_plug = plug_start;
21936 //verify that we are at then end of the ephemeral segment
21937 assert (generation_allocation_segment (consing_gen) ==
21938 ephemeral_heap_segment);
21939 //verify that we are near the end
21940 assert ((generation_allocation_pointer (consing_gen) + Align (ps)) <
21941 heap_segment_allocated (ephemeral_heap_segment));
21942 assert ((generation_allocation_pointer (consing_gen) + Align (ps)) >
21943 (heap_segment_allocated (ephemeral_heap_segment) + Align (min_obj_size)));
21947 #ifdef SIMPLE_DPRINTF
21948 dprintf (3, ("(%Ix)[%Ix->%Ix, NA: [%Ix(%Id), %Ix[: %Ix(%d)",
21949 (size_t)(node_gap_size (plug_start)),
21950 plug_start, plug_end, (size_t)new_address, (size_t)(plug_start - new_address),
21951 (size_t)new_address + ps, ps,
21952 (is_plug_padded (plug_start) ? 1 : 0)));
21953 #endif //SIMPLE_DPRINTF
21956 if (is_plug_padded (plug_start))
21958 dprintf (3, ("%Ix was padded", plug_start));
21959 dd_padding_size (dd_active_old) += Align (min_obj_size);
21961 #endif //SHORT_PLUGS
21968 if (fire_pinned_plug_events_p)
21969 FireEtwPinPlugAtGCTime(plug_start, plug_end,
21970 (merge_with_last_pin_p ? 0 : (uint8_t*)node_gap_size (plug_start)),
21971 GetClrInstanceId());
21973 if (merge_with_last_pin_p)
21975 merge_with_last_pinned_plug (last_pinned_plug, ps);
21979 assert (last_pinned_plug == plug_start);
21980 set_pinned_info (plug_start, ps, consing_gen);
21983 new_address = plug_start;
21985 dprintf (3, ( "(%Ix)PP: [%Ix, %Ix[%Ix](m:%d)",
21986 (size_t)(node_gap_size (plug_start)), (size_t)plug_start,
21987 (size_t)plug_end, ps,
21988 (merge_with_last_pin_p ? 1 : 0)));
21990 dprintf (3, ("adding %Id to gen%d pinned surv", plug_end - plug_start, active_old_gen_number));
21991 dd_pinned_survived_size (dd_active_old) += plug_end - plug_start;
21992 dd_added_pinned_size (dd_active_old) += added_pinning_size;
21993 dd_artificial_pinned_survived_size (dd_active_old) += artificial_pinned_size;
21995 if (!demote_gen1_p && (active_old_gen_number == (max_generation - 1)))
21997 last_gen1_pin_end = plug_end;
22002 // detect forward allocation in the same segment
22003 assert (!((new_address > plug_start) &&
22004 (new_address < heap_segment_reserved (seg1))));
22007 if (!merge_with_last_pin_p)
22009 if (current_brick != brick_of (plug_start))
22011 current_brick = update_brick_table (tree, current_brick, plug_start, saved_plug_end);
22012 sequence_number = 0;
22016 set_node_relocation_distance (plug_start, (new_address - plug_start));
22017 if (last_node && (node_relocation_distance (last_node) ==
22018 (node_relocation_distance (plug_start) +
22019 node_gap_size (plug_start))))
22021 //dprintf(3,( " Lb"));
22022 dprintf (3, ("%Ix Lb", plug_start));
22023 set_node_left (plug_start);
22025 if (0 == sequence_number)
22027 dprintf (2, ("sn: 0, tree is set to %Ix", plug_start));
22031 verify_pins_with_post_plug_info("before insert node");
22033 tree = insert_node (plug_start, ++sequence_number, tree, last_node);
22034 dprintf (3, ("tree is %Ix (b: %Ix) after insert_node", tree, brick_of (tree)));
22035 last_node = plug_start;
22038 // If we detect if the last plug is pinned plug right before us, we should save this gap info
22039 if (!pinned_plug_p)
22041 if (mark_stack_tos > 0)
22043 mark& m = mark_stack_array[mark_stack_tos - 1];
22044 if (m.has_post_plug_info())
22046 uint8_t* post_plug_info_start = m.saved_post_plug_info_start;
22047 size_t* current_plug_gap_start = (size_t*)(plug_start - sizeof (plug_and_gap));
22048 if ((uint8_t*)current_plug_gap_start == post_plug_info_start)
22050 dprintf (3, ("Ginfo: %Ix, %Ix, %Ix",
22051 *current_plug_gap_start, *(current_plug_gap_start + 1),
22052 *(current_plug_gap_start + 2)));
22053 memcpy (&(m.saved_post_plug_debug), current_plug_gap_start, sizeof (gap_reloc_pair));
22060 verify_pins_with_post_plug_info("after insert node");
22064 if (num_pinned_plugs_in_plug > 1)
22066 dprintf (3, ("more than %Id pinned plugs in this plug", num_pinned_plugs_in_plug));
22073 while ((mark_list_next < mark_list_index) &&
22074 (*mark_list_next <= x))
22078 if ((mark_list_next < mark_list_index)
22079 #ifdef MULTIPLE_HEAPS
22080 && (*mark_list_next < end) //for multiple segments
22081 #endif //MULTIPLE_HEAPS
22083 x = *mark_list_next;
22091 #ifdef BACKGROUND_GC
22092 if (current_c_gc_state == c_gc_state_marking)
22094 assert (recursive_gc_sync::background_running_p());
22095 while ((xl < end) && !marked (xl))
22097 dprintf (4, ("-%Ix-", (size_t)xl));
22098 assert ((size (xl) > 0));
22099 background_object_marked (xl, TRUE);
22100 xl = xl + Align (size (xl));
22105 #endif //BACKGROUND_GC
22107 while ((xl < end) && !marked (xl))
22109 dprintf (4, ("-%Ix-", (size_t)xl));
22110 assert ((size (xl) > 0));
22111 xl = xl + Align (size (xl));
22115 assert (xl <= end);
22121 while (!pinned_plug_que_empty_p())
22123 if (settings.promotion)
22125 uint8_t* pplug = pinned_plug (oldest_pin());
22126 if (in_range_for_segment (pplug, ephemeral_heap_segment))
22128 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
22129 //allocate all of the generation gaps
22130 while (active_new_gen_number > 0)
22132 active_new_gen_number--;
22134 if (active_new_gen_number == (max_generation - 1))
22136 maxgen_pinned_compact_before_advance = generation_pinned_allocation_compact_size (generation_of (max_generation));
22137 if (!demote_gen1_p)
22138 advance_pins_for_demotion (consing_gen);
22141 generation* gen = generation_of (active_new_gen_number);
22142 plan_generation_start (gen, consing_gen, 0);
22144 if (demotion_low == MAX_PTR)
22146 demotion_low = pplug;
22147 dprintf (3, ("end plan: dlow->%Ix", demotion_low));
22150 dprintf (2, ("(%d)gen%d plan start: %Ix",
22151 heap_number, active_new_gen_number, (size_t)generation_plan_allocation_start (gen)));
22152 assert (generation_plan_allocation_start (gen));
22157 if (pinned_plug_que_empty_p())
22160 size_t entry = deque_pinned_plug();
22161 mark* m = pinned_plug_of (entry);
22162 uint8_t* plug = pinned_plug (m);
22163 size_t len = pinned_len (m);
22165 // detect pinned block in different segment (later) than
22166 // allocation segment
22167 heap_segment* nseg = heap_segment_rw (generation_allocation_segment (consing_gen));
22169 while ((plug < generation_allocation_pointer (consing_gen)) ||
22170 (plug >= heap_segment_allocated (nseg)))
22172 assert ((plug < heap_segment_mem (nseg)) ||
22173 (plug > heap_segment_reserved (nseg)));
22174 //adjust the end of the segment to be the end of the plug
22175 assert (generation_allocation_pointer (consing_gen)>=
22176 heap_segment_mem (nseg));
22177 assert (generation_allocation_pointer (consing_gen)<=
22178 heap_segment_committed (nseg));
22180 heap_segment_plan_allocated (nseg) =
22181 generation_allocation_pointer (consing_gen);
22182 //switch allocation segment
22183 nseg = heap_segment_next_rw (nseg);
22184 generation_allocation_segment (consing_gen) = nseg;
22185 //reset the allocation pointer and limits
22186 generation_allocation_pointer (consing_gen) =
22187 heap_segment_mem (nseg);
22190 set_new_pin_info (m, generation_allocation_pointer (consing_gen));
22191 dprintf (2, ("pin %Ix b: %Ix->%Ix", plug, brick_of (plug),
22192 (size_t)(brick_table[brick_of (plug)])));
22194 generation_allocation_pointer (consing_gen) = plug + len;
22195 generation_allocation_limit (consing_gen) =
22196 generation_allocation_pointer (consing_gen);
22197 //Add the size of the pinned plug to the right pinned allocations
22198 //find out which gen this pinned plug came from
22199 int frgn = object_gennum (plug);
22200 if ((frgn != (int)max_generation) && settings.promotion)
22202 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
22207 plan_generation_starts (consing_gen);
22208 print_free_and_plug ("AP");
22211 #ifdef SIMPLE_DPRINTF
22212 for (int gen_idx = 0; gen_idx <= max_generation; gen_idx++)
22214 generation* temp_gen = generation_of (gen_idx);
22215 dynamic_data* temp_dd = dynamic_data_of (gen_idx);
22217 int added_pinning_ratio = 0;
22218 int artificial_pinned_ratio = 0;
22220 if (dd_pinned_survived_size (temp_dd) != 0)
22222 added_pinning_ratio = (int)((float)dd_added_pinned_size (temp_dd) * 100 / (float)dd_pinned_survived_size (temp_dd));
22223 artificial_pinned_ratio = (int)((float)dd_artificial_pinned_survived_size (temp_dd) * 100 / (float)dd_pinned_survived_size (temp_dd));
22226 size_t padding_size =
22228 dd_padding_size (temp_dd);
22231 #endif //SHORT_PLUGS
22232 dprintf (1, ("gen%d: %Ix, %Ix(%Id), NON PIN alloc: %Id, pin com: %Id, sweep: %Id, surv: %Id, pinsurv: %Id(%d%% added, %d%% art), np surv: %Id, pad: %Id",
22234 generation_allocation_start (temp_gen),
22235 generation_plan_allocation_start (temp_gen),
22236 (size_t)(generation_plan_allocation_start (temp_gen) - generation_allocation_start (temp_gen)),
22237 generation_allocation_size (temp_gen),
22238 generation_pinned_allocation_compact_size (temp_gen),
22239 generation_pinned_allocation_sweep_size (temp_gen),
22240 dd_survived_size (temp_dd),
22241 dd_pinned_survived_size (temp_dd),
22242 added_pinning_ratio,
22243 artificial_pinned_ratio,
22244 (dd_survived_size (temp_dd) - dd_pinned_survived_size (temp_dd)),
22247 #endif //SIMPLE_DPRINTF
22250 if (settings.condemned_generation == (max_generation - 1 ))
22252 size_t plan_gen2_size = generation_plan_size (max_generation);
22253 size_t growth = plan_gen2_size - old_gen2_size;
22257 dprintf (1, ("gen2 grew %Id (end seg alloc: %Id, gen1 c alloc: %Id",
22258 growth, generation_end_seg_allocated (generation_of (max_generation)),
22259 generation_condemned_allocated (generation_of (max_generation - 1))));
22263 dprintf (1, ("gen2 shrank %Id (end seg alloc: %Id, gen1 c alloc: %Id",
22264 (old_gen2_size - plan_gen2_size), generation_end_seg_allocated (generation_of (max_generation)),
22265 generation_condemned_allocated (generation_of (max_generation - 1))));
22268 generation* older_gen = generation_of (settings.condemned_generation + 1);
22269 size_t rejected_free_space = generation_free_obj_space (older_gen) - r_free_obj_space;
22270 size_t free_list_allocated = generation_free_list_allocated (older_gen) - r_older_gen_free_list_allocated;
22271 size_t end_seg_allocated = generation_end_seg_allocated (older_gen) - r_older_gen_end_seg_allocated;
22272 size_t condemned_allocated = generation_condemned_allocated (older_gen) - r_older_gen_condemned_allocated;
22274 dprintf (1, ("older gen's free alloc: %Id->%Id, seg alloc: %Id->%Id, condemned alloc: %Id->%Id",
22275 r_older_gen_free_list_allocated, generation_free_list_allocated (older_gen),
22276 r_older_gen_end_seg_allocated, generation_end_seg_allocated (older_gen),
22277 r_older_gen_condemned_allocated, generation_condemned_allocated (older_gen)));
22279 dprintf (1, ("this GC did %Id free list alloc(%Id bytes free space rejected), %Id seg alloc and %Id condemned alloc, gen1 condemned alloc is %Id",
22280 free_list_allocated, rejected_free_space, end_seg_allocated,
22281 condemned_allocated, generation_condemned_allocated (generation_of (settings.condemned_generation))));
22283 maxgen_size_increase* maxgen_size_info = &(get_gc_data_per_heap()->maxgen_size_info);
22284 maxgen_size_info->free_list_allocated = free_list_allocated;
22285 maxgen_size_info->free_list_rejected = rejected_free_space;
22286 maxgen_size_info->end_seg_allocated = end_seg_allocated;
22287 maxgen_size_info->condemned_allocated = condemned_allocated;
22288 maxgen_size_info->pinned_allocated = maxgen_pinned_compact_before_advance;
22289 maxgen_size_info->pinned_allocated_advance = generation_pinned_allocation_compact_size (generation_of (max_generation)) - maxgen_pinned_compact_before_advance;
22291 #ifdef FREE_USAGE_STATS
22292 int free_list_efficiency = 0;
22293 if ((free_list_allocated + rejected_free_space) != 0)
22294 free_list_efficiency = (int)(((float) (free_list_allocated) / (float)(free_list_allocated + rejected_free_space)) * (float)100);
22296 int running_free_list_efficiency = (int)(generation_allocator_efficiency(older_gen)*100);
22298 dprintf (1, ("gen%d free list alloc effi: %d%%, current effi: %d%%",
22299 older_gen->gen_num,
22300 free_list_efficiency, running_free_list_efficiency));
22302 dprintf (1, ("gen2 free list change"));
22303 for (int j = 0; j < NUM_GEN_POWER2; j++)
22305 dprintf (1, ("[h%d][#%Id]: 2^%d: F: %Id->%Id(%Id), P: %Id",
22308 (j + 10), r_older_gen_free_space[j], older_gen->gen_free_spaces[j],
22309 (ptrdiff_t)(r_older_gen_free_space[j] - older_gen->gen_free_spaces[j]),
22310 (generation_of(max_generation - 1))->gen_plugs[j]));
22312 #endif //FREE_USAGE_STATS
22315 size_t fragmentation =
22316 generation_fragmentation (generation_of (condemned_gen_number),
22318 heap_segment_allocated (ephemeral_heap_segment));
22320 dprintf (2,("Fragmentation: %Id", fragmentation));
22321 dprintf (2,("---- End of Plan phase ----"));
22324 finish = GetCycleCount32();
22325 plan_time = finish - start;
22328 // We may update write barrier code. We assume here EE has been suspended if we are on a GC thread.
22329 assert(IsGCInProgress());
22331 BOOL should_expand = FALSE;
22332 BOOL should_compact= FALSE;
22333 ephemeral_promotion = FALSE;
22336 if ((!settings.concurrent) &&
22337 ((condemned_gen_number < max_generation) &&
22338 ((settings.gen0_reduction_count > 0) || (settings.entry_memory_load >= 95))))
22340 dprintf (2, ("gen0 reduction count is %d, condemning %d, mem load %d",
22341 settings.gen0_reduction_count,
22342 condemned_gen_number,
22343 settings.entry_memory_load));
22344 should_compact = TRUE;
22346 get_gc_data_per_heap()->set_mechanism (gc_heap_compact,
22347 ((settings.gen0_reduction_count > 0) ? compact_fragmented_gen0 : compact_high_mem_load));
22349 if ((condemned_gen_number >= (max_generation - 1)) &&
22350 dt_low_ephemeral_space_p (tuning_deciding_expansion))
22352 dprintf (2, ("Not enough space for all ephemeral generations with compaction"));
22353 should_expand = TRUE;
22359 should_compact = decide_on_compacting (condemned_gen_number, fragmentation, should_expand);
22364 #ifdef FEATURE_LOH_COMPACTION
22365 loh_compacted_p = FALSE;
22366 #endif //FEATURE_LOH_COMPACTION
22368 if (condemned_gen_number == max_generation)
22370 #ifdef FEATURE_LOH_COMPACTION
22371 if (settings.loh_compaction)
22375 should_compact = TRUE;
22376 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_loh_forced);
22377 loh_compacted_p = TRUE;
22382 if ((heap_number == 0) && (loh_pinned_queue))
22384 loh_pinned_queue_decay--;
22386 if (!loh_pinned_queue_decay)
22388 delete loh_pinned_queue;
22389 loh_pinned_queue = 0;
22394 if (!loh_compacted_p)
22395 #endif //FEATURE_LOH_COMPACTION
22397 GCToEEInterface::DiagWalkLOHSurvivors(__this);
22398 sweep_large_objects();
22403 settings.loh_compaction = FALSE;
22406 #ifdef MULTIPLE_HEAPS
22408 new_heap_segment = NULL;
22410 if (should_compact && should_expand)
22411 gc_policy = policy_expand;
22412 else if (should_compact)
22413 gc_policy = policy_compact;
22415 gc_policy = policy_sweep;
22417 //vote for result of should_compact
22418 dprintf (3, ("Joining for compaction decision"));
22419 gc_t_join.join(this, gc_join_decide_on_compaction);
22420 if (gc_t_join.joined())
22422 //safe place to delete large heap segments
22423 if (condemned_gen_number == max_generation)
22425 for (int i = 0; i < n_heaps; i++)
22427 g_heaps [i]->rearrange_large_heap_segments ();
22431 settings.demotion = FALSE;
22432 int pol_max = policy_sweep;
22433 #ifdef GC_CONFIG_DRIVEN
22434 BOOL is_compaction_mandatory = FALSE;
22435 #endif //GC_CONFIG_DRIVEN
22438 for (i = 0; i < n_heaps; i++)
22440 if (pol_max < g_heaps[i]->gc_policy)
22441 pol_max = policy_compact;
22442 // set the demotion flag is any of the heap has demotion
22443 if (g_heaps[i]->demotion_high >= g_heaps[i]->demotion_low)
22445 (g_heaps[i]->get_gc_data_per_heap())->set_mechanism_bit (gc_demotion_bit);
22446 settings.demotion = TRUE;
22449 #ifdef GC_CONFIG_DRIVEN
22450 if (!is_compaction_mandatory)
22452 int compact_reason = (g_heaps[i]->get_gc_data_per_heap())->get_mechanism (gc_heap_compact);
22453 if (compact_reason >= 0)
22455 if (gc_heap_compact_reason_mandatory_p[compact_reason])
22456 is_compaction_mandatory = TRUE;
22459 #endif //GC_CONFIG_DRIVEN
22462 #ifdef GC_CONFIG_DRIVEN
22463 if (!is_compaction_mandatory)
22465 // If compaction is not mandatory we can feel free to change it to a sweeping GC.
22466 // Note that we may want to change this to only checking every so often instead of every single GC.
22467 if (should_do_sweeping_gc (pol_max >= policy_compact))
22469 pol_max = policy_sweep;
22473 if (pol_max == policy_sweep)
22474 pol_max = policy_compact;
22477 #endif //GC_CONFIG_DRIVEN
22479 for (i = 0; i < n_heaps; i++)
22481 if (pol_max > g_heaps[i]->gc_policy)
22482 g_heaps[i]->gc_policy = pol_max;
22483 //get the segment while we are serialized
22484 if (g_heaps[i]->gc_policy == policy_expand)
22486 g_heaps[i]->new_heap_segment =
22487 g_heaps[i]->soh_get_segment_to_expand();
22488 if (!g_heaps[i]->new_heap_segment)
22490 set_expand_in_full_gc (condemned_gen_number);
22491 //we are out of memory, cancel the expansion
22492 g_heaps[i]->gc_policy = policy_compact;
22497 BOOL is_full_compacting_gc = FALSE;
22499 if ((gc_policy >= policy_compact) && (condemned_gen_number == max_generation))
22501 full_gc_counts[gc_type_compacting]++;
22502 is_full_compacting_gc = TRUE;
22505 for (i = 0; i < n_heaps; i++)
22507 //copy the card and brick tables
22508 if (g_gc_card_table!= g_heaps[i]->card_table)
22510 g_heaps[i]->copy_brick_card_table();
22513 if (is_full_compacting_gc)
22515 g_heaps[i]->loh_alloc_since_cg = 0;
22519 //start all threads on the roots.
22520 dprintf(3, ("Starting all gc threads after compaction decision"));
22521 gc_t_join.restart();
22524 //reset the local variable accordingly
22525 should_compact = (gc_policy >= policy_compact);
22526 should_expand = (gc_policy >= policy_expand);
22528 #else //MULTIPLE_HEAPS
22530 //safe place to delete large heap segments
22531 if (condemned_gen_number == max_generation)
22533 rearrange_large_heap_segments ();
22536 settings.demotion = ((demotion_high >= demotion_low) ? TRUE : FALSE);
22537 if (settings.demotion)
22538 get_gc_data_per_heap()->set_mechanism_bit (gc_demotion_bit);
22540 #ifdef GC_CONFIG_DRIVEN
22541 BOOL is_compaction_mandatory = FALSE;
22542 int compact_reason = get_gc_data_per_heap()->get_mechanism (gc_heap_compact);
22543 if (compact_reason >= 0)
22544 is_compaction_mandatory = gc_heap_compact_reason_mandatory_p[compact_reason];
22546 if (!is_compaction_mandatory)
22548 if (should_do_sweeping_gc (should_compact))
22549 should_compact = FALSE;
22551 should_compact = TRUE;
22553 #endif //GC_CONFIG_DRIVEN
22555 if (should_compact && (condemned_gen_number == max_generation))
22557 full_gc_counts[gc_type_compacting]++;
22558 loh_alloc_since_cg = 0;
22560 #endif //MULTIPLE_HEAPS
22562 if (should_compact)
22564 dprintf (2,( "**** Doing Compacting GC ****"));
22568 #ifndef MULTIPLE_HEAPS
22569 heap_segment* new_heap_segment = soh_get_segment_to_expand();
22570 #endif //!MULTIPLE_HEAPS
22571 if (new_heap_segment)
22573 consing_gen = expand_heap(condemned_gen_number,
22578 // If we couldn't get a new segment, or we were able to
22579 // reserve one but no space to commit, we couldn't
22581 if (ephemeral_heap_segment != new_heap_segment)
22583 set_expand_in_full_gc (condemned_gen_number);
22584 should_expand = FALSE;
22587 generation_allocation_limit (condemned_gen1) =
22588 generation_allocation_pointer (condemned_gen1);
22589 if ((condemned_gen_number < max_generation))
22591 generation_allocator (older_gen)->commit_alloc_list_changes();
22593 // Fix the allocation area of the older generation
22594 fix_older_allocation_area (older_gen);
22596 assert (generation_allocation_segment (consing_gen) ==
22597 ephemeral_heap_segment);
22599 GCToEEInterface::DiagWalkSurvivors(__this);
22601 relocate_phase (condemned_gen_number, first_condemned_address);
22602 compact_phase (condemned_gen_number, first_condemned_address,
22603 (!settings.demotion && settings.promotion));
22604 fix_generation_bounds (condemned_gen_number, consing_gen);
22605 assert (generation_allocation_limit (youngest_generation) ==
22606 generation_allocation_pointer (youngest_generation));
22607 if (condemned_gen_number >= (max_generation -1))
22609 #ifdef MULTIPLE_HEAPS
22610 // this needs be serialized just because we have one
22611 // segment_standby_list/seg_table for all heaps. We should make it at least
22612 // so that when hoarding is not on we don't need this join because
22613 // decommitting memory can take a long time.
22614 //must serialize on deleting segments
22615 gc_t_join.join(this, gc_join_rearrange_segs_compaction);
22616 if (gc_t_join.joined())
22618 for (int i = 0; i < n_heaps; i++)
22620 g_heaps[i]->rearrange_heap_segments(TRUE);
22622 gc_t_join.restart();
22625 rearrange_heap_segments(TRUE);
22626 #endif //MULTIPLE_HEAPS
22630 //fix the start_segment for the ephemeral generations
22631 for (int i = 0; i < max_generation; i++)
22633 generation* gen = generation_of (i);
22634 generation_start_segment (gen) = ephemeral_heap_segment;
22635 generation_allocation_segment (gen) = ephemeral_heap_segment;
22641 #ifdef FEATURE_PREMORTEM_FINALIZATION
22642 finalize_queue->UpdatePromotedGenerations (condemned_gen_number,
22643 (!settings.demotion && settings.promotion));
22644 #endif // FEATURE_PREMORTEM_FINALIZATION
22646 #ifdef MULTIPLE_HEAPS
22647 dprintf(3, ("Joining after end of compaction"));
22648 gc_t_join.join(this, gc_join_adjust_handle_age_compact);
22649 if (gc_t_join.joined())
22650 #endif //MULTIPLE_HEAPS
22652 #ifdef MULTIPLE_HEAPS
22653 //join all threads to make sure they are synchronized
22654 dprintf(3, ("Restarting after Promotion granted"));
22655 gc_t_join.restart();
22656 #endif //MULTIPLE_HEAPS
22660 sc.thread_number = heap_number;
22661 sc.promotion = FALSE;
22662 sc.concurrent = FALSE;
22663 // new generations bounds are set can call this guy
22664 if (settings.promotion && !settings.demotion)
22666 dprintf (2, ("Promoting EE roots for gen %d",
22667 condemned_gen_number));
22668 GCScan::GcPromotionsGranted(condemned_gen_number,
22669 max_generation, &sc);
22671 else if (settings.demotion)
22673 dprintf (2, ("Demoting EE roots for gen %d",
22674 condemned_gen_number));
22675 GCScan::GcDemote (condemned_gen_number, max_generation, &sc);
22680 gen0_big_free_spaces = 0;
22682 reset_pinned_queue_bos();
22683 unsigned int gen_number = min (max_generation, 1 + condemned_gen_number);
22684 generation* gen = generation_of (gen_number);
22685 uint8_t* low = generation_allocation_start (generation_of (gen_number-1));
22686 uint8_t* high = heap_segment_allocated (ephemeral_heap_segment);
22688 while (!pinned_plug_que_empty_p())
22690 mark* m = pinned_plug_of (deque_pinned_plug());
22691 size_t len = pinned_len (m);
22692 uint8_t* arr = (pinned_plug (m) - len);
22693 dprintf(3,("free [%Ix %Ix[ pin",
22694 (size_t)arr, (size_t)arr + len));
22697 assert (len >= Align (min_obj_size));
22698 make_unused_array (arr, len);
22699 // fix fully contained bricks + first one
22700 // if the array goes beyond the first brick
22701 size_t start_brick = brick_of (arr);
22702 size_t end_brick = brick_of (arr + len);
22703 if (end_brick != start_brick)
22706 ("Fixing bricks [%Ix, %Ix[ to point to unused array %Ix",
22707 start_brick, end_brick, (size_t)arr));
22708 set_brick (start_brick,
22709 arr - brick_address (start_brick));
22710 size_t brick = start_brick+1;
22711 while (brick < end_brick)
22713 set_brick (brick, start_brick - brick);
22718 //when we take an old segment to make the new
22719 //ephemeral segment. we can have a bunch of
22720 //pinned plugs out of order going to the new ephemeral seg
22721 //and then the next plugs go back to max_generation
22722 if ((heap_segment_mem (ephemeral_heap_segment) <= arr) &&
22723 (heap_segment_reserved (ephemeral_heap_segment) > arr))
22726 while ((low <= arr) && (high > arr))
22729 assert ((gen_number >= 1) || (demotion_low != MAX_PTR) ||
22730 settings.demotion || !settings.promotion);
22731 dprintf (3, ("new free list generation %d", gen_number));
22733 gen = generation_of (gen_number);
22734 if (gen_number >= 1)
22735 low = generation_allocation_start (generation_of (gen_number-1));
22742 dprintf (3, ("new free list generation %d", max_generation));
22743 gen_number = max_generation;
22744 gen = generation_of (gen_number);
22747 dprintf(3,("threading it into generation %d", gen_number));
22748 thread_gap (arr, len, gen);
22749 add_gen_free (gen_number, len);
22755 for (int x = 0; x <= max_generation; x++)
22757 assert (generation_allocation_start (generation_of (x)));
22761 if (!settings.demotion && settings.promotion)
22763 //clear card for generation 1. generation 0 is empty
22764 clear_card_for_addresses (
22765 generation_allocation_start (generation_of (1)),
22766 generation_allocation_start (generation_of (0)));
22768 if (settings.promotion && !settings.demotion)
22770 uint8_t* start = generation_allocation_start (youngest_generation);
22771 MAYBE_UNUSED_VAR(start);
22772 assert (heap_segment_allocated (ephemeral_heap_segment) ==
22773 (start + Align (size (start))));
22778 //force promotion for sweep
22779 settings.promotion = TRUE;
22780 settings.compaction = FALSE;
22783 sc.thread_number = heap_number;
22784 sc.promotion = FALSE;
22785 sc.concurrent = FALSE;
22787 dprintf (2, ("**** Doing Mark and Sweep GC****"));
22789 if ((condemned_gen_number < max_generation))
22791 generation_allocator (older_gen)->copy_from_alloc_list (r_free_list);
22792 generation_free_list_space (older_gen) = r_free_list_space;
22793 generation_free_obj_space (older_gen) = r_free_obj_space;
22794 generation_free_list_allocated (older_gen) = r_older_gen_free_list_allocated;
22795 generation_end_seg_allocated (older_gen) = r_older_gen_end_seg_allocated;
22796 generation_condemned_allocated (older_gen) = r_older_gen_condemned_allocated;
22797 generation_allocation_limit (older_gen) = r_allocation_limit;
22798 generation_allocation_pointer (older_gen) = r_allocation_pointer;
22799 generation_allocation_context_start_region (older_gen) = r_allocation_start_region;
22800 generation_allocation_segment (older_gen) = r_allocation_segment;
22803 if ((condemned_gen_number < max_generation))
22805 // Fix the allocation area of the older generation
22806 fix_older_allocation_area (older_gen);
22809 GCToEEInterface::DiagWalkSurvivors(__this);
22811 gen0_big_free_spaces = 0;
22812 make_free_lists (condemned_gen_number);
22813 recover_saved_pinned_info();
22815 #ifdef FEATURE_PREMORTEM_FINALIZATION
22816 finalize_queue->UpdatePromotedGenerations (condemned_gen_number, TRUE);
22817 #endif // FEATURE_PREMORTEM_FINALIZATION
22818 // MTHTS: leave single thread for HT processing on plan_phase
22819 #ifdef MULTIPLE_HEAPS
22820 dprintf(3, ("Joining after end of sweep"));
22821 gc_t_join.join(this, gc_join_adjust_handle_age_sweep);
22822 if (gc_t_join.joined())
22823 #endif //MULTIPLE_HEAPS
22825 GCScan::GcPromotionsGranted(condemned_gen_number,
22826 max_generation, &sc);
22827 if (condemned_gen_number >= (max_generation -1))
22829 #ifdef MULTIPLE_HEAPS
22830 for (int i = 0; i < n_heaps; i++)
22832 g_heaps[i]->rearrange_heap_segments(FALSE);
22835 rearrange_heap_segments(FALSE);
22836 #endif //MULTIPLE_HEAPS
22839 #ifdef MULTIPLE_HEAPS
22840 //join all threads to make sure they are synchronized
22841 dprintf(3, ("Restarting after Promotion granted"));
22842 gc_t_join.restart();
22843 #endif //MULTIPLE_HEAPS
22847 for (int x = 0; x <= max_generation; x++)
22849 assert (generation_allocation_start (generation_of (x)));
22853 //clear card for generation 1. generation 0 is empty
22854 clear_card_for_addresses (
22855 generation_allocation_start (generation_of (1)),
22856 generation_allocation_start (generation_of (0)));
22857 assert ((heap_segment_allocated (ephemeral_heap_segment) ==
22858 (generation_allocation_start (youngest_generation) +
22859 Align (min_obj_size))));
22862 //verify_partial();
22865 #pragma warning(pop)
22869 /*****************************
22870 Called after compact phase to fix all generation gaps
22871 ********************************/
22872 void gc_heap::fix_generation_bounds (int condemned_gen_number,
22873 generation* consing_gen)
22875 UNREFERENCED_PARAMETER(consing_gen);
22877 assert (generation_allocation_segment (consing_gen) ==
22878 ephemeral_heap_segment);
22880 //assign the planned allocation start to the generation
22881 int gen_number = condemned_gen_number;
22882 int bottom_gen = 0;
22884 while (gen_number >= bottom_gen)
22886 generation* gen = generation_of (gen_number);
22887 dprintf(3,("Fixing generation pointers for %Ix", gen_number));
22888 if ((gen_number < max_generation) && ephemeral_promotion)
22890 make_unused_array (saved_ephemeral_plan_start[gen_number],
22891 saved_ephemeral_plan_start_size[gen_number]);
22893 reset_allocation_pointers (gen, generation_plan_allocation_start (gen));
22894 make_unused_array (generation_allocation_start (gen), generation_plan_allocation_start_size (gen));
22895 dprintf(3,(" start %Ix", (size_t)generation_allocation_start (gen)));
22898 #ifdef MULTIPLE_HEAPS
22899 if (ephemeral_promotion)
22901 //we are creating a generation fault. set the cards.
22902 // and we are only doing this for multiple heaps because in the single heap scenario the
22903 // new ephemeral generations will be empty and there'll be no need to set cards for the
22904 // old ephemeral generations that got promoted into max_generation.
22905 ptrdiff_t delta = 0;
22906 #ifdef SEG_MAPPING_TABLE
22907 heap_segment* old_ephemeral_seg = seg_mapping_table_segment_of (saved_ephemeral_plan_start[max_generation-1]);
22908 #else //SEG_MAPPING_TABLE
22909 heap_segment* old_ephemeral_seg = segment_of (saved_ephemeral_plan_start[max_generation-1], delta);
22910 #endif //SEG_MAPPING_TABLE
22912 assert (in_range_for_segment (saved_ephemeral_plan_start[max_generation-1], old_ephemeral_seg));
22913 size_t end_card = card_of (align_on_card (heap_segment_plan_allocated (old_ephemeral_seg)));
22914 size_t card = card_of (saved_ephemeral_plan_start[max_generation-1]);
22915 while (card != end_card)
22921 #endif //MULTIPLE_HEAPS
22923 alloc_allocated = heap_segment_plan_allocated(ephemeral_heap_segment);
22924 //reset the allocated size
22925 uint8_t* start = generation_allocation_start (youngest_generation);
22926 MAYBE_UNUSED_VAR(start);
22927 if (settings.promotion && !settings.demotion)
22929 assert ((start + Align (size (start))) ==
22930 heap_segment_plan_allocated(ephemeral_heap_segment));
22933 heap_segment_allocated(ephemeral_heap_segment)=
22934 heap_segment_plan_allocated(ephemeral_heap_segment);
22938 uint8_t* gc_heap::generation_limit (int gen_number)
22940 if (settings.promotion)
22942 if (gen_number <= 1)
22943 return heap_segment_reserved (ephemeral_heap_segment);
22945 return generation_allocation_start (generation_of ((gen_number - 2)));
22949 if (gen_number <= 0)
22950 return heap_segment_reserved (ephemeral_heap_segment);
22952 return generation_allocation_start (generation_of ((gen_number - 1)));
22956 BOOL gc_heap::ensure_gap_allocation (int condemned_gen_number)
22958 uint8_t* start = heap_segment_allocated (ephemeral_heap_segment);
22959 size_t size = Align (min_obj_size)*(condemned_gen_number+1);
22960 assert ((start + size) <=
22961 heap_segment_reserved (ephemeral_heap_segment));
22962 if ((start + size) >
22963 heap_segment_committed (ephemeral_heap_segment))
22965 if (!grow_heap_segment (ephemeral_heap_segment, start + size))
22973 uint8_t* gc_heap::allocate_at_end (size_t size)
22975 uint8_t* start = heap_segment_allocated (ephemeral_heap_segment);
22976 size = Align (size);
22977 uint8_t* result = start;
22978 // only called to allocate a min obj so can't overflow here.
22979 assert ((start + size) <=
22980 heap_segment_reserved (ephemeral_heap_segment));
22981 //ensure_gap_allocation took care of it
22982 assert ((start + size) <=
22983 heap_segment_committed (ephemeral_heap_segment));
22984 heap_segment_allocated (ephemeral_heap_segment) += size;
22989 void gc_heap::make_free_lists (int condemned_gen_number)
22994 start = GetCycleCount32();
22997 //Promotion has to happen in sweep case.
22998 assert (settings.promotion);
23000 generation* condemned_gen = generation_of (condemned_gen_number);
23001 uint8_t* start_address = generation_allocation_start (condemned_gen);
23003 size_t current_brick = brick_of (start_address);
23004 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
23006 PREFIX_ASSUME(current_heap_segment != NULL);
23008 uint8_t* end_address = heap_segment_allocated (current_heap_segment);
23009 size_t end_brick = brick_of (end_address-1);
23010 make_free_args args;
23011 args.free_list_gen_number = min (max_generation, 1 + condemned_gen_number);
23012 args.current_gen_limit = (((condemned_gen_number == max_generation)) ?
23014 (generation_limit (args.free_list_gen_number)));
23015 args.free_list_gen = generation_of (args.free_list_gen_number);
23016 args.highest_plug = 0;
23018 if ((start_address < end_address) ||
23019 (condemned_gen_number == max_generation))
23023 if ((current_brick > end_brick))
23025 if (args.current_gen_limit == MAX_PTR)
23027 //We had an empty segment
23028 //need to allocate the generation start
23030 generation* gen = generation_of (max_generation);
23032 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
23034 PREFIX_ASSUME(start_seg != NULL);
23036 uint8_t* gap = heap_segment_mem (start_seg);
23038 generation_allocation_start (gen) = gap;
23039 heap_segment_allocated (start_seg) = gap + Align (min_obj_size);
23040 make_unused_array (gap, Align (min_obj_size));
23041 reset_allocation_pointers (gen, gap);
23042 dprintf (3, ("Start segment empty, fixing generation start of %d to: %Ix",
23043 max_generation, (size_t)gap));
23044 args.current_gen_limit = generation_limit (args.free_list_gen_number);
23046 if (heap_segment_next_rw (current_heap_segment))
23048 current_heap_segment = heap_segment_next_rw (current_heap_segment);
23049 current_brick = brick_of (heap_segment_mem (current_heap_segment));
23050 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
23060 int brick_entry = brick_table [ current_brick ];
23061 if ((brick_entry >= 0))
23063 make_free_list_in_brick (brick_address (current_brick) + brick_entry-1, &args);
23064 dprintf(3,("Fixing brick entry %Ix to %Ix",
23065 current_brick, (size_t)args.highest_plug));
23066 set_brick (current_brick,
23067 (args.highest_plug - brick_address (current_brick)));
23071 if ((brick_entry > -32768))
23075 ptrdiff_t offset = brick_of (args.highest_plug) - current_brick;
23076 if ((brick_entry != -32767) && (! ((offset == brick_entry))))
23078 assert ((brick_entry == -1));
23081 //init to -1 for faster find_first_object
23082 set_brick (current_brick, -1);
23090 int bottom_gen = 0;
23091 args.free_list_gen_number--;
23092 while (args.free_list_gen_number >= bottom_gen)
23095 generation* gen2 = generation_of (args.free_list_gen_number);
23096 gap = allocate_at_end (Align(min_obj_size));
23097 generation_allocation_start (gen2) = gap;
23098 reset_allocation_pointers (gen2, gap);
23099 dprintf(3,("Fixing generation start of %d to: %Ix",
23100 args.free_list_gen_number, (size_t)gap));
23101 PREFIX_ASSUME(gap != NULL);
23102 make_unused_array (gap, Align (min_obj_size));
23104 args.free_list_gen_number--;
23107 //reset the allocated size
23108 uint8_t* start2 = generation_allocation_start (youngest_generation);
23109 alloc_allocated = start2 + Align (size (start2));
23113 finish = GetCycleCount32();
23114 sweep_time = finish - start;
23118 void gc_heap::make_free_list_in_brick (uint8_t* tree, make_free_args* args)
23120 assert ((tree != NULL));
23122 int right_node = node_right_child (tree);
23123 int left_node = node_left_child (tree);
23124 args->highest_plug = 0;
23127 if (! (0 == left_node))
23129 make_free_list_in_brick (tree + left_node, args);
23133 uint8_t* plug = tree;
23134 size_t gap_size = node_gap_size (tree);
23135 uint8_t* gap = (plug - gap_size);
23136 dprintf (3,("Making free list %Ix len %d in %d",
23137 //dprintf (3,("F: %Ix len %Ix in %d",
23138 (size_t)gap, gap_size, args->free_list_gen_number));
23139 args->highest_plug = tree;
23141 if (is_plug_padded (plug))
23143 dprintf (3, ("%Ix padded", plug));
23144 clear_plug_padded (plug);
23146 #endif //SHORT_PLUGS
23149 if ((args->current_gen_limit == MAX_PTR) ||
23150 ((plug >= args->current_gen_limit) &&
23151 ephemeral_pointer_p (plug)))
23153 dprintf(3,(" Crossing Generation boundary at %Ix",
23154 (size_t)args->current_gen_limit));
23155 if (!(args->current_gen_limit == MAX_PTR))
23157 args->free_list_gen_number--;
23158 args->free_list_gen = generation_of (args->free_list_gen_number);
23160 dprintf(3,( " Fixing generation start of %d to: %Ix",
23161 args->free_list_gen_number, (size_t)gap));
23163 reset_allocation_pointers (args->free_list_gen, gap);
23164 args->current_gen_limit = generation_limit (args->free_list_gen_number);
23166 if ((gap_size >= (2*Align (min_obj_size))))
23168 dprintf(3,(" Splitting the gap in two %Id left",
23170 make_unused_array (gap, Align(min_obj_size));
23171 gap_size = (gap_size - Align(min_obj_size));
23172 gap = (gap + Align(min_obj_size));
23176 make_unused_array (gap, gap_size);
23183 thread_gap (gap, gap_size, args->free_list_gen);
23184 add_gen_free (args->free_list_gen->gen_num, gap_size);
23186 if (! (0 == right_node))
23188 make_free_list_in_brick (tree + right_node, args);
23194 void gc_heap::thread_gap (uint8_t* gap_start, size_t size, generation* gen)
23196 assert (generation_allocation_start (gen));
23199 if ((gen->gen_num == 0) && (size > CLR_SIZE))
23201 gen0_big_free_spaces += size;
23204 assert ((heap_segment_rw (generation_start_segment (gen))!=
23205 ephemeral_heap_segment) ||
23206 (gap_start > generation_allocation_start (gen)));
23207 // The beginning of a segment gap is not aligned
23208 assert (size >= Align (min_obj_size));
23209 make_unused_array (gap_start, size,
23210 (!settings.concurrent && (gen != youngest_generation)),
23211 (gen->gen_num == max_generation));
23212 dprintf (3, ("fr: [%Ix, %Ix[", (size_t)gap_start, (size_t)gap_start+size));
23214 if ((size >= min_free_list))
23216 generation_free_list_space (gen) += size;
23217 generation_allocator (gen)->thread_item (gap_start, size);
23221 generation_free_obj_space (gen) += size;
23226 void gc_heap::loh_thread_gap_front (uint8_t* gap_start, size_t size, generation* gen)
23228 assert (generation_allocation_start (gen));
23229 if (size >= min_free_list)
23231 generation_free_list_space (gen) += size;
23232 generation_allocator (gen)->thread_item_front (gap_start, size);
23236 void gc_heap::make_unused_array (uint8_t* x, size_t size, BOOL clearp, BOOL resetp)
23238 dprintf (3, ("Making unused array [%Ix, %Ix[",
23239 (size_t)x, (size_t)(x+size)));
23240 assert (size >= Align (min_obj_size));
23242 //#if defined (VERIFY_HEAP) && defined (BACKGROUND_GC)
23243 // check_batch_mark_array_bits (x, x+size);
23244 //#endif //VERIFY_HEAP && BACKGROUND_GC
23247 reset_memory (x, size);
23249 ((CObjectHeader*)x)->SetFree(size);
23254 #error "This won't work on big endian platforms"
23257 size_t size_as_object = (uint32_t)(size - free_object_base_size) + free_object_base_size;
23259 if (size_as_object < size)
23262 // If the size is more than 4GB, we need to create multiple objects because of
23263 // the Array::m_NumComponents is uint32_t and the high 32 bits of unused array
23264 // size is ignored in regular object size computation.
23266 uint8_t * tmp = x + size_as_object;
23267 size_t remaining_size = size - size_as_object;
23269 while (remaining_size > UINT32_MAX)
23271 // Make sure that there will be at least Align(min_obj_size) left
23272 size_t current_size = UINT32_MAX - get_alignment_constant (FALSE)
23273 - Align (min_obj_size, get_alignment_constant (FALSE));
23275 ((CObjectHeader*)tmp)->SetFree(current_size);
23277 remaining_size -= current_size;
23278 tmp += current_size;
23281 ((CObjectHeader*)tmp)->SetFree(remaining_size);
23286 clear_card_for_addresses (x, x + Align(size));
23289 // Clear memory set by make_unused_array.
23290 void gc_heap::clear_unused_array (uint8_t* x, size_t size)
23292 // Also clear the sync block
23293 *(((PTR_PTR)x)-1) = 0;
23295 ((CObjectHeader*)x)->UnsetFree();
23300 #error "This won't work on big endian platforms"
23303 // The memory could have been cleared in the meantime. We have to mirror the algorithm
23304 // from make_unused_array since we cannot depend on the object sizes in memory.
23305 size_t size_as_object = (uint32_t)(size - free_object_base_size) + free_object_base_size;
23307 if (size_as_object < size)
23309 uint8_t * tmp = x + size_as_object;
23310 size_t remaining_size = size - size_as_object;
23312 while (remaining_size > UINT32_MAX)
23314 size_t current_size = UINT32_MAX - get_alignment_constant (FALSE)
23315 - Align (min_obj_size, get_alignment_constant (FALSE));
23317 ((CObjectHeader*)tmp)->UnsetFree();
23319 remaining_size -= current_size;
23320 tmp += current_size;
23323 ((CObjectHeader*)tmp)->UnsetFree();
23326 UNREFERENCED_PARAMETER(size);
23331 uint8_t* tree_search (uint8_t* tree, uint8_t* old_address)
23333 uint8_t* candidate = 0;
23337 if (tree < old_address)
23339 if ((cn = node_right_child (tree)) != 0)
23341 assert (candidate < tree);
23344 Prefetch (tree - 8);
23350 else if (tree > old_address)
23352 if ((cn = node_left_child (tree)) != 0)
23355 Prefetch (tree - 8);
23363 if (tree <= old_address)
23365 else if (candidate)
23371 #ifdef FEATURE_BASICFREEZE
23372 bool gc_heap::frozen_object_p (Object* obj)
23374 heap_segment* pSegment = gc_heap::find_segment ((uint8_t*)obj, FALSE);
23375 _ASSERTE(pSegment);
23377 return heap_segment_read_only_p(pSegment);
23379 #endif // FEATURE_BASICFREEZE
23381 #ifdef FEATURE_REDHAWK
23382 // TODO: this was added on RH, we have not done perf runs to see if this is the right
23383 // thing to do for other versions of the CLR.
23385 #endif // FEATURE_REDHAWK
23386 void gc_heap::relocate_address (uint8_t** pold_address THREAD_NUMBER_DCL)
23388 uint8_t* old_address = *pold_address;
23389 if (!((old_address >= gc_low) && (old_address < gc_high)))
23390 #ifdef MULTIPLE_HEAPS
23392 UNREFERENCED_PARAMETER(thread);
23393 if (old_address == 0)
23395 gc_heap* hp = heap_of (old_address);
23396 if ((hp == this) ||
23397 !((old_address >= hp->gc_low) && (old_address < hp->gc_high)))
23400 #else //MULTIPLE_HEAPS
23402 #endif //MULTIPLE_HEAPS
23403 // delta translates old_address into address_gc (old_address);
23404 size_t brick = brick_of (old_address);
23405 int brick_entry = brick_table [ brick ];
23406 uint8_t* new_address = old_address;
23407 if (! ((brick_entry == 0)))
23411 while (brick_entry < 0)
23413 brick = (brick + brick_entry);
23414 brick_entry = brick_table [ brick ];
23416 uint8_t* old_loc = old_address;
23418 uint8_t* node = tree_search ((brick_address (brick) + brick_entry-1),
23420 if ((node <= old_loc))
23421 new_address = (old_address + node_relocation_distance (node));
23424 if (node_left_p (node))
23426 dprintf(3,(" L: %Ix", (size_t)node));
23427 new_address = (old_address +
23428 (node_relocation_distance (node) +
23429 node_gap_size (node)));
23434 brick_entry = brick_table [ brick ];
23440 *pold_address = new_address;
23444 #ifdef FEATURE_LOH_COMPACTION
23445 if (loh_compacted_p
23446 #ifdef FEATURE_BASICFREEZE
23447 && !frozen_object_p((Object*)old_address)
23448 #endif // FEATURE_BASICFREEZE
23451 *pold_address = old_address + loh_node_relocation_distance (old_address);
23454 #endif //FEATURE_LOH_COMPACTION
23456 *pold_address = new_address;
23461 gc_heap::check_class_object_demotion (uint8_t* obj)
23463 #ifdef COLLECTIBLE_CLASS
23464 if (is_collectible(obj))
23466 check_class_object_demotion_internal (obj);
23469 UNREFERENCED_PARAMETER(obj);
23470 #endif //COLLECTIBLE_CLASS
23473 #ifdef COLLECTIBLE_CLASS
23475 gc_heap::check_class_object_demotion_internal (uint8_t* obj)
23477 if (settings.demotion)
23479 #ifdef MULTIPLE_HEAPS
23480 // We set the card without checking the demotion range 'cause at this point
23481 // the handle that points to the loader allocator object may or may not have
23482 // been relocated by other GC threads.
23483 set_card (card_of (obj));
23486 uint8_t* class_obj = get_class_object (obj);
23487 dprintf (3, ("%Ix: got classobj %Ix", obj, class_obj));
23488 uint8_t* temp_class_obj = class_obj;
23489 uint8_t** temp = &temp_class_obj;
23490 relocate_address (temp THREAD_NUMBER_ARG);
23492 check_demotion_helper (temp, obj);
23493 #endif //MULTIPLE_HEAPS
23497 #endif //COLLECTIBLE_CLASS
23500 gc_heap::check_demotion_helper (uint8_t** pval, uint8_t* parent_obj)
23502 // detect if we are demoting an object
23503 if ((*pval < demotion_high) &&
23504 (*pval >= demotion_low))
23506 dprintf(3, ("setting card %Ix:%Ix",
23507 card_of((uint8_t*)pval),
23510 set_card (card_of (parent_obj));
23512 #ifdef MULTIPLE_HEAPS
23513 else if (settings.demotion)
23515 dprintf (4, ("Demotion active, computing heap_of object"));
23516 gc_heap* hp = heap_of (*pval);
23517 if ((*pval < hp->demotion_high) &&
23518 (*pval >= hp->demotion_low))
23520 dprintf(3, ("setting card %Ix:%Ix",
23521 card_of((uint8_t*)pval),
23524 set_card (card_of (parent_obj));
23527 #endif //MULTIPLE_HEAPS
23531 gc_heap::reloc_survivor_helper (uint8_t** pval)
23534 relocate_address (pval THREAD_NUMBER_ARG);
23536 check_demotion_helper (pval, (uint8_t*)pval);
23540 gc_heap::relocate_obj_helper (uint8_t* x, size_t s)
23543 if (contain_pointers (x))
23545 dprintf (3, ("$%Ix$", (size_t)x));
23547 go_through_object_nostart (method_table(x), x, s, pval,
23549 uint8_t* child = *pval;
23550 reloc_survivor_helper (pval);
23553 dprintf (3, ("%Ix->%Ix->%Ix", (uint8_t*)pval, child, *pval));
23558 check_class_object_demotion (x);
23562 void gc_heap::reloc_ref_in_shortened_obj (uint8_t** address_to_set_card, uint8_t** address_to_reloc)
23566 uint8_t* old_val = (address_to_reloc ? *address_to_reloc : 0);
23567 relocate_address (address_to_reloc THREAD_NUMBER_ARG);
23568 if (address_to_reloc)
23570 dprintf (3, ("SR %Ix: %Ix->%Ix", (uint8_t*)address_to_reloc, old_val, *address_to_reloc));
23573 //check_demotion_helper (current_saved_info_to_relocate, (uint8_t*)pval);
23574 uint8_t* relocated_addr = *address_to_reloc;
23575 if ((relocated_addr < demotion_high) &&
23576 (relocated_addr >= demotion_low))
23578 dprintf (3, ("set card for location %Ix(%Ix)",
23579 (size_t)address_to_set_card, card_of((uint8_t*)address_to_set_card)));
23581 set_card (card_of ((uint8_t*)address_to_set_card));
23583 #ifdef MULTIPLE_HEAPS
23584 else if (settings.demotion)
23586 gc_heap* hp = heap_of (relocated_addr);
23587 if ((relocated_addr < hp->demotion_high) &&
23588 (relocated_addr >= hp->demotion_low))
23590 dprintf (3, ("%Ix on h%d, set card for location %Ix(%Ix)",
23591 relocated_addr, hp->heap_number, (size_t)address_to_set_card, card_of((uint8_t*)address_to_set_card)));
23593 set_card (card_of ((uint8_t*)address_to_set_card));
23596 #endif //MULTIPLE_HEAPS
23599 void gc_heap::relocate_pre_plug_info (mark* pinned_plug_entry)
23602 uint8_t* plug = pinned_plug (pinned_plug_entry);
23603 uint8_t* pre_plug_start = plug - sizeof (plug_and_gap);
23604 // Note that we need to add one ptr size here otherwise we may not be able to find the relocated
23605 // address. Consider this scenario:
23606 // gen1 start | 3-ptr sized NP | PP
23608 // If we are asking for the reloc address of 0x10 we will AV in relocate_address because
23609 // the first plug we saw in the brick is 0x18 which means 0x10 will cause us to go back a brick
23610 // which is 0, and then we'll AV in tree_search when we try to do node_right_child (tree).
23611 pre_plug_start += sizeof (uint8_t*);
23612 uint8_t** old_address = &pre_plug_start;
23614 uint8_t* old_val = (old_address ? *old_address : 0);
23615 relocate_address (old_address THREAD_NUMBER_ARG);
23618 dprintf (3, ("PreR %Ix: %Ix->%Ix, set reloc: %Ix",
23619 (uint8_t*)old_address, old_val, *old_address, (pre_plug_start - sizeof (uint8_t*))));
23622 pinned_plug_entry->set_pre_plug_info_reloc_start (pre_plug_start - sizeof (uint8_t*));
23626 void gc_heap::relocate_shortened_obj_helper (uint8_t* x, size_t s, uint8_t* end, mark* pinned_plug_entry, BOOL is_pinned)
23629 uint8_t* plug = pinned_plug (pinned_plug_entry);
23633 //// Temporary - we just wanna make sure we are doing things right when padding is needed.
23634 //if ((x + s) < plug)
23636 // dprintf (3, ("obj %Ix needed padding: end %Ix is %d bytes from pinned obj %Ix",
23637 // x, (x + s), (plug- (x + s)), plug));
23638 // GCToOSInterface::DebugBreak();
23641 relocate_pre_plug_info (pinned_plug_entry);
23644 verify_pins_with_post_plug_info("after relocate_pre_plug_info");
23646 uint8_t* saved_plug_info_start = 0;
23647 uint8_t** saved_info_to_relocate = 0;
23651 saved_plug_info_start = (uint8_t*)(pinned_plug_entry->get_post_plug_info_start());
23652 saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_post_plug_reloc_info());
23656 saved_plug_info_start = (plug - sizeof (plug_and_gap));
23657 saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_pre_plug_reloc_info());
23660 uint8_t** current_saved_info_to_relocate = 0;
23661 uint8_t* child = 0;
23663 dprintf (3, ("x: %Ix, pp: %Ix, end: %Ix", x, plug, end));
23665 if (contain_pointers (x))
23667 dprintf (3,("$%Ix$", (size_t)x));
23669 go_through_object_nostart (method_table(x), x, s, pval,
23671 dprintf (3, ("obj %Ix, member: %Ix->%Ix", x, (uint8_t*)pval, *pval));
23673 if ((uint8_t*)pval >= end)
23675 current_saved_info_to_relocate = saved_info_to_relocate + ((uint8_t*)pval - saved_plug_info_start) / sizeof (uint8_t**);
23676 child = *current_saved_info_to_relocate;
23677 reloc_ref_in_shortened_obj (pval, current_saved_info_to_relocate);
23678 dprintf (3, ("last part: R-%Ix(saved: %Ix)->%Ix ->%Ix",
23679 (uint8_t*)pval, current_saved_info_to_relocate, child, *current_saved_info_to_relocate));
23683 reloc_survivor_helper (pval);
23688 check_class_object_demotion (x);
23691 void gc_heap::relocate_survivor_helper (uint8_t* plug, uint8_t* plug_end)
23694 while (x < plug_end)
23696 size_t s = size (x);
23697 uint8_t* next_obj = x + Align (s);
23698 Prefetch (next_obj);
23699 relocate_obj_helper (x, s);
23705 // if we expanded, right now we are not handling it as We are not saving the new reloc info.
23706 void gc_heap::verify_pins_with_post_plug_info (const char* msg)
23708 #if defined (_DEBUG) && defined (VERIFY_HEAP)
23709 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
23711 if (!verify_pinned_queue_p)
23714 if (settings.heap_expansion)
23717 for (size_t i = 0; i < mark_stack_tos; i++)
23719 mark& m = mark_stack_array[i];
23721 mark* pinned_plug_entry = pinned_plug_of(i);
23723 if (pinned_plug_entry->has_post_plug_info() &&
23724 pinned_plug_entry->post_short_p() &&
23725 (pinned_plug_entry->saved_post_plug_debug.gap != 1))
23727 uint8_t* next_obj = pinned_plug_entry->get_post_plug_info_start() + sizeof (plug_and_gap);
23728 // object after pin
23729 dprintf (3, ("OFP: %Ix, G: %Ix, R: %Ix, LC: %d, RC: %d",
23730 next_obj, node_gap_size (next_obj), node_relocation_distance (next_obj),
23731 (int)node_left_child (next_obj), (int)node_right_child (next_obj)));
23733 size_t* post_plug_debug = (size_t*)(&m.saved_post_plug_debug);
23735 if (node_gap_size (next_obj) != *post_plug_debug)
23737 dprintf (3, ("obj: %Ix gap should be %Ix but it is %Ix",
23738 next_obj, *post_plug_debug, (size_t)(node_gap_size (next_obj))));
23742 // can't do node_relocation_distance here as it clears the left bit.
23743 //if (node_relocation_distance (next_obj) != *post_plug_debug)
23744 if (*((size_t*)(next_obj - 3 * sizeof (size_t))) != *post_plug_debug)
23746 dprintf (3, ("obj: %Ix reloc should be %Ix but it is %Ix",
23747 next_obj, *post_plug_debug, (size_t)(node_relocation_distance (next_obj))));
23750 if (node_left_child (next_obj) > 0)
23752 dprintf (3, ("obj: %Ix, vLC: %d\n", next_obj, (int)(node_left_child (next_obj))));
23758 dprintf (3, ("%s verified", msg));
23760 #else // _DEBUG && VERIFY_HEAP
23761 UNREFERENCED_PARAMETER(msg);
23762 #endif // _DEBUG && VERIFY_HEAP
23765 #ifdef COLLECTIBLE_CLASS
23766 // We don't want to burn another ptr size space for pinned plugs to record this so just
23767 // set the card unconditionally for collectible objects if we are demoting.
23769 gc_heap::unconditional_set_card_collectible (uint8_t* obj)
23771 if (settings.demotion)
23773 set_card (card_of (obj));
23776 #endif //COLLECTIBLE_CLASS
23778 void gc_heap::relocate_shortened_survivor_helper (uint8_t* plug, uint8_t* plug_end, mark* pinned_plug_entry)
23781 uint8_t* p_plug = pinned_plug (pinned_plug_entry);
23782 BOOL is_pinned = (plug == p_plug);
23783 BOOL check_short_obj_p = (is_pinned ? pinned_plug_entry->post_short_p() : pinned_plug_entry->pre_short_p());
23785 plug_end += sizeof (gap_reloc_pair);
23787 //dprintf (3, ("%s %Ix is shortened, and last object %s overwritten", (is_pinned ? "PP" : "NP"), plug, (check_short_obj_p ? "is" : "is not")));
23788 dprintf (3, ("%s %Ix-%Ix short, LO: %s OW", (is_pinned ? "PP" : "NP"), plug, plug_end, (check_short_obj_p ? "is" : "is not")));
23790 verify_pins_with_post_plug_info("begin reloc short surv");
23792 while (x < plug_end)
23794 if (check_short_obj_p && ((plug_end - x) < min_pre_pin_obj_size))
23796 dprintf (3, ("last obj %Ix is short", x));
23800 #ifdef COLLECTIBLE_CLASS
23801 if (pinned_plug_entry->post_short_collectible_p())
23802 unconditional_set_card_collectible (x);
23803 #endif //COLLECTIBLE_CLASS
23805 // Relocate the saved references based on bits set.
23806 uint8_t** saved_plug_info_start = (uint8_t**)(pinned_plug_entry->get_post_plug_info_start());
23807 uint8_t** saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_post_plug_reloc_info());
23808 for (size_t i = 0; i < pinned_plug_entry->get_max_short_bits(); i++)
23810 if (pinned_plug_entry->post_short_bit_p (i))
23812 reloc_ref_in_shortened_obj ((saved_plug_info_start + i), (saved_info_to_relocate + i));
23818 #ifdef COLLECTIBLE_CLASS
23819 if (pinned_plug_entry->pre_short_collectible_p())
23820 unconditional_set_card_collectible (x);
23821 #endif //COLLECTIBLE_CLASS
23823 relocate_pre_plug_info (pinned_plug_entry);
23825 // Relocate the saved references based on bits set.
23826 uint8_t** saved_plug_info_start = (uint8_t**)(p_plug - sizeof (plug_and_gap));
23827 uint8_t** saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_pre_plug_reloc_info());
23828 for (size_t i = 0; i < pinned_plug_entry->get_max_short_bits(); i++)
23830 if (pinned_plug_entry->pre_short_bit_p (i))
23832 reloc_ref_in_shortened_obj ((saved_plug_info_start + i), (saved_info_to_relocate + i));
23840 size_t s = size (x);
23841 uint8_t* next_obj = x + Align (s);
23842 Prefetch (next_obj);
23844 if (next_obj >= plug_end)
23846 dprintf (3, ("object %Ix is at the end of the plug %Ix->%Ix",
23847 next_obj, plug, plug_end));
23849 verify_pins_with_post_plug_info("before reloc short obj");
23851 relocate_shortened_obj_helper (x, s, (x + Align (s) - sizeof (plug_and_gap)), pinned_plug_entry, is_pinned);
23855 relocate_obj_helper (x, s);
23862 verify_pins_with_post_plug_info("end reloc short surv");
23865 void gc_heap::relocate_survivors_in_plug (uint8_t* plug, uint8_t* plug_end,
23866 BOOL check_last_object_p,
23867 mark* pinned_plug_entry)
23869 //dprintf(3,("Relocating pointers in Plug [%Ix,%Ix[", (size_t)plug, (size_t)plug_end));
23870 dprintf (3,("RP: [%Ix,%Ix[", (size_t)plug, (size_t)plug_end));
23872 if (check_last_object_p)
23874 relocate_shortened_survivor_helper (plug, plug_end, pinned_plug_entry);
23878 relocate_survivor_helper (plug, plug_end);
23882 void gc_heap::relocate_survivors_in_brick (uint8_t* tree, relocate_args* args)
23884 assert ((tree != NULL));
23886 dprintf (3, ("tree: %Ix, args->last_plug: %Ix, left: %Ix, right: %Ix, gap(t): %Ix",
23887 tree, args->last_plug,
23888 (tree + node_left_child (tree)),
23889 (tree + node_right_child (tree)),
23890 node_gap_size (tree)));
23892 if (node_left_child (tree))
23894 relocate_survivors_in_brick (tree + node_left_child (tree), args);
23897 uint8_t* plug = tree;
23898 BOOL has_post_plug_info_p = FALSE;
23899 BOOL has_pre_plug_info_p = FALSE;
23901 if (tree == oldest_pinned_plug)
23903 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
23904 &has_post_plug_info_p);
23905 assert (tree == pinned_plug (args->pinned_plug_entry));
23907 dprintf (3, ("tree is the oldest pin: %Ix", tree));
23909 if (args->last_plug)
23911 size_t gap_size = node_gap_size (tree);
23912 uint8_t* gap = (plug - gap_size);
23913 dprintf (3, ("tree: %Ix, gap: %Ix (%Ix)", tree, gap, gap_size));
23914 assert (gap_size >= Align (min_obj_size));
23915 uint8_t* last_plug_end = gap;
23917 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
23920 relocate_survivors_in_plug (args->last_plug, last_plug_end, check_last_object_p, args->pinned_plug_entry);
23925 assert (!has_pre_plug_info_p);
23928 args->last_plug = plug;
23929 args->is_shortened = has_post_plug_info_p;
23930 if (has_post_plug_info_p)
23932 dprintf (3, ("setting %Ix as shortened", plug));
23934 dprintf (3, ("last_plug: %Ix(shortened: %d)", plug, (args->is_shortened ? 1 : 0)));
23936 if (node_right_child (tree))
23938 relocate_survivors_in_brick (tree + node_right_child (tree), args);
23943 void gc_heap::update_oldest_pinned_plug()
23945 oldest_pinned_plug = (pinned_plug_que_empty_p() ? 0 : pinned_plug (oldest_pin()));
23948 void gc_heap::relocate_survivors (int condemned_gen_number,
23949 uint8_t* first_condemned_address)
23951 generation* condemned_gen = generation_of (condemned_gen_number);
23952 uint8_t* start_address = first_condemned_address;
23953 size_t current_brick = brick_of (start_address);
23954 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
23956 PREFIX_ASSUME(current_heap_segment != NULL);
23958 uint8_t* end_address = 0;
23960 reset_pinned_queue_bos();
23961 update_oldest_pinned_plug();
23963 end_address = heap_segment_allocated (current_heap_segment);
23965 size_t end_brick = brick_of (end_address - 1);
23966 relocate_args args;
23968 args.high = gc_high;
23969 args.is_shortened = FALSE;
23970 args.pinned_plug_entry = 0;
23971 args.last_plug = 0;
23974 if (current_brick > end_brick)
23976 if (args.last_plug)
23979 assert (!(args.is_shortened));
23980 relocate_survivors_in_plug (args.last_plug,
23981 heap_segment_allocated (current_heap_segment),
23983 args.pinned_plug_entry);
23986 args.last_plug = 0;
23989 if (heap_segment_next_rw (current_heap_segment))
23991 current_heap_segment = heap_segment_next_rw (current_heap_segment);
23992 current_brick = brick_of (heap_segment_mem (current_heap_segment));
23993 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24002 int brick_entry = brick_table [ current_brick ];
24004 if (brick_entry >= 0)
24006 relocate_survivors_in_brick (brick_address (current_brick) +
24015 void gc_heap::walk_plug (uint8_t* plug, size_t size, BOOL check_last_object_p, walk_relocate_args* args)
24017 if (check_last_object_p)
24019 size += sizeof (gap_reloc_pair);
24020 mark* entry = args->pinned_plug_entry;
24022 if (args->is_shortened)
24024 assert (entry->has_post_plug_info());
24025 entry->swap_post_plug_and_saved_for_profiler();
24029 assert (entry->has_pre_plug_info());
24030 entry->swap_pre_plug_and_saved_for_profiler();
24034 ptrdiff_t last_plug_relocation = node_relocation_distance (plug);
24035 STRESS_LOG_PLUG_MOVE(plug, (plug + size), -last_plug_relocation);
24036 ptrdiff_t reloc = settings.compaction ? last_plug_relocation : 0;
24038 (args->fn) (plug, (plug + size), reloc, args->profiling_context, !!settings.compaction, false);
24040 if (check_last_object_p)
24042 mark* entry = args->pinned_plug_entry;
24044 if (args->is_shortened)
24046 entry->swap_post_plug_and_saved_for_profiler();
24050 entry->swap_pre_plug_and_saved_for_profiler();
24055 void gc_heap::walk_relocation_in_brick (uint8_t* tree, walk_relocate_args* args)
24057 assert ((tree != NULL));
24058 if (node_left_child (tree))
24060 walk_relocation_in_brick (tree + node_left_child (tree), args);
24063 uint8_t* plug = tree;
24064 BOOL has_pre_plug_info_p = FALSE;
24065 BOOL has_post_plug_info_p = FALSE;
24067 if (tree == oldest_pinned_plug)
24069 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
24070 &has_post_plug_info_p);
24071 assert (tree == pinned_plug (args->pinned_plug_entry));
24074 if (args->last_plug != 0)
24076 size_t gap_size = node_gap_size (tree);
24077 uint8_t* gap = (plug - gap_size);
24078 uint8_t* last_plug_end = gap;
24079 size_t last_plug_size = (last_plug_end - args->last_plug);
24080 dprintf (3, ("tree: %Ix, last_plug: %Ix, gap: %Ix(%Ix), last_plug_end: %Ix, size: %Ix",
24081 tree, args->last_plug, gap, gap_size, last_plug_end, last_plug_size));
24083 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
24084 if (!check_last_object_p)
24086 assert (last_plug_size >= Align (min_obj_size));
24089 walk_plug (args->last_plug, last_plug_size, check_last_object_p, args);
24093 assert (!has_pre_plug_info_p);
24096 dprintf (3, ("set args last plug to plug: %Ix", plug));
24097 args->last_plug = plug;
24098 args->is_shortened = has_post_plug_info_p;
24100 if (node_right_child (tree))
24102 walk_relocation_in_brick (tree + node_right_child (tree), args);
24106 void gc_heap::walk_relocation (void* profiling_context, record_surv_fn fn)
24108 generation* condemned_gen = generation_of (settings.condemned_generation);
24109 uint8_t* start_address = generation_allocation_start (condemned_gen);
24110 size_t current_brick = brick_of (start_address);
24111 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
24113 PREFIX_ASSUME(current_heap_segment != NULL);
24115 reset_pinned_queue_bos();
24116 update_oldest_pinned_plug();
24117 size_t end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24118 walk_relocate_args args;
24119 args.is_shortened = FALSE;
24120 args.pinned_plug_entry = 0;
24121 args.last_plug = 0;
24122 args.profiling_context = profiling_context;
24127 if (current_brick > end_brick)
24129 if (args.last_plug)
24131 walk_plug (args.last_plug,
24132 (heap_segment_allocated (current_heap_segment) - args.last_plug),
24135 args.last_plug = 0;
24137 if (heap_segment_next_rw (current_heap_segment))
24139 current_heap_segment = heap_segment_next_rw (current_heap_segment);
24140 current_brick = brick_of (heap_segment_mem (current_heap_segment));
24141 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24150 int brick_entry = brick_table [ current_brick ];
24151 if (brick_entry >= 0)
24153 walk_relocation_in_brick (brick_address (current_brick) +
24162 void gc_heap::walk_survivors (record_surv_fn fn, void* context, walk_surv_type type)
24164 if (type == walk_for_gc)
24165 walk_survivors_relocation (context, fn);
24166 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24167 else if (type == walk_for_bgc)
24168 walk_survivors_for_bgc (context, fn);
24169 #endif //BACKGROUND_GC && FEATURE_EVENT_TRACE
24170 else if (type == walk_for_loh)
24171 walk_survivors_for_loh (context, fn);
24173 assert (!"unknown type!");
24176 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24177 void gc_heap::walk_survivors_for_bgc (void* profiling_context, record_surv_fn fn)
24179 // This should only be called for BGCs
24180 assert(settings.concurrent);
24182 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
24184 BOOL small_object_segments = TRUE;
24185 int align_const = get_alignment_constant (small_object_segments);
24191 if (small_object_segments)
24193 //switch to large segment
24194 small_object_segments = FALSE;
24196 align_const = get_alignment_constant (small_object_segments);
24197 seg = heap_segment_rw (generation_start_segment (large_object_generation));
24199 PREFIX_ASSUME(seg != NULL);
24207 uint8_t* o = heap_segment_mem (seg);
24208 uint8_t* end = heap_segment_allocated (seg);
24212 if (method_table(o) == g_gc_pFreeObjectMethodTable)
24214 o += Align (size (o), align_const);
24218 // It's survived. Make a fake plug, starting at o,
24219 // and send the event
24221 uint8_t* plug_start = o;
24223 while (method_table(o) != g_gc_pFreeObjectMethodTable)
24225 o += Align (size (o), align_const);
24232 uint8_t* plug_end = o;
24236 0, // Reloc distance == 0 as this is non-compacting
24238 false, // Non-compacting
24242 seg = heap_segment_next (seg);
24245 #endif // defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24247 void gc_heap::relocate_phase (int condemned_gen_number,
24248 uint8_t* first_condemned_address)
24251 sc.thread_number = heap_number;
24252 sc.promotion = FALSE;
24253 sc.concurrent = FALSE;
24259 start = GetCycleCount32();
24262 // %type% category = quote (relocate);
24263 dprintf (2,("---- Relocate phase -----"));
24265 #ifdef MULTIPLE_HEAPS
24266 //join all threads to make sure they are synchronized
24267 dprintf(3, ("Joining after end of plan"));
24268 gc_t_join.join(this, gc_join_begin_relocate_phase);
24269 if (gc_t_join.joined())
24270 #endif //MULTIPLE_HEAPS
24273 #ifdef MULTIPLE_HEAPS
24275 //join all threads to make sure they are synchronized
24276 dprintf(3, ("Restarting for relocation"));
24277 gc_t_join.restart();
24278 #endif //MULTIPLE_HEAPS
24281 dprintf(3,("Relocating roots"));
24282 GCScan::GcScanRoots(GCHeap::Relocate,
24283 condemned_gen_number, max_generation, &sc);
24285 verify_pins_with_post_plug_info("after reloc stack");
24287 #ifdef BACKGROUND_GC
24288 if (recursive_gc_sync::background_running_p())
24290 scan_background_roots (GCHeap::Relocate, heap_number, &sc);
24292 #endif //BACKGROUND_GC
24294 if (condemned_gen_number != max_generation)
24296 dprintf(3,("Relocating cross generation pointers"));
24297 mark_through_cards_for_segments (&gc_heap::relocate_address, TRUE);
24298 verify_pins_with_post_plug_info("after reloc cards");
24300 if (condemned_gen_number != max_generation)
24302 dprintf(3,("Relocating cross generation pointers for large objects"));
24303 mark_through_cards_for_large_objects (&gc_heap::relocate_address, TRUE);
24307 #ifdef FEATURE_LOH_COMPACTION
24308 if (loh_compacted_p)
24310 assert (settings.condemned_generation == max_generation);
24311 relocate_in_loh_compact();
24314 #endif //FEATURE_LOH_COMPACTION
24316 relocate_in_large_objects ();
24320 dprintf(3,("Relocating survivors"));
24321 relocate_survivors (condemned_gen_number,
24322 first_condemned_address);
24325 #ifdef FEATURE_PREMORTEM_FINALIZATION
24326 dprintf(3,("Relocating finalization data"));
24327 finalize_queue->RelocateFinalizationData (condemned_gen_number,
24329 #endif // FEATURE_PREMORTEM_FINALIZATION
24334 dprintf(3,("Relocating handle table"));
24335 GCScan::GcScanHandles(GCHeap::Relocate,
24336 condemned_gen_number, max_generation, &sc);
24339 #ifdef MULTIPLE_HEAPS
24340 //join all threads to make sure they are synchronized
24341 dprintf(3, ("Joining after end of relocation"));
24342 gc_t_join.join(this, gc_join_relocate_phase_done);
24344 #endif //MULTIPLE_HEAPS
24347 finish = GetCycleCount32();
24348 reloc_time = finish - start;
24351 dprintf(2,( "---- End of Relocate phase ----"));
24354 // This compares to see if tree is the current pinned plug and returns info
24355 // for this pinned plug. Also advances the pinned queue if that's the case.
24357 // We don't change the values of the plug info if tree is not the same as
24358 // the current pinned plug - the caller is responsible for setting the right
24359 // values to begin with.
24361 // POPO TODO: We are keeping this temporarily as this is also used by realloc
24362 // where it passes FALSE to deque_p, change it to use the same optimization
24363 // as relocate. Not as essential since realloc is already a slow path.
24364 mark* gc_heap::get_next_pinned_entry (uint8_t* tree,
24365 BOOL* has_pre_plug_info_p,
24366 BOOL* has_post_plug_info_p,
24369 if (!pinned_plug_que_empty_p())
24371 mark* oldest_entry = oldest_pin();
24372 uint8_t* oldest_plug = pinned_plug (oldest_entry);
24373 if (tree == oldest_plug)
24375 *has_pre_plug_info_p = oldest_entry->has_pre_plug_info();
24376 *has_post_plug_info_p = oldest_entry->has_post_plug_info();
24380 deque_pinned_plug();
24383 dprintf (3, ("found a pinned plug %Ix, pre: %d, post: %d",
24385 (*has_pre_plug_info_p ? 1 : 0),
24386 (*has_post_plug_info_p ? 1 : 0)));
24388 return oldest_entry;
24395 // This also deques the oldest entry and update the oldest plug
24396 mark* gc_heap::get_oldest_pinned_entry (BOOL* has_pre_plug_info_p,
24397 BOOL* has_post_plug_info_p)
24399 mark* oldest_entry = oldest_pin();
24400 *has_pre_plug_info_p = oldest_entry->has_pre_plug_info();
24401 *has_post_plug_info_p = oldest_entry->has_post_plug_info();
24403 deque_pinned_plug();
24404 update_oldest_pinned_plug();
24405 return oldest_entry;
24409 void gc_heap::copy_cards_range (uint8_t* dest, uint8_t* src, size_t len, BOOL copy_cards_p)
24412 copy_cards_for_addresses (dest, src, len);
24414 clear_card_for_addresses (dest, dest + len);
24417 // POPO TODO: We should actually just recover the artifically made gaps here..because when we copy
24418 // we always copy the earlier plugs first which means we won't need the gap sizes anymore. This way
24419 // we won't need to individually recover each overwritten part of plugs.
24421 void gc_heap::gcmemcopy (uint8_t* dest, uint8_t* src, size_t len, BOOL copy_cards_p)
24425 #ifdef BACKGROUND_GC
24426 if (current_c_gc_state == c_gc_state_marking)
24428 //TODO: should look to see whether we should consider changing this
24429 // to copy a consecutive region of the mark array instead.
24430 copy_mark_bits_for_addresses (dest, src, len);
24432 #endif //BACKGROUND_GC
24433 //dprintf(3,(" Memcopy [%Ix->%Ix, %Ix->%Ix[", (size_t)src, (size_t)dest, (size_t)src+len, (size_t)dest+len));
24434 dprintf(3,(" mc: [%Ix->%Ix, %Ix->%Ix[", (size_t)src, (size_t)dest, (size_t)src+len, (size_t)dest+len));
24435 memcopy (dest - plug_skew, src - plug_skew, len);
24436 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
24437 if (SoftwareWriteWatch::IsEnabledForGCHeap())
24439 // The ranges [src - plug_kew .. src[ and [src + len - plug_skew .. src + len[ are ObjHeaders, which don't have GC
24440 // references, and are not relevant for write watch. The latter range actually corresponds to the ObjHeader for the
24441 // object at (src + len), so it can be ignored anyway.
24442 SoftwareWriteWatch::SetDirtyRegion(dest, len - plug_skew);
24444 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
24445 copy_cards_range (dest, src, len, copy_cards_p);
24449 void gc_heap::compact_plug (uint8_t* plug, size_t size, BOOL check_last_object_p, compact_args* args)
24452 uint8_t* reloc_plug = plug + args->last_plug_relocation;
24454 if (check_last_object_p)
24456 size += sizeof (gap_reloc_pair);
24457 mark* entry = args->pinned_plug_entry;
24459 if (args->is_shortened)
24461 assert (entry->has_post_plug_info());
24462 entry->swap_post_plug_and_saved();
24466 assert (entry->has_pre_plug_info());
24467 entry->swap_pre_plug_and_saved();
24471 int old_brick_entry = brick_table [brick_of (plug)];
24473 assert (node_relocation_distance (plug) == args->last_plug_relocation);
24475 #ifdef FEATURE_STRUCTALIGN
24476 ptrdiff_t alignpad = node_alignpad(plug);
24479 make_unused_array (reloc_plug - alignpad, alignpad);
24480 if (brick_of (reloc_plug - alignpad) != brick_of (reloc_plug))
24482 // The alignment padding is straddling one or more bricks;
24483 // it has to be the last "object" of its first brick.
24484 fix_brick_to_highest (reloc_plug - alignpad, reloc_plug);
24487 #else // FEATURE_STRUCTALIGN
24488 size_t unused_arr_size = 0;
24489 BOOL already_padded_p = FALSE;
24491 if (is_plug_padded (plug))
24493 already_padded_p = TRUE;
24494 clear_plug_padded (plug);
24495 unused_arr_size = Align (min_obj_size);
24497 #endif //SHORT_PLUGS
24498 if (node_realigned (plug))
24500 unused_arr_size += switch_alignment_size (already_padded_p);
24503 if (unused_arr_size != 0)
24505 make_unused_array (reloc_plug - unused_arr_size, unused_arr_size);
24507 if (brick_of (reloc_plug - unused_arr_size) != brick_of (reloc_plug))
24509 dprintf (3, ("fix B for padding: %Id: %Ix->%Ix",
24510 unused_arr_size, (reloc_plug - unused_arr_size), reloc_plug));
24511 // The alignment padding is straddling one or more bricks;
24512 // it has to be the last "object" of its first brick.
24513 fix_brick_to_highest (reloc_plug - unused_arr_size, reloc_plug);
24516 #endif // FEATURE_STRUCTALIGN
24519 if (is_plug_padded (plug))
24521 make_unused_array (reloc_plug - Align (min_obj_size), Align (min_obj_size));
24523 if (brick_of (reloc_plug - Align (min_obj_size)) != brick_of (reloc_plug))
24525 // The alignment padding is straddling one or more bricks;
24526 // it has to be the last "object" of its first brick.
24527 fix_brick_to_highest (reloc_plug - Align (min_obj_size), reloc_plug);
24530 #endif //SHORT_PLUGS
24532 gcmemcopy (reloc_plug, plug, size, args->copy_cards_p);
24534 if (args->check_gennum_p)
24536 int src_gennum = args->src_gennum;
24537 if (src_gennum == -1)
24539 src_gennum = object_gennum (plug);
24542 int dest_gennum = object_gennum_plan (reloc_plug);
24544 if (src_gennum < dest_gennum)
24546 generation_allocation_size (generation_of (dest_gennum)) += size;
24550 size_t current_reloc_brick = args->current_compacted_brick;
24552 if (brick_of (reloc_plug) != current_reloc_brick)
24554 dprintf (3, ("last reloc B: %Ix, current reloc B: %Ix",
24555 current_reloc_brick, brick_of (reloc_plug)));
24557 if (args->before_last_plug)
24559 dprintf (3,(" fixing last brick %Ix to point to last plug %Ix(%Ix)",
24560 current_reloc_brick,
24561 args->before_last_plug,
24562 (args->before_last_plug - brick_address (current_reloc_brick))));
24565 set_brick (current_reloc_brick,
24566 args->before_last_plug - brick_address (current_reloc_brick));
24569 current_reloc_brick = brick_of (reloc_plug);
24571 size_t end_brick = brick_of (reloc_plug + size-1);
24572 if (end_brick != current_reloc_brick)
24574 // The plug is straddling one or more bricks
24575 // It has to be the last plug of its first brick
24576 dprintf (3,("plug spanning multiple bricks, fixing first brick %Ix to %Ix(%Ix)",
24577 current_reloc_brick, (size_t)reloc_plug,
24578 (reloc_plug - brick_address (current_reloc_brick))));
24581 set_brick (current_reloc_brick,
24582 reloc_plug - brick_address (current_reloc_brick));
24584 // update all intervening brick
24585 size_t brick = current_reloc_brick + 1;
24586 dprintf (3,("setting intervening bricks %Ix->%Ix to -1",
24587 brick, (end_brick - 1)));
24588 while (brick < end_brick)
24590 set_brick (brick, -1);
24593 // code last brick offset as a plug address
24594 args->before_last_plug = brick_address (end_brick) -1;
24595 current_reloc_brick = end_brick;
24596 dprintf (3, ("setting before last to %Ix, last brick to %Ix",
24597 args->before_last_plug, current_reloc_brick));
24601 dprintf (3, ("still in the same brick: %Ix", end_brick));
24602 args->before_last_plug = reloc_plug;
24604 args->current_compacted_brick = current_reloc_brick;
24606 if (check_last_object_p)
24608 mark* entry = args->pinned_plug_entry;
24610 if (args->is_shortened)
24612 entry->swap_post_plug_and_saved();
24616 entry->swap_pre_plug_and_saved();
24621 void gc_heap::compact_in_brick (uint8_t* tree, compact_args* args)
24623 assert (tree != NULL);
24624 int left_node = node_left_child (tree);
24625 int right_node = node_right_child (tree);
24626 ptrdiff_t relocation = node_relocation_distance (tree);
24632 dprintf (3, ("B: L: %d->%Ix", left_node, (tree + left_node)));
24633 compact_in_brick ((tree + left_node), args);
24636 uint8_t* plug = tree;
24637 BOOL has_pre_plug_info_p = FALSE;
24638 BOOL has_post_plug_info_p = FALSE;
24640 if (tree == oldest_pinned_plug)
24642 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
24643 &has_post_plug_info_p);
24644 assert (tree == pinned_plug (args->pinned_plug_entry));
24647 if (args->last_plug != 0)
24649 size_t gap_size = node_gap_size (tree);
24650 uint8_t* gap = (plug - gap_size);
24651 uint8_t* last_plug_end = gap;
24652 size_t last_plug_size = (last_plug_end - args->last_plug);
24653 dprintf (3, ("tree: %Ix, last_plug: %Ix, gap: %Ix(%Ix), last_plug_end: %Ix, size: %Ix",
24654 tree, args->last_plug, gap, gap_size, last_plug_end, last_plug_size));
24656 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
24657 if (!check_last_object_p)
24659 assert (last_plug_size >= Align (min_obj_size));
24662 compact_plug (args->last_plug, last_plug_size, check_last_object_p, args);
24666 assert (!has_pre_plug_info_p);
24669 dprintf (3, ("set args last plug to plug: %Ix, reloc: %Ix", plug, relocation));
24670 args->last_plug = plug;
24671 args->last_plug_relocation = relocation;
24672 args->is_shortened = has_post_plug_info_p;
24676 dprintf (3, ("B: R: %d->%Ix", right_node, (tree + right_node)));
24677 compact_in_brick ((tree + right_node), args);
24681 void gc_heap::recover_saved_pinned_info()
24683 reset_pinned_queue_bos();
24685 while (!(pinned_plug_que_empty_p()))
24687 mark* oldest_entry = oldest_pin();
24688 oldest_entry->recover_plug_info();
24689 #ifdef GC_CONFIG_DRIVEN
24690 if (oldest_entry->has_pre_plug_info() && oldest_entry->has_post_plug_info())
24691 record_interesting_data_point (idp_pre_and_post_pin);
24692 else if (oldest_entry->has_pre_plug_info())
24693 record_interesting_data_point (idp_pre_pin);
24694 else if (oldest_entry->has_post_plug_info())
24695 record_interesting_data_point (idp_post_pin);
24696 #endif //GC_CONFIG_DRIVEN
24698 deque_pinned_plug();
24702 void gc_heap::compact_phase (int condemned_gen_number,
24703 uint8_t* first_condemned_address,
24706 // %type% category = quote (compact);
24710 start = GetCycleCount32();
24712 generation* condemned_gen = generation_of (condemned_gen_number);
24713 uint8_t* start_address = first_condemned_address;
24714 size_t current_brick = brick_of (start_address);
24715 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
24717 PREFIX_ASSUME(current_heap_segment != NULL);
24719 reset_pinned_queue_bos();
24720 update_oldest_pinned_plug();
24722 BOOL reused_seg = expand_reused_seg_p();
24725 for (int i = 1; i <= max_generation; i++)
24727 generation_allocation_size (generation_of (i)) = 0;
24731 uint8_t* end_address = heap_segment_allocated (current_heap_segment);
24733 size_t end_brick = brick_of (end_address-1);
24735 args.last_plug = 0;
24736 args.before_last_plug = 0;
24737 args.current_compacted_brick = ~((size_t)1);
24738 args.is_shortened = FALSE;
24739 args.pinned_plug_entry = 0;
24740 args.copy_cards_p = (condemned_gen_number >= 1) || !clear_cards;
24741 args.check_gennum_p = reused_seg;
24742 if (args.check_gennum_p)
24744 args.src_gennum = ((current_heap_segment == ephemeral_heap_segment) ? -1 : 2);
24747 dprintf (2,("---- Compact Phase: %Ix(%Ix)----",
24748 first_condemned_address, brick_of (first_condemned_address)));
24750 #ifdef MULTIPLE_HEAPS
24752 if (gc_t_join.joined())
24754 #endif //MULTIPLE_HEAPS
24756 #ifdef MULTIPLE_HEAPS
24757 dprintf(3, ("Restarting for compaction"));
24758 gc_t_join.restart();
24760 #endif //MULTIPLE_HEAPS
24762 reset_pinned_queue_bos();
24764 #ifdef FEATURE_LOH_COMPACTION
24765 if (loh_compacted_p)
24769 #endif //FEATURE_LOH_COMPACTION
24771 if ((start_address < end_address) ||
24772 (condemned_gen_number == max_generation))
24776 if (current_brick > end_brick)
24778 if (args.last_plug != 0)
24780 dprintf (3, ("compacting last plug: %Ix", args.last_plug))
24781 compact_plug (args.last_plug,
24782 (heap_segment_allocated (current_heap_segment) - args.last_plug),
24787 if (heap_segment_next_rw (current_heap_segment))
24789 current_heap_segment = heap_segment_next_rw (current_heap_segment);
24790 current_brick = brick_of (heap_segment_mem (current_heap_segment));
24791 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24792 args.last_plug = 0;
24793 if (args.check_gennum_p)
24795 args.src_gennum = ((current_heap_segment == ephemeral_heap_segment) ? -1 : 2);
24801 if (args.before_last_plug !=0)
24803 dprintf (3, ("Fixing last brick %Ix to point to plug %Ix",
24804 args.current_compacted_brick, (size_t)args.before_last_plug));
24805 assert (args.current_compacted_brick != ~1u);
24806 set_brick (args.current_compacted_brick,
24807 args.before_last_plug - brick_address (args.current_compacted_brick));
24813 int brick_entry = brick_table [ current_brick ];
24814 dprintf (3, ("B: %Ix(%Ix)->%Ix",
24815 current_brick, (size_t)brick_entry, (brick_address (current_brick) + brick_entry - 1)));
24817 if (brick_entry >= 0)
24819 compact_in_brick ((brick_address (current_brick) + brick_entry -1),
24828 recover_saved_pinned_info();
24831 finish = GetCycleCount32();
24832 compact_time = finish - start;
24835 concurrent_print_time_delta ("compact end");
24837 dprintf(2,("---- End of Compact phase ----"));
24840 #ifdef MULTIPLE_HEAPS
24843 #pragma warning(push)
24844 #pragma warning(disable:4702) // C4702: unreachable code: gc_thread_function may not return
24846 void gc_heap::gc_thread_stub (void* arg)
24848 gc_heap* heap = (gc_heap*)arg;
24849 if (!gc_thread_no_affinitize_p)
24851 GCThreadAffinity affinity;
24852 affinity.Group = GCThreadAffinity::None;
24853 affinity.Processor = GCThreadAffinity::None;
24855 // We are about to set affinity for GC threads. It is a good place to set up NUMA and
24856 // CPU groups because the process mask, processor number, and group number are all
24857 // readily available.
24858 if (GCToOSInterface::CanEnableGCCPUGroups())
24859 set_thread_group_affinity_for_heap(heap->heap_number, &affinity);
24861 set_thread_affinity_mask_for_heap(heap->heap_number, &affinity);
24863 if (!GCToOSInterface::SetThreadAffinity(&affinity))
24865 dprintf(1, ("Failed to set thread affinity for server GC thread"));
24869 // server GC threads run at a higher priority than normal.
24870 GCToOSInterface::BoostThreadPriority();
24871 _alloca (256*heap->heap_number);
24872 heap->gc_thread_function();
24875 #pragma warning(pop)
24878 #endif //MULTIPLE_HEAPS
24880 #ifdef BACKGROUND_GC
24883 #pragma warning(push)
24884 #pragma warning(disable:4702) // C4702: unreachable code: gc_thread_function may not return
24886 void gc_heap::bgc_thread_stub (void* arg)
24888 gc_heap* heap = (gc_heap*)arg;
24889 heap->bgc_thread = GCToEEInterface::GetThread();
24890 assert(heap->bgc_thread != nullptr);
24891 heap->bgc_thread_function();
24894 #pragma warning(pop)
24897 #endif //BACKGROUND_GC
24899 /*------------------ Background GC ----------------------------*/
24901 #ifdef BACKGROUND_GC
24903 void gc_heap::background_drain_mark_list (int thread)
24905 UNREFERENCED_PARAMETER(thread);
24907 size_t saved_c_mark_list_index = c_mark_list_index;
24909 if (saved_c_mark_list_index)
24911 concurrent_print_time_delta ("SML");
24913 while (c_mark_list_index != 0)
24915 size_t current_index = c_mark_list_index - 1;
24916 uint8_t* o = c_mark_list [current_index];
24917 background_mark_object (o THREAD_NUMBER_ARG);
24918 c_mark_list_index--;
24920 if (saved_c_mark_list_index)
24923 concurrent_print_time_delta ("EML");
24926 fire_drain_mark_list_event (saved_c_mark_list_index);
24930 // The background GC version of scan_dependent_handles (see that method for a more in-depth comment).
24931 #ifdef MULTIPLE_HEAPS
24932 // Since we only scan dependent handles while we are stopped we'll never interfere with FGCs scanning
24933 // them. So we can use the same static variables.
24934 void gc_heap::background_scan_dependent_handles (ScanContext *sc)
24936 // Whenever we call this method there may have been preceding object promotions. So set
24937 // s_fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
24938 // based on the how the scanning proceeded).
24939 s_fUnscannedPromotions = TRUE;
24941 // We don't know how many times we need to loop yet. In particular we can't base the loop condition on
24942 // the state of this thread's portion of the dependent handle table. That's because promotions on other
24943 // threads could cause handle promotions to become necessary here. Even if there are definitely no more
24944 // promotions possible in this thread's handles, we still have to stay in lock-step with those worker
24945 // threads that haven't finished yet (each GC worker thread has to join exactly the same number of times
24946 // as all the others or they'll get out of step).
24949 // The various worker threads are all currently racing in this code. We need to work out if at least
24950 // one of them think they have work to do this cycle. Each thread needs to rescan its portion of the
24951 // dependent handle table when both of the following conditions apply:
24952 // 1) At least one (arbitrary) object might have been promoted since the last scan (because if this
24953 // object happens to correspond to a primary in one of our handles we might potentially have to
24954 // promote the associated secondary).
24955 // 2) The table for this thread has at least one handle with a secondary that isn't promoted yet.
24957 // The first condition is represented by s_fUnscannedPromotions. This is always non-zero for the first
24958 // iteration of this loop (see comment above) and in subsequent cycles each thread updates this
24959 // whenever a mark stack overflow occurs or scanning their dependent handles results in a secondary
24960 // being promoted. This value is cleared back to zero in a synchronized fashion in the join that
24961 // follows below. Note that we can't read this outside of the join since on any iteration apart from
24962 // the first threads will be racing between reading this value and completing their previous
24963 // iteration's table scan.
24965 // The second condition is tracked by the dependent handle code itself on a per worker thread basis
24966 // (and updated by the GcDhReScan() method). We call GcDhUnpromotedHandlesExist() on each thread to
24967 // determine the local value and collect the results into the s_fUnpromotedHandles variable in what is
24968 // effectively an OR operation. As per s_fUnscannedPromotions we can't read the final result until
24969 // we're safely joined.
24970 if (GCScan::GcDhUnpromotedHandlesExist(sc))
24971 s_fUnpromotedHandles = TRUE;
24973 // Synchronize all the threads so we can read our state variables safely. The following shared
24974 // variable (indicating whether we should scan the tables or terminate the loop) will be set by a
24975 // single thread inside the join.
24976 bgc_t_join.join(this, gc_join_scan_dependent_handles);
24977 if (bgc_t_join.joined())
24979 // We're synchronized so it's safe to read our shared state variables. We update another shared
24980 // variable to indicate to all threads whether we'll be scanning for another cycle or terminating
24981 // the loop. We scan if there has been at least one object promotion since last time and at least
24982 // one thread has a dependent handle table with a potential handle promotion possible.
24983 s_fScanRequired = s_fUnscannedPromotions && s_fUnpromotedHandles;
24985 // Reset our shared state variables (ready to be set again on this scan or with a good initial
24986 // value for the next call if we're terminating the loop).
24987 s_fUnscannedPromotions = FALSE;
24988 s_fUnpromotedHandles = FALSE;
24990 if (!s_fScanRequired)
24992 uint8_t* all_heaps_max = 0;
24993 uint8_t* all_heaps_min = MAX_PTR;
24995 for (i = 0; i < n_heaps; i++)
24997 if (all_heaps_max < g_heaps[i]->background_max_overflow_address)
24998 all_heaps_max = g_heaps[i]->background_max_overflow_address;
24999 if (all_heaps_min > g_heaps[i]->background_min_overflow_address)
25000 all_heaps_min = g_heaps[i]->background_min_overflow_address;
25002 for (i = 0; i < n_heaps; i++)
25004 g_heaps[i]->background_max_overflow_address = all_heaps_max;
25005 g_heaps[i]->background_min_overflow_address = all_heaps_min;
25009 // Restart all the workers.
25010 dprintf(2, ("Starting all gc thread mark stack overflow processing"));
25011 bgc_t_join.restart();
25014 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
25015 // being visible. If there really was an overflow (process_mark_overflow returns true) then set the
25016 // global flag indicating that at least one object promotion may have occurred (the usual comment
25017 // about races applies). (Note it's OK to set this flag even if we're about to terminate the loop and
25018 // exit the method since we unconditionally set this variable on method entry anyway).
25019 if (background_process_mark_overflow (sc->concurrent))
25020 s_fUnscannedPromotions = TRUE;
25022 // If we decided that no scan was required we can terminate the loop now.
25023 if (!s_fScanRequired)
25026 // Otherwise we must join with the other workers to ensure that all mark stack overflows have been
25027 // processed before we start scanning dependent handle tables (if overflows remain while we scan we
25028 // could miss noting the promotion of some primary objects).
25029 bgc_t_join.join(this, gc_join_rescan_dependent_handles);
25030 if (bgc_t_join.joined())
25032 // Restart all the workers.
25033 dprintf(3, ("Starting all gc thread for dependent handle promotion"));
25034 bgc_t_join.restart();
25037 // If the portion of the dependent handle table managed by this worker has handles that could still be
25038 // promoted perform a rescan. If the rescan resulted in at least one promotion note this fact since it
25039 // could require a rescan of handles on this or other workers.
25040 if (GCScan::GcDhUnpromotedHandlesExist(sc))
25041 if (GCScan::GcDhReScan(sc))
25042 s_fUnscannedPromotions = TRUE;
25046 void gc_heap::background_scan_dependent_handles (ScanContext *sc)
25048 // Whenever we call this method there may have been preceding object promotions. So set
25049 // fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
25050 // based on the how the scanning proceeded).
25051 bool fUnscannedPromotions = true;
25053 // Scan dependent handles repeatedly until there are no further promotions that can be made or we made a
25054 // scan without performing any new promotions.
25055 while (GCScan::GcDhUnpromotedHandlesExist(sc) && fUnscannedPromotions)
25057 // On each iteration of the loop start with the assumption that no further objects have been promoted.
25058 fUnscannedPromotions = false;
25060 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
25061 // being visible. If there was an overflow (background_process_mark_overflow returned true) then
25062 // additional objects now appear to be promoted and we should set the flag.
25063 if (background_process_mark_overflow (sc->concurrent))
25064 fUnscannedPromotions = true;
25066 // Perform the scan and set the flag if any promotions resulted.
25067 if (GCScan::GcDhReScan (sc))
25068 fUnscannedPromotions = true;
25071 // Perform a last processing of any overflowed mark stack.
25072 background_process_mark_overflow (sc->concurrent);
25074 #endif //MULTIPLE_HEAPS
25076 void gc_heap::recover_bgc_settings()
25078 if ((settings.condemned_generation < max_generation) && recursive_gc_sync::background_running_p())
25080 dprintf (2, ("restoring bgc settings"));
25081 settings = saved_bgc_settings;
25082 GCHeap::GcCondemnedGeneration = gc_heap::settings.condemned_generation;
25086 void gc_heap::allow_fgc()
25088 assert (bgc_thread == GCToEEInterface::GetThread());
25089 bool bToggleGC = false;
25091 if (g_fSuspensionPending > 0)
25093 bToggleGC = GCToEEInterface::EnablePreemptiveGC();
25096 GCToEEInterface::DisablePreemptiveGC();
25101 BOOL gc_heap::should_commit_mark_array()
25103 return (recursive_gc_sync::background_running_p() || (current_bgc_state == bgc_initialized));
25106 void gc_heap::clear_commit_flag()
25108 generation* gen = generation_of (max_generation);
25109 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
25114 if (gen != large_object_generation)
25116 gen = large_object_generation;
25117 seg = heap_segment_in_range (generation_start_segment (gen));
25125 if (seg->flags & heap_segment_flags_ma_committed)
25127 seg->flags &= ~heap_segment_flags_ma_committed;
25130 if (seg->flags & heap_segment_flags_ma_pcommitted)
25132 seg->flags &= ~heap_segment_flags_ma_pcommitted;
25135 seg = heap_segment_next (seg);
25139 void gc_heap::clear_commit_flag_global()
25141 #ifdef MULTIPLE_HEAPS
25142 for (int i = 0; i < n_heaps; i++)
25144 g_heaps[i]->clear_commit_flag();
25147 clear_commit_flag();
25148 #endif //MULTIPLE_HEAPS
25151 void gc_heap::verify_mark_array_cleared (uint8_t* begin, uint8_t* end, uint32_t* mark_array_addr)
25154 size_t markw = mark_word_of (begin);
25155 size_t markw_end = mark_word_of (end);
25157 while (markw < markw_end)
25159 if (mark_array_addr[markw])
25161 dprintf (1, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
25162 markw, mark_array_addr[markw], mark_word_address (markw)));
25168 UNREFERENCED_PARAMETER(begin);
25169 UNREFERENCED_PARAMETER(end);
25170 UNREFERENCED_PARAMETER(mark_array_addr);
25174 void gc_heap::verify_mark_array_cleared (heap_segment* seg, uint32_t* mark_array_addr)
25176 verify_mark_array_cleared (heap_segment_mem (seg), heap_segment_reserved (seg), mark_array_addr);
25179 BOOL gc_heap::commit_mark_array_new_seg (gc_heap* hp,
25181 uint32_t* new_card_table,
25182 uint8_t* new_lowest_address)
25184 UNREFERENCED_PARAMETER(hp); // compiler bug? -- this *is*, indeed, referenced
25186 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
25187 uint8_t* end = heap_segment_reserved (seg);
25189 uint8_t* lowest = hp->background_saved_lowest_address;
25190 uint8_t* highest = hp->background_saved_highest_address;
25192 uint8_t* commit_start = NULL;
25193 uint8_t* commit_end = NULL;
25194 size_t commit_flag = 0;
25196 if ((highest >= start) &&
25199 if ((start >= lowest) && (end <= highest))
25201 dprintf (GC_TABLE_LOG, ("completely in bgc range: seg %Ix-%Ix, bgc: %Ix-%Ix",
25202 start, end, lowest, highest));
25203 commit_flag = heap_segment_flags_ma_committed;
25207 dprintf (GC_TABLE_LOG, ("partially in bgc range: seg %Ix-%Ix, bgc: %Ix-%Ix",
25208 start, end, lowest, highest));
25209 commit_flag = heap_segment_flags_ma_pcommitted;
25212 commit_start = max (lowest, start);
25213 commit_end = min (highest, end);
25215 if (!commit_mark_array_by_range (commit_start, commit_end, hp->mark_array))
25220 if (new_card_table == 0)
25222 new_card_table = g_gc_card_table;
25225 if (hp->card_table != new_card_table)
25227 if (new_lowest_address == 0)
25229 new_lowest_address = g_gc_lowest_address;
25232 uint32_t* ct = &new_card_table[card_word (gcard_of (new_lowest_address))];
25233 uint32_t* ma = (uint32_t*)((uint8_t*)card_table_mark_array (ct) - size_mark_array_of (0, new_lowest_address));
25235 dprintf (GC_TABLE_LOG, ("table realloc-ed: %Ix->%Ix, MA: %Ix->%Ix",
25236 hp->card_table, new_card_table,
25237 hp->mark_array, ma));
25239 if (!commit_mark_array_by_range (commit_start, commit_end, ma))
25245 seg->flags |= commit_flag;
25251 BOOL gc_heap::commit_mark_array_by_range (uint8_t* begin, uint8_t* end, uint32_t* mark_array_addr)
25253 size_t beg_word = mark_word_of (begin);
25254 size_t end_word = mark_word_of (align_on_mark_word (end));
25255 uint8_t* commit_start = align_lower_page ((uint8_t*)&mark_array_addr[beg_word]);
25256 uint8_t* commit_end = align_on_page ((uint8_t*)&mark_array_addr[end_word]);
25257 size_t size = (size_t)(commit_end - commit_start);
25259 #ifdef SIMPLE_DPRINTF
25260 dprintf (GC_TABLE_LOG, ("range: %Ix->%Ix mark word: %Ix->%Ix(%Id), mark array: %Ix->%Ix(%Id), commit %Ix->%Ix(%Id)",
25262 beg_word, end_word,
25263 (end_word - beg_word) * sizeof (uint32_t),
25264 &mark_array_addr[beg_word],
25265 &mark_array_addr[end_word],
25266 (size_t)(&mark_array_addr[end_word] - &mark_array_addr[beg_word]),
25267 commit_start, commit_end,
25269 #endif //SIMPLE_DPRINTF
25271 if (GCToOSInterface::VirtualCommit (commit_start, size))
25273 // We can only verify the mark array is cleared from begin to end, the first and the last
25274 // page aren't necessarily all cleared 'cause they could be used by other segments or
25276 verify_mark_array_cleared (begin, end, mark_array_addr);
25281 dprintf (GC_TABLE_LOG, ("failed to commit %Id bytes", (end_word - beg_word) * sizeof (uint32_t)));
25286 BOOL gc_heap::commit_mark_array_with_check (heap_segment* seg, uint32_t* new_mark_array_addr)
25288 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
25289 uint8_t* end = heap_segment_reserved (seg);
25291 #ifdef MULTIPLE_HEAPS
25292 uint8_t* lowest = heap_segment_heap (seg)->background_saved_lowest_address;
25293 uint8_t* highest = heap_segment_heap (seg)->background_saved_highest_address;
25295 uint8_t* lowest = background_saved_lowest_address;
25296 uint8_t* highest = background_saved_highest_address;
25297 #endif //MULTIPLE_HEAPS
25299 if ((highest >= start) &&
25302 start = max (lowest, start);
25303 end = min (highest, end);
25304 if (!commit_mark_array_by_range (start, end, new_mark_array_addr))
25313 BOOL gc_heap::commit_mark_array_by_seg (heap_segment* seg, uint32_t* mark_array_addr)
25315 dprintf (GC_TABLE_LOG, ("seg: %Ix->%Ix; MA: %Ix",
25317 heap_segment_reserved (seg),
25319 uint8_t* start = (heap_segment_read_only_p (seg) ? heap_segment_mem (seg) : (uint8_t*)seg);
25321 return commit_mark_array_by_range (start, heap_segment_reserved (seg), mark_array_addr);
25324 BOOL gc_heap::commit_mark_array_bgc_init (uint32_t* mark_array_addr)
25326 UNREFERENCED_PARAMETER(mark_array_addr);
25328 dprintf (GC_TABLE_LOG, ("BGC init commit: lowest: %Ix, highest: %Ix, mark_array: %Ix",
25329 lowest_address, highest_address, mark_array));
25331 generation* gen = generation_of (max_generation);
25332 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
25337 if (gen != large_object_generation)
25339 gen = large_object_generation;
25340 seg = heap_segment_in_range (generation_start_segment (gen));
25348 dprintf (GC_TABLE_LOG, ("seg: %Ix, flags: %Id", seg, seg->flags));
25350 if (!(seg->flags & heap_segment_flags_ma_committed))
25352 // For ro segments they could always be only partially in range so we'd
25353 // be calling this at the beginning of every BGC. We are not making this
25354 // more efficient right now - ro segments are currently only used by redhawk.
25355 if (heap_segment_read_only_p (seg))
25357 if ((heap_segment_mem (seg) >= lowest_address) &&
25358 (heap_segment_reserved (seg) <= highest_address))
25360 if (commit_mark_array_by_seg (seg, mark_array))
25362 seg->flags |= heap_segment_flags_ma_committed;
25371 uint8_t* start = max (lowest_address, heap_segment_mem (seg));
25372 uint8_t* end = min (highest_address, heap_segment_reserved (seg));
25373 if (commit_mark_array_by_range (start, end, mark_array))
25375 seg->flags |= heap_segment_flags_ma_pcommitted;
25385 // For normal segments they are by design completely in range so just
25386 // commit the whole mark array for each seg.
25387 if (commit_mark_array_by_seg (seg, mark_array))
25389 if (seg->flags & heap_segment_flags_ma_pcommitted)
25391 seg->flags &= ~heap_segment_flags_ma_pcommitted;
25393 seg->flags |= heap_segment_flags_ma_committed;
25402 seg = heap_segment_next (seg);
25408 // This function doesn't check the commit flag since it's for a new array -
25409 // the mark_array flag for these segments will remain the same.
25410 BOOL gc_heap::commit_new_mark_array (uint32_t* new_mark_array_addr)
25412 dprintf (GC_TABLE_LOG, ("commiting existing segs on MA %Ix", new_mark_array_addr));
25413 generation* gen = generation_of (max_generation);
25414 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
25419 if (gen != large_object_generation)
25421 gen = large_object_generation;
25422 seg = heap_segment_in_range (generation_start_segment (gen));
25430 if (!commit_mark_array_with_check (seg, new_mark_array_addr))
25435 seg = heap_segment_next (seg);
25438 #ifdef MULTIPLE_HEAPS
25439 if (new_heap_segment)
25441 if (!commit_mark_array_with_check (new_heap_segment, new_mark_array_addr))
25446 #endif //MULTIPLE_HEAPS
25451 BOOL gc_heap::commit_new_mark_array_global (uint32_t* new_mark_array)
25453 #ifdef MULTIPLE_HEAPS
25454 for (int i = 0; i < n_heaps; i++)
25456 if (!g_heaps[i]->commit_new_mark_array (new_mark_array))
25462 if (!commit_new_mark_array (new_mark_array))
25466 #endif //MULTIPLE_HEAPS
25471 void gc_heap::decommit_mark_array_by_seg (heap_segment* seg)
25473 // if BGC is disabled (the finalize watchdog does this at shutdown), the mark array could have
25474 // been set to NULL.
25475 if (mark_array == NULL)
25480 dprintf (GC_TABLE_LOG, ("decommitting seg %Ix(%Ix), MA: %Ix", seg, seg->flags, mark_array));
25482 size_t flags = seg->flags;
25484 if ((flags & heap_segment_flags_ma_committed) ||
25485 (flags & heap_segment_flags_ma_pcommitted))
25487 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
25488 uint8_t* end = heap_segment_reserved (seg);
25490 if (flags & heap_segment_flags_ma_pcommitted)
25492 start = max (lowest_address, start);
25493 end = min (highest_address, end);
25496 size_t beg_word = mark_word_of (start);
25497 size_t end_word = mark_word_of (align_on_mark_word (end));
25498 uint8_t* decommit_start = align_on_page ((uint8_t*)&mark_array[beg_word]);
25499 uint8_t* decommit_end = align_lower_page ((uint8_t*)&mark_array[end_word]);
25500 size_t size = (size_t)(decommit_end - decommit_start);
25502 #ifdef SIMPLE_DPRINTF
25503 dprintf (GC_TABLE_LOG, ("seg: %Ix mark word: %Ix->%Ix(%Id), mark array: %Ix->%Ix(%Id), decommit %Ix->%Ix(%Id)",
25505 beg_word, end_word,
25506 (end_word - beg_word) * sizeof (uint32_t),
25507 &mark_array[beg_word],
25508 &mark_array[end_word],
25509 (size_t)(&mark_array[end_word] - &mark_array[beg_word]),
25510 decommit_start, decommit_end,
25512 #endif //SIMPLE_DPRINTF
25514 if (decommit_start < decommit_end)
25516 if (!GCToOSInterface::VirtualDecommit (decommit_start, size))
25518 dprintf (GC_TABLE_LOG, ("GCToOSInterface::VirtualDecommit on %Ix for %Id bytes failed",
25519 decommit_start, size));
25520 assert (!"decommit failed");
25524 dprintf (GC_TABLE_LOG, ("decommited [%Ix for address [%Ix", beg_word, seg));
25528 void gc_heap::background_mark_phase ()
25530 verify_mark_array_cleared();
25533 sc.thread_number = heap_number;
25534 sc.promotion = TRUE;
25535 sc.concurrent = FALSE;
25538 BOOL cooperative_mode = TRUE;
25539 #ifndef MULTIPLE_HEAPS
25540 const int thread = heap_number;
25541 #endif //!MULTIPLE_HEAPS
25543 dprintf(2,("-(GC%d)BMark-", VolatileLoad(&settings.gc_index)));
25545 assert (settings.concurrent);
25550 start = GetCycleCount32();
25553 #ifdef FFIND_OBJECT
25554 if (gen0_must_clear_bricks > 0)
25555 gen0_must_clear_bricks--;
25556 #endif //FFIND_OBJECT
25558 background_soh_alloc_count = 0;
25559 background_loh_alloc_count = 0;
25560 bgc_overflow_count = 0;
25562 bpromoted_bytes (heap_number) = 0;
25563 static uint32_t num_sizedrefs = 0;
25565 background_min_overflow_address = MAX_PTR;
25566 background_max_overflow_address = 0;
25567 background_min_soh_overflow_address = MAX_PTR;
25568 background_max_soh_overflow_address = 0;
25569 processed_soh_overflow_p = FALSE;
25572 //set up the mark lists from g_mark_list
25573 assert (g_mark_list);
25574 mark_list = g_mark_list;
25575 //dont use the mark list for full gc
25576 //because multiple segments are more complex to handle and the list
25577 //is likely to overflow
25578 mark_list_end = &mark_list [0];
25579 mark_list_index = &mark_list [0];
25581 c_mark_list_index = 0;
25583 shigh = (uint8_t*) 0;
25586 generation* gen = generation_of (max_generation);
25588 dprintf(3,("BGC: stack marking"));
25589 sc.concurrent = TRUE;
25591 GCScan::GcScanRoots(background_promote_callback,
25592 max_generation, max_generation,
25597 dprintf(3,("BGC: finalization marking"));
25598 finalize_queue->GcScanRoots(background_promote_callback, heap_number, 0);
25601 size_t total_loh_size = generation_size (max_generation + 1);
25602 bgc_begin_loh_size = total_loh_size;
25603 bgc_alloc_spin_loh = 0;
25604 bgc_loh_size_increased = 0;
25605 bgc_loh_allocated_in_free = 0;
25606 size_t total_soh_size = generation_sizes (generation_of (max_generation));
25608 dprintf (GTC_LOG, ("BM: h%d: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
25611 //concurrent_print_time_delta ("copying stack roots");
25612 concurrent_print_time_delta ("CS");
25614 FIRE_EVENT(BGC1stNonConEnd);
25616 expanded_in_fgc = FALSE;
25617 saved_overflow_ephemeral_seg = 0;
25618 current_bgc_state = bgc_reset_ww;
25620 // we don't need a join here - just whichever thread that gets here
25621 // first can change the states and call restart_vm.
25622 // this is not true - we can't let the EE run when we are scanning stack.
25623 // since we now allow reset ww to run concurrently and have a join for it,
25624 // we can do restart ee on the 1st thread that got here. Make sure we handle the
25625 // sizedref handles correctly.
25626 #ifdef MULTIPLE_HEAPS
25627 bgc_t_join.join(this, gc_join_restart_ee);
25628 if (bgc_t_join.joined())
25629 #endif //MULTIPLE_HEAPS
25631 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25632 // Resetting write watch for software write watch is pretty fast, much faster than for hardware write watch. Reset
25633 // can be done while the runtime is suspended or after the runtime is restarted, the preference was to reset while
25634 // the runtime is suspended. The reset for hardware write watch is done after the runtime is restarted below.
25636 concurrent_print_time_delta ("CRWW begin");
25638 #ifdef MULTIPLE_HEAPS
25639 for (int i = 0; i < n_heaps; i++)
25641 g_heaps[i]->reset_write_watch (FALSE);
25644 reset_write_watch (FALSE);
25645 #endif //MULTIPLE_HEAPS
25647 concurrent_print_time_delta ("CRWW");
25648 #endif //WRITE_WATCH
25649 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25651 num_sizedrefs = GCToEEInterface::GetTotalNumSizedRefHandles();
25653 // this c_write is not really necessary because restart_vm
25654 // has an instruction that will flush the cpu cache (interlocked
25655 // or whatever) but we don't want to rely on that.
25656 dprintf (BGC_LOG, ("setting cm_in_progress"));
25657 c_write (cm_in_progress, TRUE);
25659 //restart all thread, doing the marking from the array
25660 assert (dont_restart_ee_p);
25661 dont_restart_ee_p = FALSE;
25664 GCToOSInterface::YieldThread (0);
25665 #ifdef MULTIPLE_HEAPS
25666 dprintf(3, ("Starting all gc threads for gc"));
25667 bgc_t_join.restart();
25668 #endif //MULTIPLE_HEAPS
25671 #ifdef MULTIPLE_HEAPS
25672 bgc_t_join.join(this, gc_join_after_reset);
25673 if (bgc_t_join.joined())
25674 #endif //MULTIPLE_HEAPS
25676 disable_preemptive (true);
25678 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25679 // When software write watch is enabled, resetting write watch is done while the runtime is suspended above. The
25680 // post-reset call to revisit_written_pages is only necessary for concurrent reset_write_watch, to discard dirtied
25681 // pages during the concurrent reset.
25684 concurrent_print_time_delta ("CRWW begin");
25686 #ifdef MULTIPLE_HEAPS
25687 for (int i = 0; i < n_heaps; i++)
25689 g_heaps[i]->reset_write_watch (TRUE);
25692 reset_write_watch (TRUE);
25693 #endif //MULTIPLE_HEAPS
25695 concurrent_print_time_delta ("CRWW");
25696 #endif //WRITE_WATCH
25698 #ifdef MULTIPLE_HEAPS
25699 for (int i = 0; i < n_heaps; i++)
25701 g_heaps[i]->revisit_written_pages (TRUE, TRUE);
25704 revisit_written_pages (TRUE, TRUE);
25705 #endif //MULTIPLE_HEAPS
25707 concurrent_print_time_delta ("CRW");
25708 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25710 #ifdef MULTIPLE_HEAPS
25711 for (int i = 0; i < n_heaps; i++)
25713 g_heaps[i]->current_bgc_state = bgc_mark_handles;
25716 current_bgc_state = bgc_mark_handles;
25717 #endif //MULTIPLE_HEAPS
25719 current_c_gc_state = c_gc_state_marking;
25721 enable_preemptive ();
25723 #ifdef MULTIPLE_HEAPS
25724 dprintf(3, ("Joining BGC threads after resetting writewatch"));
25725 bgc_t_join.restart();
25726 #endif //MULTIPLE_HEAPS
25729 disable_preemptive (true);
25731 if (num_sizedrefs > 0)
25733 GCScan::GcScanSizedRefs(background_promote, max_generation, max_generation, &sc);
25735 enable_preemptive ();
25737 #ifdef MULTIPLE_HEAPS
25738 bgc_t_join.join(this, gc_join_scan_sizedref_done);
25739 if (bgc_t_join.joined())
25741 dprintf(3, ("Done with marking all sized refs. Starting all bgc thread for marking other strong roots"));
25742 bgc_t_join.restart();
25744 #endif //MULTIPLE_HEAPS
25746 disable_preemptive (true);
25749 dprintf (3,("BGC: handle table marking"));
25750 GCScan::GcScanHandles(background_promote,
25751 max_generation, max_generation,
25753 //concurrent_print_time_delta ("concurrent marking handle table");
25754 concurrent_print_time_delta ("CRH");
25756 current_bgc_state = bgc_mark_stack;
25757 dprintf (2,("concurrent draining mark list"));
25758 background_drain_mark_list (thread);
25759 //concurrent_print_time_delta ("concurrent marking stack roots");
25760 concurrent_print_time_delta ("CRS");
25762 dprintf (2,("concurrent revisiting dirtied pages"));
25763 revisit_written_pages (TRUE);
25764 revisit_written_pages (TRUE);
25765 //concurrent_print_time_delta ("concurrent marking dirtied pages on LOH");
25766 concurrent_print_time_delta ("CRre");
25768 enable_preemptive ();
25770 #ifdef MULTIPLE_HEAPS
25771 bgc_t_join.join(this, gc_join_concurrent_overflow);
25772 if (bgc_t_join.joined())
25774 uint8_t* all_heaps_max = 0;
25775 uint8_t* all_heaps_min = MAX_PTR;
25777 for (i = 0; i < n_heaps; i++)
25779 dprintf (3, ("heap %d overflow max is %Ix, min is %Ix",
25781 g_heaps[i]->background_max_overflow_address,
25782 g_heaps[i]->background_min_overflow_address));
25783 if (all_heaps_max < g_heaps[i]->background_max_overflow_address)
25784 all_heaps_max = g_heaps[i]->background_max_overflow_address;
25785 if (all_heaps_min > g_heaps[i]->background_min_overflow_address)
25786 all_heaps_min = g_heaps[i]->background_min_overflow_address;
25788 for (i = 0; i < n_heaps; i++)
25790 g_heaps[i]->background_max_overflow_address = all_heaps_max;
25791 g_heaps[i]->background_min_overflow_address = all_heaps_min;
25793 dprintf(3, ("Starting all bgc threads after updating the overflow info"));
25794 bgc_t_join.restart();
25796 #endif //MULTIPLE_HEAPS
25798 disable_preemptive (true);
25800 dprintf (2, ("before CRov count: %d", bgc_overflow_count));
25801 bgc_overflow_count = 0;
25802 background_process_mark_overflow (TRUE);
25803 dprintf (2, ("after CRov count: %d", bgc_overflow_count));
25804 bgc_overflow_count = 0;
25805 //concurrent_print_time_delta ("concurrent processing mark overflow");
25806 concurrent_print_time_delta ("CRov");
25808 // Stop all threads, crawl all stacks and revisit changed pages.
25809 FIRE_EVENT(BGC1stConEnd);
25811 dprintf (2, ("Stopping the EE"));
25813 enable_preemptive ();
25815 #ifdef MULTIPLE_HEAPS
25816 bgc_t_join.join(this, gc_join_suspend_ee);
25817 if (bgc_t_join.joined())
25819 bgc_threads_sync_event.Reset();
25821 dprintf(3, ("Joining BGC threads for non concurrent final marking"));
25822 bgc_t_join.restart();
25824 #endif //MULTIPLE_HEAPS
25826 if (heap_number == 0)
25828 enter_spin_lock (&gc_lock);
25832 bgc_threads_sync_event.Set();
25836 bgc_threads_sync_event.Wait(INFINITE, FALSE);
25837 dprintf (2, ("bgc_threads_sync_event is signalled"));
25840 assert (settings.concurrent);
25841 assert (settings.condemned_generation == max_generation);
25843 dprintf (2, ("clearing cm_in_progress"));
25844 c_write (cm_in_progress, FALSE);
25846 bgc_alloc_lock->check();
25848 current_bgc_state = bgc_final_marking;
25850 //concurrent_print_time_delta ("concurrent marking ended");
25851 concurrent_print_time_delta ("CR");
25853 FIRE_EVENT(BGC2ndNonConBegin);
25855 mark_absorb_new_alloc();
25857 // We need a join here 'cause find_object would complain if the gen0
25858 // bricks of another heap haven't been fixed up. So we need to make sure
25859 // that every heap's gen0 bricks are fixed up before we proceed.
25860 #ifdef MULTIPLE_HEAPS
25861 bgc_t_join.join(this, gc_join_after_absorb);
25862 if (bgc_t_join.joined())
25864 dprintf(3, ("Joining BGC threads after absorb"));
25865 bgc_t_join.restart();
25867 #endif //MULTIPLE_HEAPS
25869 // give VM a chance to do work
25870 GCToEEInterface::GcBeforeBGCSweepWork();
25872 //reset the flag, indicating that the EE no longer expect concurrent
25874 sc.concurrent = FALSE;
25876 total_loh_size = generation_size (max_generation + 1);
25877 total_soh_size = generation_sizes (generation_of (max_generation));
25879 dprintf (GTC_LOG, ("FM: h%d: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
25881 dprintf (2, ("nonconcurrent marking stack roots"));
25882 GCScan::GcScanRoots(background_promote,
25883 max_generation, max_generation,
25885 //concurrent_print_time_delta ("nonconcurrent marking stack roots");
25886 concurrent_print_time_delta ("NRS");
25888 // finalize_queue->EnterFinalizeLock();
25889 finalize_queue->GcScanRoots(background_promote, heap_number, 0);
25890 // finalize_queue->LeaveFinalizeLock();
25892 dprintf (2, ("nonconcurrent marking handle table"));
25893 GCScan::GcScanHandles(background_promote,
25894 max_generation, max_generation,
25896 //concurrent_print_time_delta ("nonconcurrent marking handle table");
25897 concurrent_print_time_delta ("NRH");
25899 dprintf (2,("---- (GC%d)final going through written pages ----", VolatileLoad(&settings.gc_index)));
25900 revisit_written_pages (FALSE);
25901 //concurrent_print_time_delta ("nonconcurrent revisit dirtied pages on LOH");
25902 concurrent_print_time_delta ("NRre LOH");
25904 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25905 #ifdef MULTIPLE_HEAPS
25906 bgc_t_join.join(this, gc_join_disable_software_write_watch);
25907 if (bgc_t_join.joined())
25908 #endif // MULTIPLE_HEAPS
25910 // The runtime is suspended, and we will be doing a final query of dirty pages, so pause tracking written pages to
25911 // avoid further perf penalty after the runtime is restarted
25912 SoftwareWriteWatch::DisableForGCHeap();
25914 #ifdef MULTIPLE_HEAPS
25915 dprintf(3, ("Restarting BGC threads after disabling software write watch"));
25916 bgc_t_join.restart();
25917 #endif // MULTIPLE_HEAPS
25919 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25921 dprintf (2, ("before NR 1st Hov count: %d", bgc_overflow_count));
25922 bgc_overflow_count = 0;
25924 // Dependent handles need to be scanned with a special algorithm (see the header comment on
25925 // scan_dependent_handles for more detail). We perform an initial scan without processing any mark
25926 // stack overflow. This is not guaranteed to complete the operation but in a common case (where there
25927 // are no dependent handles that are due to be collected) it allows us to optimize away further scans.
25928 // The call to background_scan_dependent_handles is what will cycle through more iterations if
25929 // required and will also perform processing of any mark stack overflow once the dependent handle
25930 // table has been fully promoted.
25931 dprintf (2, ("1st dependent handle scan and process mark overflow"));
25932 GCScan::GcDhInitialScan(background_promote, max_generation, max_generation, &sc);
25933 background_scan_dependent_handles (&sc);
25934 //concurrent_print_time_delta ("1st nonconcurrent dependent handle scan and process mark overflow");
25935 concurrent_print_time_delta ("NR 1st Hov");
25937 dprintf (2, ("after NR 1st Hov count: %d", bgc_overflow_count));
25938 bgc_overflow_count = 0;
25940 #ifdef MULTIPLE_HEAPS
25941 bgc_t_join.join(this, gc_join_null_dead_short_weak);
25942 if (bgc_t_join.joined())
25943 #endif //MULTIPLE_HEAPS
25945 GCToEEInterface::AfterGcScanRoots (max_generation, max_generation, &sc);
25947 #ifdef MULTIPLE_HEAPS
25948 dprintf(3, ("Joining BGC threads for short weak handle scan"));
25949 bgc_t_join.restart();
25950 #endif //MULTIPLE_HEAPS
25953 // null out the target of short weakref that were not promoted.
25954 GCScan::GcShortWeakPtrScan(background_promote, max_generation, max_generation,&sc);
25956 //concurrent_print_time_delta ("bgc GcShortWeakPtrScan");
25957 concurrent_print_time_delta ("NR GcShortWeakPtrScan");
25961 #ifdef MULTIPLE_HEAPS
25962 bgc_t_join.join(this, gc_join_scan_finalization);
25963 if (bgc_t_join.joined())
25965 dprintf(3, ("Joining BGC threads for finalization"));
25966 bgc_t_join.restart();
25968 #endif //MULTIPLE_HEAPS
25970 //Handle finalization.
25971 dprintf(3,("Marking finalization data"));
25972 //concurrent_print_time_delta ("bgc joined to mark finalization");
25973 concurrent_print_time_delta ("NRj");
25975 // finalize_queue->EnterFinalizeLock();
25976 finalize_queue->ScanForFinalization (background_promote, max_generation, FALSE, __this);
25977 // finalize_queue->LeaveFinalizeLock();
25979 concurrent_print_time_delta ("NRF");
25982 dprintf (2, ("before NR 2nd Hov count: %d", bgc_overflow_count));
25983 bgc_overflow_count = 0;
25985 // Scan dependent handles again to promote any secondaries associated with primaries that were promoted
25986 // for finalization. As before background_scan_dependent_handles will also process any mark stack
25988 dprintf (2, ("2nd dependent handle scan and process mark overflow"));
25989 background_scan_dependent_handles (&sc);
25990 //concurrent_print_time_delta ("2nd nonconcurrent dependent handle scan and process mark overflow");
25991 concurrent_print_time_delta ("NR 2nd Hov");
25993 #ifdef MULTIPLE_HEAPS
25994 bgc_t_join.join(this, gc_join_null_dead_long_weak);
25995 if (bgc_t_join.joined())
25997 dprintf(2, ("Joining BGC threads for weak pointer deletion"));
25998 bgc_t_join.restart();
26000 #endif //MULTIPLE_HEAPS
26002 // null out the target of long weakref that were not promoted.
26003 GCScan::GcWeakPtrScan (background_promote, max_generation, max_generation, &sc);
26004 concurrent_print_time_delta ("NR GcWeakPtrScan");
26006 #ifdef MULTIPLE_HEAPS
26007 bgc_t_join.join(this, gc_join_null_dead_syncblk);
26008 if (bgc_t_join.joined())
26009 #endif //MULTIPLE_HEAPS
26011 dprintf (2, ("calling GcWeakPtrScanBySingleThread"));
26012 // scan for deleted entries in the syncblk cache
26013 GCScan::GcWeakPtrScanBySingleThread (max_generation, max_generation, &sc);
26014 concurrent_print_time_delta ("NR GcWeakPtrScanBySingleThread");
26015 #ifdef MULTIPLE_HEAPS
26016 dprintf(2, ("Starting BGC threads for end of background mark phase"));
26017 bgc_t_join.restart();
26018 #endif //MULTIPLE_HEAPS
26021 gen0_bricks_cleared = FALSE;
26023 dprintf (2, ("end of bgc mark: loh: %d, soh: %d",
26024 generation_size (max_generation + 1),
26025 generation_sizes (generation_of (max_generation))));
26027 for (int gen_idx = max_generation; gen_idx <= (max_generation + 1); gen_idx++)
26029 generation* gen = generation_of (gen_idx);
26030 dynamic_data* dd = dynamic_data_of (gen_idx);
26031 dd_begin_data_size (dd) = generation_size (gen_idx) -
26032 (generation_free_list_space (gen) + generation_free_obj_space (gen)) -
26033 Align (size (generation_allocation_start (gen)));
26034 dd_survived_size (dd) = 0;
26035 dd_pinned_survived_size (dd) = 0;
26036 dd_artificial_pinned_survived_size (dd) = 0;
26037 dd_added_pinned_size (dd) = 0;
26040 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
26041 PREFIX_ASSUME(seg != NULL);
26045 seg->flags &= ~heap_segment_flags_swept;
26047 if (heap_segment_allocated (seg) == heap_segment_mem (seg))
26049 // This can't happen...
26053 if (seg == ephemeral_heap_segment)
26055 heap_segment_background_allocated (seg) = generation_allocation_start (generation_of (max_generation - 1));
26059 heap_segment_background_allocated (seg) = heap_segment_allocated (seg);
26062 dprintf (2, ("seg %Ix background allocated is %Ix",
26063 heap_segment_mem (seg),
26064 heap_segment_background_allocated (seg)));
26065 seg = heap_segment_next_rw (seg);
26068 // We need to void alloc contexts here 'cause while background_ephemeral_sweep is running
26069 // we can't let the user code consume the left over parts in these alloc contexts.
26070 repair_allocation_contexts (FALSE);
26073 finish = GetCycleCount32();
26074 mark_time = finish - start;
26077 dprintf (2, ("end of bgc mark: gen2 free list space: %d, free obj space: %d",
26078 generation_free_list_space (generation_of (max_generation)),
26079 generation_free_obj_space (generation_of (max_generation))));
26081 dprintf(2,("---- (GC%d)End of background mark phase ----", VolatileLoad(&settings.gc_index)));
26085 gc_heap::suspend_EE ()
26087 dprintf (2, ("suspend_EE"));
26088 #ifdef MULTIPLE_HEAPS
26089 gc_heap* hp = gc_heap::g_heaps[0];
26090 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26092 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26093 #endif //MULTIPLE_HEAPS
26096 #ifdef MULTIPLE_HEAPS
26098 gc_heap::bgc_suspend_EE ()
26100 for (int i = 0; i < n_heaps; i++)
26102 gc_heap::g_heaps[i]->reset_gc_done();
26105 dprintf (2, ("bgc_suspend_EE"));
26106 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26108 gc_started = FALSE;
26109 for (int i = 0; i < n_heaps; i++)
26111 gc_heap::g_heaps[i]->set_gc_done();
26116 gc_heap::bgc_suspend_EE ()
26120 dprintf (2, ("bgc_suspend_EE"));
26121 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26122 gc_started = FALSE;
26125 #endif //MULTIPLE_HEAPS
26128 gc_heap::restart_EE ()
26130 dprintf (2, ("restart_EE"));
26131 #ifdef MULTIPLE_HEAPS
26132 GCToEEInterface::RestartEE(FALSE);
26134 GCToEEInterface::RestartEE(FALSE);
26135 #endif //MULTIPLE_HEAPS
26138 inline uint8_t* gc_heap::high_page ( heap_segment* seg, BOOL concurrent_p)
26142 uint8_t* end = ((seg == ephemeral_heap_segment) ?
26143 generation_allocation_start (generation_of (max_generation-1)) :
26144 heap_segment_allocated (seg));
26145 return align_lower_page (end);
26149 return heap_segment_allocated (seg);
26153 void gc_heap::revisit_written_page (uint8_t* page,
26157 uint8_t*& last_page,
26158 uint8_t*& last_object,
26159 BOOL large_objects_p,
26160 size_t& num_marked_objects)
26162 UNREFERENCED_PARAMETER(seg);
26164 uint8_t* start_address = page;
26166 int align_const = get_alignment_constant (!large_objects_p);
26167 uint8_t* high_address = end;
26168 uint8_t* current_lowest_address = background_saved_lowest_address;
26169 uint8_t* current_highest_address = background_saved_highest_address;
26170 BOOL no_more_loop_p = FALSE;
26173 #ifndef MULTIPLE_HEAPS
26174 const int thread = heap_number;
26175 #endif //!MULTIPLE_HEAPS
26177 if (large_objects_p)
26183 if (((last_page + WRITE_WATCH_UNIT_SIZE) == page)
26184 || (start_address <= last_object))
26190 o = find_first_object (start_address, last_object);
26191 // We can visit the same object again, but on a different page.
26192 assert (o >= last_object);
26196 dprintf (3,("page %Ix start: %Ix, %Ix[ ",
26197 (size_t)page, (size_t)o,
26198 (size_t)(min (high_address, page + WRITE_WATCH_UNIT_SIZE))));
26200 while (o < (min (high_address, page + WRITE_WATCH_UNIT_SIZE)))
26204 if (concurrent_p && large_objects_p)
26206 bgc_alloc_lock->bgc_mark_set (o);
26208 if (((CObjectHeader*)o)->IsFree())
26210 s = unused_array_size (o);
26222 dprintf (3,("Considering object %Ix(%s)", (size_t)o, (background_object_marked (o, FALSE) ? "bm" : "nbm")));
26224 assert (Align (s) >= Align (min_obj_size));
26226 uint8_t* next_o = o + Align (s, align_const);
26228 if (next_o >= start_address)
26230 #ifdef MULTIPLE_HEAPS
26233 // We set last_object here for SVR BGC here because SVR BGC has more than
26234 // one GC thread. When we have more than one GC thread we would run into this
26235 // situation if we skipped unmarked objects:
26236 // bgc thread 1 calls GWW, and detect object X not marked so it would skip it
26238 // bgc thread 2 marks X and all its current children.
26239 // user thread comes along and dirties more (and later) pages in X.
26240 // bgc thread 1 calls GWW again and gets those later pages but it will not mark anything
26241 // on them because it had already skipped X. We need to detect that this object is now
26242 // marked and mark the children on the dirtied pages.
26243 // In the future if we have less BGC threads than we have heaps we should add
26244 // the check to the number of BGC threads.
26247 #endif //MULTIPLE_HEAPS
26249 if (contain_pointers (o) &&
26250 (!((o >= current_lowest_address) && (o < current_highest_address)) ||
26251 background_marked (o)))
26253 dprintf (3, ("going through %Ix", (size_t)o));
26254 go_through_object (method_table(o), o, s, poo, start_address, use_start, (o + s),
26255 if ((uint8_t*)poo >= min (high_address, page + WRITE_WATCH_UNIT_SIZE))
26257 no_more_loop_p = TRUE;
26260 uint8_t* oo = *poo;
26262 num_marked_objects++;
26263 background_mark_object (oo THREAD_NUMBER_ARG);
26268 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP // see comment below
26270 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26271 ((CObjectHeader*)o)->IsFree() &&
26272 (next_o > min (high_address, page + WRITE_WATCH_UNIT_SIZE)))
26274 // We need to not skip the object here because of this corner scenario:
26275 // A large object was being allocated during BGC mark so we first made it
26276 // into a free object, then cleared its memory. In this loop we would detect
26277 // that it's a free object which normally we would skip. But by the next time
26278 // we call GetWriteWatch we could still be on this object and the object had
26279 // been made into a valid object and some of its memory was changed. We need
26280 // to be sure to process those written pages so we can't skip the object just
26283 // Similarly, when using software write watch, don't advance last_object when
26284 // the current object is a free object that spans beyond the current page or
26285 // high_address. Software write watch acquires gc_lock before the concurrent
26286 // GetWriteWatch() call during revisit_written_pages(). A foreground GC may
26287 // happen at that point and allocate from this free region, so when
26288 // revisit_written_pages() continues, it cannot skip now-valid objects in this
26290 no_more_loop_p = TRUE;
26295 if (concurrent_p && large_objects_p)
26297 bgc_alloc_lock->bgc_mark_done ();
26299 if (no_more_loop_p)
26306 #ifdef MULTIPLE_HEAPS
26309 assert (last_object < (min (high_address, page + WRITE_WATCH_UNIT_SIZE)));
26312 #endif //MULTIPLE_HEAPS
26317 dprintf (3,("Last object: %Ix", (size_t)last_object));
26318 last_page = align_write_watch_lower_page (o);
26321 // When reset_only_p is TRUE, we should only reset pages that are in range
26322 // because we need to consider the segments or part of segments that were
26323 // allocated out of range all live.
26324 void gc_heap::revisit_written_pages (BOOL concurrent_p, BOOL reset_only_p)
26327 if (concurrent_p && !reset_only_p)
26329 current_bgc_state = bgc_revisit_soh;
26332 size_t total_dirtied_pages = 0;
26333 size_t total_marked_objects = 0;
26335 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
26337 PREFIX_ASSUME(seg != NULL);
26339 bool reset_watch_state = !!concurrent_p;
26340 bool is_runtime_suspended = !concurrent_p;
26341 BOOL small_object_segments = TRUE;
26342 int align_const = get_alignment_constant (small_object_segments);
26348 if (small_object_segments)
26350 //switch to large segment
26351 if (concurrent_p && !reset_only_p)
26353 current_bgc_state = bgc_revisit_loh;
26358 dprintf (GTC_LOG, ("h%d: SOH: dp:%Id; mo: %Id", heap_number, total_dirtied_pages, total_marked_objects));
26359 fire_revisit_event (total_dirtied_pages, total_marked_objects, !small_object_segments);
26360 concurrent_print_time_delta (concurrent_p ? "CR SOH" : "NR SOH");
26361 total_dirtied_pages = 0;
26362 total_marked_objects = 0;
26365 small_object_segments = FALSE;
26366 //concurrent_print_time_delta (concurrent_p ? "concurrent marking dirtied pages on SOH" : "nonconcurrent marking dirtied pages on SOH");
26368 dprintf (3, ("now revisiting large object segments"));
26369 align_const = get_alignment_constant (small_object_segments);
26370 seg = heap_segment_rw (generation_start_segment (large_object_generation));
26372 PREFIX_ASSUME(seg != NULL);
26380 dprintf (GTC_LOG, ("h%d: tdp: %Id", heap_number, total_dirtied_pages));
26384 dprintf (GTC_LOG, ("h%d: LOH: dp:%Id; mo: %Id", heap_number, total_dirtied_pages, total_marked_objects));
26385 fire_revisit_event (total_dirtied_pages, total_marked_objects, !small_object_segments);
26390 uint8_t* base_address = (uint8_t*)heap_segment_mem (seg);
26391 //we need to truncate to the base of the page because
26392 //some newly allocated could exist beyond heap_segment_allocated
26393 //and if we reset the last page write watch status,
26394 // they wouldn't be guaranteed to be visited -> gc hole.
26395 uintptr_t bcount = array_size;
26396 uint8_t* last_page = 0;
26397 uint8_t* last_object = heap_segment_mem (seg);
26398 uint8_t* high_address = 0;
26400 BOOL skip_seg_p = FALSE;
26404 if ((heap_segment_mem (seg) >= background_saved_lowest_address) ||
26405 (heap_segment_reserved (seg) <= background_saved_highest_address))
26407 dprintf (3, ("h%d: sseg: %Ix(-%Ix)", heap_number,
26408 heap_segment_mem (seg), heap_segment_reserved (seg)));
26415 dprintf (3, ("looking at seg %Ix", (size_t)last_object));
26419 base_address = max (base_address, background_saved_lowest_address);
26420 dprintf (3, ("h%d: reset only starting %Ix", heap_number, base_address));
26423 dprintf (3, ("h%d: starting: %Ix, seg %Ix-%Ix", heap_number, base_address,
26424 heap_segment_mem (seg), heap_segment_reserved (seg)));
26431 high_address = ((seg == ephemeral_heap_segment) ? alloc_allocated : heap_segment_allocated (seg));
26432 high_address = min (high_address, background_saved_highest_address);
26436 high_address = high_page (seg, concurrent_p);
26439 if ((base_address < high_address) &&
26440 (bcount >= array_size))
26442 ptrdiff_t region_size = high_address - base_address;
26443 dprintf (3, ("h%d: gw: [%Ix(%Id)", heap_number, (size_t)base_address, (size_t)region_size));
26445 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26446 // When the runtime is not suspended, it's possible for the table to be resized concurrently with the scan
26447 // for dirty pages below. Prevent that by synchronizing with grow_brick_card_tables(). When the runtime is
26448 // suspended, it's ok to scan for dirty pages concurrently from multiple background GC threads for disjoint
26450 if (!is_runtime_suspended)
26452 enter_spin_lock(&gc_lock);
26454 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26456 get_write_watch_for_gc_heap (reset_watch_state, base_address, region_size,
26457 (void**)background_written_addresses,
26458 &bcount, is_runtime_suspended);
26460 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26461 if (!is_runtime_suspended)
26463 leave_spin_lock(&gc_lock);
26465 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26469 total_dirtied_pages += bcount;
26471 dprintf (3, ("Found %d pages [%Ix, %Ix[",
26472 bcount, (size_t)base_address, (size_t)high_address));
26477 for (unsigned i = 0; i < bcount; i++)
26479 uint8_t* page = (uint8_t*)background_written_addresses[i];
26480 dprintf (3, ("looking at page %d at %Ix(h: %Ix)", i,
26481 (size_t)page, (size_t)high_address));
26482 if (page < high_address)
26484 //search for marked objects in the page
26485 revisit_written_page (page, high_address, concurrent_p,
26486 seg, last_page, last_object,
26487 !small_object_segments,
26488 total_marked_objects);
26492 dprintf (3, ("page %d at %Ix is >= %Ix!", i, (size_t)page, (size_t)high_address));
26493 assert (!"page shouldn't have exceeded limit");
26498 if (bcount >= array_size){
26499 base_address = background_written_addresses [array_size-1] + WRITE_WATCH_UNIT_SIZE;
26500 bcount = array_size;
26510 seg = heap_segment_next_rw (seg);
26513 #endif //WRITE_WATCH
26516 void gc_heap::background_grow_c_mark_list()
26518 assert (c_mark_list_index >= c_mark_list_length);
26519 BOOL should_drain_p = FALSE;
26521 #ifndef MULTIPLE_HEAPS
26522 const int thread = heap_number;
26523 #endif //!MULTIPLE_HEAPS
26525 dprintf (2, ("stack copy buffer overflow"));
26526 uint8_t** new_c_mark_list = 0;
26529 if (c_mark_list_length >= (SIZE_T_MAX / (2 * sizeof (uint8_t*))))
26531 should_drain_p = TRUE;
26535 new_c_mark_list = new (nothrow) uint8_t*[c_mark_list_length*2];
26536 if (new_c_mark_list == 0)
26538 should_drain_p = TRUE;
26542 if (should_drain_p)
26545 dprintf (2, ("No more memory for the stacks copy, draining.."));
26546 //drain the list by marking its elements
26547 background_drain_mark_list (thread);
26551 assert (new_c_mark_list);
26552 memcpy (new_c_mark_list, c_mark_list, c_mark_list_length*sizeof(uint8_t*));
26553 c_mark_list_length = c_mark_list_length*2;
26554 delete c_mark_list;
26555 c_mark_list = new_c_mark_list;
26559 void gc_heap::background_promote_callback (Object** ppObject, ScanContext* sc,
26562 UNREFERENCED_PARAMETER(sc);
26563 //in order to save space on the array, mark the object,
26564 //knowing that it will be visited later
26565 assert (settings.concurrent);
26567 THREAD_NUMBER_FROM_CONTEXT;
26568 #ifndef MULTIPLE_HEAPS
26569 const int thread = 0;
26570 #endif //!MULTIPLE_HEAPS
26572 uint8_t* o = (uint8_t*)*ppObject;
26579 gc_heap* hp = gc_heap::heap_of (o);
26581 if ((o < hp->background_saved_lowest_address) || (o >= hp->background_saved_highest_address))
26586 #ifdef INTERIOR_POINTERS
26587 if (flags & GC_CALL_INTERIOR)
26589 o = hp->find_object (o, hp->background_saved_lowest_address);
26593 #endif //INTERIOR_POINTERS
26595 #ifdef FEATURE_CONSERVATIVE_GC
26596 // For conservative GC, a value on stack may point to middle of a free object.
26597 // In this case, we don't need to promote the pointer.
26598 if (GCConfig::GetConservativeGC() && ((CObjectHeader*)o)->IsFree())
26602 #endif //FEATURE_CONSERVATIVE_GC
26605 ((CObjectHeader*)o)->Validate();
26608 dprintf (3, ("Concurrent Background Promote %Ix", (size_t)o));
26609 if (o && (size (o) > LARGE_OBJECT_SIZE))
26611 dprintf (3, ("Brc %Ix", (size_t)o));
26614 if (hpt->c_mark_list_index >= hpt->c_mark_list_length)
26616 hpt->background_grow_c_mark_list();
26618 dprintf (3, ("pushing %08x into mark_list", (size_t)o));
26619 hpt->c_mark_list [hpt->c_mark_list_index++] = o;
26621 STRESS_LOG3(LF_GC|LF_GCROOTS, LL_INFO1000000, " GCHeap::Background Promote: Promote GC Root *%p = %p MT = %pT", ppObject, o, o ? ((Object*) o)->GetGCSafeMethodTable() : NULL);
26624 void gc_heap::mark_absorb_new_alloc()
26626 fix_allocation_contexts (FALSE);
26628 gen0_bricks_cleared = FALSE;
26630 clear_gen0_bricks();
26633 BOOL gc_heap::prepare_bgc_thread(gc_heap* gh)
26635 BOOL success = FALSE;
26636 BOOL thread_created = FALSE;
26637 dprintf (2, ("Preparing gc thread"));
26638 gh->bgc_threads_timeout_cs.Enter();
26639 if (!(gh->bgc_thread_running))
26641 dprintf (2, ("GC thread not runnning"));
26642 if ((gh->bgc_thread == 0) && create_bgc_thread(gh))
26645 thread_created = TRUE;
26650 dprintf (3, ("GC thread already running"));
26653 gh->bgc_threads_timeout_cs.Leave();
26656 FIRE_EVENT(GCCreateConcurrentThread_V1);
26661 BOOL gc_heap::create_bgc_thread(gc_heap* gh)
26663 assert (background_gc_done_event.IsValid());
26665 //dprintf (2, ("Creating BGC thread"));
26667 gh->bgc_thread_running = GCToEEInterface::CreateThread(gh->bgc_thread_stub, gh, true, ".NET Background GC");
26668 return gh->bgc_thread_running;
26671 BOOL gc_heap::create_bgc_threads_support (int number_of_heaps)
26674 dprintf (3, ("Creating concurrent GC thread for the first time"));
26675 if (!background_gc_done_event.CreateManualEventNoThrow(TRUE))
26679 if (!bgc_threads_sync_event.CreateManualEventNoThrow(FALSE))
26683 if (!ee_proceed_event.CreateAutoEventNoThrow(FALSE))
26687 if (!bgc_start_event.CreateManualEventNoThrow(FALSE))
26692 #ifdef MULTIPLE_HEAPS
26693 bgc_t_join.init (number_of_heaps, join_flavor_bgc);
26695 UNREFERENCED_PARAMETER(number_of_heaps);
26696 #endif //MULTIPLE_HEAPS
26704 if (background_gc_done_event.IsValid())
26706 background_gc_done_event.CloseEvent();
26708 if (bgc_threads_sync_event.IsValid())
26710 bgc_threads_sync_event.CloseEvent();
26712 if (ee_proceed_event.IsValid())
26714 ee_proceed_event.CloseEvent();
26716 if (bgc_start_event.IsValid())
26718 bgc_start_event.CloseEvent();
26725 BOOL gc_heap::create_bgc_thread_support()
26730 if (!gc_lh_block_event.CreateManualEventNoThrow(FALSE))
26735 //needs to have room for enough smallest objects fitting on a page
26736 parr = new (nothrow) uint8_t*[1 + OS_PAGE_SIZE / MIN_OBJECT_SIZE];
26742 make_c_mark_list (parr);
26750 if (gc_lh_block_event.IsValid())
26752 gc_lh_block_event.CloseEvent();
26759 int gc_heap::check_for_ephemeral_alloc()
26761 int gen = ((settings.reason == reason_oos_soh) ? (max_generation - 1) : -1);
26765 #ifdef MULTIPLE_HEAPS
26766 for (int heap_index = 0; heap_index < n_heaps; heap_index++)
26767 #endif //MULTIPLE_HEAPS
26769 for (int i = 0; i <= (max_generation - 1); i++)
26771 #ifdef MULTIPLE_HEAPS
26772 if (g_heaps[heap_index]->get_new_allocation (i) <= 0)
26774 if (get_new_allocation (i) <= 0)
26775 #endif //MULTIPLE_HEAPS
26777 gen = max (gen, i);
26788 // Wait for gc to finish sequential part
26789 void gc_heap::wait_to_proceed()
26791 assert (background_gc_done_event.IsValid());
26792 assert (bgc_start_event.IsValid());
26794 user_thread_wait(&ee_proceed_event, FALSE);
26797 // Start a new concurrent gc
26798 void gc_heap::start_c_gc()
26800 assert (background_gc_done_event.IsValid());
26801 assert (bgc_start_event.IsValid());
26803 //Need to make sure that the gc thread is in the right place.
26804 background_gc_done_event.Wait(INFINITE, FALSE);
26805 background_gc_done_event.Reset();
26806 bgc_start_event.Set();
26809 void gc_heap::do_background_gc()
26811 dprintf (2, ("starting a BGC"));
26812 #ifdef MULTIPLE_HEAPS
26813 for (int i = 0; i < n_heaps; i++)
26815 g_heaps[i]->init_background_gc();
26818 init_background_gc();
26819 #endif //MULTIPLE_HEAPS
26820 //start the background gc
26823 //wait until we get restarted by the BGC.
26827 void gc_heap::kill_gc_thread()
26829 //assert (settings.concurrent == FALSE);
26831 // We are doing a two-stage shutdown now.
26832 // In the first stage, we do minimum work, and call ExitProcess at the end.
26833 // In the secodn stage, we have the Loader lock and only one thread is
26834 // alive. Hence we do not need to kill gc thread.
26835 background_gc_done_event.CloseEvent();
26836 gc_lh_block_event.CloseEvent();
26837 bgc_start_event.CloseEvent();
26838 bgc_threads_timeout_cs.Destroy();
26840 recursive_gc_sync::shutdown();
26843 void gc_heap::bgc_thread_function()
26845 assert (background_gc_done_event.IsValid());
26846 assert (bgc_start_event.IsValid());
26848 dprintf (3, ("gc_thread thread starting..."));
26850 BOOL do_exit = FALSE;
26852 bool cooperative_mode = true;
26853 bgc_thread_id.SetToCurrentThread();
26854 dprintf (1, ("bgc_thread_id is set to %x", (uint32_t)GCToOSInterface::GetCurrentThreadIdForLogging()));
26857 // Wait for work to do...
26858 dprintf (3, ("bgc thread: waiting..."));
26860 cooperative_mode = enable_preemptive ();
26861 //current_thread->m_fPreemptiveGCDisabled = 0;
26863 uint32_t result = bgc_start_event.Wait(
26865 #ifdef MULTIPLE_HEAPS
26869 #endif //MULTIPLE_HEAPS
26871 #ifdef MULTIPLE_HEAPS
26875 #endif //MULTIPLE_HEAPS
26878 dprintf (2, ("gc thread: finished waiting"));
26880 // not calling disable_preemptive here 'cause we
26881 // can't wait for GC complete here - RestartEE will be called
26882 // when we've done the init work.
26884 if (result == WAIT_TIMEOUT)
26886 // Should join the bgc threads and terminate all of them
26888 dprintf (1, ("GC thread timeout"));
26889 bgc_threads_timeout_cs.Enter();
26890 if (!keep_bgc_threads_p)
26892 dprintf (2, ("GC thread exiting"));
26893 bgc_thread_running = FALSE;
26895 bgc_thread_id.Clear();
26898 bgc_threads_timeout_cs.Leave();
26903 dprintf (3, ("GC thread needed, not exiting"));
26907 // if we signal the thread with no concurrent work to do -> exit
26908 if (!settings.concurrent)
26910 dprintf (3, ("no concurrent GC needed, exiting"));
26916 recursive_gc_sync::begin_background();
26917 dprintf (2, ("beginning of bgc: gen2 FL: %d, FO: %d, frag: %d",
26918 generation_free_list_space (generation_of (max_generation)),
26919 generation_free_obj_space (generation_of (max_generation)),
26920 dd_fragmentation (dynamic_data_of (max_generation))));
26924 current_bgc_state = bgc_not_in_process;
26927 //trace_gc = FALSE;
26930 enable_preemptive ();
26931 #ifdef MULTIPLE_HEAPS
26932 bgc_t_join.join(this, gc_join_done);
26933 if (bgc_t_join.joined())
26934 #endif //MULTIPLE_HEAPS
26936 enter_spin_lock (&gc_lock);
26937 dprintf (SPINLOCK_LOG, ("bgc Egc"));
26939 bgc_start_event.Reset();
26941 #ifdef MULTIPLE_HEAPS
26942 for (int gen = max_generation; gen <= (max_generation + 1); gen++)
26944 size_t desired_per_heap = 0;
26945 size_t total_desired = 0;
26948 for (int i = 0; i < n_heaps; i++)
26951 dd = hp->dynamic_data_of (gen);
26952 size_t temp_total_desired = total_desired + dd_desired_allocation (dd);
26953 if (temp_total_desired < total_desired)
26956 total_desired = (size_t)MAX_PTR;
26959 total_desired = temp_total_desired;
26962 desired_per_heap = Align ((total_desired/n_heaps), get_alignment_constant (FALSE));
26964 for (int i = 0; i < n_heaps; i++)
26966 hp = gc_heap::g_heaps[i];
26967 dd = hp->dynamic_data_of (gen);
26968 dd_desired_allocation (dd) = desired_per_heap;
26969 dd_gc_new_allocation (dd) = desired_per_heap;
26970 dd_new_allocation (dd) = desired_per_heap;
26973 #endif //MULTIPLE_HEAPS
26974 #ifdef MULTIPLE_HEAPS
26976 #endif //MULTIPLE_HEAPS
26978 c_write (settings.concurrent, FALSE);
26979 recursive_gc_sync::end_background();
26980 keep_bgc_threads_p = FALSE;
26981 background_gc_done_event.Set();
26983 dprintf (SPINLOCK_LOG, ("bgc Lgc"));
26984 leave_spin_lock (&gc_lock);
26985 #ifdef MULTIPLE_HEAPS
26986 dprintf(1, ("End of BGC - starting all BGC threads"));
26987 bgc_t_join.restart();
26988 #endif //MULTIPLE_HEAPS
26990 // We can't disable preempt here because there might've been a GC already
26991 // started and decided to do a BGC and waiting for a BGC thread to restart
26992 // vm. That GC will be waiting in wait_to_proceed and we are waiting for it
26993 // to restart the VM so we deadlock.
26994 //gc_heap::disable_preemptive (current_thread, TRUE);
26997 FIRE_EVENT(GCTerminateConcurrentThread_V1);
26999 dprintf (3, ("bgc_thread thread exiting"));
27003 #endif //BACKGROUND_GC
27005 //Clear the cards [start_card, end_card[
27006 void gc_heap::clear_cards (size_t start_card, size_t end_card)
27008 if (start_card < end_card)
27010 size_t start_word = card_word (start_card);
27011 size_t end_word = card_word (end_card);
27012 if (start_word < end_word)
27014 // Figure out the bit positions of the cards within their words
27015 unsigned bits = card_bit (start_card);
27016 card_table [start_word] &= lowbits (~0, bits);
27017 for (size_t i = start_word+1; i < end_word; i++)
27018 card_table [i] = 0;
27019 bits = card_bit (end_card);
27020 // Don't write beyond end_card (and possibly uncommitted card table space).
27023 card_table [end_word] &= highbits (~0, bits);
27028 // If the start and end cards are in the same word, just clear the appropriate card
27029 // bits in that word.
27030 card_table [start_word] &= (lowbits (~0, card_bit (start_card)) |
27031 highbits (~0, card_bit (end_card)));
27033 #ifdef VERYSLOWDEBUG
27034 size_t card = start_card;
27035 while (card < end_card)
27037 assert (! (card_set_p (card)));
27040 #endif //VERYSLOWDEBUG
27041 dprintf (3,("Cleared cards [%Ix:%Ix, %Ix:%Ix[",
27042 start_card, (size_t)card_address (start_card),
27043 end_card, (size_t)card_address (end_card)));
27047 void gc_heap::clear_card_for_addresses (uint8_t* start_address, uint8_t* end_address)
27049 size_t start_card = card_of (align_on_card (start_address));
27050 size_t end_card = card_of (align_lower_card (end_address));
27051 clear_cards (start_card, end_card);
27054 // copy [srccard, ...[ to [dst_card, end_card[
27055 // This will set the same bit twice. Can be optimized.
27057 void gc_heap::copy_cards (size_t dst_card,
27062 // If the range is empty, this function is a no-op - with the subtlety that
27063 // either of the accesses card_table[srcwrd] or card_table[dstwrd] could be
27064 // outside the committed region. To avoid the access, leave early.
27065 if (!(dst_card < end_card))
27068 unsigned int srcbit = card_bit (src_card);
27069 unsigned int dstbit = card_bit (dst_card);
27070 size_t srcwrd = card_word (src_card);
27071 size_t dstwrd = card_word (dst_card);
27072 unsigned int srctmp = card_table[srcwrd];
27073 unsigned int dsttmp = card_table[dstwrd];
27075 for (size_t card = dst_card; card < end_card; card++)
27077 if (srctmp & (1 << srcbit))
27078 dsttmp |= 1 << dstbit;
27080 dsttmp &= ~(1 << dstbit);
27082 if (!(++srcbit % 32))
27084 srctmp = card_table[++srcwrd];
27090 if (srctmp & (1 << srcbit))
27091 dsttmp |= 1 << dstbit;
27094 if (!(++dstbit % 32))
27096 card_table[dstwrd] = dsttmp;
27098 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27101 card_bundle_set(cardw_card_bundle(dstwrd));
27106 dsttmp = card_table[dstwrd];
27111 card_table[dstwrd] = dsttmp;
27113 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27116 card_bundle_set(cardw_card_bundle(dstwrd));
27121 void gc_heap::copy_cards_for_addresses (uint8_t* dest, uint8_t* src, size_t len)
27123 ptrdiff_t relocation_distance = src - dest;
27124 size_t start_dest_card = card_of (align_on_card (dest));
27125 size_t end_dest_card = card_of (dest + len - 1);
27126 size_t dest_card = start_dest_card;
27127 size_t src_card = card_of (card_address (dest_card)+relocation_distance);
27128 dprintf (3,("Copying cards [%Ix:%Ix->%Ix:%Ix, ",
27129 src_card, (size_t)src, dest_card, (size_t)dest));
27130 dprintf (3,(" %Ix->%Ix:%Ix[",
27131 (size_t)src+len, end_dest_card, (size_t)dest+len));
27133 dprintf (3, ("dest: %Ix, src: %Ix, len: %Ix, reloc: %Ix, align_on_card(dest) is %Ix",
27134 dest, src, len, relocation_distance, (align_on_card (dest))));
27136 dprintf (3, ("start_dest_card: %Ix (address: %Ix), end_dest_card: %Ix(addr: %Ix), card_of (dest): %Ix",
27137 start_dest_card, card_address (start_dest_card), end_dest_card, card_address (end_dest_card), card_of (dest)));
27139 //First card has two boundaries
27140 if (start_dest_card != card_of (dest))
27142 if ((card_of (card_address (start_dest_card) + relocation_distance) <= card_of (src + len - 1))&&
27143 card_set_p (card_of (card_address (start_dest_card) + relocation_distance)))
27145 dprintf (3, ("card_address (start_dest_card) + reloc is %Ix, card: %Ix(set), src+len-1: %Ix, card: %Ix",
27146 (card_address (start_dest_card) + relocation_distance),
27147 card_of (card_address (start_dest_card) + relocation_distance),
27149 card_of (src + len - 1)));
27151 dprintf (3, ("setting card: %Ix", card_of (dest)));
27152 set_card (card_of (dest));
27156 if (card_set_p (card_of (src)))
27157 set_card (card_of (dest));
27160 copy_cards (dest_card, src_card, end_dest_card,
27161 ((dest - align_lower_card (dest)) != (src - align_lower_card (src))));
27163 //Last card has two boundaries.
27164 if ((card_of (card_address (end_dest_card) + relocation_distance) >= card_of (src)) &&
27165 card_set_p (card_of (card_address (end_dest_card) + relocation_distance)))
27167 dprintf (3, ("card_address (end_dest_card) + reloc is %Ix, card: %Ix(set), src: %Ix, card: %Ix",
27168 (card_address (end_dest_card) + relocation_distance),
27169 card_of (card_address (end_dest_card) + relocation_distance),
27173 dprintf (3, ("setting card: %Ix", end_dest_card));
27174 set_card (end_dest_card);
27177 if (card_set_p (card_of (src + len - 1)))
27178 set_card (end_dest_card);
27180 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27181 card_bundles_set(cardw_card_bundle(card_word(card_of(dest))), cardw_card_bundle(align_cardw_on_bundle(card_word(end_dest_card))));
27185 #ifdef BACKGROUND_GC
27186 // this does not need the Interlocked version of mark_array_set_marked.
27187 void gc_heap::copy_mark_bits_for_addresses (uint8_t* dest, uint8_t* src, size_t len)
27189 dprintf (3, ("Copying mark_bits for addresses [%Ix->%Ix, %Ix->%Ix[",
27190 (size_t)src, (size_t)dest,
27191 (size_t)src+len, (size_t)dest+len));
27193 uint8_t* src_o = src;
27195 uint8_t* src_end = src + len;
27196 int align_const = get_alignment_constant (TRUE);
27197 ptrdiff_t reloc = dest - src;
27199 while (src_o < src_end)
27201 uint8_t* next_o = src_o + Align (size (src_o), align_const);
27203 if (background_object_marked (src_o, TRUE))
27205 dest_o = src_o + reloc;
27207 //if (background_object_marked (dest_o, FALSE))
27209 // dprintf (3, ("*%Ix shouldn't have already been marked!", (size_t)(dest_o)));
27210 // FATAL_GC_ERROR();
27213 background_mark (dest_o,
27214 background_saved_lowest_address,
27215 background_saved_highest_address);
27216 dprintf (3, ("bc*%Ix*bc, b*%Ix*b", (size_t)src_o, (size_t)(dest_o)));
27222 #endif //BACKGROUND_GC
27224 void gc_heap::fix_brick_to_highest (uint8_t* o, uint8_t* next_o)
27226 size_t new_current_brick = brick_of (o);
27227 set_brick (new_current_brick,
27228 (o - brick_address (new_current_brick)));
27229 size_t b = 1 + new_current_brick;
27230 size_t limit = brick_of (next_o);
27231 //dprintf(3,(" fixing brick %Ix to point to object %Ix, till %Ix(%Ix)",
27232 dprintf(3,("b:%Ix->%Ix-%Ix",
27233 new_current_brick, (size_t)o, (size_t)next_o));
27236 set_brick (b,(new_current_brick - b));
27241 // start can not be >= heap_segment_allocated for the segment.
27242 uint8_t* gc_heap::find_first_object (uint8_t* start, uint8_t* first_object)
27244 size_t brick = brick_of (start);
27246 //last_object == null -> no search shortcut needed
27247 if ((brick == brick_of (first_object) || (start <= first_object)))
27253 ptrdiff_t min_brick = (ptrdiff_t)brick_of (first_object);
27254 ptrdiff_t prev_brick = (ptrdiff_t)brick - 1;
27255 int brick_entry = 0;
27258 if (prev_brick < min_brick)
27262 if ((brick_entry = get_brick_entry(prev_brick)) >= 0)
27266 assert (! ((brick_entry == 0)));
27267 prev_brick = (brick_entry + prev_brick);
27270 o = ((prev_brick < min_brick) ? first_object :
27271 brick_address (prev_brick) + brick_entry - 1);
27272 assert (o <= start);
27275 assert (Align (size (o)) >= Align (min_obj_size));
27276 uint8_t* next_o = o + Align (size (o));
27277 size_t curr_cl = (size_t)next_o / brick_size;
27278 size_t min_cl = (size_t)first_object / brick_size;
27280 //dprintf (3,( "Looking for intersection with %Ix from %Ix", (size_t)start, (size_t)o));
27282 unsigned int n_o = 1;
27285 uint8_t* next_b = min (align_lower_brick (next_o) + brick_size, start+1);
27287 while (next_o <= start)
27295 assert (Align (size (o)) >= Align (min_obj_size));
27296 next_o = o + Align (size (o));
27298 }while (next_o < next_b);
27300 if (((size_t)next_o / brick_size) != curr_cl)
27302 if (curr_cl >= min_cl)
27304 fix_brick_to_highest (o, next_o);
27306 curr_cl = (size_t) next_o / brick_size;
27308 next_b = min (align_lower_brick (next_o) + brick_size, start+1);
27311 size_t bo = brick_of (o);
27312 //dprintf (3, ("Looked at %Id objects, fixing brick [%Ix-[%Ix",
27313 dprintf (3, ("%Id o, [%Ix-[%Ix",
27317 set_brick (bo, (o - brick_address(bo)));
27332 // Find the first non-zero card word between cardw and cardw_end.
27333 // The index of the word we find is returned in cardw.
27334 BOOL gc_heap::find_card_dword (size_t& cardw, size_t cardw_end)
27336 dprintf (3, ("gc: %d, find_card_dword cardw: %Ix, cardw_end: %Ix",
27337 dd_collection_count (dynamic_data_of (0)), cardw, cardw_end));
27339 if (card_bundles_enabled())
27341 size_t cardb = cardw_card_bundle (cardw);
27342 size_t end_cardb = cardw_card_bundle (align_cardw_on_bundle (cardw_end));
27345 // Find a non-zero bundle
27346 while ((cardb < end_cardb) && (card_bundle_set_p (cardb) == 0))
27351 if (cardb == end_cardb)
27354 // We found a bundle, so go through its words and find a non-zero card word
27355 uint32_t* card_word = &card_table[max(card_bundle_cardw (cardb),cardw)];
27356 uint32_t* card_word_end = &card_table[min(card_bundle_cardw (cardb+1),cardw_end)];
27357 while ((card_word < card_word_end) && !(*card_word))
27362 if (card_word != card_word_end)
27364 cardw = (card_word - &card_table[0]);
27367 else if ((cardw <= card_bundle_cardw (cardb)) &&
27368 (card_word == &card_table [card_bundle_cardw (cardb+1)]))
27370 // a whole bundle was explored and is empty
27371 dprintf (3, ("gc: %d, find_card_dword clear bundle: %Ix cardw:[%Ix,%Ix[",
27372 dd_collection_count (dynamic_data_of (0)),
27373 cardb, card_bundle_cardw (cardb),
27374 card_bundle_cardw (cardb+1)));
27375 card_bundle_clear (cardb);
27383 uint32_t* card_word = &card_table[cardw];
27384 uint32_t* card_word_end = &card_table [cardw_end];
27386 while (card_word < card_word_end)
27388 if (*card_word != 0)
27390 cardw = (card_word - &card_table [0]);
27401 #endif //CARD_BUNDLE
27403 // Find cards that are set between two points in a card table.
27405 // card_table : The card table.
27406 // card : [in/out] As input, the card to start searching from.
27407 // As output, the first card that's set.
27408 // card_word_end : The card word at which to stop looking.
27409 // end_card : [out] The last card which is set.
27410 BOOL gc_heap::find_card(uint32_t* card_table,
27412 size_t card_word_end,
27415 uint32_t* last_card_word;
27416 uint32_t card_word_value;
27417 uint32_t bit_position;
27419 // Find the first card which is set
27420 last_card_word = &card_table [card_word (card)];
27421 bit_position = card_bit (card);
27422 card_word_value = (*last_card_word) >> bit_position;
27423 if (!card_word_value)
27427 // Using the card bundle, go through the remaining card words between here and
27428 // card_word_end until we find one that is non-zero.
27429 size_t lcw = card_word(card) + 1;
27430 if (gc_heap::find_card_dword (lcw, card_word_end) == FALSE)
27436 last_card_word = &card_table [lcw];
27437 card_word_value = *last_card_word;
27440 #else //CARD_BUNDLE
27441 // Go through the remaining card words between here and card_word_end until we find
27442 // one that is non-zero.
27447 while ((last_card_word < &card_table [card_word_end]) && !(*last_card_word));
27449 if (last_card_word < &card_table [card_word_end])
27451 card_word_value = *last_card_word;
27455 // We failed to find any non-zero card words before we got to card_word_end
27458 #endif //CARD_BUNDLE
27461 // Look for the lowest bit set
27462 if (card_word_value)
27464 while (!(card_word_value & 1))
27467 card_word_value = card_word_value / 2;
27471 // card is the card word index * card size + the bit index within the card
27472 card = (last_card_word - &card_table[0]) * card_word_width + bit_position;
27476 // Keep going until we get to an un-set card.
27478 card_word_value = card_word_value / 2;
27480 // If we reach the end of the card word and haven't hit a 0 yet, start going
27481 // card word by card word until we get to one that's not fully set (0xFFFF...)
27482 // or we reach card_word_end.
27483 if ((bit_position == card_word_width) && (last_card_word < &card_table [card_word_end]))
27487 card_word_value = *(++last_card_word);
27488 } while ((last_card_word < &card_table [card_word_end]) &&
27491 (card_word_value == (1 << card_word_width)-1)
27493 // if left shift count >= width of type,
27494 // gcc reports error.
27495 (card_word_value == ~0u)
27500 } while (card_word_value & 1);
27502 end_card = (last_card_word - &card_table [0])* card_word_width + bit_position;
27504 //dprintf (3, ("find_card: [%Ix, %Ix[ set", card, end_card));
27505 dprintf (3, ("fc: [%Ix, %Ix[", card, end_card));
27510 //because of heap expansion, computing end is complicated.
27511 uint8_t* compute_next_end (heap_segment* seg, uint8_t* low)
27513 if ((low >= heap_segment_mem (seg)) &&
27514 (low < heap_segment_allocated (seg)))
27517 return heap_segment_allocated (seg);
27521 gc_heap::compute_next_boundary (uint8_t* low, int gen_number,
27524 UNREFERENCED_PARAMETER(low);
27526 //when relocating, the fault line is the plan start of the younger
27527 //generation because the generation is promoted.
27528 if (relocating && (gen_number == (settings.condemned_generation + 1)))
27530 generation* gen = generation_of (gen_number - 1);
27531 uint8_t* gen_alloc = generation_plan_allocation_start (gen);
27532 assert (gen_alloc);
27537 assert (gen_number > settings.condemned_generation);
27538 return generation_allocation_start (generation_of (gen_number - 1 ));
27544 gc_heap::keep_card_live (uint8_t* o, size_t& n_gen,
27545 size_t& cg_pointers_found)
27548 if ((gc_low <= o) && (gc_high > o))
27552 #ifdef MULTIPLE_HEAPS
27555 gc_heap* hp = heap_of (o);
27558 if ((hp->gc_low <= o) &&
27565 #endif //MULTIPLE_HEAPS
27566 cg_pointers_found ++;
27567 dprintf (4, ("keep card live for %Ix", o));
27571 gc_heap::mark_through_cards_helper (uint8_t** poo, size_t& n_gen,
27572 size_t& cg_pointers_found,
27573 card_fn fn, uint8_t* nhigh,
27574 uint8_t* next_boundary)
27577 if ((gc_low <= *poo) && (gc_high > *poo))
27580 call_fn(fn) (poo THREAD_NUMBER_ARG);
27582 #ifdef MULTIPLE_HEAPS
27585 gc_heap* hp = heap_of_gc (*poo);
27588 if ((hp->gc_low <= *poo) &&
27589 (hp->gc_high > *poo))
27592 call_fn(fn) (poo THREAD_NUMBER_ARG);
27594 if ((fn == &gc_heap::relocate_address) ||
27595 ((hp->ephemeral_low <= *poo) &&
27596 (hp->ephemeral_high > *poo)))
27598 cg_pointers_found++;
27602 #endif //MULTIPLE_HEAPS
27603 if ((next_boundary <= *poo) && (nhigh > *poo))
27605 cg_pointers_found ++;
27606 dprintf (4, ("cg pointer %Ix found, %Id so far",
27607 (size_t)*poo, cg_pointers_found ));
27612 BOOL gc_heap::card_transition (uint8_t* po, uint8_t* end, size_t card_word_end,
27613 size_t& cg_pointers_found,
27614 size_t& n_eph, size_t& n_card_set,
27615 size_t& card, size_t& end_card,
27616 BOOL& foundp, uint8_t*& start_address,
27617 uint8_t*& limit, size_t& n_cards_cleared)
27619 dprintf (3, ("pointer %Ix past card %Ix", (size_t)po, (size_t)card));
27620 dprintf (3, ("ct: %Id cg", cg_pointers_found));
27621 BOOL passed_end_card_p = FALSE;
27624 if (cg_pointers_found == 0)
27626 //dprintf(3,(" Clearing cards [%Ix, %Ix[ ",
27627 dprintf(3,(" CC [%Ix, %Ix[ ",
27628 (size_t)card_address(card), (size_t)po));
27629 clear_cards (card, card_of(po));
27630 n_card_set -= (card_of (po) - card);
27631 n_cards_cleared += (card_of (po) - card);
27634 n_eph +=cg_pointers_found;
27635 cg_pointers_found = 0;
27636 card = card_of (po);
27637 if (card >= end_card)
27639 passed_end_card_p = TRUE;
27640 dprintf (3, ("card %Ix exceeding end_card %Ix",
27641 (size_t)card, (size_t)end_card));
27642 foundp = find_card (card_table, card, card_word_end, end_card);
27645 n_card_set+= end_card - card;
27646 start_address = card_address (card);
27647 dprintf (3, ("NewC: %Ix, start: %Ix, end: %Ix",
27648 (size_t)card, (size_t)start_address,
27649 (size_t)card_address (end_card)));
27651 limit = min (end, card_address (end_card));
27653 assert (!((limit == card_address (end_card))&&
27654 card_set_p (end_card)));
27657 return passed_end_card_p;
27660 void gc_heap::mark_through_cards_for_segments (card_fn fn, BOOL relocating)
27662 #ifdef BACKGROUND_GC
27663 dprintf (3, ("current_sweep_pos is %Ix, saved_sweep_ephemeral_seg is %Ix(%Ix)",
27664 current_sweep_pos, saved_sweep_ephemeral_seg, saved_sweep_ephemeral_start));
27666 heap_segment* soh_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
27667 PREFIX_ASSUME(soh_seg != NULL);
27671 dprintf (3, ("seg %Ix, bgc_alloc: %Ix, alloc: %Ix",
27673 heap_segment_background_allocated (soh_seg),
27674 heap_segment_allocated (soh_seg)));
27676 soh_seg = heap_segment_next_rw (soh_seg);
27678 #endif //BACKGROUND_GC
27680 uint8_t* low = gc_low;
27681 uint8_t* high = gc_high;
27682 size_t end_card = 0;
27684 generation* oldest_gen = generation_of (max_generation);
27685 int curr_gen_number = max_generation;
27686 uint8_t* gen_boundary = generation_allocation_start(generation_of(curr_gen_number - 1));
27687 uint8_t* next_boundary = compute_next_boundary(gc_low, curr_gen_number, relocating);
27689 heap_segment* seg = heap_segment_rw (generation_start_segment (oldest_gen));
27690 PREFIX_ASSUME(seg != NULL);
27692 uint8_t* beg = generation_allocation_start (oldest_gen);
27693 uint8_t* end = compute_next_end (seg, low);
27694 uint8_t* last_object = beg;
27696 size_t cg_pointers_found = 0;
27698 size_t card_word_end = (card_of (align_on_card_word (end)) / card_word_width);
27702 size_t n_card_set = 0;
27703 uint8_t* nhigh = (relocating ? heap_segment_plan_allocated (ephemeral_heap_segment) : high);
27705 BOOL foundp = FALSE;
27706 uint8_t* start_address = 0;
27707 uint8_t* limit = 0;
27708 size_t card = card_of (beg);
27709 #ifdef BACKGROUND_GC
27710 BOOL consider_bgc_mark_p = FALSE;
27711 BOOL check_current_sweep_p = FALSE;
27712 BOOL check_saved_sweep_p = FALSE;
27713 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
27714 #endif //BACKGROUND_GC
27716 dprintf(3, ("CMs: %Ix->%Ix", (size_t)beg, (size_t)end));
27717 size_t total_cards_cleared = 0;
27721 if (card_of(last_object) > card)
27723 // cg means cross-generational
27724 dprintf (3, ("Found %Id cg pointers", cg_pointers_found));
27725 if (cg_pointers_found == 0)
27727 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card), (size_t)last_object));
27728 clear_cards (card, card_of(last_object));
27729 n_card_set -= (card_of (last_object) - card);
27730 total_cards_cleared += (card_of (last_object) - card);
27733 n_eph += cg_pointers_found;
27734 cg_pointers_found = 0;
27735 card = card_of (last_object);
27738 if (card >= end_card)
27740 // Find the first card that's set (between card and card_word_end)
27741 foundp = find_card(card_table, card, card_word_end, end_card);
27744 // We found card(s) set.
27745 n_card_set += end_card - card;
27746 start_address = max (beg, card_address (card));
27749 limit = min (end, card_address (end_card));
27752 if (!foundp || (last_object >= end) || (card_address (card) >= end))
27754 if (foundp && (cg_pointers_found == 0))
27756 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card),
27758 clear_cards (card, card_of (end));
27759 n_card_set -= (card_of (end) - card);
27760 total_cards_cleared += (card_of (end) - card);
27763 n_eph += cg_pointers_found;
27764 cg_pointers_found = 0;
27766 if ((seg = heap_segment_next_in_range (seg)) != 0)
27768 #ifdef BACKGROUND_GC
27769 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
27770 #endif //BACKGROUND_GC
27771 beg = heap_segment_mem (seg);
27772 end = compute_next_end (seg, low);
27773 card_word_end = card_of (align_on_card_word (end)) / card_word_width;
27774 card = card_of (beg);
27785 // We've found a card and will now go through the objects in it.
27786 assert (card_set_p (card));
27788 uint8_t* o = last_object;
27789 o = find_first_object (start_address, last_object);
27790 // Never visit an object twice.
27791 assert (o >= last_object);
27793 //dprintf(3,("Considering card %Ix start object: %Ix, %Ix[ boundary: %Ix",
27794 dprintf(3, ("c: %Ix, o: %Ix, l: %Ix[ boundary: %Ix",
27795 card, (size_t)o, (size_t)limit, (size_t)gen_boundary));
27799 assert (Align (size (o)) >= Align (min_obj_size));
27800 size_t s = size (o);
27802 uint8_t* next_o = o + Align (s);
27805 if ((o >= gen_boundary) &&
27806 (seg == ephemeral_heap_segment))
27808 dprintf (3, ("switching gen boundary %Ix", (size_t)gen_boundary));
27810 assert ((curr_gen_number > 0));
27811 gen_boundary = generation_allocation_start
27812 (generation_of (curr_gen_number - 1));
27813 next_boundary = (compute_next_boundary
27814 (low, curr_gen_number, relocating));
27817 dprintf (4, ("|%Ix|", (size_t)o));
27819 if (next_o < start_address)
27824 #ifdef BACKGROUND_GC
27825 if (!fgc_should_consider_object (o, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p))
27829 #endif //BACKGROUND_GC
27831 #ifdef COLLECTIBLE_CLASS
27832 if (is_collectible(o))
27834 BOOL passed_end_card_p = FALSE;
27836 if (card_of (o) > card)
27838 passed_end_card_p = card_transition (o, end, card_word_end,
27842 foundp, start_address,
27843 limit, total_cards_cleared);
27846 if ((!passed_end_card_p || foundp) && (card_of (o) == card))
27848 // card is valid and it covers the head of the object
27849 if (fn == &gc_heap::relocate_address)
27851 keep_card_live (o, n_gen, cg_pointers_found);
27855 uint8_t* class_obj = get_class_object (o);
27856 mark_through_cards_helper (&class_obj, n_gen,
27857 cg_pointers_found, fn,
27858 nhigh, next_boundary);
27862 if (passed_end_card_p)
27864 if (foundp && (card_address (card) < next_o))
27866 goto go_through_refs;
27868 else if (foundp && (start_address < limit))
27870 next_o = find_first_object (start_address, o);
27879 #endif //COLLECTIBLE_CLASS
27881 if (contain_pointers (o))
27883 dprintf(3,("Going through %Ix start_address: %Ix", (size_t)o, (size_t)start_address));
27886 dprintf (4, ("normal object path"));
27888 (method_table(o), o, s, poo,
27889 start_address, use_start, (o + s),
27891 dprintf (4, ("<%Ix>:%Ix", (size_t)poo, (size_t)*poo));
27892 if (card_of ((uint8_t*)poo) > card)
27894 BOOL passed_end_card_p = card_transition ((uint8_t*)poo, end,
27899 foundp, start_address,
27900 limit, total_cards_cleared);
27902 if (passed_end_card_p)
27904 if (foundp && (card_address (card) < next_o))
27908 if (ppstop <= (uint8_t**)start_address)
27910 else if (poo < (uint8_t**)start_address)
27911 {poo = (uint8_t**)start_address;}
27914 else if (foundp && (start_address < limit))
27916 next_o = find_first_object (start_address, o);
27924 mark_through_cards_helper (poo, n_gen,
27925 cg_pointers_found, fn,
27926 nhigh, next_boundary);
27933 if (((size_t)next_o / brick_size) != ((size_t) o / brick_size))
27935 if (brick_table [brick_of (o)] <0)
27936 fix_brick_to_highest (o, next_o);
27944 // compute the efficiency ratio of the card table
27947 generation_skip_ratio = ((n_eph > 400)? (int)(((float)n_gen / (float)n_eph) * 100) : 100);
27948 dprintf (3, ("Msoh: cross: %Id, useful: %Id, cards set: %Id, cards cleared: %Id, ratio: %d",
27949 n_eph, n_gen , n_card_set, total_cards_cleared, generation_skip_ratio));
27953 dprintf (3, ("R: Msoh: cross: %Id, useful: %Id, cards set: %Id, cards cleared: %Id, ratio: %d",
27954 n_gen, n_eph, n_card_set, total_cards_cleared, generation_skip_ratio));
27958 #ifdef SEG_REUSE_STATS
27959 size_t gc_heap::dump_buckets (size_t* ordered_indices, int count, size_t* total_size)
27961 size_t total_items = 0;
27963 for (int i = 0; i < count; i++)
27965 total_items += ordered_indices[i];
27966 *total_size += ordered_indices[i] << (MIN_INDEX_POWER2 + i);
27967 dprintf (SEG_REUSE_LOG_0, ("[%d]%4d 2^%2d", heap_number, ordered_indices[i], (MIN_INDEX_POWER2 + i)));
27969 dprintf (SEG_REUSE_LOG_0, ("[%d]Total %d items, total size is 0x%Ix", heap_number, total_items, *total_size));
27970 return total_items;
27972 #endif // SEG_REUSE_STATS
27974 void gc_heap::count_plug (size_t last_plug_size, uint8_t*& last_plug)
27976 // detect pinned plugs
27977 if (!pinned_plug_que_empty_p() && (last_plug == pinned_plug (oldest_pin())))
27979 deque_pinned_plug();
27980 update_oldest_pinned_plug();
27981 dprintf (3, ("dequed pin,now oldest pin is %Ix", pinned_plug (oldest_pin())));
27985 size_t plug_size = last_plug_size + Align(min_obj_size);
27986 BOOL is_padded = FALSE;
27989 plug_size += Align (min_obj_size);
27991 #endif //SHORT_PLUGS
27993 #ifdef RESPECT_LARGE_ALIGNMENT
27994 plug_size += switch_alignment_size (is_padded);
27995 #endif //RESPECT_LARGE_ALIGNMENT
27997 total_ephemeral_plugs += plug_size;
27998 size_t plug_size_power2 = round_up_power2 (plug_size);
27999 ordered_plug_indices[relative_index_power2_plug (plug_size_power2)]++;
28000 dprintf (SEG_REUSE_LOG_1, ("[%d]count_plug: adding 0x%Ix - %Id (2^%d) to ordered plug array",
28004 (relative_index_power2_plug (plug_size_power2) + MIN_INDEX_POWER2)));
28008 void gc_heap::count_plugs_in_brick (uint8_t* tree, uint8_t*& last_plug)
28010 assert ((tree != NULL));
28011 if (node_left_child (tree))
28013 count_plugs_in_brick (tree + node_left_child (tree), last_plug);
28016 if (last_plug != 0)
28018 uint8_t* plug = tree;
28019 size_t gap_size = node_gap_size (plug);
28020 uint8_t* gap = (plug - gap_size);
28021 uint8_t* last_plug_end = gap;
28022 size_t last_plug_size = (last_plug_end - last_plug);
28023 dprintf (3, ("tree: %Ix, last plug: %Ix, gap size: %Ix, gap: %Ix, last plug size: %Ix",
28024 tree, last_plug, gap_size, gap, last_plug_size));
28026 if (tree == oldest_pinned_plug)
28028 dprintf (3, ("tree %Ix is pinned, last plug is %Ix, size is %Ix",
28029 tree, last_plug, last_plug_size));
28030 mark* m = oldest_pin();
28031 if (m->has_pre_plug_info())
28033 last_plug_size += sizeof (gap_reloc_pair);
28034 dprintf (3, ("pin %Ix has pre plug, adjusting plug size to %Ix", tree, last_plug_size));
28037 // Can't assert here - if it's a pinned plug it can be less.
28038 //assert (last_plug_size >= Align (min_obj_size));
28040 count_plug (last_plug_size, last_plug);
28045 if (node_right_child (tree))
28047 count_plugs_in_brick (tree + node_right_child (tree), last_plug);
28051 void gc_heap::build_ordered_plug_indices ()
28053 memset (ordered_plug_indices, 0, sizeof(ordered_plug_indices));
28054 memset (saved_ordered_plug_indices, 0, sizeof(saved_ordered_plug_indices));
28056 uint8_t* start_address = generation_limit (max_generation);
28057 uint8_t* end_address = heap_segment_allocated (ephemeral_heap_segment);
28058 size_t current_brick = brick_of (start_address);
28059 size_t end_brick = brick_of (end_address - 1);
28060 uint8_t* last_plug = 0;
28062 //Look for the right pinned plug to start from.
28063 reset_pinned_queue_bos();
28064 while (!pinned_plug_que_empty_p())
28066 mark* m = oldest_pin();
28067 if ((m->first >= start_address) && (m->first < end_address))
28069 dprintf (3, ("found a pin %Ix between %Ix and %Ix", m->first, start_address, end_address));
28074 deque_pinned_plug();
28077 update_oldest_pinned_plug();
28079 while (current_brick <= end_brick)
28081 int brick_entry = brick_table [ current_brick ];
28082 if (brick_entry >= 0)
28084 count_plugs_in_brick (brick_address (current_brick) + brick_entry -1, last_plug);
28092 count_plug (end_address - last_plug, last_plug);
28095 // we need to make sure that after fitting all the existing plugs, we
28096 // have big enough free space left to guarantee that the next allocation
28098 size_t extra_size = END_SPACE_AFTER_GC + Align (min_obj_size);
28099 total_ephemeral_plugs += extra_size;
28100 dprintf (SEG_REUSE_LOG_0, ("Making sure we can fit a large object after fitting all plugs"));
28101 ordered_plug_indices[relative_index_power2_plug (round_up_power2 (extra_size))]++;
28103 memcpy (saved_ordered_plug_indices, ordered_plug_indices, sizeof(ordered_plug_indices));
28105 #ifdef SEG_REUSE_STATS
28106 dprintf (SEG_REUSE_LOG_0, ("Plugs:"));
28107 size_t total_plug_power2 = 0;
28108 dump_buckets (ordered_plug_indices, MAX_NUM_BUCKETS, &total_plug_power2);
28109 dprintf (SEG_REUSE_LOG_0, ("plugs: 0x%Ix (rounded up to 0x%Ix (%d%%))",
28110 total_ephemeral_plugs,
28112 (total_ephemeral_plugs ?
28113 (total_plug_power2 * 100 / total_ephemeral_plugs) :
28115 dprintf (SEG_REUSE_LOG_0, ("-------------------"));
28116 #endif // SEG_REUSE_STATS
28119 void gc_heap::init_ordered_free_space_indices ()
28121 memset (ordered_free_space_indices, 0, sizeof(ordered_free_space_indices));
28122 memset (saved_ordered_free_space_indices, 0, sizeof(saved_ordered_free_space_indices));
28125 void gc_heap::trim_free_spaces_indices ()
28127 trimmed_free_space_index = -1;
28128 size_t max_count = max_free_space_items - 1;
28131 for (i = (MAX_NUM_BUCKETS - 1); i >= 0; i--)
28133 count += ordered_free_space_indices[i];
28135 if (count >= max_count)
28141 ptrdiff_t extra_free_space_items = count - max_count;
28143 if (extra_free_space_items > 0)
28145 ordered_free_space_indices[i] -= extra_free_space_items;
28146 free_space_items = max_count;
28147 trimmed_free_space_index = i;
28151 free_space_items = count;
28159 free_space_buckets = MAX_NUM_BUCKETS - i;
28161 for (--i; i >= 0; i--)
28163 ordered_free_space_indices[i] = 0;
28166 memcpy (saved_ordered_free_space_indices,
28167 ordered_free_space_indices,
28168 sizeof(ordered_free_space_indices));
28171 // We fit as many plugs as we can and update the number of plugs left and the number
28172 // of free spaces left.
28173 BOOL gc_heap::can_fit_in_spaces_p (size_t* ordered_blocks, int small_index, size_t* ordered_spaces, int big_index)
28175 assert (small_index <= big_index);
28176 assert (big_index < MAX_NUM_BUCKETS);
28178 size_t small_blocks = ordered_blocks[small_index];
28180 if (small_blocks == 0)
28185 size_t big_spaces = ordered_spaces[big_index];
28187 if (big_spaces == 0)
28192 dprintf (SEG_REUSE_LOG_1, ("[%d]Fitting %Id 2^%d plugs into %Id 2^%d free spaces",
28194 small_blocks, (small_index + MIN_INDEX_POWER2),
28195 big_spaces, (big_index + MIN_INDEX_POWER2)));
28197 size_t big_to_small = big_spaces << (big_index - small_index);
28199 ptrdiff_t extra_small_spaces = big_to_small - small_blocks;
28200 dprintf (SEG_REUSE_LOG_1, ("[%d]%d 2^%d spaces can fit %d 2^%d blocks",
28202 big_spaces, (big_index + MIN_INDEX_POWER2), big_to_small, (small_index + MIN_INDEX_POWER2)));
28203 BOOL can_fit = (extra_small_spaces >= 0);
28207 dprintf (SEG_REUSE_LOG_1, ("[%d]Can fit with %d 2^%d extras blocks",
28209 extra_small_spaces, (small_index + MIN_INDEX_POWER2)));
28214 dprintf (SEG_REUSE_LOG_1, ("[%d]Setting # of 2^%d spaces to 0", heap_number, (big_index + MIN_INDEX_POWER2)));
28215 ordered_spaces[big_index] = 0;
28216 if (extra_small_spaces > 0)
28218 dprintf (SEG_REUSE_LOG_1, ("[%d]Setting # of 2^%d blocks to 0", heap_number, (small_index + MIN_INDEX_POWER2)));
28219 ordered_blocks[small_index] = 0;
28220 for (i = small_index; i < big_index; i++)
28222 if (extra_small_spaces & 1)
28224 dprintf (SEG_REUSE_LOG_1, ("[%d]Increasing # of 2^%d spaces from %d to %d",
28226 (i + MIN_INDEX_POWER2), ordered_spaces[i], (ordered_spaces[i] + 1)));
28227 ordered_spaces[i] += 1;
28229 extra_small_spaces >>= 1;
28232 dprintf (SEG_REUSE_LOG_1, ("[%d]Finally increasing # of 2^%d spaces from %d to %d",
28234 (i + MIN_INDEX_POWER2), ordered_spaces[i], (ordered_spaces[i] + extra_small_spaces)));
28235 ordered_spaces[i] += extra_small_spaces;
28239 dprintf (SEG_REUSE_LOG_1, ("[%d]Decreasing # of 2^%d blocks from %d to %d",
28241 (small_index + MIN_INDEX_POWER2),
28242 ordered_blocks[small_index],
28243 (ordered_blocks[small_index] - big_to_small)));
28244 ordered_blocks[small_index] -= big_to_small;
28247 #ifdef SEG_REUSE_STATS
28249 dprintf (SEG_REUSE_LOG_1, ("[%d]Plugs became:", heap_number));
28250 dump_buckets (ordered_blocks, MAX_NUM_BUCKETS, &temp);
28252 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces became:", heap_number));
28253 dump_buckets (ordered_spaces, MAX_NUM_BUCKETS, &temp);
28254 #endif //SEG_REUSE_STATS
28259 // space_index gets updated to the biggest available space index.
28260 BOOL gc_heap::can_fit_blocks_p (size_t* ordered_blocks, int block_index, size_t* ordered_spaces, int* space_index)
28262 assert (*space_index >= block_index);
28264 while (!can_fit_in_spaces_p (ordered_blocks, block_index, ordered_spaces, *space_index))
28267 if (*space_index < block_index)
28276 BOOL gc_heap::can_fit_all_blocks_p (size_t* ordered_blocks, size_t* ordered_spaces, int count)
28278 #ifdef FEATURE_STRUCTALIGN
28279 // BARTOKTODO (4841): reenable when can_fit_in_spaces_p takes alignment requirements into account
28281 #endif // FEATURE_STRUCTALIGN
28282 int space_index = count - 1;
28283 for (int block_index = (count - 1); block_index >= 0; block_index--)
28285 if (!can_fit_blocks_p (ordered_blocks, block_index, ordered_spaces, &space_index))
28294 void gc_heap::build_ordered_free_spaces (heap_segment* seg)
28296 assert (bestfit_seg);
28298 //bestfit_seg->add_buckets (MAX_NUM_BUCKETS - free_space_buckets + MIN_INDEX_POWER2,
28299 // ordered_free_space_indices + (MAX_NUM_BUCKETS - free_space_buckets),
28300 // free_space_buckets,
28301 // free_space_items);
28303 bestfit_seg->add_buckets (MIN_INDEX_POWER2,
28304 ordered_free_space_indices,
28308 assert (settings.condemned_generation == max_generation);
28310 uint8_t* first_address = heap_segment_mem (seg);
28311 uint8_t* end_address = heap_segment_reserved (seg);
28312 //look through the pinned plugs for relevant ones.
28313 //Look for the right pinned plug to start from.
28314 reset_pinned_queue_bos();
28316 // See comment in can_expand_into_p why we need (max_generation + 1).
28317 size_t eph_gen_starts = (Align (min_obj_size)) * (max_generation + 1);
28318 BOOL has_fit_gen_starts = FALSE;
28320 while (!pinned_plug_que_empty_p())
28323 if ((pinned_plug (m) >= first_address) &&
28324 (pinned_plug (m) < end_address) &&
28325 (pinned_len (m) >= eph_gen_starts))
28328 assert ((pinned_plug (m) - pinned_len (m)) == bestfit_first_pin);
28333 deque_pinned_plug();
28337 if (!pinned_plug_que_empty_p())
28339 bestfit_seg->add ((void*)m, TRUE, TRUE);
28340 deque_pinned_plug();
28342 has_fit_gen_starts = TRUE;
28345 while (!pinned_plug_que_empty_p() &&
28346 ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address)))
28348 bestfit_seg->add ((void*)m, TRUE, FALSE);
28349 deque_pinned_plug();
28353 if (commit_end_of_seg)
28355 if (!has_fit_gen_starts)
28357 assert (bestfit_first_pin == heap_segment_plan_allocated (seg));
28359 bestfit_seg->add ((void*)seg, FALSE, (!has_fit_gen_starts));
28363 bestfit_seg->check();
28367 BOOL gc_heap::try_best_fit (BOOL end_of_segment_p)
28369 if (!end_of_segment_p)
28371 trim_free_spaces_indices ();
28374 BOOL can_bestfit = can_fit_all_blocks_p (ordered_plug_indices,
28375 ordered_free_space_indices,
28378 return can_bestfit;
28381 BOOL gc_heap::best_fit (size_t free_space,
28382 size_t largest_free_space,
28383 size_t additional_space,
28384 BOOL* use_additional_space)
28386 dprintf (SEG_REUSE_LOG_0, ("gen%d: trying best fit mechanism", settings.condemned_generation));
28388 assert (!additional_space || (additional_space && use_additional_space));
28389 if (use_additional_space)
28391 *use_additional_space = FALSE;
28394 if (ordered_plug_indices_init == FALSE)
28396 total_ephemeral_plugs = 0;
28397 build_ordered_plug_indices();
28398 ordered_plug_indices_init = TRUE;
28402 memcpy (ordered_plug_indices, saved_ordered_plug_indices, sizeof(ordered_plug_indices));
28405 if (total_ephemeral_plugs == (END_SPACE_AFTER_GC + Align (min_obj_size)))
28407 dprintf (SEG_REUSE_LOG_0, ("No ephemeral plugs to realloc, done"));
28408 size_t empty_eph = (END_SPACE_AFTER_GC + Align (min_obj_size) + (Align (min_obj_size)) * (max_generation + 1));
28409 BOOL can_fit_empty_eph = (largest_free_space >= empty_eph);
28410 if (!can_fit_empty_eph)
28412 can_fit_empty_eph = (additional_space >= empty_eph);
28414 if (can_fit_empty_eph)
28416 *use_additional_space = TRUE;
28420 return can_fit_empty_eph;
28423 if ((total_ephemeral_plugs + approximate_new_allocation()) >= (free_space + additional_space))
28425 dprintf (SEG_REUSE_LOG_0, ("We won't have enough free space left in this segment after fitting, done"));
28429 if ((free_space + additional_space) == 0)
28431 dprintf (SEG_REUSE_LOG_0, ("No free space in this segment, done"));
28435 #ifdef SEG_REUSE_STATS
28436 dprintf (SEG_REUSE_LOG_0, ("Free spaces:"));
28437 size_t total_free_space_power2 = 0;
28438 size_t total_free_space_items =
28439 dump_buckets (ordered_free_space_indices,
28441 &total_free_space_power2);
28442 dprintf (SEG_REUSE_LOG_0, ("currently max free spaces is %Id", max_free_space_items));
28444 dprintf (SEG_REUSE_LOG_0, ("Ephemeral plugs: 0x%Ix, free space: 0x%Ix (rounded down to 0x%Ix (%Id%%)), additional free_space: 0x%Ix",
28445 total_ephemeral_plugs,
28447 total_free_space_power2,
28448 (free_space ? (total_free_space_power2 * 100 / free_space) : 0),
28449 additional_space));
28451 size_t saved_all_free_space_indices[MAX_NUM_BUCKETS];
28452 memcpy (saved_all_free_space_indices,
28453 ordered_free_space_indices,
28454 sizeof(saved_all_free_space_indices));
28456 #endif // SEG_REUSE_STATS
28458 if (total_ephemeral_plugs > (free_space + additional_space))
28463 use_bestfit = try_best_fit(FALSE);
28465 if (!use_bestfit && additional_space)
28467 int relative_free_space_index = relative_index_power2_free_space (round_down_power2 (additional_space));
28469 if (relative_free_space_index != -1)
28471 int relative_plug_index = 0;
28472 size_t plugs_to_fit = 0;
28474 for (relative_plug_index = (MAX_NUM_BUCKETS - 1); relative_plug_index >= 0; relative_plug_index--)
28476 plugs_to_fit = ordered_plug_indices[relative_plug_index];
28477 if (plugs_to_fit != 0)
28483 if ((relative_plug_index > relative_free_space_index) ||
28484 ((relative_plug_index == relative_free_space_index) &&
28485 (plugs_to_fit > 1)))
28487 #ifdef SEG_REUSE_STATS
28488 dprintf (SEG_REUSE_LOG_0, ("additional space is 2^%d but we stopped at %d 2^%d plug(s)",
28489 (relative_free_space_index + MIN_INDEX_POWER2),
28491 (relative_plug_index + MIN_INDEX_POWER2)));
28492 #endif // SEG_REUSE_STATS
28496 dprintf (SEG_REUSE_LOG_0, ("Adding end of segment (2^%d)", (relative_free_space_index + MIN_INDEX_POWER2)));
28497 ordered_free_space_indices[relative_free_space_index]++;
28498 use_bestfit = try_best_fit(TRUE);
28501 free_space_items++;
28502 // Since we might've trimmed away some of the free spaces we had, we should see
28503 // if we really need to use end of seg space - if it's the same or smaller than
28504 // the largest space we trimmed we can just add that one back instead of
28505 // using end of seg.
28506 if (relative_free_space_index > trimmed_free_space_index)
28508 *use_additional_space = TRUE;
28512 // If the addition space is <= than the last trimmed space, we
28513 // should just use that last trimmed space instead.
28514 saved_ordered_free_space_indices[trimmed_free_space_index]++;
28524 dprintf (SEG_REUSE_LOG_0, ("couldn't fit..."));
28526 #ifdef SEG_REUSE_STATS
28527 size_t saved_max = max_free_space_items;
28528 BOOL temp_bestfit = FALSE;
28530 dprintf (SEG_REUSE_LOG_0, ("----Starting experiment process----"));
28531 dprintf (SEG_REUSE_LOG_0, ("----Couldn't fit with max free items %Id", max_free_space_items));
28533 // TODO: need to take the end of segment into consideration.
28534 while (max_free_space_items <= total_free_space_items)
28536 max_free_space_items += max_free_space_items / 2;
28537 dprintf (SEG_REUSE_LOG_0, ("----Temporarily increasing max free spaces to %Id", max_free_space_items));
28538 memcpy (ordered_free_space_indices,
28539 saved_all_free_space_indices,
28540 sizeof(ordered_free_space_indices));
28541 if (try_best_fit(FALSE))
28543 temp_bestfit = TRUE;
28550 dprintf (SEG_REUSE_LOG_0, ("----With %Id max free spaces we could fit", max_free_space_items));
28554 dprintf (SEG_REUSE_LOG_0, ("----Tried all free spaces and still couldn't fit, lost too much space"));
28557 dprintf (SEG_REUSE_LOG_0, ("----Restoring max free spaces to %Id", saved_max));
28558 max_free_space_items = saved_max;
28559 #endif // SEG_REUSE_STATS
28560 if (free_space_items)
28562 max_free_space_items = min (MAX_NUM_FREE_SPACES, free_space_items * 2);
28563 max_free_space_items = max (max_free_space_items, MIN_NUM_FREE_SPACES);
28567 max_free_space_items = MAX_NUM_FREE_SPACES;
28571 dprintf (SEG_REUSE_LOG_0, ("Adjusted number of max free spaces to %Id", max_free_space_items));
28572 dprintf (SEG_REUSE_LOG_0, ("------End of best fitting process------\n"));
28574 return use_bestfit;
28577 BOOL gc_heap::process_free_space (heap_segment* seg,
28579 size_t min_free_size,
28580 size_t min_cont_size,
28581 size_t* total_free_space,
28582 size_t* largest_free_space)
28584 *total_free_space += free_space;
28585 *largest_free_space = max (*largest_free_space, free_space);
28587 #ifdef SIMPLE_DPRINTF
28588 dprintf (SEG_REUSE_LOG_1, ("free space len: %Ix, total free space: %Ix, largest free space: %Ix",
28589 free_space, *total_free_space, *largest_free_space));
28590 #endif //SIMPLE_DPRINTF
28592 if ((*total_free_space >= min_free_size) && (*largest_free_space >= min_cont_size))
28594 #ifdef SIMPLE_DPRINTF
28595 dprintf (SEG_REUSE_LOG_0, ("(gen%d)total free: %Ix(min: %Ix), largest free: %Ix(min: %Ix). Found segment %Ix to reuse without bestfit",
28596 settings.condemned_generation,
28597 *total_free_space, min_free_size, *largest_free_space, min_cont_size,
28600 UNREFERENCED_PARAMETER(seg);
28601 #endif //SIMPLE_DPRINTF
28605 int free_space_index = relative_index_power2_free_space (round_down_power2 (free_space));
28606 if (free_space_index != -1)
28608 ordered_free_space_indices[free_space_index]++;
28613 BOOL gc_heap::expand_reused_seg_p()
28615 BOOL reused_seg = FALSE;
28616 int heap_expand_mechanism = gc_data_per_heap.get_mechanism (gc_heap_expand);
28617 if ((heap_expand_mechanism == expand_reuse_bestfit) ||
28618 (heap_expand_mechanism == expand_reuse_normal))
28626 BOOL gc_heap::can_expand_into_p (heap_segment* seg, size_t min_free_size, size_t min_cont_size,
28627 allocator* gen_allocator)
28629 min_cont_size += END_SPACE_AFTER_GC;
28630 use_bestfit = FALSE;
28631 commit_end_of_seg = FALSE;
28632 bestfit_first_pin = 0;
28633 uint8_t* first_address = heap_segment_mem (seg);
28634 uint8_t* end_address = heap_segment_reserved (seg);
28635 size_t end_extra_space = end_space_after_gc();
28637 if ((heap_segment_reserved (seg) - end_extra_space) <= heap_segment_plan_allocated (seg))
28639 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: can't use segment [%Ix %Ix, has less than %d bytes at the end",
28640 first_address, end_address, end_extra_space));
28644 end_address -= end_extra_space;
28646 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p(gen%d): min free: %Ix, min continuous: %Ix",
28647 settings.condemned_generation, min_free_size, min_cont_size));
28648 size_t eph_gen_starts = eph_gen_starts_size;
28650 if (settings.condemned_generation == max_generation)
28652 size_t free_space = 0;
28653 size_t largest_free_space = free_space;
28654 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: gen2: testing segment [%Ix %Ix", first_address, end_address));
28655 //Look through the pinned plugs for relevant ones and Look for the right pinned plug to start from.
28656 //We are going to allocate the generation starts in the 1st free space,
28657 //so start from the first free space that's big enough for gen starts and a min object size.
28658 // If we see a free space that is >= gen starts but < gen starts + min obj size we just don't use it -
28659 // we could use it by allocating the last generation start a bit bigger but
28660 // the complexity isn't worth the effort (those plugs are from gen2
28661 // already anyway).
28662 reset_pinned_queue_bos();
28664 BOOL has_fit_gen_starts = FALSE;
28666 init_ordered_free_space_indices ();
28667 while (!pinned_plug_que_empty_p())
28670 if ((pinned_plug (m) >= first_address) &&
28671 (pinned_plug (m) < end_address) &&
28672 (pinned_len (m) >= (eph_gen_starts + Align (min_obj_size))))
28678 deque_pinned_plug();
28682 if (!pinned_plug_que_empty_p())
28684 bestfit_first_pin = pinned_plug (m) - pinned_len (m);
28686 if (process_free_space (seg,
28687 pinned_len (m) - eph_gen_starts,
28688 min_free_size, min_cont_size,
28689 &free_space, &largest_free_space))
28694 deque_pinned_plug();
28696 has_fit_gen_starts = TRUE;
28699 dprintf (3, ("first pin is %Ix", pinned_plug (m)));
28701 //tally up free space
28702 while (!pinned_plug_que_empty_p() &&
28703 ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address)))
28705 dprintf (3, ("looking at pin %Ix", pinned_plug (m)));
28706 if (process_free_space (seg,
28708 min_free_size, min_cont_size,
28709 &free_space, &largest_free_space))
28714 deque_pinned_plug();
28718 //try to find space at the end of the segment.
28719 size_t end_space = (end_address - heap_segment_plan_allocated (seg));
28720 size_t additional_space = ((min_free_size > free_space) ? (min_free_size - free_space) : 0);
28721 dprintf (SEG_REUSE_LOG_0, ("end space: %Ix; additional: %Ix", end_space, additional_space));
28722 if (end_space >= additional_space)
28724 BOOL can_fit = TRUE;
28725 commit_end_of_seg = TRUE;
28727 if (largest_free_space < min_cont_size)
28729 if (end_space >= min_cont_size)
28731 additional_space = max (min_cont_size, additional_space);
28732 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse without bestfit, with committing end of seg for eph",
28737 if (settings.concurrent)
28740 commit_end_of_seg = FALSE;
28744 size_t additional_space_bestfit = additional_space;
28745 if (!has_fit_gen_starts)
28747 if (additional_space_bestfit < (eph_gen_starts + Align (min_obj_size)))
28749 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, gen starts not allocated yet and end space is too small: %Id",
28750 additional_space_bestfit));
28754 bestfit_first_pin = heap_segment_plan_allocated (seg);
28755 additional_space_bestfit -= eph_gen_starts;
28758 can_fit = best_fit (free_space,
28759 largest_free_space,
28760 additional_space_bestfit,
28761 &commit_end_of_seg);
28765 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse with bestfit, %s committing end of seg",
28766 seg, (commit_end_of_seg ? "with" : "without")));
28770 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, total free space is %Ix", (free_space + end_space)));
28777 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse without bestfit, with committing end of seg", seg));
28780 assert (additional_space <= end_space);
28781 if (commit_end_of_seg)
28783 if (!grow_heap_segment (seg, heap_segment_plan_allocated (seg) + additional_space))
28785 dprintf (2, ("Couldn't commit end of segment?!"));
28786 use_bestfit = FALSE;
28793 // We increase the index here because growing heap segment could create a discrepency with
28794 // the additional space we used (could be bigger).
28795 size_t free_space_end_of_seg =
28796 heap_segment_committed (seg) - heap_segment_plan_allocated (seg);
28797 int relative_free_space_index = relative_index_power2_free_space (round_down_power2 (free_space_end_of_seg));
28798 saved_ordered_free_space_indices[relative_free_space_index]++;
28804 memcpy (ordered_free_space_indices,
28805 saved_ordered_free_space_indices,
28806 sizeof(ordered_free_space_indices));
28807 max_free_space_items = max (MIN_NUM_FREE_SPACES, free_space_items * 3 / 2);
28808 max_free_space_items = min (MAX_NUM_FREE_SPACES, max_free_space_items);
28809 dprintf (SEG_REUSE_LOG_0, ("could fit! %Id free spaces, %Id max", free_space_items, max_free_space_items));
28815 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, total free space is %Ix", (free_space + end_space)));
28820 assert (settings.condemned_generation == (max_generation-1));
28821 size_t free_space = (end_address - heap_segment_plan_allocated (seg));
28822 size_t largest_free_space = free_space;
28823 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: gen1: testing segment [%Ix %Ix", first_address, end_address));
28824 //find the first free list in range of the current segment
28825 size_t sz_list = gen_allocator->first_bucket_size();
28826 unsigned int a_l_idx = 0;
28827 uint8_t* free_list = 0;
28828 for (; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
28830 if ((eph_gen_starts <= sz_list) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
28832 free_list = gen_allocator->alloc_list_head_of (a_l_idx);
28835 if ((free_list >= first_address) &&
28836 (free_list < end_address) &&
28837 (unused_array_size (free_list) >= eph_gen_starts))
28843 free_list = free_list_slot (free_list);
28851 init_ordered_free_space_indices ();
28852 if (process_free_space (seg,
28853 unused_array_size (free_list) - eph_gen_starts + Align (min_obj_size),
28854 min_free_size, min_cont_size,
28855 &free_space, &largest_free_space))
28860 free_list = free_list_slot (free_list);
28864 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, no free list"));
28868 //tally up free space
28874 if ((free_list >= first_address) && (free_list < end_address) &&
28875 process_free_space (seg,
28876 unused_array_size (free_list),
28877 min_free_size, min_cont_size,
28878 &free_space, &largest_free_space))
28883 free_list = free_list_slot (free_list);
28886 if (a_l_idx < gen_allocator->number_of_buckets())
28888 free_list = gen_allocator->alloc_list_head_of (a_l_idx);
28894 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, total free space is %Ix", free_space));
28898 BOOL can_fit = best_fit (free_space, 0, NULL);
28901 dprintf (SEG_REUSE_LOG_0, ("(gen1)Found segment %Ix to reuse with bestfit", seg));
28905 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, total free space is %Ix", free_space));
28913 void gc_heap::realloc_plug (size_t last_plug_size, uint8_t*& last_plug,
28914 generation* gen, uint8_t* start_address,
28915 unsigned int& active_new_gen_number,
28916 uint8_t*& last_pinned_gap, BOOL& leftp,
28919 , mark* pinned_plug_entry
28920 #endif //SHORT_PLUGS
28923 // detect generation boundaries
28924 // make sure that active_new_gen_number is not the youngest generation.
28925 // because the generation_limit wouldn't return the right thing in this case.
28928 if ((active_new_gen_number > 1) &&
28929 (last_plug >= generation_limit (active_new_gen_number)))
28931 assert (last_plug >= start_address);
28932 active_new_gen_number--;
28933 realloc_plan_generation_start (generation_of (active_new_gen_number), gen);
28934 assert (generation_plan_allocation_start (generation_of (active_new_gen_number)));
28939 // detect pinned plugs
28940 if (!pinned_plug_que_empty_p() && (last_plug == pinned_plug (oldest_pin())))
28942 size_t entry = deque_pinned_plug();
28943 mark* m = pinned_plug_of (entry);
28945 size_t saved_pinned_len = pinned_len(m);
28946 pinned_len(m) = last_plug - last_pinned_gap;
28947 //dprintf (3,("Adjusting pinned gap: [%Ix, %Ix[", (size_t)last_pinned_gap, (size_t)last_plug));
28949 if (m->has_post_plug_info())
28951 last_plug_size += sizeof (gap_reloc_pair);
28952 dprintf (3, ("ra pinned %Ix was shortened, adjusting plug size to %Ix", last_plug, last_plug_size))
28955 last_pinned_gap = last_plug + last_plug_size;
28956 dprintf (3, ("ra found pin %Ix, len: %Ix->%Ix, last_p: %Ix, last_p_size: %Ix",
28957 pinned_plug (m), saved_pinned_len, pinned_len (m), last_plug, last_plug_size));
28960 //we are creating a generation fault. set the cards.
28962 size_t end_card = card_of (align_on_card (last_plug + last_plug_size));
28963 size_t card = card_of (last_plug);
28964 while (card != end_card)
28971 else if (last_plug >= start_address)
28973 #ifdef FEATURE_STRUCTALIGN
28974 int requiredAlignment;
28976 node_aligninfo (last_plug, requiredAlignment, pad);
28978 // from how we previously aligned the plug's destination address,
28979 // compute the actual alignment offset.
28980 uint8_t* reloc_plug = last_plug + node_relocation_distance (last_plug);
28981 ptrdiff_t alignmentOffset = ComputeStructAlignPad(reloc_plug, requiredAlignment, 0);
28982 if (!alignmentOffset)
28984 // allocate_in_expanded_heap doesn't expect alignmentOffset to be zero.
28985 alignmentOffset = requiredAlignment;
28988 //clear the alignment info because we are reallocating
28989 clear_node_aligninfo (last_plug);
28990 #else // FEATURE_STRUCTALIGN
28991 //clear the realignment flag because we are reallocating
28992 clear_node_realigned (last_plug);
28993 #endif // FEATURE_STRUCTALIGN
28994 BOOL adjacentp = FALSE;
28995 BOOL set_padding_on_saved_p = FALSE;
28999 last_plug_size += sizeof (gap_reloc_pair);
29002 assert (pinned_plug_entry != NULL);
29003 if (last_plug_size <= sizeof (plug_and_gap))
29005 set_padding_on_saved_p = TRUE;
29007 #endif //SHORT_PLUGS
29009 dprintf (3, ("ra plug %Ix was shortened, adjusting plug size to %Ix", last_plug, last_plug_size))
29013 clear_padding_in_expand (last_plug, set_padding_on_saved_p, pinned_plug_entry);
29014 #endif //SHORT_PLUGS
29016 uint8_t* new_address = allocate_in_expanded_heap(gen, last_plug_size, adjacentp, last_plug,
29018 set_padding_on_saved_p,
29020 #endif //SHORT_PLUGS
29021 TRUE, active_new_gen_number REQD_ALIGN_AND_OFFSET_ARG);
29023 dprintf (3, ("ra NA: [%Ix, %Ix[: %Ix", new_address, (new_address + last_plug_size), last_plug_size));
29024 assert (new_address);
29025 set_node_relocation_distance (last_plug, new_address - last_plug);
29026 #ifdef FEATURE_STRUCTALIGN
29027 if (leftp && node_alignpad (last_plug) == 0)
29028 #else // FEATURE_STRUCTALIGN
29029 if (leftp && !node_realigned (last_plug))
29030 #endif // FEATURE_STRUCTALIGN
29032 // TODO - temporarily disable L optimization because of a bug in it.
29033 //set_node_left (last_plug);
29035 dprintf (3,(" Re-allocating %Ix->%Ix len %Id", (size_t)last_plug, (size_t)new_address, last_plug_size));
29040 void gc_heap::realloc_in_brick (uint8_t* tree, uint8_t*& last_plug,
29041 uint8_t* start_address,
29043 unsigned int& active_new_gen_number,
29044 uint8_t*& last_pinned_gap, BOOL& leftp)
29046 assert (tree != NULL);
29047 int left_node = node_left_child (tree);
29048 int right_node = node_right_child (tree);
29050 dprintf (3, ("ra: tree: %Ix, last_pin_gap: %Ix, last_p: %Ix, L: %d, R: %d",
29051 tree, last_pinned_gap, last_plug, left_node, right_node));
29055 dprintf (3, ("LN: realloc %Ix(%Ix)", (tree + left_node), last_plug));
29056 realloc_in_brick ((tree + left_node), last_plug, start_address,
29057 gen, active_new_gen_number, last_pinned_gap,
29061 if (last_plug != 0)
29063 uint8_t* plug = tree;
29065 BOOL has_pre_plug_info_p = FALSE;
29066 BOOL has_post_plug_info_p = FALSE;
29067 mark* pinned_plug_entry = get_next_pinned_entry (tree,
29068 &has_pre_plug_info_p,
29069 &has_post_plug_info_p,
29072 // We only care about the pre plug info 'cause that's what decides if the last plug is shortened.
29073 // The pinned plugs are handled in realloc_plug.
29074 size_t gap_size = node_gap_size (plug);
29075 uint8_t* gap = (plug - gap_size);
29076 uint8_t* last_plug_end = gap;
29077 size_t last_plug_size = (last_plug_end - last_plug);
29078 // Cannot assert this - a plug could be less than that due to the shortened ones.
29079 //assert (last_plug_size >= Align (min_obj_size));
29080 dprintf (3, ("ra: plug %Ix, gap size: %Ix, last_pin_gap: %Ix, last_p: %Ix, last_p_end: %Ix, shortened: %d",
29081 plug, gap_size, last_pinned_gap, last_plug, last_plug_end, (has_pre_plug_info_p ? 1 : 0)));
29082 realloc_plug (last_plug_size, last_plug, gen, start_address,
29083 active_new_gen_number, last_pinned_gap,
29084 leftp, has_pre_plug_info_p
29086 , pinned_plug_entry
29087 #endif //SHORT_PLUGS
29095 dprintf (3, ("RN: realloc %Ix(%Ix)", (tree + right_node), last_plug));
29096 realloc_in_brick ((tree + right_node), last_plug, start_address,
29097 gen, active_new_gen_number, last_pinned_gap,
29103 gc_heap::realloc_plugs (generation* consing_gen, heap_segment* seg,
29104 uint8_t* start_address, uint8_t* end_address,
29105 unsigned active_new_gen_number)
29107 dprintf (3, ("--- Reallocing ---"));
29111 //make sure that every generation has a planned allocation start
29112 int gen_number = max_generation - 1;
29113 while (gen_number >= 0)
29115 generation* gen = generation_of (gen_number);
29116 if (0 == generation_plan_allocation_start (gen))
29118 generation_plan_allocation_start (gen) =
29119 bestfit_first_pin + (max_generation - gen_number - 1) * Align (min_obj_size);
29120 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
29121 assert (generation_plan_allocation_start (gen));
29127 uint8_t* first_address = start_address;
29128 //Look for the right pinned plug to start from.
29129 reset_pinned_queue_bos();
29130 uint8_t* planned_ephemeral_seg_end = heap_segment_plan_allocated (seg);
29131 while (!pinned_plug_que_empty_p())
29133 mark* m = oldest_pin();
29134 if ((pinned_plug (m) >= planned_ephemeral_seg_end) && (pinned_plug (m) < end_address))
29136 if (pinned_plug (m) < first_address)
29138 first_address = pinned_plug (m);
29143 deque_pinned_plug();
29146 size_t current_brick = brick_of (first_address);
29147 size_t end_brick = brick_of (end_address-1);
29148 uint8_t* last_plug = 0;
29150 uint8_t* last_pinned_gap = heap_segment_plan_allocated (seg);
29151 BOOL leftp = FALSE;
29153 dprintf (3, ("start addr: %Ix, first addr: %Ix, current oldest pin: %Ix",
29154 start_address, first_address, pinned_plug (oldest_pin())));
29156 while (current_brick <= end_brick)
29158 int brick_entry = brick_table [ current_brick ];
29159 if (brick_entry >= 0)
29161 realloc_in_brick ((brick_address (current_brick) + brick_entry - 1),
29162 last_plug, start_address, consing_gen,
29163 active_new_gen_number, last_pinned_gap,
29169 if (last_plug != 0)
29171 realloc_plug (end_address - last_plug, last_plug, consing_gen,
29173 active_new_gen_number, last_pinned_gap,
29177 #endif //SHORT_PLUGS
29181 //Fix the old segment allocated size
29182 assert (last_pinned_gap >= heap_segment_mem (seg));
29183 assert (last_pinned_gap <= heap_segment_committed (seg));
29184 heap_segment_plan_allocated (seg) = last_pinned_gap;
29187 void gc_heap::verify_no_pins (uint8_t* start, uint8_t* end)
29190 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
29192 BOOL contains_pinned_plugs = FALSE;
29195 while (mi != mark_stack_tos)
29197 m = pinned_plug_of (mi);
29198 if ((pinned_plug (m) >= start) && (pinned_plug (m) < end))
29200 contains_pinned_plugs = TRUE;
29207 if (contains_pinned_plugs)
29212 #endif //VERIFY_HEAP
29215 void gc_heap::set_expand_in_full_gc (int condemned_gen_number)
29217 if (!should_expand_in_full_gc)
29219 if ((condemned_gen_number != max_generation) &&
29220 (settings.pause_mode != pause_low_latency) &&
29221 (settings.pause_mode != pause_sustained_low_latency))
29223 should_expand_in_full_gc = TRUE;
29228 void gc_heap::save_ephemeral_generation_starts()
29230 for (int ephemeral_generation = 0; ephemeral_generation < max_generation; ephemeral_generation++)
29232 saved_ephemeral_plan_start[ephemeral_generation] =
29233 generation_plan_allocation_start (generation_of (ephemeral_generation));
29234 saved_ephemeral_plan_start_size[ephemeral_generation] =
29235 generation_plan_allocation_start_size (generation_of (ephemeral_generation));
29239 generation* gc_heap::expand_heap (int condemned_generation,
29240 generation* consing_gen,
29241 heap_segment* new_heap_segment)
29243 UNREFERENCED_PARAMETER(condemned_generation);
29244 assert (condemned_generation >= (max_generation -1));
29245 unsigned int active_new_gen_number = max_generation; //Set one too high to get generation gap
29246 uint8_t* start_address = generation_limit (max_generation);
29247 uint8_t* end_address = heap_segment_allocated (ephemeral_heap_segment);
29248 BOOL should_promote_ephemeral = FALSE;
29249 ptrdiff_t eph_size = total_ephemeral_size;
29250 #ifdef BACKGROUND_GC
29251 dprintf(2,("%s: ---- Heap Expansion ----", (recursive_gc_sync::background_running_p() ? "FGC" : "NGC")));
29252 #endif //BACKGROUND_GC
29253 settings.heap_expansion = TRUE;
29255 #ifdef BACKGROUND_GC
29256 if (cm_in_progress)
29258 if (!expanded_in_fgc)
29260 expanded_in_fgc = TRUE;
29263 #endif //BACKGROUND_GC
29265 //reset the elevation state for next time.
29266 dprintf (2, ("Elevation: elevation = el_none"));
29267 if (settings.should_lock_elevation && !expand_reused_seg_p())
29268 settings.should_lock_elevation = FALSE;
29270 heap_segment* new_seg = new_heap_segment;
29273 return consing_gen;
29275 //copy the card and brick tables
29276 if (g_gc_card_table!= card_table)
29277 copy_brick_card_table();
29279 BOOL new_segment_p = (heap_segment_next (new_seg) == 0);
29280 dprintf (2, ("new_segment_p %Ix", (size_t)new_segment_p));
29282 assert (generation_plan_allocation_start (generation_of (max_generation-1)));
29283 assert (generation_plan_allocation_start (generation_of (max_generation-1)) >=
29284 heap_segment_mem (ephemeral_heap_segment));
29285 assert (generation_plan_allocation_start (generation_of (max_generation-1)) <=
29286 heap_segment_committed (ephemeral_heap_segment));
29288 assert (generation_plan_allocation_start (youngest_generation));
29289 assert (generation_plan_allocation_start (youngest_generation) <
29290 heap_segment_plan_allocated (ephemeral_heap_segment));
29292 if (settings.pause_mode == pause_no_gc)
29294 // We don't reuse for no gc, so the size used on the new eph seg is eph_size.
29295 if ((size_t)(heap_segment_reserved (new_seg) - heap_segment_mem (new_seg)) < (eph_size + soh_allocation_no_gc))
29296 should_promote_ephemeral = TRUE;
29302 should_promote_ephemeral = dt_low_ephemeral_space_p (tuning_deciding_promote_ephemeral);
29306 if (should_promote_ephemeral)
29308 ephemeral_promotion = TRUE;
29309 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, expand_new_seg_ep);
29310 dprintf (2, ("promoting ephemeral"));
29311 save_ephemeral_generation_starts();
29315 // commit the new ephemeral segment all at once if it is a new one.
29316 if ((eph_size > 0) && new_segment_p)
29318 #ifdef FEATURE_STRUCTALIGN
29319 // The destination may require a larger alignment padding than the source.
29320 // Assume the worst possible alignment padding.
29321 eph_size += ComputeStructAlignPad(heap_segment_mem (new_seg), MAX_STRUCTALIGN, OBJECT_ALIGNMENT_OFFSET);
29322 #endif // FEATURE_STRUCTALIGN
29323 #ifdef RESPECT_LARGE_ALIGNMENT
29324 //Since the generation start can be larger than min_obj_size
29325 //The alignment could be switched.
29326 eph_size += switch_alignment_size(FALSE);
29327 #endif //RESPECT_LARGE_ALIGNMENT
29328 //Since the generation start can be larger than min_obj_size
29329 //Compare the alignment of the first object in gen1
29330 if (grow_heap_segment (new_seg, heap_segment_mem (new_seg) + eph_size) == 0)
29332 fgm_result.set_fgm (fgm_commit_eph_segment, eph_size, FALSE);
29333 return consing_gen;
29335 heap_segment_used (new_seg) = heap_segment_committed (new_seg);
29338 //Fix the end of the old ephemeral heap segment
29339 heap_segment_plan_allocated (ephemeral_heap_segment) =
29340 generation_plan_allocation_start (generation_of (max_generation-1));
29342 dprintf (3, ("Old ephemeral allocated set to %Ix",
29343 (size_t)heap_segment_plan_allocated (ephemeral_heap_segment)));
29348 // TODO - Is this really necessary? We should think about it.
29349 //initialize the first brick
29350 size_t first_brick = brick_of (heap_segment_mem (new_seg));
29351 set_brick (first_brick,
29352 heap_segment_mem (new_seg) - brick_address (first_brick));
29355 //From this point on, we cannot run out of memory
29357 //reset the allocation of the consing generation back to the end of the
29358 //old ephemeral segment
29359 generation_allocation_limit (consing_gen) =
29360 heap_segment_plan_allocated (ephemeral_heap_segment);
29361 generation_allocation_pointer (consing_gen) = generation_allocation_limit (consing_gen);
29362 generation_allocation_segment (consing_gen) = ephemeral_heap_segment;
29364 //clear the generation gap for all of the ephemeral generations
29366 int generation_num = max_generation-1;
29367 while (generation_num >= 0)
29369 generation* gen = generation_of (generation_num);
29370 generation_plan_allocation_start (gen) = 0;
29375 heap_segment* old_seg = ephemeral_heap_segment;
29376 ephemeral_heap_segment = new_seg;
29378 //Note: the ephemeral segment shouldn't be threaded onto the segment chain
29379 //because the relocation and compact phases shouldn't see it
29381 // set the generation members used by allocate_in_expanded_heap
29382 // and switch to ephemeral generation
29383 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
29385 if (!should_promote_ephemeral)
29387 realloc_plugs (consing_gen, old_seg, start_address, end_address,
29388 active_new_gen_number);
29393 repair_allocation_in_expanded_heap (consing_gen);
29396 // assert that the generation gap for all of the ephemeral generations were allocated.
29399 int generation_num = max_generation-1;
29400 while (generation_num >= 0)
29402 generation* gen = generation_of (generation_num);
29403 assert (generation_plan_allocation_start (gen));
29409 if (!new_segment_p)
29411 dprintf (2, ("Demoting ephemeral segment"));
29412 //demote the entire segment.
29413 settings.demotion = TRUE;
29414 get_gc_data_per_heap()->set_mechanism_bit (gc_demotion_bit);
29415 demotion_low = heap_segment_mem (ephemeral_heap_segment);
29416 demotion_high = heap_segment_reserved (ephemeral_heap_segment);
29420 demotion_low = MAX_PTR;
29422 #ifndef MULTIPLE_HEAPS
29423 settings.demotion = FALSE;
29424 get_gc_data_per_heap()->clear_mechanism_bit (gc_demotion_bit);
29425 #endif //!MULTIPLE_HEAPS
29427 ptrdiff_t eph_size1 = total_ephemeral_size;
29428 MAYBE_UNUSED_VAR(eph_size1);
29430 if (!should_promote_ephemeral && new_segment_p)
29432 assert (eph_size1 <= eph_size);
29435 if (heap_segment_mem (old_seg) == heap_segment_plan_allocated (old_seg))
29437 // This is to catch when we accidently delete a segment that has pins.
29438 verify_no_pins (heap_segment_mem (old_seg), heap_segment_reserved (old_seg));
29441 verify_no_pins (heap_segment_plan_allocated (old_seg), heap_segment_reserved(old_seg));
29443 dprintf(2,("---- End of Heap Expansion ----"));
29444 return consing_gen;
29447 void gc_heap::set_static_data()
29449 static_data* pause_mode_sdata = static_data_table[latency_level];
29450 for (int i = 0; i < NUMBERGENERATIONS; i++)
29452 dynamic_data* dd = dynamic_data_of (i);
29453 static_data* sdata = &pause_mode_sdata[i];
29456 dd->min_size = sdata->min_size;
29458 dprintf (GTC_LOG, ("PM: %d - min: %Id, max: %Id, fr_l: %Id, fr_b: %d%%",
29459 settings.pause_mode,
29460 dd->min_size, dd_max_size,
29461 dd->fragmentation_limit, (int)(dd->fragmentation_burden_limit * 100)));
29465 // Initialize the values that are not const.
29466 void gc_heap::init_static_data()
29468 size_t gen0size = GCHeap::GetValidGen0MaxSize(get_valid_segment_size());
29469 size_t gen0_min_size = Align(gen0size / 8 * 5);
29471 size_t gen0_max_size =
29472 #ifdef MULTIPLE_HEAPS
29473 max (6*1024*1024, min ( Align(soh_segment_size/2), 200*1024*1024));
29474 #else //MULTIPLE_HEAPS
29475 (gc_can_use_concurrent ?
29477 max (6*1024*1024, min ( Align(soh_segment_size/2), 200*1024*1024)));
29478 #endif //MULTIPLE_HEAPS
29480 // TODO: gen0_max_size has a 200mb cap; gen1_max_size should also have a cap.
29481 size_t gen1_max_size =
29482 #ifdef MULTIPLE_HEAPS
29483 max (6*1024*1024, Align(soh_segment_size/2));
29484 #else //MULTIPLE_HEAPS
29485 (gc_can_use_concurrent ?
29487 max (6*1024*1024, Align(soh_segment_size/2)));
29488 #endif //MULTIPLE_HEAPS
29490 dprintf (GTC_LOG, ("gen0size: %Id, gen0 min: %Id, max: %Id, gen1 max: %Id",
29491 gen0size, gen0_min_size, gen0_max_size, gen1_max_size));
29493 for (int i = latency_level_first; i <= latency_level_last; i++)
29495 static_data_table[i][0].min_size = gen0_min_size;
29496 static_data_table[i][0].max_size = gen0_max_size;
29497 static_data_table[i][1].max_size = gen1_max_size;
29501 bool gc_heap::init_dynamic_data()
29503 qpf = GCToOSInterface::QueryPerformanceFrequency();
29505 uint32_t now = (uint32_t)GetHighPrecisionTimeStamp();
29509 for (int i = 0; i <= max_generation+1; i++)
29511 dynamic_data* dd = dynamic_data_of (i);
29513 dd->time_clock = now;
29514 dd->current_size = 0;
29515 dd->promoted_size = 0;
29516 dd->collection_count = 0;
29517 dd->new_allocation = dd->min_size;
29518 dd->gc_new_allocation = dd->new_allocation;
29519 dd->desired_allocation = dd->new_allocation;
29520 dd->fragmentation = 0;
29523 #ifdef GC_CONFIG_DRIVEN
29524 if (heap_number == 0)
29526 #endif //GC_CONFIG_DRIVEN
29531 float gc_heap::surv_to_growth (float cst, float limit, float max_limit)
29533 if (cst < ((max_limit - limit ) / (limit * (max_limit-1.0f))))
29534 return ((limit - limit*cst) / (1.0f - (cst * limit)));
29540 //if the allocation budget wasn't exhausted, the new budget may be wrong because the survival may
29541 //not be correct (collection happened too soon). Correct with a linear estimation based on the previous
29542 //value of the budget
29543 static size_t linear_allocation_model (float allocation_fraction, size_t new_allocation,
29544 size_t previous_desired_allocation, size_t collection_count)
29546 if ((allocation_fraction < 0.95) && (allocation_fraction > 0.0))
29548 dprintf (2, ("allocation fraction: %d", (int)(allocation_fraction/100.0)));
29549 new_allocation = (size_t)(allocation_fraction*new_allocation + (1.0-allocation_fraction)*previous_desired_allocation);
29552 size_t smoothing = 3; // exponential smoothing factor
29553 if (smoothing > collection_count)
29554 smoothing = collection_count;
29555 new_allocation = new_allocation / smoothing + ((previous_desired_allocation / smoothing) * (smoothing-1));
29557 UNREFERENCED_PARAMETER(collection_count);
29559 return new_allocation;
29562 size_t gc_heap::desired_new_allocation (dynamic_data* dd,
29563 size_t out, int gen_number,
29566 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
29568 if (dd_begin_data_size (dd) == 0)
29570 size_t new_allocation = dd_min_size (dd);
29571 current_gc_data_per_heap->gen_data[gen_number].new_allocation = new_allocation;
29572 return new_allocation;
29577 size_t previous_desired_allocation = dd_desired_allocation (dd);
29578 size_t current_size = dd_current_size (dd);
29579 float max_limit = dd_max_limit (dd);
29580 float limit = dd_limit (dd);
29581 size_t min_gc_size = dd_min_size (dd);
29583 size_t max_size = dd_max_size (dd);
29584 size_t new_allocation = 0;
29585 float allocation_fraction = (float) (dd_desired_allocation (dd) - dd_gc_new_allocation (dd)) / (float) (dd_desired_allocation (dd));
29586 if (gen_number >= max_generation)
29588 size_t new_size = 0;
29590 cst = min (1.0f, float (out) / float (dd_begin_data_size (dd)));
29592 f = surv_to_growth (cst, limit, max_limit);
29593 size_t max_growth_size = (size_t)(max_size / f);
29594 if (current_size >= max_growth_size)
29596 new_size = max_size;
29600 new_size = (size_t) min (max ( (f * current_size), min_gc_size), max_size);
29603 assert ((new_size >= current_size) || (new_size == max_size));
29605 if (gen_number == max_generation)
29607 new_allocation = max((new_size - current_size), min_gc_size);
29609 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
29610 dd_desired_allocation (dd), dd_collection_count (dd));
29612 if ((dd_fragmentation (dd) > ((size_t)((f-1)*current_size))))
29614 //reducing allocation in case of fragmentation
29615 size_t new_allocation1 = max (min_gc_size,
29617 (size_t)((float)new_allocation * current_size /
29618 ((float)current_size + 2*dd_fragmentation (dd))));
29619 dprintf (2, ("Reducing max_gen allocation due to fragmentation from %Id to %Id",
29620 new_allocation, new_allocation1));
29621 new_allocation = new_allocation1;
29624 else //large object heap
29626 uint32_t memory_load = 0;
29627 uint64_t available_physical = 0;
29628 get_memory_info (&memory_load, &available_physical);
29629 if (heap_number == 0)
29630 settings.exit_memory_load = memory_load;
29631 if (available_physical > 1024*1024)
29632 available_physical -= 1024*1024;
29634 uint64_t available_free = available_physical + (uint64_t)generation_free_list_space (generation_of (gen_number));
29635 if (available_free > (uint64_t)MAX_PTR)
29637 available_free = (uint64_t)MAX_PTR;
29640 //try to avoid OOM during large object allocation
29641 new_allocation = max (min(max((new_size - current_size), dd_desired_allocation (dynamic_data_of (max_generation))),
29642 (size_t)available_free),
29643 max ((current_size/4), min_gc_size));
29645 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
29646 dd_desired_allocation (dd), dd_collection_count (dd));
29652 size_t survivors = out;
29653 cst = float (survivors) / float (dd_begin_data_size (dd));
29654 f = surv_to_growth (cst, limit, max_limit);
29655 new_allocation = (size_t) min (max ((f * (survivors)), min_gc_size), max_size);
29657 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
29658 dd_desired_allocation (dd), dd_collection_count (dd));
29660 if (gen_number == 0)
29665 //printf ("%f, %Id\n", cst, new_allocation);
29666 size_t free_space = generation_free_list_space (generation_of (gen_number));
29667 // DTREVIEW - is min_gc_size really a good choice?
29668 // on 64-bit this will almost always be true.
29669 dprintf (GTC_LOG, ("frag: %Id, min: %Id", free_space, min_gc_size));
29670 if (free_space > min_gc_size)
29672 settings.gen0_reduction_count = 2;
29676 if (settings.gen0_reduction_count > 0)
29677 settings.gen0_reduction_count--;
29680 if (settings.gen0_reduction_count > 0)
29682 dprintf (2, ("Reducing new allocation based on fragmentation"));
29683 new_allocation = min (new_allocation,
29684 max (min_gc_size, (max_size/3)));
29689 size_t new_allocation_ret =
29690 Align (new_allocation, get_alignment_constant (!(gen_number == (max_generation+1))));
29691 int gen_data_index = gen_number;
29692 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_data_index]);
29693 gen_data->new_allocation = new_allocation_ret;
29695 dd_surv (dd) = cst;
29697 #ifdef SIMPLE_DPRINTF
29698 dprintf (1, ("h%d g%d surv: %Id current: %Id alloc: %Id (%d%%) f: %d%% new-size: %Id new-alloc: %Id",
29699 heap_number, gen_number, out, current_size, (dd_desired_allocation (dd) - dd_gc_new_allocation (dd)),
29700 (int)(cst*100), (int)(f*100), current_size + new_allocation, new_allocation));
29702 dprintf (1,("gen: %d in: %Id out: %Id ", gen_number, generation_allocation_size (generation_of (gen_number)), out));
29703 dprintf (1,("current: %Id alloc: %Id ", current_size, (dd_desired_allocation (dd) - dd_gc_new_allocation (dd))));
29704 dprintf (1,(" surv: %d%% f: %d%% new-size: %Id new-alloc: %Id",
29705 (int)(cst*100), (int)(f*100), current_size + new_allocation, new_allocation));
29706 #endif //SIMPLE_DPRINTF
29708 return new_allocation_ret;
29712 //returns the planned size of a generation (including free list element)
29713 size_t gc_heap::generation_plan_size (int gen_number)
29715 if (0 == gen_number)
29716 return max((heap_segment_plan_allocated (ephemeral_heap_segment) -
29717 generation_plan_allocation_start (generation_of (gen_number))),
29718 (int)Align (min_obj_size));
29721 generation* gen = generation_of (gen_number);
29722 if (heap_segment_rw (generation_start_segment (gen)) == ephemeral_heap_segment)
29723 return (generation_plan_allocation_start (generation_of (gen_number - 1)) -
29724 generation_plan_allocation_start (generation_of (gen_number)));
29727 size_t gensize = 0;
29728 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
29730 PREFIX_ASSUME(seg != NULL);
29732 while (seg && (seg != ephemeral_heap_segment))
29734 gensize += heap_segment_plan_allocated (seg) -
29735 heap_segment_mem (seg);
29736 seg = heap_segment_next_rw (seg);
29740 gensize += (generation_plan_allocation_start (generation_of (gen_number - 1)) -
29741 heap_segment_mem (ephemeral_heap_segment));
29749 //returns the size of a generation (including free list element)
29750 size_t gc_heap::generation_size (int gen_number)
29752 if (0 == gen_number)
29753 return max((heap_segment_allocated (ephemeral_heap_segment) -
29754 generation_allocation_start (generation_of (gen_number))),
29755 (int)Align (min_obj_size));
29758 generation* gen = generation_of (gen_number);
29759 if (heap_segment_rw (generation_start_segment (gen)) == ephemeral_heap_segment)
29760 return (generation_allocation_start (generation_of (gen_number - 1)) -
29761 generation_allocation_start (generation_of (gen_number)));
29764 size_t gensize = 0;
29765 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
29767 PREFIX_ASSUME(seg != NULL);
29769 while (seg && (seg != ephemeral_heap_segment))
29771 gensize += heap_segment_allocated (seg) -
29772 heap_segment_mem (seg);
29773 seg = heap_segment_next_rw (seg);
29777 gensize += (generation_allocation_start (generation_of (gen_number - 1)) -
29778 heap_segment_mem (ephemeral_heap_segment));
29787 size_t gc_heap::compute_in (int gen_number)
29789 assert (gen_number != 0);
29790 dynamic_data* dd = dynamic_data_of (gen_number);
29792 size_t in = generation_allocation_size (generation_of (gen_number));
29794 if (gen_number == max_generation && ephemeral_promotion)
29797 for (int i = 0; i <= max_generation; i++)
29799 dynamic_data* dd = dynamic_data_of (i);
29800 in += dd_survived_size (dd);
29801 if (i != max_generation)
29803 generation_condemned_allocated (generation_of (gen_number)) += dd_survived_size (dd);
29808 dd_gc_new_allocation (dd) -= in;
29809 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
29811 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
29812 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_number]);
29815 generation_allocation_size (generation_of (gen_number)) = 0;
29819 void gc_heap::compute_promoted_allocation (int gen_number)
29821 compute_in (gen_number);
29826 size_t gc_heap::trim_youngest_desired (uint32_t memory_load,
29827 size_t total_new_allocation,
29828 size_t total_min_allocation)
29830 if (memory_load < MAX_ALLOWED_MEM_LOAD)
29832 // If the total of memory load and gen0 budget exceeds
29833 // our max memory load limit, trim the gen0 budget so the total
29834 // is the max memory load limit.
29835 size_t remain_memory_load = (MAX_ALLOWED_MEM_LOAD - memory_load) * mem_one_percent;
29836 return min (total_new_allocation, remain_memory_load);
29840 return max (mem_one_percent, total_min_allocation);
29844 size_t gc_heap::joined_youngest_desired (size_t new_allocation)
29846 dprintf (2, ("Entry memory load: %d; gen0 new_alloc: %Id", settings.entry_memory_load, new_allocation));
29848 size_t final_new_allocation = new_allocation;
29849 if (new_allocation > MIN_YOUNGEST_GEN_DESIRED)
29851 uint32_t num_heaps = 1;
29853 #ifdef MULTIPLE_HEAPS
29854 num_heaps = gc_heap::n_heaps;
29855 #endif //MULTIPLE_HEAPS
29857 size_t total_new_allocation = new_allocation * num_heaps;
29858 size_t total_min_allocation = MIN_YOUNGEST_GEN_DESIRED * num_heaps;
29860 if ((settings.entry_memory_load >= MAX_ALLOWED_MEM_LOAD) ||
29861 (total_new_allocation > max (youngest_gen_desired_th, total_min_allocation)))
29863 uint32_t memory_load = 0;
29864 get_memory_info (&memory_load);
29865 settings.exit_memory_load = memory_load;
29866 dprintf (2, ("Current emory load: %d", memory_load));
29868 size_t final_total =
29869 trim_youngest_desired (memory_load, total_new_allocation, total_min_allocation);
29870 size_t max_new_allocation =
29871 #ifdef MULTIPLE_HEAPS
29872 dd_max_size (g_heaps[0]->dynamic_data_of (0));
29873 #else //MULTIPLE_HEAPS
29874 dd_max_size (dynamic_data_of (0));
29875 #endif //MULTIPLE_HEAPS
29877 final_new_allocation = min (Align ((final_total / num_heaps), get_alignment_constant (TRUE)), max_new_allocation);
29881 if (final_new_allocation < new_allocation)
29883 settings.gen0_reduction_count = 2;
29886 return final_new_allocation;
29891 gc_history_per_heap* gc_heap::get_gc_data_per_heap()
29893 #ifdef BACKGROUND_GC
29894 return (settings.concurrent ? &bgc_data_per_heap : &gc_data_per_heap);
29896 return &gc_data_per_heap;
29897 #endif //BACKGROUND_GC
29900 void gc_heap::compute_new_dynamic_data (int gen_number)
29902 PREFIX_ASSUME(gen_number >= 0);
29903 PREFIX_ASSUME(gen_number <= max_generation);
29905 dynamic_data* dd = dynamic_data_of (gen_number);
29906 generation* gen = generation_of (gen_number);
29907 size_t in = (gen_number==0) ? 0 : compute_in (gen_number);
29909 size_t total_gen_size = generation_size (gen_number);
29910 //keep track of fragmentation
29911 dd_fragmentation (dd) = generation_free_list_space (gen) + generation_free_obj_space (gen);
29912 dd_current_size (dd) = total_gen_size - dd_fragmentation (dd);
29914 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
29916 size_t out = dd_survived_size (dd);
29918 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_number]);
29919 gen_data->size_after = total_gen_size;
29920 gen_data->free_list_space_after = generation_free_list_space (gen);
29921 gen_data->free_obj_space_after = generation_free_obj_space (gen);
29923 if ((settings.pause_mode == pause_low_latency) && (gen_number <= 1))
29925 // When we are in the low latency mode, we can still be
29926 // condemning more than gen1's 'cause of induced GCs.
29927 dd_desired_allocation (dd) = low_latency_alloc;
29931 if (gen_number == 0)
29933 //compensate for dead finalizable objects promotion.
29934 //they shoudn't be counted for growth.
29935 size_t final_promoted = 0;
29936 final_promoted = min (promoted_bytes (heap_number), out);
29937 // Prefast: this is clear from above but prefast needs to be told explicitly
29938 PREFIX_ASSUME(final_promoted <= out);
29940 dprintf (2, ("gen: %d final promoted: %Id", gen_number, final_promoted));
29941 dd_freach_previous_promotion (dd) = final_promoted;
29942 size_t lower_bound = desired_new_allocation (dd, out-final_promoted, gen_number, 0);
29944 if (settings.condemned_generation == 0)
29946 //there is no noise.
29947 dd_desired_allocation (dd) = lower_bound;
29951 size_t higher_bound = desired_new_allocation (dd, out, gen_number, 1);
29953 // <TODO>This assert was causing AppDomains\unload\test1n\test1nrun.bat to fail</TODO>
29954 //assert ( lower_bound <= higher_bound);
29956 //discount the noise. Change the desired allocation
29957 //only if the previous value is outside of the range.
29958 if (dd_desired_allocation (dd) < lower_bound)
29960 dd_desired_allocation (dd) = lower_bound;
29962 else if (dd_desired_allocation (dd) > higher_bound)
29964 dd_desired_allocation (dd) = higher_bound;
29966 #if defined (BIT64) && !defined (MULTIPLE_HEAPS)
29967 dd_desired_allocation (dd) = joined_youngest_desired (dd_desired_allocation (dd));
29968 #endif // BIT64 && !MULTIPLE_HEAPS
29969 trim_youngest_desired_low_memory();
29970 dprintf (2, ("final gen0 new_alloc: %Id", dd_desired_allocation (dd)));
29975 dd_desired_allocation (dd) = desired_new_allocation (dd, out, gen_number, 0);
29979 gen_data->pinned_surv = dd_pinned_survived_size (dd);
29980 gen_data->npinned_surv = dd_survived_size (dd) - dd_pinned_survived_size (dd);
29982 dd_gc_new_allocation (dd) = dd_desired_allocation (dd);
29983 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
29986 dd_promoted_size (dd) = out;
29987 if (gen_number == max_generation)
29989 dd = dynamic_data_of (max_generation+1);
29990 total_gen_size = generation_size (max_generation + 1);
29991 dd_fragmentation (dd) = generation_free_list_space (large_object_generation) +
29992 generation_free_obj_space (large_object_generation);
29993 dd_current_size (dd) = total_gen_size - dd_fragmentation (dd);
29994 dd_survived_size (dd) = dd_current_size (dd);
29996 out = dd_current_size (dd);
29997 dd_desired_allocation (dd) = desired_new_allocation (dd, out, max_generation+1, 0);
29998 dd_gc_new_allocation (dd) = Align (dd_desired_allocation (dd),
29999 get_alignment_constant (FALSE));
30000 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
30002 gen_data = &(current_gc_data_per_heap->gen_data[max_generation+1]);
30003 gen_data->size_after = total_gen_size;
30004 gen_data->free_list_space_after = generation_free_list_space (large_object_generation);
30005 gen_data->free_obj_space_after = generation_free_obj_space (large_object_generation);
30006 gen_data->npinned_surv = out;
30007 #ifdef BACKGROUND_GC
30008 end_loh_size = total_gen_size;
30009 #endif //BACKGROUND_GC
30011 dd_promoted_size (dd) = out;
30015 void gc_heap::trim_youngest_desired_low_memory()
30017 if (g_low_memory_status)
30019 size_t committed_mem = 0;
30020 heap_segment* seg = generation_start_segment (generation_of (max_generation));
30023 committed_mem += heap_segment_committed (seg) - heap_segment_mem (seg);
30024 seg = heap_segment_next (seg);
30026 seg = generation_start_segment (generation_of (max_generation + 1));
30029 committed_mem += heap_segment_committed (seg) - heap_segment_mem (seg);
30030 seg = heap_segment_next (seg);
30033 dynamic_data* dd = dynamic_data_of (0);
30034 size_t current = dd_desired_allocation (dd);
30035 size_t candidate = max (Align ((committed_mem / 10), get_alignment_constant(FALSE)), dd_min_size (dd));
30037 dd_desired_allocation (dd) = min (current, candidate);
30041 void gc_heap::decommit_ephemeral_segment_pages()
30043 if (settings.concurrent)
30048 size_t slack_space = heap_segment_committed (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment);
30049 dynamic_data* dd = dynamic_data_of (0);
30051 #ifndef MULTIPLE_HEAPS
30052 size_t extra_space = (g_low_memory_status ? 0 : (512 * 1024));
30053 size_t decommit_timeout = (g_low_memory_status ? 0 : GC_EPHEMERAL_DECOMMIT_TIMEOUT);
30054 size_t ephemeral_elapsed = dd_time_clock(dd) - gc_last_ephemeral_decommit_time;
30056 if (dd_desired_allocation (dd) > gc_gen0_desired_high)
30058 gc_gen0_desired_high = dd_desired_allocation (dd) + extra_space;
30061 if (ephemeral_elapsed >= decommit_timeout)
30063 slack_space = min (slack_space, gc_gen0_desired_high);
30065 gc_last_ephemeral_decommit_time = dd_time_clock(dd);
30066 gc_gen0_desired_high = 0;
30068 #endif //!MULTIPLE_HEAPS
30070 if (settings.condemned_generation >= (max_generation-1))
30072 size_t new_slack_space =
30074 max(min(min(soh_segment_size/32, dd_max_size(dd)), (generation_size (max_generation) / 10)), dd_desired_allocation(dd));
30076 #ifdef FEATURE_CORECLR
30077 dd_desired_allocation (dd);
30080 #endif //FEATURE_CORECLR
30083 slack_space = min (slack_space, new_slack_space);
30086 decommit_heap_segment_pages (ephemeral_heap_segment, slack_space);
30088 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
30089 current_gc_data_per_heap->extra_gen0_committed = heap_segment_committed (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment);
30092 size_t gc_heap::new_allocation_limit (size_t size, size_t free_size, int gen_number)
30094 dynamic_data* dd = dynamic_data_of (gen_number);
30095 ptrdiff_t new_alloc = dd_new_allocation (dd);
30096 assert (new_alloc == (ptrdiff_t)Align (new_alloc,
30097 get_alignment_constant (!(gen_number == (max_generation+1)))));
30098 size_t limit = min (max (new_alloc, (ptrdiff_t)size), (ptrdiff_t)free_size);
30099 assert (limit == Align (limit, get_alignment_constant (!(gen_number == (max_generation+1)))));
30100 dd_new_allocation (dd) = (new_alloc - limit );
30104 //This is meant to be called by decide_on_compacting.
30106 size_t gc_heap::generation_fragmentation (generation* gen,
30107 generation* consing_gen,
30111 uint8_t* alloc = generation_allocation_pointer (consing_gen);
30112 // If the allocation pointer has reached the ephemeral segment
30113 // fine, otherwise the whole ephemeral segment is considered
30115 if (in_range_for_segment (alloc, ephemeral_heap_segment))
30117 if (alloc <= heap_segment_allocated(ephemeral_heap_segment))
30118 frag = end - alloc;
30121 // case when no survivors, allocated set to beginning
30124 dprintf (3, ("ephemeral frag: %Id", frag));
30127 frag = (heap_segment_allocated (ephemeral_heap_segment) -
30128 heap_segment_mem (ephemeral_heap_segment));
30129 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
30131 PREFIX_ASSUME(seg != NULL);
30133 while (seg != ephemeral_heap_segment)
30135 frag += (heap_segment_allocated (seg) -
30136 heap_segment_plan_allocated (seg));
30137 dprintf (3, ("seg: %Ix, frag: %Id", (size_t)seg,
30138 (heap_segment_allocated (seg) -
30139 heap_segment_plan_allocated (seg))));
30141 seg = heap_segment_next_rw (seg);
30144 dprintf (3, ("frag: %Id discounting pinned plugs", frag));
30145 //add the length of the dequeued plug free space
30147 while (bos < mark_stack_bos)
30149 frag += (pinned_len (pinned_plug_of (bos)));
30156 // for SOH this returns the total sizes of the generation and its
30157 // younger generation(s).
30158 // for LOH this returns just LOH size.
30159 size_t gc_heap::generation_sizes (generation* gen)
30162 if (generation_start_segment (gen ) == ephemeral_heap_segment)
30163 result = (heap_segment_allocated (ephemeral_heap_segment) -
30164 generation_allocation_start (gen));
30167 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
30169 PREFIX_ASSUME(seg != NULL);
30173 result += (heap_segment_allocated (seg) -
30174 heap_segment_mem (seg));
30175 seg = heap_segment_next_in_range (seg);
30182 BOOL gc_heap::decide_on_compacting (int condemned_gen_number,
30183 size_t fragmentation,
30184 BOOL& should_expand)
30186 BOOL should_compact = FALSE;
30187 should_expand = FALSE;
30188 generation* gen = generation_of (condemned_gen_number);
30189 dynamic_data* dd = dynamic_data_of (condemned_gen_number);
30190 size_t gen_sizes = generation_sizes(gen);
30191 float fragmentation_burden = ( ((0 == fragmentation) || (0 == gen_sizes)) ? (0.0f) :
30192 (float (fragmentation) / gen_sizes) );
30194 dprintf (GTC_LOG, ("fragmentation: %Id (%d%%)", fragmentation, (int)(fragmentation_burden * 100.0)));
30197 // for pure GC stress runs we need compaction, for GC stress "mix"
30198 // we need to ensure a better mix of compacting and sweeping collections
30199 if (GCStress<cfg_any>::IsEnabled() && !settings.concurrent
30200 && !g_pConfig->IsGCStressMix())
30201 should_compact = TRUE;
30204 // in GC stress "mix" mode, for stress induced collections make sure we
30205 // keep sweeps and compactions relatively balanced. do not (yet) force sweeps
30206 // against the GC's determination, as it may lead to premature OOMs.
30207 if (g_pConfig->IsGCStressMix() && settings.stress_induced)
30209 int compactions = g_GCStatistics.cntCompactFGC+g_GCStatistics.cntCompactNGC;
30210 int sweeps = g_GCStatistics.cntFGC + g_GCStatistics.cntNGC - compactions;
30211 if (compactions < sweeps / 10)
30213 should_compact = TRUE;
30217 #endif //STRESS_HEAP
30219 if (GCConfig::GetForceCompact())
30220 should_compact = TRUE;
30222 if ((condemned_gen_number == max_generation) && last_gc_before_oom)
30224 should_compact = TRUE;
30225 last_gc_before_oom = FALSE;
30226 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_last_gc);
30229 if (settings.reason == reason_induced_compacting)
30231 dprintf (2, ("induced compacting GC"));
30232 should_compact = TRUE;
30233 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_induced_compacting);
30236 dprintf (2, ("Fragmentation: %d Fragmentation burden %d%%",
30237 fragmentation, (int) (100*fragmentation_burden)));
30239 if (!should_compact)
30241 if (dt_low_ephemeral_space_p (tuning_deciding_compaction))
30243 dprintf(GTC_LOG, ("compacting due to low ephemeral"));
30244 should_compact = TRUE;
30245 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_low_ephemeral);
30249 if (should_compact)
30251 if ((condemned_gen_number >= (max_generation - 1)))
30253 if (dt_low_ephemeral_space_p (tuning_deciding_expansion))
30255 dprintf (GTC_LOG,("Not enough space for all ephemeral generations with compaction"));
30256 should_expand = TRUE;
30262 BOOL high_memory = FALSE;
30265 if (!should_compact)
30267 // We are not putting this in dt_high_frag_p because it's not exactly
30268 // high fragmentation - it's just enough planned fragmentation for us to
30269 // want to compact. Also the "fragmentation" we are talking about here
30270 // is different from anywhere else.
30271 BOOL frag_exceeded = ((fragmentation >= dd_fragmentation_limit (dd)) &&
30272 (fragmentation_burden >= dd_fragmentation_burden_limit (dd)));
30276 #ifdef BACKGROUND_GC
30277 // do not force compaction if this was a stress-induced GC
30278 IN_STRESS_HEAP(if (!settings.stress_induced))
30280 #endif // BACKGROUND_GC
30281 assert (settings.concurrent == FALSE);
30282 should_compact = TRUE;
30283 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_high_frag);
30284 #ifdef BACKGROUND_GC
30286 #endif // BACKGROUND_GC
30290 // check for high memory situation
30291 if(!should_compact)
30293 uint32_t num_heaps = 1;
30294 #ifdef MULTIPLE_HEAPS
30295 num_heaps = gc_heap::n_heaps;
30296 #endif // MULTIPLE_HEAPS
30298 ptrdiff_t reclaim_space = generation_size(max_generation) - generation_plan_size(max_generation);
30299 if((settings.entry_memory_load >= high_memory_load_th) && (settings.entry_memory_load < v_high_memory_load_th))
30301 if(reclaim_space > (int64_t)(min_high_fragmentation_threshold (entry_available_physical_mem, num_heaps)))
30303 dprintf(GTC_LOG,("compacting due to fragmentation in high memory"));
30304 should_compact = TRUE;
30305 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_high_mem_frag);
30307 high_memory = TRUE;
30309 else if(settings.entry_memory_load >= v_high_memory_load_th)
30311 if(reclaim_space > (ptrdiff_t)(min_reclaim_fragmentation_threshold (num_heaps)))
30313 dprintf(GTC_LOG,("compacting due to fragmentation in very high memory"));
30314 should_compact = TRUE;
30315 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_vhigh_mem_frag);
30317 high_memory = TRUE;
30323 // The purpose of calling ensure_gap_allocation here is to make sure
30324 // that we actually are able to commit the memory to allocate generation
30326 if ((should_compact == FALSE) &&
30327 (ensure_gap_allocation (condemned_gen_number) == FALSE))
30329 should_compact = TRUE;
30330 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_no_gaps);
30333 if (settings.condemned_generation == max_generation)
30335 //check the progress
30338 (high_memory && !should_compact) ||
30340 (generation_plan_allocation_start (generation_of (max_generation - 1)) >=
30341 generation_allocation_start (generation_of (max_generation - 1))))
30343 dprintf (2, (" Elevation: gen2 size: %d, gen2 plan size: %d, no progress, elevation = locked",
30344 generation_size (max_generation),
30345 generation_plan_size (max_generation)));
30346 //no progress -> lock
30347 settings.should_lock_elevation = TRUE;
30351 if (settings.pause_mode == pause_no_gc)
30353 should_compact = TRUE;
30354 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_plan_allocated (ephemeral_heap_segment))
30355 < soh_allocation_no_gc)
30357 should_expand = TRUE;
30361 dprintf (2, ("will %s", (should_compact ? "compact" : "sweep")));
30362 return should_compact;
30365 size_t align_lower_good_size_allocation (size_t size)
30367 return (size/64)*64;
30370 size_t gc_heap::approximate_new_allocation()
30372 dynamic_data* dd0 = dynamic_data_of (0);
30373 return max (2*dd_min_size (dd0), ((dd_desired_allocation (dd0)*2)/3));
30376 // After we did a GC we expect to have at least this
30377 // much space at the end of the segment to satisfy
30378 // a reasonable amount of allocation requests.
30379 size_t gc_heap::end_space_after_gc()
30381 return max ((dd_min_size (dynamic_data_of (0))/2), (END_SPACE_AFTER_GC + Align (min_obj_size)));
30384 BOOL gc_heap::ephemeral_gen_fit_p (gc_tuning_point tp)
30386 uint8_t* start = 0;
30388 if ((tp == tuning_deciding_condemned_gen) ||
30389 (tp == tuning_deciding_compaction))
30391 start = (settings.concurrent ? alloc_allocated : heap_segment_allocated (ephemeral_heap_segment));
30392 if (settings.concurrent)
30394 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment (alloc_allocated)",
30395 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated)));
30399 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment (allocated)",
30400 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment))));
30403 else if (tp == tuning_deciding_expansion)
30405 start = heap_segment_plan_allocated (ephemeral_heap_segment);
30406 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment based on plan",
30407 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - start)));
30411 assert (tp == tuning_deciding_full_gc);
30412 dprintf (GTC_LOG, ("FGC: %Id left at the end of ephemeral segment (alloc_allocated)",
30413 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated)));
30414 start = alloc_allocated;
30417 if (start == 0) // empty ephemeral generations
30419 assert (tp == tuning_deciding_expansion);
30420 // if there are no survivors in the ephemeral segment,
30421 // this should be the beginning of ephemeral segment.
30422 start = generation_allocation_pointer (generation_of (max_generation));
30423 assert (start == heap_segment_mem (ephemeral_heap_segment));
30426 if (tp == tuning_deciding_expansion)
30428 assert (settings.condemned_generation >= (max_generation-1));
30429 size_t gen0size = approximate_new_allocation();
30430 size_t eph_size = gen0size;
30432 for (int j = 1; j <= max_generation-1; j++)
30434 eph_size += 2*dd_min_size (dynamic_data_of(j));
30437 // We must find room for one large object and enough room for gen0size
30438 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - start) > eph_size)
30440 dprintf (3, ("Enough room before end of segment"));
30445 size_t room = align_lower_good_size_allocation
30446 (heap_segment_reserved (ephemeral_heap_segment) - start);
30447 size_t end_seg = room;
30449 //look at the plug free space
30450 size_t largest_alloc = END_SPACE_AFTER_GC + Align (min_obj_size);
30451 bool large_chunk_found = FALSE;
30453 uint8_t* gen0start = generation_plan_allocation_start (youngest_generation);
30454 dprintf (3, ("ephemeral_gen_fit_p: gen0 plan start: %Ix", (size_t)gen0start));
30455 if (gen0start == 0)
30457 dprintf (3, ("ephemeral_gen_fit_p: room before free list search %Id, needed: %Id",
30459 while ((bos < mark_stack_bos) &&
30460 !((room >= gen0size) && large_chunk_found))
30462 uint8_t* plug = pinned_plug (pinned_plug_of (bos));
30463 if (in_range_for_segment (plug, ephemeral_heap_segment))
30465 if (plug >= gen0start)
30467 size_t chunk = align_lower_good_size_allocation (pinned_len (pinned_plug_of (bos)));
30469 if (!large_chunk_found)
30471 large_chunk_found = (chunk >= largest_alloc);
30473 dprintf (3, ("ephemeral_gen_fit_p: room now %Id, large chunk: %Id",
30474 room, large_chunk_found));
30480 if (room >= gen0size)
30482 if (large_chunk_found)
30484 dprintf (3, ("Enough room"));
30489 // now we need to find largest_alloc at the end of the segment.
30490 if (end_seg >= end_space_after_gc())
30492 dprintf (3, ("Enough room (may need end of seg)"));
30498 dprintf (3, ("Not enough room"));
30504 size_t end_space = 0;
30505 dynamic_data* dd = dynamic_data_of (0);
30506 if ((tp == tuning_deciding_condemned_gen) ||
30507 (tp == tuning_deciding_full_gc))
30509 end_space = 2*dd_min_size (dd);
30513 assert (tp == tuning_deciding_compaction);
30514 end_space = approximate_new_allocation();
30517 if (!((size_t)(heap_segment_reserved (ephemeral_heap_segment) - start) > end_space))
30519 dprintf (GTC_LOG, ("ephemeral_gen_fit_p: does not fit without compaction"));
30521 return ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - start) > end_space);
30525 CObjectHeader* gc_heap::allocate_large_object (size_t jsize, int64_t& alloc_bytes)
30527 //create a new alloc context because gen3context is shared.
30528 alloc_context acontext;
30529 acontext.alloc_ptr = 0;
30530 acontext.alloc_limit = 0;
30531 acontext.alloc_bytes = 0;
30532 #ifdef MULTIPLE_HEAPS
30533 acontext.set_alloc_heap(vm_heap);
30534 #endif //MULTIPLE_HEAPS
30537 uint8_t* current_lowest_address = lowest_address;
30538 uint8_t* current_highest_address = highest_address;
30539 #ifdef BACKGROUND_GC
30540 if (recursive_gc_sync::background_running_p())
30542 current_lowest_address = background_saved_lowest_address;
30543 current_highest_address = background_saved_highest_address;
30545 #endif //BACKGROUND_GC
30546 #endif // MARK_ARRAY
30549 size_t maxObjectSize = (INT64_MAX - 7 - Align(min_obj_size));
30551 size_t maxObjectSize = (INT32_MAX - 7 - Align(min_obj_size));
30554 if (jsize >= maxObjectSize)
30556 if (GCConfig::GetBreakOnOOM())
30558 GCToOSInterface::DebugBreak();
30563 size_t size = AlignQword (jsize);
30564 int align_const = get_alignment_constant (FALSE);
30565 #ifdef FEATURE_LOH_COMPACTION
30566 size_t pad = Align (loh_padding_obj_size, align_const);
30569 #endif //FEATURE_LOH_COMPACTION
30571 assert (size >= Align (min_obj_size, align_const));
30573 #pragma inline_depth(0)
30575 if (! allocate_more_space (&acontext, (size + pad), max_generation+1))
30581 #pragma inline_depth(20)
30584 #ifdef FEATURE_LOH_COMPACTION
30585 // The GC allocator made a free object already in this alloc context and
30586 // adjusted the alloc_ptr accordingly.
30587 #endif //FEATURE_LOH_COMPACTION
30589 uint8_t* result = acontext.alloc_ptr;
30591 assert ((size_t)(acontext.alloc_limit - acontext.alloc_ptr) == size);
30592 alloc_bytes += size;
30594 CObjectHeader* obj = (CObjectHeader*)result;
30597 if (recursive_gc_sync::background_running_p())
30599 if ((result < current_highest_address) && (result >= current_lowest_address))
30601 dprintf (3, ("Clearing mark bit at address %Ix",
30602 (size_t)(&mark_array [mark_word_of (result)])));
30604 mark_array_clear_marked (result);
30606 #ifdef BACKGROUND_GC
30607 //the object has to cover one full mark uint32_t
30608 assert (size > mark_word_size);
30609 if (current_c_gc_state == c_gc_state_marking)
30611 dprintf (3, ("Concurrent allocation of a large object %Ix",
30613 //mark the new block specially so we know it is a new object
30614 if ((result < current_highest_address) && (result >= current_lowest_address))
30616 dprintf (3, ("Setting mark bit at address %Ix",
30617 (size_t)(&mark_array [mark_word_of (result)])));
30619 mark_array_set_marked (result);
30622 #endif //BACKGROUND_GC
30624 #endif //MARK_ARRAY
30627 assert ((size_t)obj == Align ((size_t)obj, align_const));
30632 void reset_memory (uint8_t* o, size_t sizeo)
30634 if (sizeo > 128 * 1024)
30636 // We cannot reset the memory for the useful part of a free object.
30637 size_t size_to_skip = min_free_list - plug_skew;
30639 size_t page_start = align_on_page ((size_t)(o + size_to_skip));
30640 size_t size = align_lower_page ((size_t)o + sizeo - size_to_skip - plug_skew) - page_start;
30641 // Note we need to compensate for an OS bug here. This bug would cause the MEM_RESET to fail
30642 // on write watched memory.
30645 #ifdef MULTIPLE_HEAPS
30646 bool unlock_p = true;
30648 // We don't do unlock because there could be many processes using workstation GC and it's
30649 // bad perf to have many threads doing unlock at the same time.
30650 bool unlock_p = false;
30651 #endif // MULTIPLE_HEAPS
30653 reset_mm_p = GCToOSInterface::VirtualReset((void*)page_start, size, unlock_p);
30658 void gc_heap::reset_large_object (uint8_t* o)
30660 // If it's a large object, allow the O/S to discard the backing store for these pages.
30661 reset_memory (o, size(o));
30664 BOOL gc_heap::large_object_marked (uint8_t* o, BOOL clearp)
30667 // It shouldn't be necessary to do these comparisons because this is only used for blocking
30668 // GCs and LOH segments cannot be out of range.
30669 if ((o >= lowest_address) && (o < highest_address))
30689 void gc_heap::walk_survivors_relocation (void* profiling_context, record_surv_fn fn)
30691 // Now walk the portion of memory that is actually being relocated.
30692 walk_relocation (profiling_context, fn);
30694 #ifdef FEATURE_LOH_COMPACTION
30695 if (loh_compacted_p)
30697 walk_relocation_for_loh (profiling_context, fn);
30699 #endif //FEATURE_LOH_COMPACTION
30702 void gc_heap::walk_survivors_for_loh (void* profiling_context, record_surv_fn fn)
30704 generation* gen = large_object_generation;
30705 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));;
30707 PREFIX_ASSUME(seg != NULL);
30709 uint8_t* o = generation_allocation_start (gen);
30710 uint8_t* plug_end = o;
30711 uint8_t* plug_start = o;
30715 if (o >= heap_segment_allocated (seg))
30717 seg = heap_segment_next (seg);
30721 o = heap_segment_mem (seg);
30723 if (large_object_marked(o, FALSE))
30730 o = o + AlignQword (size (o));
30731 if (o >= heap_segment_allocated (seg))
30735 m = large_object_marked (o, FALSE);
30740 fn (plug_start, plug_end, 0, profiling_context, false, false);
30744 while (o < heap_segment_allocated (seg) && !large_object_marked(o, FALSE))
30746 o = o + AlignQword (size (o));
30752 #ifdef BACKGROUND_GC
30754 BOOL gc_heap::background_object_marked (uint8_t* o, BOOL clearp)
30757 if ((o >= background_saved_lowest_address) && (o < background_saved_highest_address))
30759 if (mark_array_marked (o))
30763 mark_array_clear_marked (o);
30764 //dprintf (3, ("mark array bit for object %Ix is cleared", o));
30765 dprintf (3, ("CM: %Ix", o));
30775 dprintf (3, ("o %Ix(%d) %s", o, size(o), (m ? "was bm" : "was NOT bm")));
30779 uint8_t* gc_heap::background_next_end (heap_segment* seg, BOOL large_objects_p)
30782 (large_objects_p ? heap_segment_allocated (seg) : heap_segment_background_allocated (seg));
30785 void gc_heap::set_mem_verify (uint8_t* start, uint8_t* end, uint8_t b)
30790 if ((GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC) &&
30791 !(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_NO_MEM_FILL))
30793 dprintf (3, ("setting mem to %c [%Ix, [%Ix", b, start, end));
30794 memset (start, b, (end - start));
30797 #endif //VERIFY_HEAP
30800 void gc_heap::generation_delete_heap_segment (generation* gen,
30802 heap_segment* prev_seg,
30803 heap_segment* next_seg)
30805 dprintf (3, ("bgc sweep: deleting seg %Ix", seg));
30806 if (gen == large_object_generation)
30808 heap_segment_next (prev_seg) = next_seg;
30810 dprintf (3, ("Preparing empty large segment %Ix for deletion", (size_t)seg));
30812 heap_segment_next (seg) = freeable_large_heap_segment;
30813 freeable_large_heap_segment = seg;
30817 if (seg == ephemeral_heap_segment)
30822 heap_segment_next (next_seg) = prev_seg;
30824 dprintf (3, ("Preparing empty small segment %Ix for deletion", (size_t)seg));
30825 heap_segment_next (seg) = freeable_small_heap_segment;
30826 freeable_small_heap_segment = seg;
30829 decommit_heap_segment (seg);
30830 seg->flags |= heap_segment_flags_decommitted;
30832 set_mem_verify (heap_segment_allocated (seg) - plug_skew, heap_segment_used (seg), 0xbb);
30835 void gc_heap::process_background_segment_end (heap_segment* seg,
30837 uint8_t* last_plug_end,
30838 heap_segment* start_seg,
30842 uint8_t* allocated = heap_segment_allocated (seg);
30843 uint8_t* background_allocated = heap_segment_background_allocated (seg);
30845 dprintf (3, ("Processing end of background segment [%Ix, %Ix[(%Ix[)",
30846 (size_t)heap_segment_mem (seg), background_allocated, allocated));
30849 if (allocated != background_allocated)
30851 if (gen == large_object_generation)
30856 dprintf (3, ("Make a free object before newly promoted objects [%Ix, %Ix[",
30857 (size_t)last_plug_end, background_allocated));
30858 thread_gap (last_plug_end, background_allocated - last_plug_end, generation_of (max_generation));
30860 fix_brick_to_highest (last_plug_end, background_allocated);
30862 // When we allowed fgc's during going through gaps, we could have erased the brick
30863 // that corresponds to bgc_allocated 'cause we had to update the brick there,
30864 // recover it here.
30865 fix_brick_to_highest (background_allocated, background_allocated);
30869 // by default, if allocated == background_allocated, it can't
30870 // be the ephemeral segment.
30871 if (seg == ephemeral_heap_segment)
30876 if (allocated == heap_segment_mem (seg))
30878 // this can happen with LOH segments when multiple threads
30879 // allocate new segments and not all of them were needed to
30880 // satisfy allocation requests.
30881 assert (gen == large_object_generation);
30884 if (last_plug_end == heap_segment_mem (seg))
30886 dprintf (3, ("Segment allocated is %Ix (beginning of this seg) - %s be deleted",
30887 (size_t)allocated, (*delete_p ? "should" : "should not")));
30889 if (seg != start_seg)
30896 dprintf (3, ("Trimming seg to %Ix[", (size_t)last_plug_end));
30897 heap_segment_allocated (seg) = last_plug_end;
30898 set_mem_verify (heap_segment_allocated (seg) - plug_skew, heap_segment_used (seg), 0xbb);
30900 decommit_heap_segment_pages (seg, 0);
30904 dprintf (3, ("verifying seg %Ix's mark array was completely cleared", seg));
30905 bgc_verify_mark_array_cleared (seg);
30908 void gc_heap::process_n_background_segments (heap_segment* seg,
30909 heap_segment* prev_seg,
30912 assert (gen != large_object_generation);
30916 dprintf (2, ("processing seg %Ix (not seen by bgc mark)", seg));
30917 heap_segment* next_seg = heap_segment_next (seg);
30919 if (heap_segment_read_only_p (seg))
30925 if (heap_segment_allocated (seg) == heap_segment_mem (seg))
30927 // This can happen - if we have a LOH segment where nothing survived
30928 // or a SOH segment allocated by a gen1 GC when BGC was going where
30929 // nothing survived last time we did a gen1 GC.
30930 generation_delete_heap_segment (gen, seg, prev_seg, next_seg);
30938 verify_soh_segment_list();
30944 BOOL gc_heap::fgc_should_consider_object (uint8_t* o,
30946 BOOL consider_bgc_mark_p,
30947 BOOL check_current_sweep_p,
30948 BOOL check_saved_sweep_p)
30950 // the logic for this function must be kept in sync with the analogous function
30951 // in ToolBox\SOS\Strike\gc.cpp
30953 // TRUE means we don't need to check the bgc mark bit
30954 // FALSE means we do.
30955 BOOL no_bgc_mark_p = FALSE;
30957 if (consider_bgc_mark_p)
30959 if (check_current_sweep_p && (o < current_sweep_pos))
30961 dprintf (3, ("no bgc mark - o: %Ix < cs: %Ix", o, current_sweep_pos));
30962 no_bgc_mark_p = TRUE;
30965 if (!no_bgc_mark_p)
30967 if(check_saved_sweep_p && (o >= saved_sweep_ephemeral_start))
30969 dprintf (3, ("no bgc mark - o: %Ix >= ss: %Ix", o, saved_sweep_ephemeral_start));
30970 no_bgc_mark_p = TRUE;
30973 if (!check_saved_sweep_p)
30975 uint8_t* background_allocated = heap_segment_background_allocated (seg);
30976 // if this was the saved ephemeral segment, check_saved_sweep_p
30977 // would've been true.
30978 assert (heap_segment_background_allocated (seg) != saved_sweep_ephemeral_start);
30979 // background_allocated could be 0 for the new segments acquired during bgc
30980 // sweep and we still want no_bgc_mark_p to be true.
30981 if (o >= background_allocated)
30983 dprintf (3, ("no bgc mark - o: %Ix >= ba: %Ix", o, background_allocated));
30984 no_bgc_mark_p = TRUE;
30991 no_bgc_mark_p = TRUE;
30994 dprintf (3, ("bgc mark %Ix: %s (bm: %s)", o, (no_bgc_mark_p ? "no" : "yes"), (background_object_marked (o, FALSE) ? "yes" : "no")));
30995 return (no_bgc_mark_p ? TRUE : background_object_marked (o, FALSE));
30998 // consider_bgc_mark_p tells you if you need to care about the bgc mark bit at all
30999 // if it's TRUE, check_current_sweep_p tells you if you should consider the
31000 // current sweep position or not.
31001 void gc_heap::should_check_bgc_mark (heap_segment* seg,
31002 BOOL* consider_bgc_mark_p,
31003 BOOL* check_current_sweep_p,
31004 BOOL* check_saved_sweep_p)
31006 // the logic for this function must be kept in sync with the analogous function
31007 // in ToolBox\SOS\Strike\gc.cpp
31008 *consider_bgc_mark_p = FALSE;
31009 *check_current_sweep_p = FALSE;
31010 *check_saved_sweep_p = FALSE;
31012 if (current_c_gc_state == c_gc_state_planning)
31014 // We are doing the current_sweep_pos comparison here because we have yet to
31015 // turn on the swept flag for the segment but in_range_for_segment will return
31016 // FALSE if the address is the same as reserved.
31017 if ((seg->flags & heap_segment_flags_swept) || (current_sweep_pos == heap_segment_reserved (seg)))
31019 dprintf (3, ("seg %Ix is already swept by bgc", seg));
31023 *consider_bgc_mark_p = TRUE;
31025 dprintf (3, ("seg %Ix hasn't been swept by bgc", seg));
31027 if (seg == saved_sweep_ephemeral_seg)
31029 dprintf (3, ("seg %Ix is the saved ephemeral seg", seg));
31030 *check_saved_sweep_p = TRUE;
31033 if (in_range_for_segment (current_sweep_pos, seg))
31035 dprintf (3, ("current sweep pos is %Ix and within seg %Ix",
31036 current_sweep_pos, seg));
31037 *check_current_sweep_p = TRUE;
31043 void gc_heap::background_ephemeral_sweep()
31045 dprintf (3, ("bgc ephemeral sweep"));
31047 int align_const = get_alignment_constant (TRUE);
31049 saved_sweep_ephemeral_seg = ephemeral_heap_segment;
31050 saved_sweep_ephemeral_start = generation_allocation_start (generation_of (max_generation - 1));
31052 // Since we don't want to interfere with gen0 allocation while we are threading gen0 free list,
31053 // we thread onto a list first then publish it when we are done.
31054 allocator youngest_free_list;
31055 size_t youngest_free_list_space = 0;
31056 size_t youngest_free_obj_space = 0;
31058 youngest_free_list.clear();
31060 for (int i = 0; i <= (max_generation - 1); i++)
31062 generation* gen_to_reset = generation_of (i);
31063 assert (generation_free_list_space (gen_to_reset) == 0);
31064 // Can only assert free_list_space is 0, not free_obj_space as the allocator could have added
31065 // something there.
31068 for (int i = (max_generation - 1); i >= 0; i--)
31070 generation* current_gen = generation_of (i);
31071 uint8_t* o = generation_allocation_start (current_gen);
31072 //Skip the generation gap object
31073 o = o + Align(size (o), align_const);
31074 uint8_t* end = ((i > 0) ?
31075 generation_allocation_start (generation_of (i - 1)) :
31076 heap_segment_allocated (ephemeral_heap_segment));
31078 uint8_t* plug_end = o;
31079 uint8_t* plug_start = o;
31080 BOOL marked_p = FALSE;
31084 marked_p = background_object_marked (o, TRUE);
31088 size_t plug_size = plug_start - plug_end;
31092 thread_gap (plug_end, plug_size, current_gen);
31098 make_unused_array (plug_end, plug_size);
31099 if (plug_size >= min_free_list)
31101 youngest_free_list_space += plug_size;
31102 youngest_free_list.thread_item (plug_end, plug_size);
31106 youngest_free_obj_space += plug_size;
31111 fix_brick_to_highest (plug_end, plug_start);
31112 fix_brick_to_highest (plug_start, plug_start);
31117 o = o + Align (size (o), align_const);
31123 m = background_object_marked (o, TRUE);
31126 dprintf (3, ("bgs: plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
31130 while ((o < end) && !background_object_marked (o, FALSE))
31132 o = o + Align (size (o), align_const);
31137 if (plug_end != end)
31141 thread_gap (plug_end, end - plug_end, current_gen);
31142 fix_brick_to_highest (plug_end, end);
31146 heap_segment_allocated (ephemeral_heap_segment) = plug_end;
31147 // the following line is temporary.
31148 heap_segment_saved_bg_allocated (ephemeral_heap_segment) = plug_end;
31150 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
31152 make_unused_array (plug_end, (end - plug_end));
31154 #endif //VERIFY_HEAP
31158 dd_fragmentation (dynamic_data_of (i)) =
31159 generation_free_list_space (current_gen) + generation_free_obj_space (current_gen);
31162 generation* youngest_gen = generation_of (0);
31163 generation_free_list_space (youngest_gen) = youngest_free_list_space;
31164 generation_free_obj_space (youngest_gen) = youngest_free_obj_space;
31165 dd_fragmentation (dynamic_data_of (0)) = youngest_free_list_space + youngest_free_obj_space;
31166 generation_allocator (youngest_gen)->copy_with_no_repair (&youngest_free_list);
31169 void gc_heap::background_sweep()
31171 generation* gen = generation_of (max_generation);
31172 dynamic_data* dd = dynamic_data_of (max_generation);
31173 // For SOH segments we go backwards.
31174 heap_segment* start_seg = ephemeral_heap_segment;
31175 PREFIX_ASSUME(start_seg != NULL);
31176 heap_segment* fseg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
31177 heap_segment* seg = start_seg;
31178 uint8_t* o = heap_segment_mem (seg);
31180 heap_segment* prev_seg = heap_segment_next (seg);
31181 int align_const = get_alignment_constant (TRUE);
31184 assert (o == generation_allocation_start (generation_of (max_generation)));
31185 o = o + Align(size (o), align_const);
31188 uint8_t* plug_end = o;
31189 uint8_t* plug_start = o;
31190 next_sweep_obj = o;
31191 current_sweep_pos = o;
31193 //uint8_t* end = background_next_end (seg, (gen == large_object_generation));
31194 uint8_t* end = heap_segment_background_allocated (seg);
31195 BOOL delete_p = FALSE;
31197 //concurrent_print_time_delta ("finished with mark and start with sweep");
31198 concurrent_print_time_delta ("Sw");
31199 dprintf (2, ("---- (GC%d)Background Sweep Phase ----", VolatileLoad(&settings.gc_index)));
31201 //block concurrent allocation for large objects
31202 dprintf (3, ("lh state: planning"));
31203 if (gc_lh_block_event.IsValid())
31205 gc_lh_block_event.Reset();
31208 for (int i = 0; i <= (max_generation + 1); i++)
31210 generation* gen_to_reset = generation_of (i);
31211 generation_allocator (gen_to_reset)->clear();
31212 generation_free_list_space (gen_to_reset) = 0;
31213 generation_free_obj_space (gen_to_reset) = 0;
31214 generation_free_list_allocated (gen_to_reset) = 0;
31215 generation_end_seg_allocated (gen_to_reset) = 0;
31216 generation_condemned_allocated (gen_to_reset) = 0;
31217 //reset the allocation so foreground gc can allocate into older generation
31218 generation_allocation_pointer (gen_to_reset)= 0;
31219 generation_allocation_limit (gen_to_reset) = 0;
31220 generation_allocation_segment (gen_to_reset) = heap_segment_rw (generation_start_segment (gen_to_reset));
31223 FIRE_EVENT(BGC2ndNonConEnd);
31225 current_bgc_state = bgc_sweep_soh;
31226 verify_soh_segment_list();
31228 #ifdef FEATURE_BASICFREEZE
31229 if ((generation_start_segment (gen) != ephemeral_heap_segment) &&
31230 ro_segments_in_range)
31232 sweep_ro_segments (generation_start_segment (gen));
31234 #endif // FEATURE_BASICFREEZE
31236 //TODO BACKGROUND_GC: can we move this to where we switch to the LOH?
31237 if (current_c_gc_state != c_gc_state_planning)
31239 current_c_gc_state = c_gc_state_planning;
31242 concurrent_print_time_delta ("Swe");
31244 heap_segment* loh_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation + 1)));
31245 PREFIX_ASSUME(loh_seg != NULL);
31248 loh_seg->flags &= ~heap_segment_flags_swept;
31249 heap_segment_background_allocated (loh_seg) = heap_segment_allocated (loh_seg);
31250 loh_seg = heap_segment_next_rw (loh_seg);
31253 #ifdef MULTIPLE_HEAPS
31254 bgc_t_join.join(this, gc_join_restart_ee);
31255 if (bgc_t_join.joined())
31256 #endif //MULTIPLE_HEAPS
31258 #ifdef MULTIPLE_HEAPS
31259 dprintf(2, ("Starting BGC threads for resuming EE"));
31260 bgc_t_join.restart();
31261 #endif //MULTIPLE_HEAPS
31264 if (heap_number == 0)
31269 FIRE_EVENT(BGC2ndConBegin);
31271 background_ephemeral_sweep();
31273 #ifdef MULTIPLE_HEAPS
31274 bgc_t_join.join(this, gc_join_after_ephemeral_sweep);
31275 if (bgc_t_join.joined())
31276 #endif //MULTIPLE_HEAPS
31278 #ifdef FEATURE_EVENT_TRACE
31279 bgc_heap_walk_for_etw_p = GCEventStatus::IsEnabled(GCEventProvider_Default,
31280 GCEventKeyword_GCHeapSurvivalAndMovement,
31281 GCEventLevel_Information);
31282 #endif //FEATURE_EVENT_TRACE
31284 leave_spin_lock (&gc_lock);
31286 #ifdef MULTIPLE_HEAPS
31287 dprintf(2, ("Starting BGC threads for BGC sweeping"));
31288 bgc_t_join.restart();
31289 #endif //MULTIPLE_HEAPS
31292 disable_preemptive (true);
31294 dprintf (2, ("bgs: sweeping gen2 objects"));
31295 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
31296 (size_t)heap_segment_mem (seg),
31297 (size_t)heap_segment_allocated (seg),
31298 (size_t)heap_segment_background_allocated (seg)));
31300 int num_objs = 256;
31301 int current_num_objs = 0;
31302 heap_segment* next_seg = 0;
31308 if (gen == large_object_generation)
31310 next_seg = heap_segment_next (seg);
31314 next_seg = heap_segment_prev (fseg, seg);
31319 if (!heap_segment_read_only_p (seg))
31321 if (gen == large_object_generation)
31323 // we can treat all LOH segments as in the bgc domain
31324 // regardless of whether we saw in bgc mark or not
31325 // because we don't allow LOH allocations during bgc
31326 // sweep anyway - the LOH segments can't change.
31327 process_background_segment_end (seg, gen, plug_end,
31328 start_seg, &delete_p);
31332 assert (heap_segment_background_allocated (seg) != 0);
31333 process_background_segment_end (seg, gen, plug_end,
31334 start_seg, &delete_p);
31336 assert (next_seg || !delete_p);
31342 generation_delete_heap_segment (gen, seg, prev_seg, next_seg);
31347 dprintf (2, ("seg %Ix has been swept", seg));
31348 seg->flags |= heap_segment_flags_swept;
31351 verify_soh_segment_list();
31355 dprintf (GTC_LOG, ("seg: %Ix, next_seg: %Ix, prev_seg: %Ix", seg, next_seg, prev_seg));
31359 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
31361 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
31363 if (gen != large_object_generation)
31365 dprintf (2, ("bgs: sweeping gen3 objects"));
31366 current_bgc_state = bgc_sweep_loh;
31367 gen = generation_of (max_generation+1);
31368 start_seg = heap_segment_rw (generation_start_segment (gen));
31370 PREFIX_ASSUME(start_seg != NULL);
31374 o = generation_allocation_start (gen);
31375 assert (method_table (o) == g_gc_pFreeObjectMethodTable);
31376 align_const = get_alignment_constant (FALSE);
31377 o = o + Align(size (o), align_const);
31379 end = heap_segment_allocated (seg);
31380 dprintf (2, ("sweeping gen3 objects"));
31381 generation_free_obj_space (gen) = 0;
31382 generation_allocator (gen)->clear();
31383 generation_free_list_space (gen) = 0;
31385 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
31386 (size_t)heap_segment_mem (seg),
31387 (size_t)heap_segment_allocated (seg),
31388 (size_t)heap_segment_background_allocated (seg)));
31395 o = heap_segment_mem (seg);
31398 assert (gen != large_object_generation);
31399 assert (o == generation_allocation_start (generation_of (max_generation)));
31400 align_const = get_alignment_constant (TRUE);
31401 o = o + Align(size (o), align_const);
31405 current_sweep_pos = o;
31406 next_sweep_obj = o;
31409 end = background_next_end (seg, (gen == large_object_generation));
31410 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
31411 (size_t)heap_segment_mem (seg),
31412 (size_t)heap_segment_allocated (seg),
31413 (size_t)heap_segment_background_allocated (seg)));
31417 if ((o < end) && background_object_marked (o, TRUE))
31420 if (gen == large_object_generation)
31422 dprintf (2, ("loh fr: [%Ix-%Ix[(%Id)", plug_end, plug_start, plug_start-plug_end));
31425 thread_gap (plug_end, plug_start-plug_end, gen);
31426 if (gen != large_object_generation)
31428 add_gen_free (max_generation, plug_start-plug_end);
31429 fix_brick_to_highest (plug_end, plug_start);
31430 // we need to fix the brick for the next plug here 'cause an FGC can
31431 // happen and can't read a stale brick.
31432 fix_brick_to_highest (plug_start, plug_start);
31439 next_sweep_obj = o + Align(size (o), align_const);
31440 current_num_objs++;
31441 if (current_num_objs >= num_objs)
31443 current_sweep_pos = next_sweep_obj;
31446 current_num_objs = 0;
31449 o = next_sweep_obj;
31455 m = background_object_marked (o, TRUE);
31458 if (gen != large_object_generation)
31460 add_gen_plug (max_generation, plug_end-plug_start);
31461 dd_survived_size (dd) += (plug_end - plug_start);
31463 dprintf (3, ("bgs: plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
31467 while ((o < end) && !background_object_marked (o, FALSE))
31469 next_sweep_obj = o + Align(size (o), align_const);;
31470 current_num_objs++;
31471 if (current_num_objs >= num_objs)
31473 current_sweep_pos = plug_end;
31474 dprintf (1234, ("f: swept till %Ix", current_sweep_pos));
31476 current_num_objs = 0;
31479 o = next_sweep_obj;
31484 size_t total_loh_size = generation_size (max_generation + 1);
31485 size_t total_soh_size = generation_sizes (generation_of (max_generation));
31487 dprintf (GTC_LOG, ("h%d: S: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
31489 dprintf (GTC_LOG, ("end of bgc sweep: gen2 FL: %Id, FO: %Id",
31490 generation_free_list_space (generation_of (max_generation)),
31491 generation_free_obj_space (generation_of (max_generation))));
31492 dprintf (GTC_LOG, ("h%d: end of bgc sweep: gen3 FL: %Id, FO: %Id",
31494 generation_free_list_space (generation_of (max_generation + 1)),
31495 generation_free_obj_space (generation_of (max_generation + 1))));
31497 FIRE_EVENT(BGC2ndConEnd);
31498 concurrent_print_time_delta ("background sweep");
31500 heap_segment* reset_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
31501 PREFIX_ASSUME(reset_seg != NULL);
31505 heap_segment_saved_bg_allocated (reset_seg) = heap_segment_background_allocated (reset_seg);
31506 heap_segment_background_allocated (reset_seg) = 0;
31507 reset_seg = heap_segment_next_rw (reset_seg);
31510 // We calculate dynamic data here because if we wait till we signal the lh event,
31511 // the allocation thread can change the fragmentation and we may read an intermediate
31512 // value (which can be greater than the generation size). Plus by that time it won't
31514 compute_new_dynamic_data (max_generation);
31516 enable_preemptive ();
31518 #ifdef MULTIPLE_HEAPS
31519 bgc_t_join.join(this, gc_join_set_state_free);
31520 if (bgc_t_join.joined())
31521 #endif //MULTIPLE_HEAPS
31523 // TODO: We are using this join just to set the state. Should
31524 // look into eliminating it - check to make sure things that use
31525 // this state can live with per heap state like should_check_bgc_mark.
31526 current_c_gc_state = c_gc_state_free;
31528 #ifdef MULTIPLE_HEAPS
31529 dprintf(2, ("Starting BGC threads after background sweep phase"));
31530 bgc_t_join.restart();
31531 #endif //MULTIPLE_HEAPS
31534 disable_preemptive (true);
31536 if (gc_lh_block_event.IsValid())
31538 gc_lh_block_event.Set();
31541 //dprintf (GTC_LOG, ("---- (GC%d)End Background Sweep Phase ----", VolatileLoad(&settings.gc_index)));
31542 dprintf (GTC_LOG, ("---- (GC%d)ESw ----", VolatileLoad(&settings.gc_index)));
31544 #endif //BACKGROUND_GC
31546 void gc_heap::sweep_large_objects ()
31548 //this min value is for the sake of the dynamic tuning.
31549 //so we know that we are not starting even if we have no
31551 generation* gen = large_object_generation;
31552 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
31554 PREFIX_ASSUME(start_seg != NULL);
31556 heap_segment* seg = start_seg;
31557 heap_segment* prev_seg = 0;
31558 uint8_t* o = generation_allocation_start (gen);
31559 int align_const = get_alignment_constant (FALSE);
31561 //Skip the generation gap object
31562 o = o + Align(size (o), align_const);
31564 uint8_t* plug_end = o;
31565 uint8_t* plug_start = o;
31567 generation_allocator (gen)->clear();
31568 generation_free_list_space (gen) = 0;
31569 generation_free_obj_space (gen) = 0;
31572 dprintf (3, ("sweeping large objects"));
31573 dprintf (3, ("seg: %Ix, [%Ix, %Ix[, starting from %Ix",
31575 (size_t)heap_segment_mem (seg),
31576 (size_t)heap_segment_allocated (seg),
31581 if (o >= heap_segment_allocated (seg))
31583 heap_segment* next_seg = heap_segment_next (seg);
31584 //delete the empty segment if not the only one
31585 if ((plug_end == heap_segment_mem (seg)) &&
31586 (seg != start_seg) && !heap_segment_read_only_p (seg))
31588 //prepare for deletion
31589 dprintf (3, ("Preparing empty large segment %Ix", (size_t)seg));
31591 heap_segment_next (prev_seg) = next_seg;
31592 heap_segment_next (seg) = freeable_large_heap_segment;
31593 freeable_large_heap_segment = seg;
31597 if (!heap_segment_read_only_p (seg))
31599 dprintf (3, ("Trimming seg to %Ix[", (size_t)plug_end));
31600 heap_segment_allocated (seg) = plug_end;
31601 decommit_heap_segment_pages (seg, 0);
31610 o = heap_segment_mem (seg);
31612 dprintf (3, ("seg: %Ix, [%Ix, %Ix[", (size_t)seg,
31613 (size_t)heap_segment_mem (seg),
31614 (size_t)heap_segment_allocated (seg)));
31617 if (large_object_marked(o, TRUE))
31620 //everything between plug_end and plug_start is free
31621 thread_gap (plug_end, plug_start-plug_end, gen);
31626 o = o + AlignQword (size (o));
31627 if (o >= heap_segment_allocated (seg))
31631 m = large_object_marked (o, TRUE);
31634 dprintf (3, ("plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
31638 while (o < heap_segment_allocated (seg) && !large_object_marked(o, FALSE))
31640 o = o + AlignQword (size (o));
31645 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
31647 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
31650 void gc_heap::relocate_in_large_objects ()
31652 relocate_args args;
31654 args.high = gc_high;
31655 args.last_plug = 0;
31657 generation* gen = large_object_generation;
31659 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
31661 PREFIX_ASSUME(seg != NULL);
31663 uint8_t* o = generation_allocation_start (gen);
31667 if (o >= heap_segment_allocated (seg))
31669 seg = heap_segment_next_rw (seg);
31674 o = heap_segment_mem (seg);
31677 while (o < heap_segment_allocated (seg))
31679 check_class_object_demotion (o);
31680 if (contain_pointers (o))
31682 dprintf(3, ("Relocating through large object %Ix", (size_t)o));
31683 go_through_object_nostart (method_table (o), o, size(o), pval,
31685 reloc_survivor_helper (pval);
31688 o = o + AlignQword (size (o));
31693 void gc_heap::mark_through_cards_for_large_objects (card_fn fn,
31696 uint8_t* low = gc_low;
31697 size_t end_card = 0;
31698 generation* oldest_gen = generation_of (max_generation+1);
31699 heap_segment* seg = heap_segment_rw (generation_start_segment (oldest_gen));
31701 PREFIX_ASSUME(seg != NULL);
31703 uint8_t* beg = generation_allocation_start (oldest_gen);
31704 uint8_t* end = heap_segment_allocated (seg);
31706 size_t cg_pointers_found = 0;
31708 size_t card_word_end = (card_of (align_on_card_word (end)) /
31713 size_t n_card_set = 0;
31714 uint8_t* next_boundary = (relocating ?
31715 generation_plan_allocation_start (generation_of (max_generation -1)) :
31718 uint8_t* nhigh = (relocating ?
31719 heap_segment_plan_allocated (ephemeral_heap_segment) :
31722 BOOL foundp = FALSE;
31723 uint8_t* start_address = 0;
31724 uint8_t* limit = 0;
31725 size_t card = card_of (beg);
31727 #ifdef BACKGROUND_GC
31728 BOOL consider_bgc_mark_p = FALSE;
31729 BOOL check_current_sweep_p = FALSE;
31730 BOOL check_saved_sweep_p = FALSE;
31731 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
31732 #endif //BACKGROUND_GC
31734 size_t total_cards_cleared = 0;
31736 //dprintf(3,( "scanning large objects from %Ix to %Ix", (size_t)beg, (size_t)end));
31737 dprintf(3, ("CMl: %Ix->%Ix", (size_t)beg, (size_t)end));
31740 if ((o < end) && (card_of(o) > card))
31742 dprintf (3, ("Found %Id cg pointers", cg_pointers_found));
31743 if (cg_pointers_found == 0)
31745 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card), (size_t)o));
31746 clear_cards (card, card_of((uint8_t*)o));
31747 total_cards_cleared += (card_of((uint8_t*)o) - card);
31749 n_eph +=cg_pointers_found;
31750 cg_pointers_found = 0;
31751 card = card_of ((uint8_t*)o);
31753 if ((o < end) &&(card >= end_card))
31755 foundp = find_card (card_table, card, card_word_end, end_card);
31758 n_card_set+= end_card - card;
31759 start_address = max (beg, card_address (card));
31761 limit = min (end, card_address (end_card));
31763 if ((!foundp) || (o >= end) || (card_address (card) >= end))
31765 if ((foundp) && (cg_pointers_found == 0))
31767 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card),
31768 (size_t)card_address(card+1)));
31769 clear_cards (card, card+1);
31770 total_cards_cleared += 1;
31772 n_eph +=cg_pointers_found;
31773 cg_pointers_found = 0;
31774 if ((seg = heap_segment_next_rw (seg)) != 0)
31776 #ifdef BACKGROUND_GC
31777 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
31778 #endif //BACKGROUND_GC
31779 beg = heap_segment_mem (seg);
31780 end = compute_next_end (seg, low);
31781 card_word_end = card_of (align_on_card_word (end)) / card_word_width;
31782 card = card_of (beg);
31793 assert (card_set_p (card));
31795 dprintf(3,("card %Ix: o: %Ix, l: %Ix[ ",
31796 card, (size_t)o, (size_t)limit));
31798 assert (Align (size (o)) >= Align (min_obj_size));
31799 size_t s = size (o);
31800 uint8_t* next_o = o + AlignQword (s);
31806 assert (Align (s) >= Align (min_obj_size));
31807 next_o = o + AlignQword (s);
31810 dprintf (4, ("|%Ix|", (size_t)o));
31811 if (next_o < start_address)
31816 #ifdef BACKGROUND_GC
31817 if (!fgc_should_consider_object (o, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p))
31821 #endif //BACKGROUND_GC
31823 #ifdef COLLECTIBLE_CLASS
31824 if (is_collectible(o))
31826 BOOL passed_end_card_p = FALSE;
31828 if (card_of (o) > card)
31830 passed_end_card_p = card_transition (o, end, card_word_end,
31834 foundp, start_address,
31835 limit, total_cards_cleared);
31838 if ((!passed_end_card_p || foundp) && (card_of (o) == card))
31840 // card is valid and it covers the head of the object
31841 if (fn == &gc_heap::relocate_address)
31843 keep_card_live (o, n_gen, cg_pointers_found);
31847 uint8_t* class_obj = get_class_object (o);
31848 mark_through_cards_helper (&class_obj, n_gen,
31849 cg_pointers_found, fn,
31850 nhigh, next_boundary);
31854 if (passed_end_card_p)
31856 if (foundp && (card_address (card) < next_o))
31858 goto go_through_refs;
31868 #endif //COLLECTIBLE_CLASS
31870 if (contain_pointers (o))
31872 dprintf(3,("Going through %Ix", (size_t)o));
31874 go_through_object (method_table(o), o, s, poo,
31875 start_address, use_start, (o + s),
31877 if (card_of ((uint8_t*)poo) > card)
31879 BOOL passed_end_card_p = card_transition ((uint8_t*)poo, end,
31884 foundp, start_address,
31885 limit, total_cards_cleared);
31887 if (passed_end_card_p)
31889 if (foundp && (card_address (card) < next_o))
31893 if (ppstop <= (uint8_t**)start_address)
31895 else if (poo < (uint8_t**)start_address)
31896 {poo = (uint8_t**)start_address;}
31906 mark_through_cards_helper (poo, n_gen,
31907 cg_pointers_found, fn,
31908 nhigh, next_boundary);
31920 // compute the efficiency ratio of the card table
31923 generation_skip_ratio = min (((n_eph > 800) ?
31924 (int)(((float)n_gen / (float)n_eph) * 100) : 100),
31925 generation_skip_ratio);
31927 dprintf (3, ("Mloh: cross: %Id, useful: %Id, cards cleared: %Id, cards set: %Id, ratio: %d",
31928 n_eph, n_gen, total_cards_cleared, n_card_set, generation_skip_ratio));
31932 dprintf (3, ("R: Mloh: cross: %Id, useful: %Id, cards set: %Id, ratio: %d",
31933 n_eph, n_gen, n_card_set, generation_skip_ratio));
31937 void gc_heap::descr_segment (heap_segment* seg )
31940 uint8_t* x = heap_segment_mem (seg);
31941 while (x < heap_segment_allocated (seg))
31943 dprintf(2, ( "%Ix: %d ", (size_t)x, size (x)));
31944 x = x + Align(size (x));
31947 UNREFERENCED_PARAMETER(seg);
31951 void gc_heap::descr_generations_to_profiler (gen_walk_fn fn, void *context)
31953 #ifdef MULTIPLE_HEAPS
31954 int n_heaps = g_theGCHeap->GetNumberOfHeaps ();
31955 for (int i = 0; i < n_heaps; i++)
31957 gc_heap* hp = GCHeap::GetHeap(i)->pGenGCHeap;
31958 #else //MULTIPLE_HEAPS
31960 gc_heap* hp = NULL;
31962 // prefix complains about us dereferencing hp in wks build even though we only access static members
31963 // this way. not sure how to shut it up except for this ugly workaround:
31964 PREFIX_ASSUME(hp != NULL);
31965 #endif // _PREFAST_
31966 #endif //MULTIPLE_HEAPS
31968 int curr_gen_number0 = max_generation+1;
31969 while (curr_gen_number0 >= 0)
31971 generation* gen = hp->generation_of (curr_gen_number0);
31972 heap_segment* seg = generation_start_segment (gen);
31973 while (seg && (seg != hp->ephemeral_heap_segment))
31975 assert (curr_gen_number0 > 0);
31977 // report bounds from heap_segment_mem (seg) to
31978 // heap_segment_allocated (seg);
31979 // for generation # curr_gen_number0
31980 // for heap # heap_no
31982 fn(context, curr_gen_number0, heap_segment_mem (seg),
31983 heap_segment_allocated (seg),
31984 curr_gen_number0 == max_generation+1 ? heap_segment_reserved (seg) : heap_segment_allocated (seg));
31986 seg = heap_segment_next (seg);
31990 assert (seg == hp->ephemeral_heap_segment);
31991 assert (curr_gen_number0 <= max_generation);
31993 if (curr_gen_number0 == max_generation)
31995 if (heap_segment_mem (seg) < generation_allocation_start (hp->generation_of (max_generation-1)))
31997 // report bounds from heap_segment_mem (seg) to
31998 // generation_allocation_start (generation_of (max_generation-1))
31999 // for heap # heap_number
32001 fn(context, curr_gen_number0, heap_segment_mem (seg),
32002 generation_allocation_start (hp->generation_of (max_generation-1)),
32003 generation_allocation_start (hp->generation_of (max_generation-1)) );
32006 else if (curr_gen_number0 != 0)
32008 //report bounds from generation_allocation_start (generation_of (curr_gen_number0))
32009 // to generation_allocation_start (generation_of (curr_gen_number0-1))
32010 // for heap # heap_number
32012 fn(context, curr_gen_number0, generation_allocation_start (hp->generation_of (curr_gen_number0)),
32013 generation_allocation_start (hp->generation_of (curr_gen_number0-1)),
32014 generation_allocation_start (hp->generation_of (curr_gen_number0-1)));
32018 //report bounds from generation_allocation_start (generation_of (curr_gen_number0))
32019 // to heap_segment_allocated (ephemeral_heap_segment);
32020 // for heap # heap_number
32022 fn(context, curr_gen_number0, generation_allocation_start (hp->generation_of (curr_gen_number0)),
32023 heap_segment_allocated (hp->ephemeral_heap_segment),
32024 heap_segment_reserved (hp->ephemeral_heap_segment) );
32027 curr_gen_number0--;
32033 // Note that when logging is on it can take a long time to go through the free items.
32034 void gc_heap::print_free_list (int gen, heap_segment* seg)
32036 UNREFERENCED_PARAMETER(gen);
32037 UNREFERENCED_PARAMETER(seg);
32039 if (settings.concurrent == FALSE)
32041 uint8_t* seg_start = heap_segment_mem (seg);
32042 uint8_t* seg_end = heap_segment_allocated (seg);
32044 dprintf (3, ("Free list in seg %Ix:", seg_start));
32046 size_t total_free_item = 0;
32048 allocator* gen_allocator = generation_allocator (generation_of (gen));
32049 for (unsigned int b = 0; b < gen_allocator->number_of_buckets(); b++)
32051 uint8_t* fo = gen_allocator->alloc_list_head_of (b);
32054 if (fo >= seg_start && fo < seg_end)
32058 size_t free_item_len = size(fo);
32060 dprintf (3, ("[%Ix, %Ix[:%Id",
32062 (size_t)(fo + free_item_len),
32066 fo = free_list_slot (fo);
32070 dprintf (3, ("total %Id free items", total_free_item));
32076 void gc_heap::descr_generations (BOOL begin_gc_p)
32078 UNREFERENCED_PARAMETER(begin_gc_p);
32080 if (StressLog::StressLogOn(LF_GC, LL_INFO10))
32083 #ifdef MULTIPLE_HEAPS
32085 #endif //MULTIPLE_HEAPS
32087 STRESS_LOG1(LF_GC, LL_INFO10, "GC Heap %p\n", hp);
32088 for (int n = max_generation; n >= 0; --n)
32090 STRESS_LOG4(LF_GC, LL_INFO10, " Generation %d [%p, %p] cur = %p\n",
32092 generation_allocation_start(generation_of(n)),
32093 generation_allocation_limit(generation_of(n)),
32094 generation_allocation_pointer(generation_of(n)));
32096 heap_segment* seg = generation_start_segment(generation_of(n));
32099 STRESS_LOG4(LF_GC, LL_INFO10, " Segment mem %p alloc = %p used %p committed %p\n",
32100 heap_segment_mem(seg),
32101 heap_segment_allocated(seg),
32102 heap_segment_used(seg),
32103 heap_segment_committed(seg));
32104 seg = heap_segment_next(seg);
32108 #endif // STRESS_LOG
32111 dprintf (2, ("lowest_address: %Ix highest_address: %Ix",
32112 (size_t) lowest_address, (size_t) highest_address));
32113 #ifdef BACKGROUND_GC
32114 dprintf (2, ("bgc lowest_address: %Ix bgc highest_address: %Ix",
32115 (size_t) background_saved_lowest_address, (size_t) background_saved_highest_address));
32116 #endif //BACKGROUND_GC
32118 if (heap_number == 0)
32120 dprintf (1, ("total heap size: %Id, commit size: %Id", get_total_heap_size(), get_total_committed_size()));
32123 int curr_gen_number = max_generation+1;
32124 while (curr_gen_number >= 0)
32126 size_t total_gen_size = generation_size (curr_gen_number);
32127 #ifdef SIMPLE_DPRINTF
32128 dprintf (GTC_LOG, ("[%s][g%d]gen %d:, size: %Id, frag: %Id(L: %Id, O: %Id), f: %d%% %s %s %s",
32129 (begin_gc_p ? "BEG" : "END"),
32130 settings.condemned_generation,
32133 dd_fragmentation (dynamic_data_of (curr_gen_number)),
32134 generation_free_list_space (generation_of (curr_gen_number)),
32135 generation_free_obj_space (generation_of (curr_gen_number)),
32137 (int)(((double)dd_fragmentation (dynamic_data_of (curr_gen_number)) / (double)total_gen_size) * 100) :
32139 (begin_gc_p ? ("") : (settings.compaction ? "(compact)" : "(sweep)")),
32140 (settings.heap_expansion ? "(EX)" : " "),
32141 (settings.promotion ? "Promotion" : "NoPromotion")));
32143 dprintf (2, ( "Generation %d: gap size: %d, generation size: %Id, fragmentation: %Id",
32145 size (generation_allocation_start (generation_of (curr_gen_number))),
32147 dd_fragmentation (dynamic_data_of (curr_gen_number))));
32148 #endif //SIMPLE_DPRINTF
32150 generation* gen = generation_of (curr_gen_number);
32151 heap_segment* seg = generation_start_segment (gen);
32152 while (seg && (seg != ephemeral_heap_segment))
32154 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[-%Ix[ (%Id) (%Id)",
32156 (size_t)heap_segment_mem (seg),
32157 (size_t)heap_segment_allocated (seg),
32158 (size_t)heap_segment_committed (seg),
32159 (size_t)(heap_segment_allocated (seg) - heap_segment_mem (seg)),
32160 (size_t)(heap_segment_committed (seg) - heap_segment_allocated (seg))));
32161 print_free_list (curr_gen_number, seg);
32162 seg = heap_segment_next (seg);
32164 if (seg && (seg != generation_start_segment (gen)))
32166 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[",
32168 (size_t)heap_segment_mem (seg),
32169 (size_t)generation_allocation_start (generation_of (curr_gen_number-1))));
32170 print_free_list (curr_gen_number, seg);
32175 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[",
32177 (size_t)generation_allocation_start (generation_of (curr_gen_number)),
32178 (size_t)(((curr_gen_number == 0)) ?
32179 (heap_segment_allocated
32180 (generation_start_segment
32181 (generation_of (curr_gen_number)))) :
32182 (generation_allocation_start
32183 (generation_of (curr_gen_number - 1))))
32185 print_free_list (curr_gen_number, seg);
32197 //-----------------------------------------------------------------------------
32199 // VM Specific support
32201 //-----------------------------------------------------------------------------
32206 unsigned int PromotedObjectCount = 0;
32207 unsigned int CreatedObjectCount = 0;
32208 unsigned int AllocDuration = 0;
32209 unsigned int AllocCount = 0;
32210 unsigned int AllocBigCount = 0;
32211 unsigned int AllocSmallCount = 0;
32212 unsigned int AllocStart = 0;
32215 //Static member variables.
32216 VOLATILE(BOOL) GCHeap::GcInProgress = FALSE;
32218 //CMCSafeLock* GCHeap::fGcLock;
32219 GCEvent *GCHeap::WaitForGCEvent = NULL;
32222 unsigned int GCHeap::GcDuration;
32224 unsigned GCHeap::GcCondemnedGeneration = 0;
32225 size_t GCHeap::totalSurvivedSize = 0;
32226 #ifdef FEATURE_PREMORTEM_FINALIZATION
32227 CFinalize* GCHeap::m_Finalize = 0;
32228 BOOL GCHeap::GcCollectClasses = FALSE;
32229 VOLATILE(int32_t) GCHeap::m_GCFLock = 0;
32231 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
32233 #ifdef BACKGROUND_GC
32234 int GCHeap::gc_stress_fgcs_in_bgc = 0;
32235 #endif // BACKGROUND_GC
32236 #ifndef MULTIPLE_HEAPS
32237 OBJECTHANDLE GCHeap::m_StressObjs[NUM_HEAP_STRESS_OBJS];
32238 int GCHeap::m_CurStressObj = 0;
32239 #endif // !MULTIPLE_HEAPS
32240 #endif // STRESS_HEAP
32241 #endif // FEATURE_REDHAWK
32243 #endif //FEATURE_PREMORTEM_FINALIZATION
32245 class NoGCRegionLockHolder
32248 NoGCRegionLockHolder()
32250 enter_spin_lock_noinstru(&g_no_gc_lock);
32253 ~NoGCRegionLockHolder()
32255 leave_spin_lock_noinstru(&g_no_gc_lock);
32259 // An explanation of locking for finalization:
32261 // Multiple threads allocate objects. During the allocation, they are serialized by
32262 // the AllocLock above. But they release that lock before they register the object
32263 // for finalization. That's because there is much contention for the alloc lock, but
32264 // finalization is presumed to be a rare case.
32266 // So registering an object for finalization must be protected by the FinalizeLock.
32268 // There is another logical queue that involves finalization. When objects registered
32269 // for finalization become unreachable, they are moved from the "registered" queue to
32270 // the "unreachable" queue. Note that this only happens inside a GC, so no other
32271 // threads can be manipulating either queue at that time. Once the GC is over and
32272 // threads are resumed, the Finalizer thread will dequeue objects from the "unreachable"
32273 // queue and call their finalizers. This dequeue operation is also protected with
32274 // the finalize lock.
32276 // At first, this seems unnecessary. Only one thread is ever enqueuing or dequeuing
32277 // on the unreachable queue (either the GC thread during a GC or the finalizer thread
32278 // when a GC is not in progress). The reason we share a lock with threads enqueuing
32279 // on the "registered" queue is that the "registered" and "unreachable" queues are
32282 // They are actually two regions of a longer list, which can only grow at one end.
32283 // So to enqueue an object to the "registered" list, you actually rotate an unreachable
32284 // object at the boundary between the logical queues, out to the other end of the
32285 // unreachable queue -- where all growing takes place. Then you move the boundary
32286 // pointer so that the gap we created at the boundary is now on the "registered"
32287 // side rather than the "unreachable" side. Now the object can be placed into the
32288 // "registered" side at that point. This is much more efficient than doing moves
32289 // of arbitrarily long regions, but it causes the two queues to require a shared lock.
32291 // Notice that Enter/LeaveFinalizeLock is not a GC-aware spin lock. Instead, it relies
32292 // on the fact that the lock will only be taken for a brief period and that it will
32293 // never provoke or allow a GC while the lock is held. This is critical. If the
32294 // FinalizeLock used enter_spin_lock (and thus sometimes enters preemptive mode to
32295 // allow a GC), then the Alloc client would have to GC protect a finalizable object
32296 // to protect against that eventuality. That is too slow!
32300 BOOL IsValidObject99(uint8_t *pObject)
32303 if (!((CObjectHeader*)pObject)->IsFree())
32304 ((CObjectHeader *) pObject)->Validate();
32305 #endif //VERIFY_HEAP
32309 #ifdef BACKGROUND_GC
32310 BOOL gc_heap::bgc_mark_array_range (heap_segment* seg,
32312 uint8_t** range_beg,
32313 uint8_t** range_end)
32315 uint8_t* seg_start = heap_segment_mem (seg);
32316 uint8_t* seg_end = (whole_seg_p ? heap_segment_reserved (seg) : align_on_mark_word (heap_segment_allocated (seg)));
32318 if ((seg_start < background_saved_highest_address) &&
32319 (seg_end > background_saved_lowest_address))
32321 *range_beg = max (seg_start, background_saved_lowest_address);
32322 *range_end = min (seg_end, background_saved_highest_address);
32331 void gc_heap::bgc_verify_mark_array_cleared (heap_segment* seg)
32333 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32334 if (recursive_gc_sync::background_running_p() && GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
32336 uint8_t* range_beg = 0;
32337 uint8_t* range_end = 0;
32339 if (bgc_mark_array_range (seg, TRUE, &range_beg, &range_end))
32341 size_t markw = mark_word_of (range_beg);
32342 size_t markw_end = mark_word_of (range_end);
32343 while (markw < markw_end)
32345 if (mark_array [markw])
32347 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
32348 markw, mark_array [markw], mark_word_address (markw)));
32353 uint8_t* p = mark_word_address (markw_end);
32354 while (p < range_end)
32356 assert (!(mark_array_marked (p)));
32361 #endif //VERIFY_HEAP && MARK_ARRAY
32364 void gc_heap::verify_mark_bits_cleared (uint8_t* obj, size_t s)
32366 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32367 size_t start_mark_bit = mark_bit_of (obj) + 1;
32368 size_t end_mark_bit = mark_bit_of (obj + s);
32369 unsigned int startbit = mark_bit_bit (start_mark_bit);
32370 unsigned int endbit = mark_bit_bit (end_mark_bit);
32371 size_t startwrd = mark_bit_word (start_mark_bit);
32372 size_t endwrd = mark_bit_word (end_mark_bit);
32373 unsigned int result = 0;
32375 unsigned int firstwrd = ~(lowbits (~0, startbit));
32376 unsigned int lastwrd = ~(highbits (~0, endbit));
32378 if (startwrd == endwrd)
32380 unsigned int wrd = firstwrd & lastwrd;
32381 result = mark_array[startwrd] & wrd;
32389 // verify the first mark word is cleared.
32392 result = mark_array[startwrd] & firstwrd;
32400 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
32402 result = mark_array[wrdtmp];
32409 // set the last mark word.
32412 result = mark_array[endwrd] & lastwrd;
32418 #endif //VERIFY_HEAP && MARK_ARRAY
32421 void gc_heap::clear_all_mark_array()
32424 //size_t num_dwords_written = 0;
32425 //size_t begin_time = GetHighPrecisionTimeStamp();
32427 generation* gen = generation_of (max_generation);
32428 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32434 if (gen != large_object_generation)
32436 gen = generation_of (max_generation+1);
32437 seg = heap_segment_rw (generation_start_segment (gen));
32445 uint8_t* range_beg = 0;
32446 uint8_t* range_end = 0;
32448 if (bgc_mark_array_range (seg, (seg == ephemeral_heap_segment), &range_beg, &range_end))
32450 size_t markw = mark_word_of (range_beg);
32451 size_t markw_end = mark_word_of (range_end);
32452 size_t size_total = (markw_end - markw) * sizeof (uint32_t);
32453 //num_dwords_written = markw_end - markw;
32455 size_t size_left = 0;
32457 assert (((size_t)&mark_array[markw] & (sizeof(PTR_PTR)-1)) == 0);
32459 if ((size_total & (sizeof(PTR_PTR) - 1)) != 0)
32461 size = (size_total & ~(sizeof(PTR_PTR) - 1));
32462 size_left = size_total - size;
32463 assert ((size_left & (sizeof (uint32_t) - 1)) == 0);
32470 memclr ((uint8_t*)&mark_array[markw], size);
32472 if (size_left != 0)
32474 uint32_t* markw_to_clear = &mark_array[markw + size / sizeof (uint32_t)];
32475 for (size_t i = 0; i < (size_left / sizeof (uint32_t)); i++)
32477 *markw_to_clear = 0;
32483 seg = heap_segment_next_rw (seg);
32486 //size_t end_time = GetHighPrecisionTimeStamp() - begin_time;
32488 //printf ("took %Id ms to clear %Id bytes\n", end_time, num_dwords_written*sizeof(uint32_t));
32490 #endif //MARK_ARRAY
32493 #endif //BACKGROUND_GC
32495 void gc_heap::verify_mark_array_cleared (heap_segment* seg)
32497 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32498 assert (card_table == g_gc_card_table);
32499 size_t markw = mark_word_of (heap_segment_mem (seg));
32500 size_t markw_end = mark_word_of (heap_segment_reserved (seg));
32502 while (markw < markw_end)
32504 if (mark_array [markw])
32506 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
32507 markw, mark_array [markw], mark_word_address (markw)));
32512 #endif //VERIFY_HEAP && MARK_ARRAY
32515 void gc_heap::verify_mark_array_cleared ()
32517 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32518 if (recursive_gc_sync::background_running_p() && GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
32520 generation* gen = generation_of (max_generation);
32521 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32527 if (gen != large_object_generation)
32529 gen = generation_of (max_generation+1);
32530 seg = heap_segment_rw (generation_start_segment (gen));
32538 bgc_verify_mark_array_cleared (seg);
32539 seg = heap_segment_next_rw (seg);
32542 #endif //VERIFY_HEAP && MARK_ARRAY
32545 void gc_heap::verify_seg_end_mark_array_cleared()
32547 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32548 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
32550 generation* gen = generation_of (max_generation);
32551 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32557 if (gen != large_object_generation)
32559 gen = generation_of (max_generation+1);
32560 seg = heap_segment_rw (generation_start_segment (gen));
32568 // We already cleared all mark array bits for ephemeral generations
32569 // at the beginning of bgc sweep
32570 uint8_t* from = ((seg == ephemeral_heap_segment) ?
32571 generation_allocation_start (generation_of (max_generation - 1)) :
32572 heap_segment_allocated (seg));
32573 size_t markw = mark_word_of (align_on_mark_word (from));
32574 size_t markw_end = mark_word_of (heap_segment_reserved (seg));
32576 while (from < mark_word_address (markw))
32578 if (is_mark_bit_set (from))
32580 dprintf (3, ("mark bit for %Ix was not cleared", from));
32584 from += mark_bit_pitch;
32587 while (markw < markw_end)
32589 if (mark_array [markw])
32591 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
32592 markw, mark_array [markw], mark_word_address (markw)));
32597 seg = heap_segment_next_rw (seg);
32600 #endif //VERIFY_HEAP && MARK_ARRAY
32603 // This function is called to make sure we don't mess up the segment list
32604 // in SOH. It's called by:
32605 // 1) begin and end of ephemeral GCs
32606 // 2) during bgc sweep when we switch segments.
32607 void gc_heap::verify_soh_segment_list()
32610 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
32612 generation* gen = generation_of (max_generation);
32613 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32614 heap_segment* last_seg = 0;
32618 seg = heap_segment_next_rw (seg);
32620 if (last_seg != ephemeral_heap_segment)
32625 #endif //VERIFY_HEAP
32628 // This function can be called at any foreground GCs or blocking GCs. For background GCs,
32629 // it can be called at the end of the final marking; and at any point during background
32631 // NOTE - to be able to call this function during background sweep, we need to temporarily
32632 // NOT clear the mark array bits as we go.
32633 void gc_heap::verify_partial ()
32635 #ifdef BACKGROUND_GC
32636 //printf ("GC#%d: Verifying loh during sweep\n", settings.gc_index);
32637 //generation* gen = large_object_generation;
32638 generation* gen = generation_of (max_generation);
32639 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32640 int align_const = get_alignment_constant (gen != large_object_generation);
32646 // Different ways to fail.
32647 BOOL mark_missed_p = FALSE;
32648 BOOL bad_ref_p = FALSE;
32649 BOOL free_ref_p = FALSE;
32655 if (gen != large_object_generation)
32658 gen = large_object_generation;
32659 align_const = get_alignment_constant (gen != large_object_generation);
32660 seg = heap_segment_rw (generation_start_segment (gen));
32669 o = heap_segment_mem (seg);
32670 end = heap_segment_allocated (seg);
32671 //printf ("validating [%Ix-[%Ix\n", o, end);
32676 BOOL marked_p = background_object_marked (o, FALSE);
32680 go_through_object_cl (method_table (o), o, s, oo,
32684 //dprintf (3, ("VOM: verifying member %Ix in obj %Ix", (size_t)*oo, o));
32685 MethodTable *pMT = method_table (*oo);
32687 if (pMT == g_gc_pFreeObjectMethodTable)
32693 if (!pMT->SanityCheck())
32696 dprintf (3, ("Bad member of %Ix %Ix",
32697 (size_t)oo, (size_t)*oo));
32701 if (current_bgc_state == bgc_final_marking)
32703 if (marked_p && !background_object_marked (*oo, FALSE))
32705 mark_missed_p = TRUE;
32714 o = o + Align(s, align_const);
32716 seg = heap_segment_next_rw (seg);
32719 //printf ("didn't find any large object large enough...\n");
32720 //printf ("finished verifying loh\n");
32721 #endif //BACKGROUND_GC
32727 gc_heap::verify_free_lists ()
32729 for (int gen_num = 0; gen_num <= max_generation+1; gen_num++)
32731 dprintf (3, ("Verifying free list for gen:%d", gen_num));
32732 allocator* gen_alloc = generation_allocator (generation_of (gen_num));
32733 size_t sz = gen_alloc->first_bucket_size();
32734 bool verify_undo_slot = (gen_num != 0) && (gen_num != max_generation+1) && !gen_alloc->discard_if_no_fit_p();
32736 for (unsigned int a_l_number = 0; a_l_number < gen_alloc->number_of_buckets(); a_l_number++)
32738 uint8_t* free_list = gen_alloc->alloc_list_head_of (a_l_number);
32742 if (!((CObjectHeader*)free_list)->IsFree())
32744 dprintf (3, ("Verifiying Heap: curr free list item %Ix isn't a free object)",
32745 (size_t)free_list));
32748 if (((a_l_number < (gen_alloc->number_of_buckets()-1))&& (unused_array_size (free_list) >= sz))
32749 || ((a_l_number != 0) && (unused_array_size (free_list) < sz/2)))
32751 dprintf (3, ("Verifiying Heap: curr free list item %Ix isn't in the right bucket",
32752 (size_t)free_list));
32755 if (verify_undo_slot && (free_list_undo (free_list) != UNDO_EMPTY))
32757 dprintf (3, ("Verifiying Heap: curr free list item %Ix has non empty undo slot",
32758 (size_t)free_list));
32761 if ((gen_num != max_generation+1)&&(object_gennum (free_list)!= gen_num))
32763 dprintf (3, ("Verifiying Heap: curr free list item %Ix is in the wrong generation free list",
32764 (size_t)free_list));
32769 free_list = free_list_slot (free_list);
32771 //verify the sanity of the tail
32772 uint8_t* tail = gen_alloc->alloc_list_tail_of (a_l_number);
32773 if (!((tail == 0) || (tail == prev)))
32775 dprintf (3, ("Verifying Heap: tail of free list is not correct"));
32780 uint8_t* head = gen_alloc->alloc_list_head_of (a_l_number);
32781 if ((head != 0) && (free_list_slot (head) != 0))
32783 dprintf (3, ("Verifying Heap: tail of free list is not correct"));
32794 gc_heap::verify_heap (BOOL begin_gc_p)
32796 int heap_verify_level = static_cast<int>(GCConfig::GetHeapVerifyLevel());
32797 size_t last_valid_brick = 0;
32798 BOOL bCurrentBrickInvalid = FALSE;
32799 BOOL large_brick_p = TRUE;
32800 size_t curr_brick = 0;
32801 size_t prev_brick = (size_t)-1;
32802 int curr_gen_num = max_generation+1;
32803 heap_segment* seg = heap_segment_in_range (generation_start_segment (generation_of (curr_gen_num ) ));
32805 PREFIX_ASSUME(seg != NULL);
32807 uint8_t* curr_object = heap_segment_mem (seg);
32808 uint8_t* prev_object = 0;
32809 uint8_t* begin_youngest = generation_allocation_start(generation_of(0));
32810 uint8_t* end_youngest = heap_segment_allocated (ephemeral_heap_segment);
32811 uint8_t* next_boundary = generation_allocation_start (generation_of (max_generation - 1));
32812 int align_const = get_alignment_constant (FALSE);
32813 size_t total_objects_verified = 0;
32814 size_t total_objects_verified_deep = 0;
32816 #ifdef BACKGROUND_GC
32817 BOOL consider_bgc_mark_p = FALSE;
32818 BOOL check_current_sweep_p = FALSE;
32819 BOOL check_saved_sweep_p = FALSE;
32820 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
32821 #endif //BACKGROUND_GC
32823 #ifdef MULTIPLE_HEAPS
32824 t_join* current_join = &gc_t_join;
32825 #ifdef BACKGROUND_GC
32826 if (settings.concurrent && (bgc_thread_id.IsCurrentThread()))
32828 // We always call verify_heap on entry of GC on the SVR GC threads.
32829 current_join = &bgc_t_join;
32831 #endif //BACKGROUND_GC
32832 #endif //MULTIPLE_HEAPS
32834 UNREFERENCED_PARAMETER(begin_gc_p);
32835 #ifdef BACKGROUND_GC
32836 dprintf (2,("[%s]GC#%d(%s): Verifying heap - begin",
32837 (begin_gc_p ? "BEG" : "END"),
32838 VolatileLoad(&settings.gc_index),
32839 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC"))));
32841 dprintf (2,("[%s]GC#%d: Verifying heap - begin",
32842 (begin_gc_p ? "BEG" : "END"), VolatileLoad(&settings.gc_index)));
32843 #endif //BACKGROUND_GC
32845 #ifndef MULTIPLE_HEAPS
32846 if ((ephemeral_low != generation_allocation_start (generation_of (max_generation - 1))) ||
32847 (ephemeral_high != heap_segment_reserved (ephemeral_heap_segment)))
32851 #endif //MULTIPLE_HEAPS
32853 #ifdef BACKGROUND_GC
32854 //don't touch the memory because the program is allocating from it.
32855 if (!settings.concurrent)
32856 #endif //BACKGROUND_GC
32858 if (!(heap_verify_level & GCConfig::HEAPVERIFY_NO_MEM_FILL))
32860 //uninit the unused portions of segments.
32861 generation* gen1 = large_object_generation;
32862 heap_segment* seg1 = heap_segment_rw (generation_start_segment (gen1));
32863 PREFIX_ASSUME(seg1 != NULL);
32869 uint8_t* clear_start = heap_segment_allocated (seg1) - plug_skew;
32870 if (heap_segment_used (seg1) > clear_start)
32872 dprintf (3, ("setting end of seg %Ix: [%Ix-[%Ix to 0xaa",
32873 heap_segment_mem (seg1),
32875 heap_segment_used (seg1)));
32876 memset (heap_segment_allocated (seg1) - plug_skew, 0xaa,
32877 (heap_segment_used (seg1) - clear_start));
32879 seg1 = heap_segment_next_rw (seg1);
32883 if (gen1 == large_object_generation)
32885 gen1 = generation_of (max_generation);
32886 seg1 = heap_segment_rw (generation_start_segment (gen1));
32887 PREFIX_ASSUME(seg1 != NULL);
32898 #ifdef MULTIPLE_HEAPS
32899 current_join->join(this, gc_join_verify_copy_table);
32900 if (current_join->joined())
32902 // in concurrent GC, new segment could be allocated when GC is working so the card brick table might not be updated at this point
32903 for (int i = 0; i < n_heaps; i++)
32905 //copy the card and brick tables
32906 if (g_gc_card_table != g_heaps[i]->card_table)
32908 g_heaps[i]->copy_brick_card_table();
32912 current_join->restart();
32915 if (g_gc_card_table != card_table)
32916 copy_brick_card_table();
32917 #endif //MULTIPLE_HEAPS
32919 //verify that the generation structures makes sense
32921 generation* gen = generation_of (max_generation);
32923 assert (generation_allocation_start (gen) ==
32924 heap_segment_mem (heap_segment_rw (generation_start_segment (gen))));
32925 int gen_num = max_generation-1;
32926 generation* prev_gen = gen;
32927 while (gen_num >= 0)
32929 gen = generation_of (gen_num);
32930 assert (generation_allocation_segment (gen) == ephemeral_heap_segment);
32931 assert (generation_allocation_start (gen) >= heap_segment_mem (ephemeral_heap_segment));
32932 assert (generation_allocation_start (gen) < heap_segment_allocated (ephemeral_heap_segment));
32934 if (generation_start_segment (prev_gen ) ==
32935 generation_start_segment (gen))
32937 assert (generation_allocation_start (prev_gen) <
32938 generation_allocation_start (gen));
32947 // Handle segment transitions
32948 if (curr_object >= heap_segment_allocated (seg))
32950 if (curr_object > heap_segment_allocated(seg))
32952 dprintf (3, ("Verifiying Heap: curr_object: %Ix > heap_segment_allocated (seg: %Ix)",
32953 (size_t)curr_object, (size_t)seg));
32956 seg = heap_segment_next_in_range (seg);
32959 #ifdef BACKGROUND_GC
32960 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
32961 #endif //BACKGROUND_GC
32962 curr_object = heap_segment_mem(seg);
32968 if (curr_gen_num == (max_generation+1))
32971 seg = heap_segment_in_range (generation_start_segment (generation_of (curr_gen_num)));
32973 PREFIX_ASSUME(seg != NULL);
32975 #ifdef BACKGROUND_GC
32976 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
32977 #endif //BACKGROUND_GC
32978 curr_object = heap_segment_mem (seg);
32980 large_brick_p = FALSE;
32981 align_const = get_alignment_constant (TRUE);
32984 break; // Done Verifying Heap -- no more segments
32988 // Are we at the end of the youngest_generation?
32989 if (seg == ephemeral_heap_segment)
32991 if (curr_object >= end_youngest)
32993 // prev_object length is too long if we hit this int3
32994 if (curr_object > end_youngest)
32996 dprintf (3, ("Verifiying Heap: curr_object: %Ix > end_youngest: %Ix",
32997 (size_t)curr_object, (size_t)end_youngest));
33003 if ((curr_object >= next_boundary) && (curr_gen_num > 0))
33006 if (curr_gen_num > 0)
33008 next_boundary = generation_allocation_start (generation_of (curr_gen_num - 1));
33013 //if (is_mark_set (curr_object))
33015 // printf ("curr_object: %Ix is marked!",(size_t)curr_object);
33016 // FATAL_GC_ERROR();
33019 size_t s = size (curr_object);
33020 dprintf (3, ("o: %Ix, s: %d", (size_t)curr_object, s));
33023 dprintf (3, ("Verifying Heap: size of current object %Ix == 0", curr_object));
33027 // If object is not in the youngest generation, then lets
33028 // verify that the brick table is correct....
33029 if (((seg != ephemeral_heap_segment) ||
33030 (brick_of(curr_object) < brick_of(begin_youngest))))
33032 curr_brick = brick_of(curr_object);
33034 // Brick Table Verification...
33036 // On brick transition
33037 // if brick is negative
33038 // verify that brick indirects to previous valid brick
33040 // set current brick invalid flag to be flipped if we
33041 // encounter an object at the correct place
33043 if (curr_brick != prev_brick)
33045 // If the last brick we were examining had positive
33046 // entry but we never found the matching object, then
33047 // we have a problem
33048 // If prev_brick was the last one of the segment
33049 // it's ok for it to be invalid because it is never looked at
33050 if (bCurrentBrickInvalid &&
33051 (curr_brick != brick_of (heap_segment_mem (seg))) &&
33052 !heap_segment_read_only_p (seg))
33054 dprintf (3, ("curr brick %Ix invalid", curr_brick));
33060 //large objects verify the table only if they are in
33062 if ((heap_segment_reserved (seg) <= highest_address) &&
33063 (heap_segment_mem (seg) >= lowest_address) &&
33064 brick_table [curr_brick] != 0)
33066 dprintf (3, ("curr_brick %Ix for large object %Ix not set to -32768",
33067 curr_brick, (size_t)curr_object));
33072 bCurrentBrickInvalid = FALSE;
33077 // If the current brick contains a negative value make sure
33078 // that the indirection terminates at the last valid brick
33079 if (brick_table [curr_brick] <= 0)
33081 if (brick_table [curr_brick] == 0)
33083 dprintf(3, ("curr_brick %Ix for object %Ix set to 0",
33084 curr_brick, (size_t)curr_object));
33087 ptrdiff_t i = curr_brick;
33088 while ((i >= ((ptrdiff_t) brick_of (heap_segment_mem (seg)))) &&
33089 (brick_table[i] < 0))
33091 i = i + brick_table[i];
33093 if (i < ((ptrdiff_t)(brick_of (heap_segment_mem (seg))) - 1))
33095 dprintf (3, ("ptrdiff i: %Ix < brick_of (heap_segment_mem (seg)):%Ix - 1. curr_brick: %Ix",
33096 i, brick_of (heap_segment_mem (seg)),
33100 // if (i != last_valid_brick)
33101 // FATAL_GC_ERROR();
33102 bCurrentBrickInvalid = FALSE;
33104 else if (!heap_segment_read_only_p (seg))
33106 bCurrentBrickInvalid = TRUE;
33111 if (bCurrentBrickInvalid)
33113 if (curr_object == (brick_address(curr_brick) + brick_table[curr_brick] - 1))
33115 bCurrentBrickInvalid = FALSE;
33116 last_valid_brick = curr_brick;
33121 if (*((uint8_t**)curr_object) != (uint8_t *) g_gc_pFreeObjectMethodTable)
33123 #ifdef FEATURE_LOH_COMPACTION
33124 if ((curr_gen_num == (max_generation+1)) && (prev_object != 0))
33126 assert (method_table (prev_object) == g_gc_pFreeObjectMethodTable);
33128 #endif //FEATURE_LOH_COMPACTION
33130 total_objects_verified++;
33132 BOOL can_verify_deep = TRUE;
33133 #ifdef BACKGROUND_GC
33134 can_verify_deep = fgc_should_consider_object (curr_object, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p);
33135 #endif //BACKGROUND_GC
33137 BOOL deep_verify_obj = can_verify_deep;
33138 if ((heap_verify_level & GCConfig::HEAPVERIFY_DEEP_ON_COMPACT) && !settings.compaction)
33139 deep_verify_obj = FALSE;
33141 ((CObjectHeader*)curr_object)->ValidateHeap((Object*)curr_object, deep_verify_obj);
33143 if (can_verify_deep)
33145 if (curr_gen_num > 0)
33147 BOOL need_card_p = FALSE;
33148 if (contain_pointers_or_collectible (curr_object))
33150 dprintf (4, ("curr_object: %Ix", (size_t)curr_object));
33151 size_t crd = card_of (curr_object);
33152 BOOL found_card_p = card_set_p (crd);
33154 #ifdef COLLECTIBLE_CLASS
33155 if (is_collectible(curr_object))
33157 uint8_t* class_obj = get_class_object (curr_object);
33158 if ((class_obj < ephemeral_high) && (class_obj >= next_boundary))
33162 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix pointing to class object %Ix",
33163 card_of (curr_object), (size_t)curr_object, class_obj));
33169 #endif //COLLECTIBLE_CLASS
33171 if (contain_pointers(curr_object))
33173 go_through_object_nostart
33174 (method_table(curr_object), curr_object, s, oo,
33176 if ((crd != card_of ((uint8_t*)oo)) && !found_card_p)
33178 crd = card_of ((uint8_t*)oo);
33179 found_card_p = card_set_p (crd);
33180 need_card_p = FALSE;
33182 if ((*oo < ephemeral_high) && (*oo >= next_boundary))
33184 need_card_p = TRUE;
33187 if (need_card_p && !found_card_p)
33190 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix, %Ix:%Ix[",
33191 card_of (curr_object), (size_t)curr_object,
33192 card_of (curr_object+Align(s, align_const)), (size_t)curr_object+Align(s, align_const)));
33198 if (need_card_p && !found_card_p)
33200 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix, %Ix:%Ix[",
33201 card_of (curr_object), (size_t)curr_object,
33202 card_of (curr_object+Align(s, align_const)), (size_t)curr_object+Align(s, align_const)));
33207 total_objects_verified_deep++;
33211 prev_object = curr_object;
33212 prev_brick = curr_brick;
33213 curr_object = curr_object + Align(s, align_const);
33214 if (curr_object < prev_object)
33216 dprintf (3, ("overflow because of a bad object size: %Ix size %Ix", prev_object, s));
33221 #ifdef BACKGROUND_GC
33222 dprintf (2, ("(%s)(%s)(%s) total_objects_verified is %Id, total_objects_verified_deep is %Id",
33223 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p () ? "FGC" : "NGC")),
33224 (begin_gc_p ? "BEG" : "END"),
33225 ((current_c_gc_state == c_gc_state_planning) ? "in plan" : "not in plan"),
33226 total_objects_verified, total_objects_verified_deep));
33227 if (current_c_gc_state != c_gc_state_planning)
33229 assert (total_objects_verified == total_objects_verified_deep);
33231 #endif //BACKGROUND_GC
33233 verify_free_lists();
33235 #ifdef FEATURE_PREMORTEM_FINALIZATION
33236 finalize_queue->CheckFinalizerObjects();
33237 #endif // FEATURE_PREMORTEM_FINALIZATION
33240 // to be consistent with handle table APIs pass a ScanContext*
33241 // to provide the heap number. the SC isn't complete though so
33242 // limit its scope to handle table verification.
33244 sc.thread_number = heap_number;
33245 GCScan::VerifyHandleTable(max_generation, max_generation, &sc);
33248 #ifdef MULTIPLE_HEAPS
33249 current_join->join(this, gc_join_verify_objects_done);
33250 if (current_join->joined())
33251 #endif //MULTIPLE_HEAPS
33253 SyncBlockCache::GetSyncBlockCache()->VerifySyncTableEntry();
33254 #ifdef MULTIPLE_HEAPS
33255 current_join->restart();
33256 #endif //MULTIPLE_HEAPS
33259 #ifdef BACKGROUND_GC
33260 if (!settings.concurrent)
33262 if (current_c_gc_state == c_gc_state_planning)
33264 // temporarily commenting this out 'cause an FGC
33265 // could be triggered before we sweep ephemeral.
33266 //verify_seg_end_mark_array_cleared();
33270 if (settings.concurrent)
33272 verify_mark_array_cleared();
33274 dprintf (2,("GC%d(%s): Verifying heap - end",
33275 VolatileLoad(&settings.gc_index),
33276 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC"))));
33278 dprintf (2,("GC#d: Verifying heap - end", VolatileLoad(&settings.gc_index)));
33279 #endif //BACKGROUND_GC
33282 #endif //VERIFY_HEAP
33285 void GCHeap::ValidateObjectMember (Object* obj)
33288 size_t s = size (obj);
33289 uint8_t* o = (uint8_t*)obj;
33291 go_through_object_cl (method_table (obj), o, s, oo,
33293 uint8_t* child_o = *oo;
33296 dprintf (3, ("VOM: m: %Ix obj %Ix", (size_t)child_o, o));
33297 MethodTable *pMT = method_table (child_o);
33299 if (!pMT->SanityCheck()) {
33300 dprintf (3, ("Bad member of %Ix %Ix",
33301 (size_t)oo, (size_t)child_o));
33306 #endif // VERIFY_HEAP
33309 void DestructObject (CObjectHeader* hdr)
33311 UNREFERENCED_PARAMETER(hdr); // compiler bug? -- this *is*, indeed, referenced
33312 hdr->~CObjectHeader();
33315 HRESULT GCHeap::Shutdown ()
33319 GCScan::GcRuntimeStructuresValid (FALSE);
33321 // Cannot assert this, since we use SuspendEE as the mechanism to quiesce all
33322 // threads except the one performing the shutdown.
33323 // ASSERT( !GcInProgress );
33325 // Guard against any more GC occurring and against any threads blocking
33326 // for GC to complete when the GC heap is gone. This fixes a race condition
33327 // where a thread in GC is destroyed as part of process destruction and
33328 // the remaining threads block for GC complete.
33331 //EnterAllocLock();
33333 //EnterFinalizeLock();
33336 // during shutdown lot of threads are suspended
33337 // on this even, we don't want to wake them up just yet
33338 //CloseHandle (WaitForGCEvent);
33340 //find out if the global card table hasn't been used yet
33341 uint32_t* ct = &g_gc_card_table[card_word (gcard_of (g_gc_lowest_address))];
33342 if (card_table_refcount (ct) == 0)
33344 destroy_card_table (ct);
33345 g_gc_card_table = nullptr;
33347 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
33348 g_gc_card_bundle_table = nullptr;
33350 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
33351 SoftwareWriteWatch::StaticClose();
33352 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
33355 //destroy all segments on the standby list
33356 while(gc_heap::segment_standby_list != 0)
33358 heap_segment* next_seg = heap_segment_next (gc_heap::segment_standby_list);
33359 #ifdef MULTIPLE_HEAPS
33360 (gc_heap::g_heaps[0])->delete_heap_segment (gc_heap::segment_standby_list, FALSE);
33361 #else //MULTIPLE_HEAPS
33362 pGenGCHeap->delete_heap_segment (gc_heap::segment_standby_list, FALSE);
33363 #endif //MULTIPLE_HEAPS
33364 gc_heap::segment_standby_list = next_seg;
33368 #ifdef MULTIPLE_HEAPS
33370 for (int i = 0; i < gc_heap::n_heaps; i ++)
33372 delete gc_heap::g_heaps[i]->vm_heap;
33373 //destroy pure GC stuff
33374 gc_heap::destroy_gc_heap (gc_heap::g_heaps[i]);
33377 gc_heap::destroy_gc_heap (pGenGCHeap);
33379 #endif //MULTIPLE_HEAPS
33380 gc_heap::shutdown_gc();
33385 // Wait until a garbage collection is complete
33386 // returns NOERROR if wait was OK, other error code if failure.
33387 // WARNING: This will not undo the must complete state. If you are
33388 // in a must complete when you call this, you'd better know what you're
33391 #ifdef FEATURE_PREMORTEM_FINALIZATION
33393 HRESULT AllocateCFinalize(CFinalize **pCFinalize)
33395 *pCFinalize = new (nothrow) CFinalize();
33396 if (*pCFinalize == NULL || !(*pCFinalize)->Initialize())
33397 return E_OUTOFMEMORY;
33401 #endif // FEATURE_PREMORTEM_FINALIZATION
33403 // init the instance heap
33404 HRESULT GCHeap::Init(size_t hn)
33406 HRESULT hres = S_OK;
33408 #ifdef MULTIPLE_HEAPS
33409 if ((pGenGCHeap = gc_heap::make_gc_heap(this, (int)hn)) == 0)
33410 hres = E_OUTOFMEMORY;
33412 UNREFERENCED_PARAMETER(hn);
33413 if (!gc_heap::make_gc_heap())
33414 hres = E_OUTOFMEMORY;
33415 #endif //MULTIPLE_HEAPS
33421 //System wide initialization
33422 HRESULT GCHeap::Initialize ()
33426 g_gc_pFreeObjectMethodTable = GCToEEInterface::GetFreeObjectMethodTable();
33427 g_num_processors = GCToOSInterface::GetTotalProcessorCount();
33428 assert(g_num_processors != 0);
33430 //Initialize the static members.
33433 CreatedObjectCount = 0;
33436 size_t seg_size = get_valid_segment_size();
33437 gc_heap::soh_segment_size = seg_size;
33438 size_t large_seg_size = get_valid_segment_size(TRUE);
33439 gc_heap::min_loh_segment_size = large_seg_size;
33440 gc_heap::min_segment_size = min (seg_size, large_seg_size);
33441 #ifdef SEG_MAPPING_TABLE
33442 gc_heap::min_segment_size_shr = index_of_highest_set_bit (gc_heap::min_segment_size);
33443 #endif //SEG_MAPPING_TABLE
33445 #ifdef MULTIPLE_HEAPS
33446 if (GCConfig::GetNoAffinitize())
33447 gc_heap::gc_thread_no_affinitize_p = true;
33449 uint32_t nhp_from_config = static_cast<uint32_t>(GCConfig::GetHeapCount());
33451 uint32_t nhp_from_process = GCToOSInterface::GetCurrentProcessCpuCount();
33453 uint32_t nhp = ((nhp_from_config == 0) ? nhp_from_process :
33454 (min (nhp_from_config, nhp_from_process)));
33456 nhp = min (nhp, MAX_SUPPORTED_CPUS);
33458 hr = gc_heap::initialize_gc (seg_size, large_seg_size /*LHEAP_ALLOC*/, nhp);
33460 hr = gc_heap::initialize_gc (seg_size, large_seg_size /*LHEAP_ALLOC*/);
33461 #endif //MULTIPLE_HEAPS
33466 gc_heap::total_physical_mem = GCToOSInterface::GetPhysicalMemoryLimit();
33468 gc_heap::mem_one_percent = gc_heap::total_physical_mem / 100;
33469 #ifndef MULTIPLE_HEAPS
33470 gc_heap::mem_one_percent /= g_num_processors;
33471 #endif //!MULTIPLE_HEAPS
33473 // We should only use this if we are in the "many process" mode which really is only applicable
33474 // to very powerful machines - before that's implemented, temporarily I am only enabling this for 80GB+ memory.
33475 // For now I am using an estimate to calculate these numbers but this should really be obtained
33476 // programmatically going forward.
33477 // I am assuming 47 processes using WKS GC and 3 using SVR GC.
33478 // I am assuming 3 in part due to the "very high memory load" is 97%.
33479 int available_mem_th = 10;
33480 if (gc_heap::total_physical_mem >= ((uint64_t)80 * 1024 * 1024 * 1024))
33482 int adjusted_available_mem_th = 3 + (int)((float)47 / (float)(g_num_processors));
33483 available_mem_th = min (available_mem_th, adjusted_available_mem_th);
33486 gc_heap::high_memory_load_th = 100 - available_mem_th;
33489 gc_heap::youngest_gen_desired_th = gc_heap::mem_one_percent;
33492 WaitForGCEvent = new (nothrow) GCEvent;
33494 if (!WaitForGCEvent)
33496 return E_OUTOFMEMORY;
33499 if (!WaitForGCEvent->CreateManualEventNoThrow(TRUE))
33504 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
33505 #if defined (STRESS_HEAP) && !defined (MULTIPLE_HEAPS)
33506 if (GCStress<cfg_any>::IsEnabled()) {
33507 for(int i = 0; i < GCHeap::NUM_HEAP_STRESS_OBJS; i++)
33508 m_StressObjs[i] = CreateGlobalHandle(0);
33509 m_CurStressObj = 0;
33511 #endif //STRESS_HEAP && !MULTIPLE_HEAPS
33512 #endif // FEATURE_REDHAWK
33514 initGCShadow(); // If we are debugging write barriers, initialize heap shadow
33516 #ifdef MULTIPLE_HEAPS
33518 for (unsigned i = 0; i < nhp; i++)
33520 GCHeap* Hp = new (nothrow) GCHeap();
33522 return E_OUTOFMEMORY;
33524 if ((hr = Hp->Init (i))!= S_OK)
33529 // initialize numa node to heap map
33530 heap_select::init_numa_node_to_heap_map(nhp);
33533 #endif //MULTIPLE_HEAPS
33537 GCScan::GcRuntimeStructuresValid (TRUE);
33539 GCToEEInterface::DiagUpdateGenerationBounds();
33546 // GC callback functions
33547 bool GCHeap::IsPromoted(Object* object)
33550 ((CObjectHeader*)object)->Validate();
33553 uint8_t* o = (uint8_t*)object;
33555 if (gc_heap::settings.condemned_generation == max_generation)
33557 #ifdef MULTIPLE_HEAPS
33558 gc_heap* hp = gc_heap::g_heaps[0];
33560 gc_heap* hp = pGenGCHeap;
33561 #endif //MULTIPLE_HEAPS
33563 #ifdef BACKGROUND_GC
33564 if (gc_heap::settings.concurrent)
33566 bool is_marked = (!((o < hp->background_saved_highest_address) && (o >= hp->background_saved_lowest_address))||
33567 hp->background_marked (o));
33571 #endif //BACKGROUND_GC
33573 return (!((o < hp->highest_address) && (o >= hp->lowest_address))
33574 || hp->is_mark_set (o));
33579 gc_heap* hp = gc_heap::heap_of (o);
33580 return (!((o < hp->gc_high) && (o >= hp->gc_low))
33581 || hp->is_mark_set (o));
33585 size_t GCHeap::GetPromotedBytes(int heap_index)
33587 #ifdef BACKGROUND_GC
33588 if (gc_heap::settings.concurrent)
33590 return gc_heap::bpromoted_bytes (heap_index);
33593 #endif //BACKGROUND_GC
33595 return gc_heap::promoted_bytes (heap_index);
33599 unsigned int GCHeap::WhichGeneration (Object* object)
33601 gc_heap* hp = gc_heap::heap_of ((uint8_t*)object);
33602 unsigned int g = hp->object_gennum ((uint8_t*)object);
33603 dprintf (3, ("%Ix is in gen %d", (size_t)object, g));
33607 bool GCHeap::IsEphemeral (Object* object)
33609 uint8_t* o = (uint8_t*)object;
33610 gc_heap* hp = gc_heap::heap_of (o);
33611 return !!hp->ephemeral_pointer_p (o);
33614 // Return NULL if can't find next object. When EE is not suspended,
33615 // the result is not accurate: if the input arg is in gen0, the function could
33616 // return zeroed out memory as next object
33617 Object * GCHeap::NextObj (Object * object)
33620 uint8_t* o = (uint8_t*)object;
33622 #ifndef FEATURE_BASICFREEZE
33623 if (!((o < g_gc_highest_address) && (o >= g_gc_lowest_address)))
33627 #endif //!FEATURE_BASICFREEZE
33629 heap_segment * hs = gc_heap::find_segment (o, FALSE);
33635 BOOL large_object_p = heap_segment_loh_p (hs);
33636 if (large_object_p)
33637 return NULL; //could be racing with another core allocating.
33638 #ifdef MULTIPLE_HEAPS
33639 gc_heap* hp = heap_segment_heap (hs);
33640 #else //MULTIPLE_HEAPS
33642 #endif //MULTIPLE_HEAPS
33643 unsigned int g = hp->object_gennum ((uint8_t*)object);
33644 if ((g == 0) && hp->settings.demotion)
33645 return NULL;//could be racing with another core allocating.
33646 int align_const = get_alignment_constant (!large_object_p);
33647 uint8_t* nextobj = o + Align (size (o), align_const);
33648 if (nextobj <= o) // either overflow or 0 sized object.
33653 if ((nextobj < heap_segment_mem(hs)) ||
33654 (nextobj >= heap_segment_allocated(hs) && hs != hp->ephemeral_heap_segment) ||
33655 (nextobj >= hp->alloc_allocated))
33660 return (Object *)nextobj;
33663 #endif // VERIFY_HEAP
33668 #ifdef FEATURE_BASICFREEZE
33669 BOOL GCHeap::IsInFrozenSegment (Object * object)
33671 uint8_t* o = (uint8_t*)object;
33672 heap_segment * hs = gc_heap::find_segment (o, FALSE);
33673 //We create a frozen object for each frozen segment before the segment is inserted
33674 //to segment list; during ngen, we could also create frozen objects in segments which
33675 //don't belong to current GC heap.
33676 //So we return true if hs is NULL. It might create a hole about detecting invalidate
33677 //object. But given all other checks present, the hole should be very small
33678 return !hs || heap_segment_read_only_p (hs);
33680 #endif //FEATURE_BASICFREEZE
33682 #endif //VERIFY_HEAP
33684 // returns TRUE if the pointer is in one of the GC heaps.
33685 bool GCHeap::IsHeapPointer (void* vpObject, bool small_heap_only)
33687 STATIC_CONTRACT_SO_TOLERANT;
33689 // removed STATIC_CONTRACT_CAN_TAKE_LOCK here because find_segment
33690 // no longer calls GCEvent::Wait which eventually takes a lock.
33692 uint8_t* object = (uint8_t*) vpObject;
33693 #ifndef FEATURE_BASICFREEZE
33694 if (!((object < g_gc_highest_address) && (object >= g_gc_lowest_address)))
33696 #endif //!FEATURE_BASICFREEZE
33698 heap_segment * hs = gc_heap::find_segment (object, small_heap_only);
33702 #ifdef STRESS_PINNING
33703 static n_promote = 0;
33704 #endif //STRESS_PINNING
33705 // promote an object
33706 void GCHeap::Promote(Object** ppObject, ScanContext* sc, uint32_t flags)
33708 THREAD_NUMBER_FROM_CONTEXT;
33709 #ifndef MULTIPLE_HEAPS
33710 const int thread = 0;
33711 #endif //!MULTIPLE_HEAPS
33713 uint8_t* o = (uint8_t*)*ppObject;
33718 #ifdef DEBUG_DestroyedHandleValue
33719 // we can race with destroy handle during concurrent scan
33720 if (o == (uint8_t*)DEBUG_DestroyedHandleValue)
33722 #endif //DEBUG_DestroyedHandleValue
33726 gc_heap* hp = gc_heap::heap_of (o);
33728 dprintf (3, ("Promote %Ix", (size_t)o));
33730 #ifdef INTERIOR_POINTERS
33731 if (flags & GC_CALL_INTERIOR)
33733 if ((o < hp->gc_low) || (o >= hp->gc_high))
33737 if ( (o = hp->find_object (o, hp->gc_low)) == 0)
33743 #endif //INTERIOR_POINTERS
33745 #ifdef FEATURE_CONSERVATIVE_GC
33746 // For conservative GC, a value on stack may point to middle of a free object.
33747 // In this case, we don't need to promote the pointer.
33748 if (GCConfig::GetConservativeGC()
33749 && ((CObjectHeader*)o)->IsFree())
33756 ((CObjectHeader*)o)->ValidatePromote(sc, flags);
33758 UNREFERENCED_PARAMETER(sc);
33761 if (flags & GC_CALL_PINNED)
33762 hp->pin_object (o, (uint8_t**) ppObject, hp->gc_low, hp->gc_high);
33764 #ifdef STRESS_PINNING
33765 if ((++n_promote % 20) == 1)
33766 hp->pin_object (o, (uint8_t**) ppObject, hp->gc_low, hp->gc_high);
33767 #endif //STRESS_PINNING
33769 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
33770 size_t promoted_size_begin = hp->promoted_bytes (thread);
33771 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
33773 if ((o >= hp->gc_low) && (o < hp->gc_high))
33775 hpt->mark_object_simple (&o THREAD_NUMBER_ARG);
33778 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
33779 size_t promoted_size_end = hp->promoted_bytes (thread);
33782 if (sc->pCurrentDomain)
33784 sc->pCurrentDomain->RecordSurvivedBytes ((promoted_size_end - promoted_size_begin), thread);
33787 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
33789 STRESS_LOG_ROOT_PROMOTE(ppObject, o, o ? header(o)->GetMethodTable() : NULL);
33792 void GCHeap::Relocate (Object** ppObject, ScanContext* sc,
33795 UNREFERENCED_PARAMETER(sc);
33797 uint8_t* object = (uint8_t*)(Object*)(*ppObject);
33799 THREAD_NUMBER_FROM_CONTEXT;
33801 //dprintf (3, ("Relocate location %Ix\n", (size_t)ppObject));
33802 dprintf (3, ("R: %Ix", (size_t)ppObject));
33807 gc_heap* hp = gc_heap::heap_of (object);
33810 if (!(flags & GC_CALL_INTERIOR))
33812 // We cannot validate this object if it's in the condemned gen because it could
33813 // be one of the objects that were overwritten by an artificial gap due to a pinned plug.
33814 if (!((object >= hp->gc_low) && (object < hp->gc_high)))
33816 ((CObjectHeader*)object)->Validate(FALSE);
33821 dprintf (3, ("Relocate %Ix\n", (size_t)object));
33825 if ((flags & GC_CALL_INTERIOR) && gc_heap::settings.loh_compaction)
33827 if (!((object >= hp->gc_low) && (object < hp->gc_high)))
33832 if (gc_heap::loh_object_p (object))
33834 pheader = hp->find_object (object, 0);
33840 ptrdiff_t ref_offset = object - pheader;
33841 hp->relocate_address(&pheader THREAD_NUMBER_ARG);
33842 *ppObject = (Object*)(pheader + ref_offset);
33849 hp->relocate_address(&pheader THREAD_NUMBER_ARG);
33850 *ppObject = (Object*)pheader;
33853 STRESS_LOG_ROOT_RELOCATE(ppObject, object, pheader, ((!(flags & GC_CALL_INTERIOR)) ? ((Object*)object)->GetGCSafeMethodTable() : 0));
33856 /*static*/ bool GCHeap::IsObjectInFixedHeap(Object *pObj)
33858 // For now we simply look at the size of the object to determine if it in the
33859 // fixed heap or not. If the bit indicating this gets set at some point
33860 // we should key off that instead.
33861 return size( pObj ) >= LARGE_OBJECT_SIZE;
33864 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
33867 void StressHeapDummy ();
33869 static int32_t GCStressStartCount = -1;
33870 static int32_t GCStressCurCount = 0;
33871 static int32_t GCStressStartAtJit = -1;
33873 // the maximum number of foreground GCs we'll induce during one BGC
33874 // (this number does not include "naturally" occuring GCs).
33875 static int32_t GCStressMaxFGCsPerBGC = -1;
33877 // CLRRandom implementation can produce FPU exceptions if
33878 // the test/application run by CLR is enabling any FPU exceptions.
33879 // We want to avoid any unexpected exception coming from stress
33880 // infrastructure, so CLRRandom is not an option.
33881 // The code below is a replicate of CRT rand() implementation.
33882 // Using CRT rand() is not an option because we will interfere with the user application
33883 // that may also use it.
33884 int StressRNG(int iMaxValue)
33886 static BOOL bisRandInit = FALSE;
33887 static int lHoldrand = 1L;
33891 lHoldrand = (int)time(NULL);
33892 bisRandInit = TRUE;
33894 int randValue = (((lHoldrand = lHoldrand * 214013L + 2531011L) >> 16) & 0x7fff);
33895 return randValue % iMaxValue;
33897 #endif // STRESS_HEAP
33898 #endif // !FEATURE_REDHAWK
33900 // free up object so that things will move and then do a GC
33901 //return TRUE if GC actually happens, otherwise FALSE
33902 bool GCHeap::StressHeap(gc_alloc_context * context)
33904 #if defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
33905 alloc_context* acontext = static_cast<alloc_context*>(context);
33906 assert(context != nullptr);
33908 // if GC stress was dynamically disabled during this run we return FALSE
33909 if (!GCStressPolicy::IsEnabled())
33913 if (g_pConfig->FastGCStressLevel() && !GCToEEInterface::GetThread()->StressHeapIsEnabled()) {
33919 if ((g_pConfig->GetGCStressLevel() & EEConfig::GCSTRESS_UNIQUE)
33921 || g_pConfig->FastGCStressLevel() > 1
33924 if (!Thread::UniqueStack(&acontext)) {
33929 #ifdef BACKGROUND_GC
33930 // don't trigger a GC from the GC threads but still trigger GCs from user threads.
33931 if (GCToEEInterface::WasCurrentThreadCreatedByGC())
33935 #endif //BACKGROUND_GC
33937 if (GCStressStartAtJit == -1 || GCStressStartCount == -1)
33939 GCStressStartCount = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_GCStressStart);
33940 GCStressStartAtJit = CLRConfig::GetConfigValue(CLRConfig::INTERNAL_GCStressStartAtJit);
33943 if (GCStressMaxFGCsPerBGC == -1)
33945 GCStressMaxFGCsPerBGC = CLRConfig::GetConfigValue(CLRConfig::INTERNAL_GCStressMaxFGCsPerBGC);
33946 if (g_pConfig->IsGCStressMix() && GCStressMaxFGCsPerBGC == -1)
33947 GCStressMaxFGCsPerBGC = 6;
33951 if (g_JitCount < GCStressStartAtJit)
33955 // Allow programmer to skip the first N Stress GCs so that you can
33956 // get to the interesting ones faster.
33957 Interlocked::Increment(&GCStressCurCount);
33958 if (GCStressCurCount < GCStressStartCount)
33961 // throttle the number of stress-induced GCs by a factor given by GCStressStep
33962 if ((GCStressCurCount % g_pConfig->GetGCStressStep()) != 0)
33967 #ifdef BACKGROUND_GC
33968 if (IsConcurrentGCEnabled() && IsConcurrentGCInProgress())
33970 // allow a maximum number of stress induced FGCs during one BGC
33971 if (gc_stress_fgcs_in_bgc >= GCStressMaxFGCsPerBGC)
33973 ++gc_stress_fgcs_in_bgc;
33975 #endif // BACKGROUND_GC
33977 if (g_pStringClass == 0)
33979 // If the String class has not been loaded, dont do any stressing. This should
33980 // be kept to a minimum to get as complete coverage as possible.
33981 _ASSERTE(g_fEEInit);
33985 #ifndef MULTIPLE_HEAPS
33986 static int32_t OneAtATime = -1;
33988 // Only bother with this if the stress level is big enough and if nobody else is
33989 // doing it right now. Note that some callers are inside the AllocLock and are
33990 // guaranteed synchronized. But others are using AllocationContexts and have no
33991 // particular synchronization.
33993 // For this latter case, we want a very high-speed way of limiting this to one
33994 // at a time. A secondary advantage is that we release part of our StressObjs
33995 // buffer sparingly but just as effectively.
33997 if (Interlocked::Increment(&OneAtATime) == 0 &&
33998 !TrackAllocations()) // Messing with object sizes can confuse the profiler (see ICorProfilerInfo::GetObjectSize)
34002 // If the current string is used up
34003 if (HndFetchHandle(m_StressObjs[m_CurStressObj]) == 0)
34005 // Populate handles with strings
34006 int i = m_CurStressObj;
34007 while(HndFetchHandle(m_StressObjs[i]) == 0)
34009 _ASSERTE(m_StressObjs[i] != 0);
34010 unsigned strLen = (LARGE_OBJECT_SIZE - 32) / sizeof(WCHAR);
34011 unsigned strSize = PtrAlign(StringObject::GetSize(strLen));
34013 // update the cached type handle before allocating
34014 SetTypeHandleOnThreadForAlloc(TypeHandle(g_pStringClass));
34015 str = (StringObject*) pGenGCHeap->allocate (strSize, acontext);
34018 str->SetMethodTable (g_pStringClass);
34019 str->SetStringLength (strLen);
34021 HndAssignHandle(m_StressObjs[i], ObjectToOBJECTREF(str));
34023 i = (i + 1) % NUM_HEAP_STRESS_OBJS;
34024 if (i == m_CurStressObj) break;
34027 // advance the current handle to the next string
34028 m_CurStressObj = (m_CurStressObj + 1) % NUM_HEAP_STRESS_OBJS;
34031 // Get the current string
34032 str = (StringObject*) OBJECTREFToObject(HndFetchHandle(m_StressObjs[m_CurStressObj]));
34035 // Chop off the end of the string and form a new object out of it.
34036 // This will 'free' an object at the begining of the heap, which will
34037 // force data movement. Note that we can only do this so many times.
34038 // before we have to move on to the next string.
34039 unsigned sizeOfNewObj = (unsigned)Align(min_obj_size * 31);
34040 if (str->GetStringLength() > sizeOfNewObj / sizeof(WCHAR))
34042 unsigned sizeToNextObj = (unsigned)Align(size(str));
34043 uint8_t* freeObj = ((uint8_t*) str) + sizeToNextObj - sizeOfNewObj;
34044 pGenGCHeap->make_unused_array (freeObj, sizeOfNewObj);
34045 str->SetStringLength(str->GetStringLength() - (sizeOfNewObj / sizeof(WCHAR)));
34049 // Let the string itself become garbage.
34050 // will be realloced next time around
34051 HndAssignHandle(m_StressObjs[m_CurStressObj], 0);
34055 Interlocked::Decrement(&OneAtATime);
34056 #endif // !MULTIPLE_HEAPS
34057 if (IsConcurrentGCEnabled())
34059 int rgen = StressRNG(10);
34061 // gen0:gen1:gen2 distribution: 40:40:20
34064 else if (rgen >= 4)
34069 GarbageCollectTry (rgen, FALSE, collection_gcstress);
34073 GarbageCollect(max_generation, FALSE, collection_gcstress);
34078 UNREFERENCED_PARAMETER(context);
34080 #endif // defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
34084 #ifdef FEATURE_PREMORTEM_FINALIZATION
34085 #define REGISTER_FOR_FINALIZATION(_object, _size) \
34086 hp->finalize_queue->RegisterForFinalization (0, (_object), (_size))
34087 #else // FEATURE_PREMORTEM_FINALIZATION
34088 #define REGISTER_FOR_FINALIZATION(_object, _size) true
34089 #endif // FEATURE_PREMORTEM_FINALIZATION
34091 #define CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(_object, _size, _register) do { \
34092 if ((_object) == NULL || ((_register) && !REGISTER_FOR_FINALIZATION(_object, _size))) \
34094 STRESS_LOG_OOM_STACK(_size); \
34100 // Small Object Allocator
34103 // Allocate small object with an alignment requirement of 8-bytes.
34105 GCHeap::AllocAlign8(gc_alloc_context* ctx, size_t size, uint32_t flags )
34107 #ifdef FEATURE_64BIT_ALIGNMENT
34113 alloc_context* acontext = static_cast<alloc_context*>(ctx);
34115 #ifdef MULTIPLE_HEAPS
34116 if (acontext->get_alloc_heap() == 0)
34118 AssignHeap (acontext);
34119 assert (acontext->get_alloc_heap());
34122 gc_heap* hp = acontext->get_alloc_heap()->pGenGCHeap;
34124 gc_heap* hp = pGenGCHeap;
34125 #endif //MULTIPLE_HEAPS
34127 return AllocAlign8Common(hp, acontext, size, flags);
34129 UNREFERENCED_PARAMETER(ctx);
34130 UNREFERENCED_PARAMETER(size);
34131 UNREFERENCED_PARAMETER(flags);
34132 assert(!"should not call GCHeap::AllocAlign8 without FEATURE_64BIT_ALIGNMENT defined!");
34134 #endif //FEATURE_64BIT_ALIGNMENT
34137 // Common code used by both variants of AllocAlign8 above.
34139 GCHeap::AllocAlign8Common(void* _hp, alloc_context* acontext, size_t size, uint32_t flags)
34141 #ifdef FEATURE_64BIT_ALIGNMENT
34147 gc_heap* hp = (gc_heap*)_hp;
34151 Object* newAlloc = NULL;
34154 #ifdef COUNT_CYCLES
34155 AllocStart = GetCycleCount32();
34157 #elif defined(ENABLE_INSTRUMENTATION)
34158 unsigned AllocStart = GetInstLogTime();
34160 #endif //COUNT_CYCLES
34163 if (size < LARGE_OBJECT_SIZE)
34169 // Depending on where in the object the payload requiring 8-byte alignment resides we might have to
34170 // align the object header on an 8-byte boundary or midway between two such boundaries. The unaligned
34171 // case is indicated to the GC via the GC_ALLOC_ALIGN8_BIAS flag.
34172 size_t desiredAlignment = (flags & GC_ALLOC_ALIGN8_BIAS) ? 4 : 0;
34174 // Retrieve the address of the next allocation from the context (note that we're inside the alloc
34175 // lock at this point).
34176 uint8_t* result = acontext->alloc_ptr;
34178 // Will an allocation at this point yield the correct alignment and fit into the remainder of the
34180 if ((((size_t)result & 7) == desiredAlignment) && ((result + size) <= acontext->alloc_limit))
34182 // Yes, we can just go ahead and make the allocation.
34183 newAlloc = (Object*) hp->allocate (size, acontext);
34184 ASSERT(((size_t)newAlloc & 7) == desiredAlignment);
34188 // No, either the next available address is not aligned in the way we require it or there's
34189 // not enough space to allocate an object of the required size. In both cases we allocate a
34190 // padding object (marked as a free object). This object's size is such that it will reverse
34191 // the alignment of the next header (asserted below).
34193 // We allocate both together then decide based on the result whether we'll format the space as
34194 // free object + real object or real object + free object.
34195 ASSERT((Align(min_obj_size) & 7) == 4);
34196 CObjectHeader *freeobj = (CObjectHeader*) hp->allocate (Align(size) + Align(min_obj_size), acontext);
34199 if (((size_t)freeobj & 7) == desiredAlignment)
34201 // New allocation has desired alignment, return this one and place the free object at the
34202 // end of the allocated space.
34203 newAlloc = (Object*)freeobj;
34204 freeobj = (CObjectHeader*)((uint8_t*)freeobj + Align(size));
34208 // New allocation is still mis-aligned, format the initial space as a free object and the
34209 // rest of the space should be correctly aligned for the real object.
34210 newAlloc = (Object*)((uint8_t*)freeobj + Align(min_obj_size));
34211 ASSERT(((size_t)newAlloc & 7) == desiredAlignment);
34213 freeobj->SetFree(min_obj_size);
34219 // The LOH always guarantees at least 8-byte alignment, regardless of platform. Moreover it doesn't
34220 // support mis-aligned object headers so we can't support biased headers as above. Luckily for us
34221 // we've managed to arrange things so the only case where we see a bias is for boxed value types and
34222 // these can never get large enough to be allocated on the LOH.
34223 ASSERT(65536 < LARGE_OBJECT_SIZE);
34224 ASSERT((flags & GC_ALLOC_ALIGN8_BIAS) == 0);
34226 alloc_context* acontext = generation_alloc_context (hp->generation_of (max_generation+1));
34228 newAlloc = (Object*) hp->allocate_large_object (size, acontext->alloc_bytes_loh);
34229 ASSERT(((size_t)newAlloc & 7) == 0);
34232 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
34235 #ifdef COUNT_CYCLES
34236 finish = GetCycleCount32();
34237 #elif defined(ENABLE_INSTRUMENTATION)
34238 finish = GetInstLogTime();
34239 #endif //COUNT_CYCLES
34240 AllocDuration += finish - AllocStart;
34245 UNREFERENCED_PARAMETER(_hp);
34246 UNREFERENCED_PARAMETER(acontext);
34247 UNREFERENCED_PARAMETER(size);
34248 UNREFERENCED_PARAMETER(flags);
34249 assert(!"Should not call GCHeap::AllocAlign8Common without FEATURE_64BIT_ALIGNMENT defined!");
34251 #endif // FEATURE_64BIT_ALIGNMENT
34255 GCHeap::AllocLHeap( size_t size, uint32_t flags REQD_ALIGN_DCL)
34264 Object* newAlloc = NULL;
34267 #ifdef COUNT_CYCLES
34268 AllocStart = GetCycleCount32();
34270 #elif defined(ENABLE_INSTRUMENTATION)
34271 unsigned AllocStart = GetInstLogTime();
34273 #endif //COUNT_CYCLES
34276 #ifdef MULTIPLE_HEAPS
34277 //take the first heap....
34278 gc_heap* hp = gc_heap::g_heaps[0];
34280 gc_heap* hp = pGenGCHeap;
34282 // prefix complains about us dereferencing hp in wks build even though we only access static members
34283 // this way. not sure how to shut it up except for this ugly workaround:
34284 PREFIX_ASSUME(hp != NULL);
34286 #endif //MULTIPLE_HEAPS
34288 alloc_context* acontext = generation_alloc_context (hp->generation_of (max_generation+1));
34290 newAlloc = (Object*) hp->allocate_large_object (size + ComputeMaxStructAlignPadLarge(requiredAlignment), acontext->alloc_bytes_loh);
34291 #ifdef FEATURE_STRUCTALIGN
34292 newAlloc = (Object*) hp->pad_for_alignment_large ((uint8_t*) newAlloc, requiredAlignment, size);
34293 #endif // FEATURE_STRUCTALIGN
34294 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
34297 #ifdef COUNT_CYCLES
34298 finish = GetCycleCount32();
34299 #elif defined(ENABLE_INSTRUMENTATION)
34300 finish = GetInstLogTime();
34301 #endif //COUNT_CYCLES
34302 AllocDuration += finish - AllocStart;
34309 GCHeap::Alloc(gc_alloc_context* context, size_t size, uint32_t flags REQD_ALIGN_DCL)
34318 Object* newAlloc = NULL;
34319 alloc_context* acontext = static_cast<alloc_context*>(context);
34322 #ifdef COUNT_CYCLES
34323 AllocStart = GetCycleCount32();
34325 #elif defined(ENABLE_INSTRUMENTATION)
34326 unsigned AllocStart = GetInstLogTime();
34328 #endif //COUNT_CYCLES
34331 #ifdef MULTIPLE_HEAPS
34332 if (acontext->get_alloc_heap() == 0)
34334 AssignHeap (acontext);
34335 assert (acontext->get_alloc_heap());
34337 #endif //MULTIPLE_HEAPS
34339 #ifdef MULTIPLE_HEAPS
34340 gc_heap* hp = acontext->get_alloc_heap()->pGenGCHeap;
34342 gc_heap* hp = pGenGCHeap;
34344 // prefix complains about us dereferencing hp in wks build even though we only access static members
34345 // this way. not sure how to shut it up except for this ugly workaround:
34346 PREFIX_ASSUME(hp != NULL);
34348 #endif //MULTIPLE_HEAPS
34350 if (size < LARGE_OBJECT_SIZE)
34356 newAlloc = (Object*) hp->allocate (size + ComputeMaxStructAlignPad(requiredAlignment), acontext);
34357 #ifdef FEATURE_STRUCTALIGN
34358 newAlloc = (Object*) hp->pad_for_alignment ((uint8_t*) newAlloc, requiredAlignment, size, acontext);
34359 #endif // FEATURE_STRUCTALIGN
34360 // ASSERT (newAlloc);
34364 newAlloc = (Object*) hp->allocate_large_object (size + ComputeMaxStructAlignPadLarge(requiredAlignment), acontext->alloc_bytes_loh);
34365 #ifdef FEATURE_STRUCTALIGN
34366 newAlloc = (Object*) hp->pad_for_alignment_large ((uint8_t*) newAlloc, requiredAlignment, size);
34367 #endif // FEATURE_STRUCTALIGN
34370 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
34373 #ifdef COUNT_CYCLES
34374 finish = GetCycleCount32();
34375 #elif defined(ENABLE_INSTRUMENTATION)
34376 finish = GetInstLogTime();
34377 #endif //COUNT_CYCLES
34378 AllocDuration += finish - AllocStart;
34385 GCHeap::FixAllocContext (gc_alloc_context* context, bool lockp, void* arg, void *heap)
34387 alloc_context* acontext = static_cast<alloc_context*>(context);
34388 #ifdef MULTIPLE_HEAPS
34391 acontext->alloc_count = 0;
34393 uint8_t * alloc_ptr = acontext->alloc_ptr;
34398 // The acontext->alloc_heap can be out of sync with the ptrs because
34399 // of heap re-assignment in allocate
34400 gc_heap* hp = gc_heap::heap_of (alloc_ptr);
34402 gc_heap* hp = pGenGCHeap;
34403 #endif //MULTIPLE_HEAPS
34405 if (heap == NULL || heap == hp)
34409 enter_spin_lock (&hp->more_space_lock);
34411 hp->fix_allocation_context (acontext, ((arg != 0)? TRUE : FALSE),
34412 get_alignment_constant(TRUE));
34415 leave_spin_lock (&hp->more_space_lock);
34421 GCHeap::GetContainingObject (void *pInteriorPtr, bool fCollectedGenOnly)
34423 uint8_t *o = (uint8_t*)pInteriorPtr;
34425 gc_heap* hp = gc_heap::heap_of (o);
34427 uint8_t* lowest = (fCollectedGenOnly ? hp->gc_low : hp->lowest_address);
34428 uint8_t* highest = (fCollectedGenOnly ? hp->gc_high : hp->highest_address);
34430 if (o >= lowest && o < highest)
34432 o = hp->find_object (o, lowest);
34439 return (Object *)o;
34442 BOOL should_collect_optimized (dynamic_data* dd, BOOL low_memory_p)
34444 if (dd_new_allocation (dd) < 0)
34449 if (((float)(dd_new_allocation (dd)) / (float)dd_desired_allocation (dd)) < (low_memory_p ? 0.7 : 0.3))
34457 //----------------------------------------------------------------------------
34458 // #GarbageCollector
34460 // API to ensure that a complete new garbage collection takes place
34463 GCHeap::GarbageCollect (int generation, bool low_memory_p, int mode)
34468 size_t total_allocated = 0;
34469 size_t total_desired = 0;
34470 #ifdef MULTIPLE_HEAPS
34472 for (hn = 0; hn < gc_heap::n_heaps; hn++)
34474 gc_heap* hp = gc_heap::g_heaps [hn];
34475 total_desired += dd_desired_allocation (hp->dynamic_data_of (0));
34476 total_allocated += dd_desired_allocation (hp->dynamic_data_of (0))-
34477 dd_new_allocation (hp->dynamic_data_of (0));
34480 gc_heap* hp = pGenGCHeap;
34481 total_desired = dd_desired_allocation (hp->dynamic_data_of (0));
34482 total_allocated = dd_desired_allocation (hp->dynamic_data_of (0))-
34483 dd_new_allocation (hp->dynamic_data_of (0));
34484 #endif //MULTIPLE_HEAPS
34486 if ((total_desired > gc_heap::mem_one_percent) && (total_allocated < gc_heap::mem_one_percent))
34488 dprintf (2, ("Async low mem but we've only allocated %d (< 10%% of physical mem) out of %d, returning",
34489 total_allocated, total_desired));
34496 #ifdef MULTIPLE_HEAPS
34497 gc_heap* hpt = gc_heap::g_heaps[0];
34500 #endif //MULTIPLE_HEAPS
34502 generation = (generation < 0) ? max_generation : min (generation, max_generation);
34503 dynamic_data* dd = hpt->dynamic_data_of (generation);
34505 #ifdef BACKGROUND_GC
34506 if (recursive_gc_sync::background_running_p())
34508 if ((mode == collection_optimized) || (mode & collection_non_blocking))
34512 if (mode & collection_blocking)
34514 pGenGCHeap->background_gc_wait();
34515 if (mode & collection_optimized)
34521 #endif //BACKGROUND_GC
34523 if (mode & collection_optimized)
34525 if (pGenGCHeap->gc_started)
34531 BOOL should_collect = FALSE;
34532 BOOL should_check_loh = (generation == max_generation);
34533 #ifdef MULTIPLE_HEAPS
34534 for (int i = 0; i < gc_heap::n_heaps; i++)
34536 dynamic_data* dd1 = gc_heap::g_heaps [i]->dynamic_data_of (generation);
34537 dynamic_data* dd2 = (should_check_loh ?
34538 (gc_heap::g_heaps [i]->dynamic_data_of (max_generation + 1)) :
34541 if (should_collect_optimized (dd1, low_memory_p))
34543 should_collect = TRUE;
34546 if (dd2 && should_collect_optimized (dd2, low_memory_p))
34548 should_collect = TRUE;
34553 should_collect = should_collect_optimized (dd, low_memory_p);
34554 if (!should_collect && should_check_loh)
34557 should_collect_optimized (hpt->dynamic_data_of (max_generation + 1), low_memory_p);
34559 #endif //MULTIPLE_HEAPS
34560 if (!should_collect)
34567 size_t CollectionCountAtEntry = dd_collection_count (dd);
34568 size_t BlockingCollectionCountAtEntry = gc_heap::full_gc_counts[gc_type_blocking];
34569 size_t CurrentCollectionCount = 0;
34573 CurrentCollectionCount = GarbageCollectTry(generation, low_memory_p, mode);
34575 if ((mode & collection_blocking) &&
34576 (generation == max_generation) &&
34577 (gc_heap::full_gc_counts[gc_type_blocking] == BlockingCollectionCountAtEntry))
34579 #ifdef BACKGROUND_GC
34580 if (recursive_gc_sync::background_running_p())
34582 pGenGCHeap->background_gc_wait();
34584 #endif //BACKGROUND_GC
34589 if (CollectionCountAtEntry == CurrentCollectionCount)
34598 GCHeap::GarbageCollectTry (int generation, BOOL low_memory_p, int mode)
34600 int gen = (generation < 0) ?
34601 max_generation : min (generation, max_generation);
34603 gc_reason reason = reason_empty;
34607 if (mode & collection_blocking)
34608 reason = reason_lowmemory_blocking;
34610 reason = reason_lowmemory;
34613 reason = reason_induced;
34615 if (reason == reason_induced)
34617 if (mode & collection_compacting)
34619 reason = reason_induced_compacting;
34621 else if (mode & collection_non_blocking)
34623 reason = reason_induced_noforce;
34626 else if (mode & collection_gcstress)
34628 reason = reason_gcstress;
34633 return GarbageCollectGeneration (gen, reason);
34636 void gc_heap::do_pre_gc()
34638 STRESS_LOG_GC_STACK;
34641 STRESS_LOG_GC_START(VolatileLoad(&settings.gc_index),
34642 (uint32_t)settings.condemned_generation,
34643 (uint32_t)settings.reason);
34644 #endif // STRESS_LOG
34646 #ifdef MULTIPLE_HEAPS
34647 gc_heap* hp = g_heaps[0];
34650 #endif //MULTIPLE_HEAPS
34652 #ifdef BACKGROUND_GC
34653 settings.b_state = hp->current_bgc_state;
34654 #endif //BACKGROUND_GC
34656 #ifdef BACKGROUND_GC
34657 dprintf (1, ("*GC* %d(gen0:%d)(%d)(%s)(%d)",
34658 VolatileLoad(&settings.gc_index),
34659 dd_collection_count (hp->dynamic_data_of (0)),
34660 settings.condemned_generation,
34661 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC")),
34662 settings.b_state));
34664 dprintf (1, ("*GC* %d(gen0:%d)(%d)",
34665 VolatileLoad(&settings.gc_index),
34666 dd_collection_count(hp->dynamic_data_of(0)),
34667 settings.condemned_generation));
34668 #endif //BACKGROUND_GC
34670 // TODO: this can happen...it's because of the way we are calling
34671 // do_pre_gc, will fix later.
34672 //if (last_gc_index > VolatileLoad(&settings.gc_index))
34674 // FATAL_GC_ERROR();
34677 last_gc_index = VolatileLoad(&settings.gc_index);
34678 GCHeap::UpdatePreGCCounters();
34680 if (settings.concurrent)
34682 #ifdef BACKGROUND_GC
34683 full_gc_counts[gc_type_background]++;
34684 #if defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
34685 GCHeap::gc_stress_fgcs_in_bgc = 0;
34686 #endif // STRESS_HEAP && !FEATURE_REDHAWK
34687 #endif // BACKGROUND_GC
34691 if (settings.condemned_generation == max_generation)
34693 full_gc_counts[gc_type_blocking]++;
34697 #ifdef BACKGROUND_GC
34698 if (settings.background_p)
34700 ephemeral_fgc_counts[settings.condemned_generation]++;
34702 #endif //BACKGROUND_GC
34706 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
34709 SystemDomain::ResetADSurvivedBytes();
34711 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
34714 #ifdef GC_CONFIG_DRIVEN
34715 void gc_heap::record_interesting_info_per_heap()
34717 // datapoints are always from the last blocking GC so don't record again
34719 if (!(settings.concurrent))
34721 for (int i = 0; i < max_idp_count; i++)
34723 interesting_data_per_heap[i] += interesting_data_per_gc[i];
34727 int compact_reason = get_gc_data_per_heap()->get_mechanism (gc_heap_compact);
34728 if (compact_reason >= 0)
34729 (compact_reasons_per_heap[compact_reason])++;
34730 int expand_mechanism = get_gc_data_per_heap()->get_mechanism (gc_heap_expand);
34731 if (expand_mechanism >= 0)
34732 (expand_mechanisms_per_heap[expand_mechanism])++;
34734 for (int i = 0; i < max_gc_mechanism_bits_count; i++)
34736 if (get_gc_data_per_heap()->is_mechanism_bit_set ((gc_mechanism_bit_per_heap)i))
34737 (interesting_mechanism_bits_per_heap[i])++;
34740 // h# | GC | gen | C | EX | NF | BF | ML | DM || PreS | PostS | Merge | Conv | Pre | Post | PrPo | PreP | PostP |
34741 cprintf (("%2d | %6d | %1d | %1s | %2s | %2s | %2s | %2s | %2s || %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id |",
34743 (size_t)settings.gc_index,
34744 settings.condemned_generation,
34745 // TEMP - I am just doing this for wks GC 'cuase I wanna see the pattern of doing C/S GCs.
34746 (settings.compaction ? (((compact_reason >= 0) && gc_heap_compact_reason_mandatory_p[compact_reason]) ? "M" : "W") : ""), // compaction
34747 ((expand_mechanism >= 0)? "X" : ""), // EX
34748 ((expand_mechanism == expand_reuse_normal) ? "X" : ""), // NF
34749 ((expand_mechanism == expand_reuse_bestfit) ? "X" : ""), // BF
34750 (get_gc_data_per_heap()->is_mechanism_bit_set (gc_mark_list_bit) ? "X" : ""), // ML
34751 (get_gc_data_per_heap()->is_mechanism_bit_set (gc_demotion_bit) ? "X" : ""), // DM
34752 interesting_data_per_gc[idp_pre_short],
34753 interesting_data_per_gc[idp_post_short],
34754 interesting_data_per_gc[idp_merged_pin],
34755 interesting_data_per_gc[idp_converted_pin],
34756 interesting_data_per_gc[idp_pre_pin],
34757 interesting_data_per_gc[idp_post_pin],
34758 interesting_data_per_gc[idp_pre_and_post_pin],
34759 interesting_data_per_gc[idp_pre_short_padded],
34760 interesting_data_per_gc[idp_post_short_padded]));
34763 void gc_heap::record_global_mechanisms()
34765 for (int i = 0; i < max_global_mechanisms_count; i++)
34767 if (gc_data_global.get_mechanism_p ((gc_global_mechanism_p)i))
34769 ::record_global_mechanism (i);
34774 BOOL gc_heap::should_do_sweeping_gc (BOOL compact_p)
34776 if (!compact_ratio)
34777 return (!compact_p);
34779 size_t compact_count = compact_or_sweep_gcs[0];
34780 size_t sweep_count = compact_or_sweep_gcs[1];
34782 size_t total_count = compact_count + sweep_count;
34783 BOOL should_compact = compact_p;
34784 if (total_count > 3)
34788 int temp_ratio = (int)((compact_count + 1) * 100 / (total_count + 1));
34789 if (temp_ratio > compact_ratio)
34791 // cprintf (("compact would be: %d, total_count: %d, ratio would be %d%% > target\n",
34792 // (compact_count + 1), (total_count + 1), temp_ratio));
34793 should_compact = FALSE;
34798 int temp_ratio = (int)((sweep_count + 1) * 100 / (total_count + 1));
34799 if (temp_ratio > (100 - compact_ratio))
34801 // cprintf (("sweep would be: %d, total_count: %d, ratio would be %d%% > target\n",
34802 // (sweep_count + 1), (total_count + 1), temp_ratio));
34803 should_compact = TRUE;
34808 return !should_compact;
34810 #endif //GC_CONFIG_DRIVEN
34812 void gc_heap::do_post_gc()
34814 if (!settings.concurrent)
34820 #ifdef COUNT_CYCLES
34821 AllocStart = GetCycleCount32();
34823 AllocStart = clock();
34824 #endif //COUNT_CYCLES
34827 #ifdef MULTIPLE_HEAPS
34828 gc_heap* hp = g_heaps[0];
34831 #endif //MULTIPLE_HEAPS
34833 GCToEEInterface::GcDone(settings.condemned_generation);
34835 GCToEEInterface::DiagGCEnd(VolatileLoad(&settings.gc_index),
34836 (uint32_t)settings.condemned_generation,
34837 (uint32_t)settings.reason,
34838 !!settings.concurrent);
34840 //dprintf (1, (" ****end of Garbage Collection**** %d(gen0:%d)(%d)",
34841 dprintf (1, ("*EGC* %d(gen0:%d)(%d)(%s)",
34842 VolatileLoad(&settings.gc_index),
34843 dd_collection_count(hp->dynamic_data_of(0)),
34844 settings.condemned_generation,
34845 (settings.concurrent ? "BGC" : "GC")));
34847 if (settings.exit_memory_load != 0)
34848 last_gc_memory_load = settings.exit_memory_load;
34849 else if (settings.entry_memory_load != 0)
34850 last_gc_memory_load = settings.entry_memory_load;
34852 last_gc_heap_size = get_total_heap_size();
34853 last_gc_fragmentation = get_total_fragmentation();
34855 GCHeap::UpdatePostGCCounters();
34856 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
34857 //if (g_fEnableARM)
34859 // SystemDomain::GetADSurvivedBytes();
34861 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
34864 STRESS_LOG_GC_END(VolatileLoad(&settings.gc_index),
34865 (uint32_t)settings.condemned_generation,
34866 (uint32_t)settings.reason);
34867 #endif // STRESS_LOG
34869 #ifdef GC_CONFIG_DRIVEN
34870 if (!settings.concurrent)
34872 if (settings.compaction)
34873 (compact_or_sweep_gcs[0])++;
34875 (compact_or_sweep_gcs[1])++;
34878 #ifdef MULTIPLE_HEAPS
34879 for (int i = 0; i < n_heaps; i++)
34880 g_heaps[i]->record_interesting_info_per_heap();
34882 record_interesting_info_per_heap();
34883 #endif //MULTIPLE_HEAPS
34884 record_global_mechanisms();
34885 #endif //GC_CONFIG_DRIVEN
34888 unsigned GCHeap::GetGcCount()
34890 return (unsigned int)VolatileLoad(&pGenGCHeap->settings.gc_index);
34894 GCHeap::GarbageCollectGeneration (unsigned int gen, gc_reason reason)
34896 dprintf (2, ("triggered a GC!"));
34898 #ifdef MULTIPLE_HEAPS
34899 gc_heap* hpt = gc_heap::g_heaps[0];
34902 #endif //MULTIPLE_HEAPS
34903 bool cooperative_mode = true;
34904 dynamic_data* dd = hpt->dynamic_data_of (gen);
34905 size_t localCount = dd_collection_count (dd);
34907 enter_spin_lock (&gc_heap::gc_lock);
34908 dprintf (SPINLOCK_LOG, ("GC Egc"));
34909 ASSERT_HOLDING_SPIN_LOCK(&gc_heap::gc_lock);
34911 //don't trigger another GC if one was already in progress
34912 //while waiting for the lock
34914 size_t col_count = dd_collection_count (dd);
34916 if (localCount != col_count)
34918 #ifdef SYNCHRONIZATION_STATS
34919 gc_lock_contended++;
34920 #endif //SYNCHRONIZATION_STATS
34921 dprintf (SPINLOCK_LOG, ("no need GC Lgc"));
34922 leave_spin_lock (&gc_heap::gc_lock);
34924 // We don't need to release msl here 'cause this means a GC
34925 // has happened and would have release all msl's.
34930 #ifdef COUNT_CYCLES
34931 int gc_start = GetCycleCount32();
34932 #endif //COUNT_CYCLES
34935 #ifdef COUNT_CYCLES
34936 AllocDuration += GetCycleCount32() - AllocStart;
34938 AllocDuration += clock() - AllocStart;
34939 #endif //COUNT_CYCLES
34942 gc_heap::g_low_memory_status = (reason == reason_lowmemory) ||
34943 (reason == reason_lowmemory_blocking) ||
34944 (gc_heap::latency_level == latency_level_memory_footprint);
34946 gc_trigger_reason = reason;
34948 #ifdef MULTIPLE_HEAPS
34949 for (int i = 0; i < gc_heap::n_heaps; i++)
34951 gc_heap::g_heaps[i]->reset_gc_done();
34954 gc_heap::reset_gc_done();
34955 #endif //MULTIPLE_HEAPS
34957 gc_heap::gc_started = TRUE;
34960 init_sync_log_stats();
34962 #ifndef MULTIPLE_HEAPS
34963 cooperative_mode = gc_heap::enable_preemptive ();
34965 dprintf (2, ("Suspending EE"));
34966 BEGIN_TIMING(suspend_ee_during_log);
34967 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC);
34968 END_TIMING(suspend_ee_during_log);
34969 gc_heap::proceed_with_gc_p = gc_heap::should_proceed_with_gc();
34970 gc_heap::disable_preemptive (cooperative_mode);
34971 if (gc_heap::proceed_with_gc_p)
34972 pGenGCHeap->settings.init_mechanisms();
34974 gc_heap::update_collection_counts_for_no_gc();
34976 #endif //!MULTIPLE_HEAPS
34979 // MAP_EVENT_MONITORS(EE_MONITOR_GARBAGE_COLLECTIONS, NotifyEvent(EE_EVENT_TYPE_GC_STARTED, 0));
34982 #ifdef COUNT_CYCLES
34985 start = GetCycleCount32();
34990 #endif //COUNT_CYCLES
34991 PromotedObjectCount = 0;
34994 unsigned int condemned_generation_number = gen;
34996 // We want to get a stack from the user thread that triggered the GC
34997 // instead of on the GC thread which is the case for Server GC.
34998 // But we are doing it for Workstation GC as well to be uniform.
34999 FIRE_EVENT(GCTriggered, static_cast<uint32_t>(reason));
35001 #ifdef MULTIPLE_HEAPS
35002 GcCondemnedGeneration = condemned_generation_number;
35004 cooperative_mode = gc_heap::enable_preemptive ();
35006 BEGIN_TIMING(gc_during_log);
35007 gc_heap::ee_suspend_event.Set();
35008 gc_heap::wait_for_gc_done();
35009 END_TIMING(gc_during_log);
35011 gc_heap::disable_preemptive (cooperative_mode);
35013 condemned_generation_number = GcCondemnedGeneration;
35015 if (gc_heap::proceed_with_gc_p)
35017 BEGIN_TIMING(gc_during_log);
35018 pGenGCHeap->garbage_collect (condemned_generation_number);
35019 END_TIMING(gc_during_log);
35021 #endif //MULTIPLE_HEAPS
35024 #ifdef COUNT_CYCLES
35025 finish = GetCycleCount32();
35028 #endif //COUNT_CYCLES
35029 GcDuration += finish - start;
35031 ("<GC# %d> Condemned: %d, Duration: %d, total: %d Alloc Avg: %d, Small Objects:%d Large Objects:%d",
35032 VolatileLoad(&pGenGCHeap->settings.gc_index), condemned_generation_number,
35033 finish - start, GcDuration,
35034 AllocCount ? (AllocDuration / AllocCount) : 0,
35035 AllocSmallCount, AllocBigCount));
35040 #ifdef BACKGROUND_GC
35041 // We are deciding whether we should fire the alloc wait end event here
35042 // because in begin_foreground we could be calling end_foreground
35043 // if we need to retry.
35044 if (gc_heap::alloc_wait_event_p)
35046 hpt->fire_alloc_wait_event_end (awr_fgc_wait_for_bgc);
35047 gc_heap::alloc_wait_event_p = FALSE;
35049 #endif //BACKGROUND_GC
35051 #ifndef MULTIPLE_HEAPS
35052 #ifdef BACKGROUND_GC
35053 if (!gc_heap::dont_restart_ee_p)
35055 #endif //BACKGROUND_GC
35056 BEGIN_TIMING(restart_ee_during_log);
35057 GCToEEInterface::RestartEE(TRUE);
35058 END_TIMING(restart_ee_during_log);
35059 #ifdef BACKGROUND_GC
35061 #endif //BACKGROUND_GC
35062 #endif //!MULTIPLE_HEAPS
35064 #ifdef COUNT_CYCLES
35065 printf ("GC: %d Time: %d\n", GcCondemnedGeneration,
35066 GetCycleCount32() - gc_start);
35067 #endif //COUNT_CYCLES
35069 #ifndef MULTIPLE_HEAPS
35070 process_sync_log_stats();
35071 gc_heap::gc_started = FALSE;
35072 gc_heap::set_gc_done();
35073 dprintf (SPINLOCK_LOG, ("GC Lgc"));
35074 leave_spin_lock (&gc_heap::gc_lock);
35075 #endif //!MULTIPLE_HEAPS
35077 #ifdef FEATURE_PREMORTEM_FINALIZATION
35078 GCToEEInterface::EnableFinalization(!pGenGCHeap->settings.concurrent && pGenGCHeap->settings.found_finalizers);
35079 #endif // FEATURE_PREMORTEM_FINALIZATION
35081 return dd_collection_count (dd);
35084 size_t GCHeap::GetTotalBytesInUse ()
35086 #ifdef MULTIPLE_HEAPS
35087 //enumarate all the heaps and get their size.
35088 size_t tot_size = 0;
35089 for (int i = 0; i < gc_heap::n_heaps; i++)
35091 GCHeap* Hp = gc_heap::g_heaps [i]->vm_heap;
35092 tot_size += Hp->ApproxTotalBytesInUse (FALSE);
35096 return ApproxTotalBytesInUse ();
35097 #endif //MULTIPLE_HEAPS
35100 int GCHeap::CollectionCount (int generation, int get_bgc_fgc_count)
35102 if (get_bgc_fgc_count != 0)
35104 #ifdef BACKGROUND_GC
35105 if (generation == max_generation)
35107 return (int)(gc_heap::full_gc_counts[gc_type_background]);
35111 return (int)(gc_heap::ephemeral_fgc_counts[generation]);
35115 #endif //BACKGROUND_GC
35118 #ifdef MULTIPLE_HEAPS
35119 gc_heap* hp = gc_heap::g_heaps [0];
35120 #else //MULTIPLE_HEAPS
35121 gc_heap* hp = pGenGCHeap;
35122 #endif //MULTIPLE_HEAPS
35123 if (generation > max_generation)
35126 return (int)dd_collection_count (hp->dynamic_data_of (generation));
35129 size_t GCHeap::ApproxTotalBytesInUse(BOOL small_heap_only)
35131 size_t totsize = 0;
35133 //ASSERT(InMustComplete());
35134 enter_spin_lock (&pGenGCHeap->gc_lock);
35136 heap_segment* eph_seg = generation_allocation_segment (pGenGCHeap->generation_of (0));
35137 // Get small block heap size info
35138 totsize = (pGenGCHeap->alloc_allocated - heap_segment_mem (eph_seg));
35139 heap_segment* seg1 = generation_start_segment (pGenGCHeap->generation_of (max_generation));
35140 while (seg1 != eph_seg)
35142 totsize += heap_segment_allocated (seg1) -
35143 heap_segment_mem (seg1);
35144 seg1 = heap_segment_next (seg1);
35147 //discount the fragmentation
35148 for (int i = 0; i <= max_generation; i++)
35150 generation* gen = pGenGCHeap->generation_of (i);
35151 totsize -= (generation_free_list_space (gen) + generation_free_obj_space (gen));
35154 if (!small_heap_only)
35156 heap_segment* seg2 = generation_start_segment (pGenGCHeap->generation_of (max_generation+1));
35160 totsize += heap_segment_allocated (seg2) -
35161 heap_segment_mem (seg2);
35162 seg2 = heap_segment_next (seg2);
35165 //discount the fragmentation
35166 generation* loh_gen = pGenGCHeap->generation_of (max_generation+1);
35167 size_t frag = generation_free_list_space (loh_gen) + generation_free_obj_space (loh_gen);
35170 leave_spin_lock (&pGenGCHeap->gc_lock);
35174 #ifdef MULTIPLE_HEAPS
35175 void GCHeap::AssignHeap (alloc_context* acontext)
35177 // Assign heap based on processor
35178 acontext->set_alloc_heap(GetHeap(heap_select::select_heap(acontext, 0)));
35179 acontext->set_home_heap(acontext->get_alloc_heap());
35181 GCHeap* GCHeap::GetHeap (int n)
35183 assert (n < gc_heap::n_heaps);
35184 return gc_heap::g_heaps [n]->vm_heap;
35186 #endif //MULTIPLE_HEAPS
35188 bool GCHeap::IsThreadUsingAllocationContextHeap(gc_alloc_context* context, int thread_number)
35190 alloc_context* acontext = static_cast<alloc_context*>(context);
35191 #ifdef MULTIPLE_HEAPS
35192 return ((acontext->get_home_heap() == GetHeap(thread_number)) ||
35193 ((acontext->get_home_heap() == 0) && (thread_number == 0)));
35195 UNREFERENCED_PARAMETER(acontext);
35196 UNREFERENCED_PARAMETER(thread_number);
35198 #endif //MULTIPLE_HEAPS
35201 // Returns the number of processors required to trigger the use of thread based allocation contexts
35202 int GCHeap::GetNumberOfHeaps ()
35204 #ifdef MULTIPLE_HEAPS
35205 return gc_heap::n_heaps;
35208 #endif //MULTIPLE_HEAPS
35212 in this way we spend extra time cycling through all the heaps while create the handle
35213 it ought to be changed by keeping alloc_context.home_heap as number (equals heap_number)
35215 int GCHeap::GetHomeHeapNumber ()
35217 #ifdef MULTIPLE_HEAPS
35218 gc_alloc_context* ctx = GCToEEInterface::GetAllocContext();
35224 GCHeap *hp = static_cast<alloc_context*>(ctx)->get_home_heap();
35225 return (hp ? hp->pGenGCHeap->heap_number : 0);
35228 #endif //MULTIPLE_HEAPS
35231 unsigned int GCHeap::GetCondemnedGeneration()
35233 return gc_heap::settings.condemned_generation;
35236 void GCHeap::GetMemoryInfo(uint32_t* highMemLoadThreshold,
35237 uint64_t* totalPhysicalMem,
35238 uint32_t* lastRecordedMemLoad,
35239 size_t* lastRecordedHeapSize,
35240 size_t* lastRecordedFragmentation)
35242 *highMemLoadThreshold = gc_heap::high_memory_load_th;
35243 *totalPhysicalMem = gc_heap::total_physical_mem;
35244 *lastRecordedMemLoad = gc_heap::last_gc_memory_load;
35245 *lastRecordedHeapSize = gc_heap::last_gc_heap_size;
35246 *lastRecordedFragmentation = gc_heap::last_gc_fragmentation;
35249 int GCHeap::GetGcLatencyMode()
35251 return (int)(pGenGCHeap->settings.pause_mode);
35254 int GCHeap::SetGcLatencyMode (int newLatencyMode)
35256 if (gc_heap::settings.pause_mode == pause_no_gc)
35257 return (int)set_pause_mode_no_gc;
35259 gc_pause_mode new_mode = (gc_pause_mode)newLatencyMode;
35261 if (new_mode == pause_low_latency)
35263 #ifndef MULTIPLE_HEAPS
35264 pGenGCHeap->settings.pause_mode = new_mode;
35265 #endif //!MULTIPLE_HEAPS
35267 else if (new_mode == pause_sustained_low_latency)
35269 #ifdef BACKGROUND_GC
35270 if (gc_heap::gc_can_use_concurrent)
35272 pGenGCHeap->settings.pause_mode = new_mode;
35274 #endif //BACKGROUND_GC
35278 pGenGCHeap->settings.pause_mode = new_mode;
35281 #ifdef BACKGROUND_GC
35282 if (recursive_gc_sync::background_running_p())
35284 // If we get here, it means we are doing an FGC. If the pause
35285 // mode was altered we will need to save it in the BGC settings.
35286 if (gc_heap::saved_bgc_settings.pause_mode != new_mode)
35288 gc_heap::saved_bgc_settings.pause_mode = new_mode;
35291 #endif //BACKGROUND_GC
35293 return (int)set_pause_mode_success;
35296 int GCHeap::GetLOHCompactionMode()
35298 return pGenGCHeap->loh_compaction_mode;
35301 void GCHeap::SetLOHCompactionMode (int newLOHCompactionyMode)
35303 #ifdef FEATURE_LOH_COMPACTION
35304 pGenGCHeap->loh_compaction_mode = (gc_loh_compaction_mode)newLOHCompactionyMode;
35305 #endif //FEATURE_LOH_COMPACTION
35308 bool GCHeap::RegisterForFullGCNotification(uint32_t gen2Percentage,
35309 uint32_t lohPercentage)
35311 #ifdef MULTIPLE_HEAPS
35312 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35314 gc_heap* hp = gc_heap::g_heaps [hn];
35315 hp->fgn_last_alloc = dd_new_allocation (hp->dynamic_data_of (0));
35317 #else //MULTIPLE_HEAPS
35318 pGenGCHeap->fgn_last_alloc = dd_new_allocation (pGenGCHeap->dynamic_data_of (0));
35319 #endif //MULTIPLE_HEAPS
35321 pGenGCHeap->full_gc_approach_event.Reset();
35322 pGenGCHeap->full_gc_end_event.Reset();
35323 pGenGCHeap->full_gc_approach_event_set = false;
35325 pGenGCHeap->fgn_maxgen_percent = gen2Percentage;
35326 pGenGCHeap->fgn_loh_percent = lohPercentage;
35331 bool GCHeap::CancelFullGCNotification()
35333 pGenGCHeap->fgn_maxgen_percent = 0;
35334 pGenGCHeap->fgn_loh_percent = 0;
35336 pGenGCHeap->full_gc_approach_event.Set();
35337 pGenGCHeap->full_gc_end_event.Set();
35342 int GCHeap::WaitForFullGCApproach(int millisecondsTimeout)
35344 dprintf (2, ("WFGA: Begin wait"));
35345 int result = gc_heap::full_gc_wait (&(pGenGCHeap->full_gc_approach_event), millisecondsTimeout);
35346 dprintf (2, ("WFGA: End wait"));
35350 int GCHeap::WaitForFullGCComplete(int millisecondsTimeout)
35352 dprintf (2, ("WFGE: Begin wait"));
35353 int result = gc_heap::full_gc_wait (&(pGenGCHeap->full_gc_end_event), millisecondsTimeout);
35354 dprintf (2, ("WFGE: End wait"));
35358 int GCHeap::StartNoGCRegion(uint64_t totalSize, bool lohSizeKnown, uint64_t lohSize, bool disallowFullBlockingGC)
35360 NoGCRegionLockHolder lh;
35362 dprintf (1, ("begin no gc called"));
35363 start_no_gc_region_status status = gc_heap::prepare_for_no_gc_region (totalSize, lohSizeKnown, lohSize, disallowFullBlockingGC);
35364 if (status == start_no_gc_success)
35366 GarbageCollect (max_generation);
35367 status = gc_heap::get_start_no_gc_region_status();
35370 if (status != start_no_gc_success)
35371 gc_heap::handle_failure_for_no_gc();
35373 return (int)status;
35376 int GCHeap::EndNoGCRegion()
35378 NoGCRegionLockHolder lh;
35379 return (int)gc_heap::end_no_gc_region();
35382 void GCHeap::PublishObject (uint8_t* Obj)
35384 #ifdef BACKGROUND_GC
35385 gc_heap* hp = gc_heap::heap_of (Obj);
35386 hp->bgc_alloc_lock->loh_alloc_done (Obj);
35387 #endif //BACKGROUND_GC
35390 // The spec for this one isn't clear. This function
35391 // returns the size that can be allocated without
35392 // triggering a GC of any kind.
35393 size_t GCHeap::ApproxFreeBytes()
35396 //ASSERT(InMustComplete());
35397 enter_spin_lock (&pGenGCHeap->gc_lock);
35399 generation* gen = pGenGCHeap->generation_of (0);
35400 size_t res = generation_allocation_limit (gen) - generation_allocation_pointer (gen);
35402 leave_spin_lock (&pGenGCHeap->gc_lock);
35407 HRESULT GCHeap::GetGcCounters(int gen, gc_counters* counters)
35409 if ((gen < 0) || (gen > max_generation))
35411 #ifdef MULTIPLE_HEAPS
35412 counters->current_size = 0;
35413 counters->promoted_size = 0;
35414 counters->collection_count = 0;
35416 //enumarate all the heaps and get their counters.
35417 for (int i = 0; i < gc_heap::n_heaps; i++)
35419 dynamic_data* dd = gc_heap::g_heaps [i]->dynamic_data_of (gen);
35421 counters->current_size += dd_current_size (dd);
35422 counters->promoted_size += dd_promoted_size (dd);
35424 counters->collection_count += dd_collection_count (dd);
35427 dynamic_data* dd = pGenGCHeap->dynamic_data_of (gen);
35428 counters->current_size = dd_current_size (dd);
35429 counters->promoted_size = dd_promoted_size (dd);
35430 counters->collection_count = dd_collection_count (dd);
35431 #endif //MULTIPLE_HEAPS
35435 // Get the segment size to use, making sure it conforms.
35436 size_t GCHeap::GetValidSegmentSize(bool large_seg)
35438 return get_valid_segment_size (large_seg);
35441 // Get the max gen0 heap size, making sure it conforms.
35442 size_t GCHeap::GetValidGen0MaxSize(size_t seg_size)
35444 size_t gen0size = static_cast<size_t>(GCConfig::GetGen0Size());
35446 if ((gen0size == 0) || !g_theGCHeap->IsValidGen0MaxSize(gen0size))
35449 // performance data seems to indicate halving the size results
35450 // in optimal perf. Ask for adjusted gen0 size.
35451 gen0size = max(GCToOSInterface::GetCacheSizePerLogicalCpu(FALSE),(256*1024));
35453 // if gen0 size is too large given the available memory, reduce it.
35454 // Get true cache size, as we don't want to reduce below this.
35455 size_t trueSize = max(GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE),(256*1024));
35456 dprintf (2, ("cache: %Id-%Id, cpu: %Id",
35457 GCToOSInterface::GetCacheSizePerLogicalCpu(FALSE),
35458 GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE)));
35460 int n_heaps = gc_heap::n_heaps;
35462 size_t trueSize = GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE);
35463 gen0size = max((4*trueSize/5),(256*1024));
35464 trueSize = max(trueSize, (256*1024));
35468 // if the total min GC across heaps will exceed 1/6th of available memory,
35469 // then reduce the min GC size until it either fits or has been reduced to cache size.
35470 while ((gen0size * n_heaps) > GCToOSInterface::GetPhysicalMemoryLimit() / 6)
35472 gen0size = gen0size / 2;
35473 if (gen0size <= trueSize)
35475 gen0size = trueSize;
35481 // Generation 0 must never be more than 1/2 the segment size.
35482 if (gen0size >= (seg_size / 2))
35483 gen0size = seg_size / 2;
35488 void GCHeap::SetReservedVMLimit (size_t vmlimit)
35490 gc_heap::reserved_memory_limit = vmlimit;
35494 //versions of same method on each heap
35496 #ifdef FEATURE_PREMORTEM_FINALIZATION
35498 Object* GCHeap::GetNextFinalizableObject()
35501 #ifdef MULTIPLE_HEAPS
35503 //return the first non critical one in the first queue.
35504 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35506 gc_heap* hp = gc_heap::g_heaps [hn];
35507 Object* O = hp->finalize_queue->GetNextFinalizableObject(TRUE);
35511 //return the first non crtitical/critical one in the first queue.
35512 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35514 gc_heap* hp = gc_heap::g_heaps [hn];
35515 Object* O = hp->finalize_queue->GetNextFinalizableObject(FALSE);
35522 #else //MULTIPLE_HEAPS
35523 return pGenGCHeap->finalize_queue->GetNextFinalizableObject();
35524 #endif //MULTIPLE_HEAPS
35528 size_t GCHeap::GetNumberFinalizableObjects()
35530 #ifdef MULTIPLE_HEAPS
35532 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35534 gc_heap* hp = gc_heap::g_heaps [hn];
35535 cnt += hp->finalize_queue->GetNumberFinalizableObjects();
35540 #else //MULTIPLE_HEAPS
35541 return pGenGCHeap->finalize_queue->GetNumberFinalizableObjects();
35542 #endif //MULTIPLE_HEAPS
35545 size_t GCHeap::GetFinalizablePromotedCount()
35547 #ifdef MULTIPLE_HEAPS
35550 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35552 gc_heap* hp = gc_heap::g_heaps [hn];
35553 cnt += hp->finalize_queue->GetPromotedCount();
35557 #else //MULTIPLE_HEAPS
35558 return pGenGCHeap->finalize_queue->GetPromotedCount();
35559 #endif //MULTIPLE_HEAPS
35562 bool GCHeap::FinalizeAppDomain(void *pDomain, bool fRunFinalizers)
35564 #ifdef MULTIPLE_HEAPS
35565 bool foundp = false;
35566 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35568 gc_heap* hp = gc_heap::g_heaps [hn];
35569 if (hp->finalize_queue->FinalizeAppDomain (pDomain, fRunFinalizers))
35574 #else //MULTIPLE_HEAPS
35575 return pGenGCHeap->finalize_queue->FinalizeAppDomain (pDomain, fRunFinalizers);
35576 #endif //MULTIPLE_HEAPS
35579 bool GCHeap::ShouldRestartFinalizerWatchDog()
35581 // This condition was historically used as part of the condition to detect finalizer thread timeouts
35582 return gc_heap::gc_lock.lock != -1;
35585 void GCHeap::SetFinalizeQueueForShutdown(bool fHasLock)
35587 #ifdef MULTIPLE_HEAPS
35588 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35590 gc_heap* hp = gc_heap::g_heaps [hn];
35591 hp->finalize_queue->SetSegForShutDown(fHasLock);
35594 #else //MULTIPLE_HEAPS
35595 pGenGCHeap->finalize_queue->SetSegForShutDown(fHasLock);
35596 #endif //MULTIPLE_HEAPS
35599 //---------------------------------------------------------------------------
35600 // Finalized class tracking
35601 //---------------------------------------------------------------------------
35603 bool GCHeap::RegisterForFinalization (int gen, Object* obj)
35607 if (((((CObjectHeader*)obj)->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN))
35609 //just reset the bit
35610 ((CObjectHeader*)obj)->GetHeader()->ClrBit(BIT_SBLK_FINALIZER_RUN);
35615 gc_heap* hp = gc_heap::heap_of ((uint8_t*)obj);
35616 return hp->finalize_queue->RegisterForFinalization (gen, obj);
35620 void GCHeap::SetFinalizationRun (Object* obj)
35622 ((CObjectHeader*)obj)->GetHeader()->SetBit(BIT_SBLK_FINALIZER_RUN);
35626 //--------------------------------------------------------------------
35628 // Support for finalization
35630 //--------------------------------------------------------------------
35633 unsigned int gen_segment (int gen)
35635 assert (((signed)NUMBERGENERATIONS - gen - 1)>=0);
35636 return (NUMBERGENERATIONS - gen - 1);
35639 bool CFinalize::Initialize()
35646 m_Array = new (nothrow)(Object*[100]);
35651 STRESS_LOG_OOM_STACK(sizeof(Object*[100]));
35652 if (GCConfig::GetBreakOnOOM())
35654 GCToOSInterface::DebugBreak();
35658 m_EndArray = &m_Array[100];
35660 for (int i =0; i < FreeList; i++)
35662 SegQueueLimit (i) = m_Array;
35664 m_PromotedCount = 0;
35667 lockowner_threadid.Clear();
35673 CFinalize::~CFinalize()
35678 size_t CFinalize::GetPromotedCount ()
35680 return m_PromotedCount;
35684 void CFinalize::EnterFinalizeLock()
35686 _ASSERTE(dbgOnly_IsSpecialEEThread() ||
35687 GCToEEInterface::GetThread() == 0 ||
35688 GCToEEInterface::IsPreemptiveGCDisabled());
35691 if (Interlocked::CompareExchange(&lock, 0, -1) >= 0)
35693 unsigned int i = 0;
35696 YieldProcessor(); // indicate to the processor that we are spining
35698 GCToOSInterface::YieldThread (0);
35700 GCToOSInterface::Sleep (5);
35706 lockowner_threadid.SetToCurrentThread();
35711 void CFinalize::LeaveFinalizeLock()
35713 _ASSERTE(dbgOnly_IsSpecialEEThread() ||
35714 GCToEEInterface::GetThread() == 0 ||
35715 GCToEEInterface::IsPreemptiveGCDisabled());
35718 lockowner_threadid.Clear();
35724 CFinalize::RegisterForFinalization (int gen, Object* obj, size_t size)
35731 EnterFinalizeLock();
35733 unsigned int dest = 0;
35735 if (g_fFinalizerRunOnShutDown)
35737 //no method table available yet,
35738 //put it in the finalizer queue and sort out when
35740 dest = FinalizerListSeg;
35744 dest = gen_segment (gen);
35746 // Adjust boundary for segments so that GC will keep objects alive.
35747 Object*** s_i = &SegQueue (FreeList);
35748 if ((*s_i) == m_EndArray)
35752 LeaveFinalizeLock();
35753 if (method_table(obj) == NULL)
35755 // If the object is uninitialized, a valid size should have been passed.
35756 assert (size >= Align (min_obj_size));
35757 dprintf (3, ("Making unused array [%Ix, %Ix[", (size_t)obj, (size_t)(obj+size)));
35758 ((CObjectHeader*)obj)->SetFree(size);
35760 STRESS_LOG_OOM_STACK(0);
35761 if (GCConfig::GetBreakOnOOM())
35763 GCToOSInterface::DebugBreak();
35768 Object*** end_si = &SegQueueLimit (dest);
35771 //is the segment empty?
35772 if (!(*s_i == *(s_i-1)))
35774 //no, swap the end elements.
35775 *(*s_i) = *(*(s_i-1));
35777 //increment the fill pointer
35779 //go to the next segment.
35781 } while (s_i > end_si);
35783 // We have reached the destination segment
35784 // store the object
35786 // increment the fill pointer
35789 LeaveFinalizeLock();
35795 CFinalize::GetNextFinalizableObject (BOOL only_non_critical)
35799 EnterFinalizeLock();
35802 if (!IsSegEmpty(FinalizerListSeg))
35804 if (g_fFinalizerRunOnShutDown)
35806 obj = *(SegQueueLimit (FinalizerListSeg)-1);
35807 if (method_table(obj)->HasCriticalFinalizer())
35809 MoveItem ((SegQueueLimit (FinalizerListSeg)-1),
35810 FinalizerListSeg, CriticalFinalizerListSeg);
35814 --SegQueueLimit (FinalizerListSeg);
35817 obj = *(--SegQueueLimit (FinalizerListSeg));
35820 else if (!only_non_critical && !IsSegEmpty(CriticalFinalizerListSeg))
35822 //the FinalizerList is empty, we can adjust both
35823 // limit instead of moving the object to the free list
35824 obj = *(--SegQueueLimit (CriticalFinalizerListSeg));
35825 --SegQueueLimit (FinalizerListSeg);
35829 dprintf (3, ("running finalizer for %Ix (mt: %Ix)", obj, method_table (obj)));
35831 LeaveFinalizeLock();
35836 CFinalize::SetSegForShutDown(BOOL fHasLock)
35841 EnterFinalizeLock();
35842 for (i = 0; i <= max_generation; i++)
35844 unsigned int seg = gen_segment (i);
35845 Object** startIndex = SegQueueLimit (seg)-1;
35846 Object** stopIndex = SegQueue (seg);
35847 for (Object** po = startIndex; po >= stopIndex; po--)
35850 if (method_table(obj)->HasCriticalFinalizer())
35852 MoveItem (po, seg, CriticalFinalizerListSeg);
35856 MoveItem (po, seg, FinalizerListSeg);
35861 LeaveFinalizeLock();
35865 CFinalize::DiscardNonCriticalObjects()
35867 //empty the finalization queue
35868 Object** startIndex = SegQueueLimit (FinalizerListSeg)-1;
35869 Object** stopIndex = SegQueue (FinalizerListSeg);
35870 for (Object** po = startIndex; po >= stopIndex; po--)
35872 MoveItem (po, FinalizerListSeg, FreeList);
35877 CFinalize::GetNumberFinalizableObjects()
35879 return SegQueueLimit (FinalizerListSeg) -
35880 (g_fFinalizerRunOnShutDown ? m_Array : SegQueue(FinalizerListSeg));
35884 CFinalize::FinalizeSegForAppDomain (void *pDomain,
35885 BOOL fRunFinalizers,
35888 BOOL finalizedFound = FALSE;
35889 Object** endIndex = SegQueue (Seg);
35890 for (Object** i = SegQueueLimit (Seg)-1; i >= endIndex ;i--)
35892 CObjectHeader* obj = (CObjectHeader*)*i;
35894 // Objects are put into the finalization queue before they are complete (ie their methodtable
35895 // may be null) so we must check that the object we found has a method table before checking
35896 // if it has the index we are looking for. If the methodtable is null, it can't be from the
35897 // unloading domain, so skip it.
35898 if (method_table(obj) == NULL)
35903 // does the EE actually want us to finalize this object?
35904 if (!GCToEEInterface::ShouldFinalizeObjectForUnload(pDomain, obj))
35909 if (!fRunFinalizers || (obj->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN)
35911 //remove the object because we don't want to
35912 //run the finalizer
35913 MoveItem (i, Seg, FreeList);
35914 //Reset the bit so it will be put back on the queue
35915 //if resurrected and re-registered.
35916 obj->GetHeader()->ClrBit (BIT_SBLK_FINALIZER_RUN);
35920 if (method_table(obj)->HasCriticalFinalizer())
35922 finalizedFound = TRUE;
35923 MoveItem (i, Seg, CriticalFinalizerListSeg);
35927 if (GCToEEInterface::AppDomainIsRudeUnload(pDomain))
35929 MoveItem (i, Seg, FreeList);
35933 finalizedFound = TRUE;
35934 MoveItem (i, Seg, FinalizerListSeg);
35940 return finalizedFound;
35944 CFinalize::FinalizeAppDomain (void *pDomain, bool fRunFinalizers)
35946 bool finalizedFound = false;
35948 unsigned int startSeg = gen_segment (max_generation);
35950 EnterFinalizeLock();
35952 for (unsigned int Seg = startSeg; Seg <= gen_segment (0); Seg++)
35954 if (FinalizeSegForAppDomain (pDomain, fRunFinalizers, Seg))
35956 finalizedFound = true;
35960 LeaveFinalizeLock();
35962 return finalizedFound;
35966 CFinalize::MoveItem (Object** fromIndex,
35967 unsigned int fromSeg,
35968 unsigned int toSeg)
35972 ASSERT (fromSeg != toSeg);
35973 if (fromSeg > toSeg)
35977 // Place the element at the boundary closest to dest
35978 Object** srcIndex = fromIndex;
35979 for (unsigned int i = fromSeg; i != toSeg; i+= step)
35981 Object**& destFill = m_FillPointers[i+(step - 1 )/2];
35982 Object** destIndex = destFill - (step + 1)/2;
35983 if (srcIndex != destIndex)
35985 Object* tmp = *srcIndex;
35986 *srcIndex = *destIndex;
35990 srcIndex = destIndex;
35995 CFinalize::GcScanRoots (promote_func* fn, int hn, ScanContext *pSC)
36001 pSC->thread_number = hn;
36003 //scan the finalization queue
36004 Object** startIndex = SegQueue (CriticalFinalizerListSeg);
36005 Object** stopIndex = SegQueueLimit (FinalizerListSeg);
36007 for (Object** po = startIndex; po < stopIndex; po++)
36010 //dprintf (3, ("scan freacheable %Ix", (size_t)o));
36011 dprintf (3, ("scan f %Ix", (size_t)o));
36012 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
36015 pSC->pCurrentDomain = SystemDomain::GetAppDomainAtIndex(o->GetAppDomainIndex());
36017 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
36023 void CFinalize::WalkFReachableObjects (fq_walk_fn fn)
36025 Object** startIndex = SegQueue (CriticalFinalizerListSeg);
36026 Object** stopCriticalIndex = SegQueueLimit (CriticalFinalizerListSeg);
36027 Object** stopIndex = SegQueueLimit (FinalizerListSeg);
36028 for (Object** po = startIndex; po < stopIndex; po++)
36031 fn(po < stopCriticalIndex, *po);
36036 CFinalize::ScanForFinalization (promote_func* pfn, int gen, BOOL mark_only_p,
36040 sc.promotion = TRUE;
36041 #ifdef MULTIPLE_HEAPS
36042 sc.thread_number = hp->heap_number;
36044 UNREFERENCED_PARAMETER(hp);
36045 #endif //MULTIPLE_HEAPS
36047 BOOL finalizedFound = FALSE;
36049 //start with gen and explore all the younger generations.
36050 unsigned int startSeg = gen_segment (gen);
36052 m_PromotedCount = 0;
36053 for (unsigned int Seg = startSeg; Seg <= gen_segment(0); Seg++)
36055 Object** endIndex = SegQueue (Seg);
36056 for (Object** i = SegQueueLimit (Seg)-1; i >= endIndex ;i--)
36058 CObjectHeader* obj = (CObjectHeader*)*i;
36059 dprintf (3, ("scanning: %Ix", (size_t)obj));
36060 if (!g_theGCHeap->IsPromoted (obj))
36062 dprintf (3, ("freacheable: %Ix", (size_t)obj));
36064 assert (method_table(obj)->HasFinalizer());
36066 if (GCToEEInterface::EagerFinalized(obj))
36068 MoveItem (i, Seg, FreeList);
36070 else if ((obj->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN)
36072 //remove the object because we don't want to
36073 //run the finalizer
36075 MoveItem (i, Seg, FreeList);
36077 //Reset the bit so it will be put back on the queue
36078 //if resurrected and re-registered.
36079 obj->GetHeader()->ClrBit (BIT_SBLK_FINALIZER_RUN);
36086 if (method_table(obj)->HasCriticalFinalizer())
36088 MoveItem (i, Seg, CriticalFinalizerListSeg);
36092 MoveItem (i, Seg, FinalizerListSeg);
36096 #ifdef BACKGROUND_GC
36099 if ((gen == max_generation) && (recursive_gc_sync::background_running_p()))
36101 // TODO - fix the following line.
36102 //assert (gc_heap::background_object_marked ((uint8_t*)obj, FALSE));
36103 dprintf (3, ("%Ix is marked", (size_t)obj));
36106 #endif //BACKGROUND_GC
36110 finalizedFound = !IsSegEmpty(FinalizerListSeg) ||
36111 !IsSegEmpty(CriticalFinalizerListSeg);
36113 if (finalizedFound)
36115 //Promote the f-reachable objects
36117 #ifdef MULTIPLE_HEAPS
36121 #endif //MULTIPLE_HEAPS
36124 hp->settings.found_finalizers = TRUE;
36126 #ifdef BACKGROUND_GC
36127 if (hp->settings.concurrent)
36129 hp->settings.found_finalizers = !(IsSegEmpty(FinalizerListSeg) && IsSegEmpty(CriticalFinalizerListSeg));
36131 #endif //BACKGROUND_GC
36132 if (hp->settings.concurrent && hp->settings.found_finalizers)
36135 GCToEEInterface::EnableFinalization(true);
36139 return finalizedFound;
36142 //Relocates all of the objects in the finalization array
36144 CFinalize::RelocateFinalizationData (int gen, gc_heap* hp)
36147 sc.promotion = FALSE;
36148 #ifdef MULTIPLE_HEAPS
36149 sc.thread_number = hp->heap_number;
36151 UNREFERENCED_PARAMETER(hp);
36152 #endif //MULTIPLE_HEAPS
36154 unsigned int Seg = gen_segment (gen);
36156 Object** startIndex = SegQueue (Seg);
36157 for (Object** po = startIndex; po < SegQueue (FreeList);po++)
36159 GCHeap::Relocate (po, &sc);
36164 CFinalize::UpdatePromotedGenerations (int gen, BOOL gen_0_empty_p)
36166 // update the generation fill pointers.
36167 // if gen_0_empty is FALSE, test each object to find out if
36168 // it was promoted or not
36171 for (int i = min (gen+1, max_generation); i > 0; i--)
36173 m_FillPointers [gen_segment(i)] = m_FillPointers [gen_segment(i-1)];
36178 //Look for demoted or promoted plugs
36180 for (int i = gen; i >= 0; i--)
36182 unsigned int Seg = gen_segment (i);
36183 Object** startIndex = SegQueue (Seg);
36185 for (Object** po = startIndex;
36186 po < SegQueueLimit (gen_segment(i)); po++)
36188 int new_gen = g_theGCHeap->WhichGeneration (*po);
36194 MoveItem (po, gen_segment (i), gen_segment (new_gen));
36199 MoveItem (po, gen_segment (i), gen_segment (new_gen));
36200 //back down in order to see all objects.
36211 CFinalize::GrowArray()
36213 size_t oldArraySize = (m_EndArray - m_Array);
36214 size_t newArraySize = (size_t)(((float)oldArraySize / 10) * 12);
36216 Object** newArray = new (nothrow) Object*[newArraySize];
36219 // It's not safe to throw here, because of the FinalizeLock. Tell our caller
36220 // to throw for us.
36221 // ASSERT (newArray);
36224 memcpy (newArray, m_Array, oldArraySize*sizeof(Object*));
36226 //adjust the fill pointers
36227 for (int i = 0; i < FreeList; i++)
36229 m_FillPointers [i] += (newArray - m_Array);
36232 m_Array = newArray;
36233 m_EndArray = &m_Array [newArraySize];
36239 void CFinalize::CheckFinalizerObjects()
36241 for (int i = 0; i <= max_generation; i++)
36243 Object **startIndex = SegQueue (gen_segment (i));
36244 Object **stopIndex = SegQueueLimit (gen_segment (i));
36246 for (Object **po = startIndex; po < stopIndex; po++)
36248 if ((int)g_theGCHeap->WhichGeneration (*po) < i)
36250 ((CObjectHeader*)*po)->Validate();
36254 #endif //VERIFY_HEAP
36256 #endif // FEATURE_PREMORTEM_FINALIZATION
36259 //------------------------------------------------------------------------------
36261 // End of VM specific support
36263 //------------------------------------------------------------------------------
36264 void gc_heap::walk_heap_per_heap (walk_fn fn, void* context, int gen_number, BOOL walk_large_object_heap_p)
36266 generation* gen = gc_heap::generation_of (gen_number);
36267 heap_segment* seg = generation_start_segment (gen);
36268 uint8_t* x = ((gen_number == max_generation) ? heap_segment_mem (seg) :
36269 generation_allocation_start (gen));
36271 uint8_t* end = heap_segment_allocated (seg);
36272 BOOL small_object_segments = TRUE;
36273 int align_const = get_alignment_constant (small_object_segments);
36280 if ((seg = heap_segment_next (seg)) != 0)
36282 x = heap_segment_mem (seg);
36283 end = heap_segment_allocated (seg);
36288 if (small_object_segments && walk_large_object_heap_p)
36291 small_object_segments = FALSE;
36292 align_const = get_alignment_constant (small_object_segments);
36293 seg = generation_start_segment (large_object_generation);
36294 x = heap_segment_mem (seg);
36295 end = heap_segment_allocated (seg);
36305 size_t s = size (x);
36306 CObjectHeader* o = (CObjectHeader*)x;
36311 _ASSERTE(((size_t)o & 0x3) == 0); // Last two bits should never be set at this point
36313 if (!fn (o->GetObjectBase(), context))
36316 x = x + Align (s, align_const);
36320 void gc_heap::walk_finalize_queue (fq_walk_fn fn)
36322 #ifdef FEATURE_PREMORTEM_FINALIZATION
36323 finalize_queue->WalkFReachableObjects (fn);
36324 #endif //FEATURE_PREMORTEM_FINALIZATION
36327 void gc_heap::walk_heap (walk_fn fn, void* context, int gen_number, BOOL walk_large_object_heap_p)
36329 #ifdef MULTIPLE_HEAPS
36330 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36332 gc_heap* hp = gc_heap::g_heaps [hn];
36334 hp->walk_heap_per_heap (fn, context, gen_number, walk_large_object_heap_p);
36337 walk_heap_per_heap(fn, context, gen_number, walk_large_object_heap_p);
36338 #endif //MULTIPLE_HEAPS
36341 void GCHeap::DiagWalkObject (Object* obj, walk_fn fn, void* context)
36343 uint8_t* o = (uint8_t*)obj;
36346 go_through_object_cl (method_table (o), o, size(o), oo,
36350 Object *oh = (Object*)*oo;
36351 if (!fn (oh, context))
36359 void GCHeap::DiagWalkSurvivorsWithType (void* gc_context, record_surv_fn fn, void* diag_context, walk_surv_type type)
36361 gc_heap* hp = (gc_heap*)gc_context;
36362 hp->walk_survivors (fn, diag_context, type);
36365 void GCHeap::DiagWalkHeap (walk_fn fn, void* context, int gen_number, bool walk_large_object_heap_p)
36367 gc_heap::walk_heap (fn, context, gen_number, walk_large_object_heap_p);
36370 void GCHeap::DiagWalkFinalizeQueue (void* gc_context, fq_walk_fn fn)
36372 gc_heap* hp = (gc_heap*)gc_context;
36373 hp->walk_finalize_queue (fn);
36376 void GCHeap::DiagScanFinalizeQueue (fq_scan_fn fn, ScanContext* sc)
36378 #ifdef MULTIPLE_HEAPS
36379 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36381 gc_heap* hp = gc_heap::g_heaps [hn];
36382 hp->finalize_queue->GcScanRoots(fn, hn, sc);
36385 pGenGCHeap->finalize_queue->GcScanRoots(fn, 0, sc);
36386 #endif //MULTIPLE_HEAPS
36389 void GCHeap::DiagScanHandles (handle_scan_fn fn, int gen_number, ScanContext* context)
36391 UNREFERENCED_PARAMETER(gen_number);
36392 GCScan::GcScanHandlesForProfilerAndETW (max_generation, context, fn);
36395 void GCHeap::DiagScanDependentHandles (handle_scan_fn fn, int gen_number, ScanContext* context)
36397 UNREFERENCED_PARAMETER(gen_number);
36398 GCScan::GcScanDependentHandlesForProfilerAndETW (max_generation, context, fn);
36401 // Go through and touch (read) each page straddled by a memory block.
36402 void TouchPages(void * pStart, size_t cb)
36404 const uint32_t pagesize = OS_PAGE_SIZE;
36405 _ASSERTE(0 == (pagesize & (pagesize-1))); // Must be a power of 2.
36408 VOLATILE(char)* pEnd = (VOLATILE(char)*)(cb + (char*)pStart);
36409 VOLATILE(char)* p = (VOLATILE(char)*)(((char*)pStart) - (((size_t)pStart) & (pagesize-1)));
36413 a = VolatileLoad(p);
36414 //printf("Touching page %lxh\n", (uint32_t)p);
36420 #if defined(WRITE_BARRIER_CHECK) && !defined (SERVER_GC)
36421 // This code is designed to catch the failure to update the write barrier
36422 // The way it works is to copy the whole heap right after every GC. The write
36423 // barrier code has been modified so that it updates the shadow as well as the
36424 // real GC heap. Before doing the next GC, we walk the heap, looking for pointers
36425 // that were updated in the real heap, but not the shadow. A mismatch indicates
36426 // an error. The offending code can be found by breaking after the correct GC,
36427 // and then placing a data breakpoint on the Heap location that was updated without
36428 // going through the write barrier.
36430 // Called at process shutdown
36431 void deleteGCShadow()
36433 if (g_GCShadow != 0)
36434 GCToOSInterface::VirtualRelease (g_GCShadow, g_GCShadowEnd - g_GCShadow);
36439 // Called at startup and right after a GC, get a snapshot of the GC Heap
36440 void initGCShadow()
36442 if (!(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_BARRIERCHECK))
36445 size_t len = g_gc_highest_address - g_gc_lowest_address;
36446 if (len > (size_t)(g_GCShadowEnd - g_GCShadow))
36449 g_GCShadowEnd = g_GCShadow = (uint8_t *)GCToOSInterface::VirtualReserve(len, 0, VirtualReserveFlags::None);
36450 if (g_GCShadow == NULL || !GCToOSInterface::VirtualCommit(g_GCShadow, len))
36452 _ASSERTE(!"Not enough memory to run HeapVerify level 2");
36453 // If after the assert we decide to allow the program to continue
36454 // running we need to be in a state that will not trigger any
36455 // additional AVs while we fail to allocate a shadow segment, i.e.
36456 // ensure calls to updateGCShadow() checkGCWriteBarrier() don't AV
36461 g_GCShadowEnd += len;
36464 // save the value of g_gc_lowest_address at this time. If this value changes before
36465 // the next call to checkGCWriteBarrier() it means we extended the heap (with a
36466 // large object segment most probably), and the whole shadow segment is inconsistent.
36467 g_shadow_lowest_address = g_gc_lowest_address;
36469 //****** Copy the whole GC heap ******
36471 // NOTE: This is the one situation where the combination of heap_segment_rw(gen_start_segment())
36472 // can produce a NULL result. This is because the initialization has not completed.
36474 generation* gen = gc_heap::generation_of (max_generation);
36475 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
36477 ptrdiff_t delta = g_GCShadow - g_gc_lowest_address;
36478 BOOL small_object_segments = TRUE;
36483 if (small_object_segments)
36485 small_object_segments = FALSE;
36486 seg = heap_segment_rw (generation_start_segment (gc_heap::generation_of (max_generation+1)));
36492 // Copy the segment
36493 uint8_t* start = heap_segment_mem(seg);
36494 uint8_t* end = heap_segment_allocated (seg);
36495 memcpy(start + delta, start, end - start);
36496 seg = heap_segment_next_rw (seg);
36500 #define INVALIDGCVALUE (void*)((size_t)0xcccccccd)
36502 // test to see if 'ptr' was only updated via the write barrier.
36503 inline void testGCShadow(Object** ptr)
36505 Object** shadow = (Object**) &g_GCShadow[((uint8_t*) ptr - g_gc_lowest_address)];
36506 if (*ptr != 0 && (uint8_t*) shadow < g_GCShadowEnd && *ptr != *shadow)
36509 // If you get this assertion, someone updated a GC pointer in the heap without
36510 // using the write barrier. To find out who, check the value of
36511 // dd_collection_count (dynamic_data_of (0)). Also
36512 // note the value of 'ptr'. Rerun the App that the previous GC just occurred.
36513 // Then put a data breakpoint for the value of 'ptr' Then check every write
36514 // to pointer between the two GCs. The last one is not using the write barrier.
36516 // If the memory of interest does not exist at system startup,
36517 // you need to set the data breakpoint right after the memory gets committed
36518 // Set a breakpoint at the end of grow_heap_segment, and put the value of 'ptr'
36519 // in the memory window. run until the memory gets mapped. Then you can set
36522 // Note a recent change, we've identified race conditions when updating the gc shadow.
36523 // Throughout the runtime, code will update an address in the gc heap, then erect the
36524 // write barrier, which calls updateGCShadow. With an app that pounds one heap location
36525 // from multiple threads, you can hit this assert even though all involved are using the
36526 // write barrier properly. Thusly, we detect the race and set this location to INVALIDGCVALUE.
36527 // TODO: the code in jithelp.asm doesn't call updateGCShadow, and hasn't been
36528 // TODO: fixed to detect the race. We've only seen this race from VolatileWritePtr,
36529 // TODO: so elect not to fix jithelp.asm at this time. It should be done if we start hitting
36530 // TODO: erroneous asserts in here.
36532 if(*shadow!=INVALIDGCVALUE)
36534 #ifdef FEATURE_BASICFREEZE
36535 // Write barriers for stores of references to frozen objects may be optimized away.
36536 if (!gc_heap::frozen_object_p(*ptr))
36537 #endif // FEATURE_BASICFREEZE
36539 _ASSERTE(!"Pointer updated without using write barrier");
36545 printf("saw a INVALIDGCVALUE. (just to let you know)\n");
36551 void testGCShadowHelper (uint8_t* x)
36553 size_t s = size (x);
36554 if (contain_pointers (x))
36556 go_through_object_nostart (method_table(x), x, s, oo,
36557 { testGCShadow((Object**) oo); });
36561 // Walk the whole heap, looking for pointers that were not updated with the write barrier.
36562 void checkGCWriteBarrier()
36564 // g_shadow_lowest_address != g_gc_lowest_address means the GC heap was extended by a segment
36565 // and the GC shadow segment did not track that change!
36566 if (g_GCShadowEnd <= g_GCShadow || g_shadow_lowest_address != g_gc_lowest_address)
36568 // No shadow stack, nothing to check.
36573 generation* gen = gc_heap::generation_of (max_generation);
36574 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
36576 PREFIX_ASSUME(seg != NULL);
36580 uint8_t* x = heap_segment_mem(seg);
36581 while (x < heap_segment_allocated (seg))
36583 size_t s = size (x);
36584 testGCShadowHelper (x);
36587 seg = heap_segment_next_rw (seg);
36592 // go through large object heap
36593 int alignment = get_alignment_constant(FALSE);
36594 generation* gen = gc_heap::generation_of (max_generation+1);
36595 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
36597 PREFIX_ASSUME(seg != NULL);
36601 uint8_t* x = heap_segment_mem(seg);
36602 while (x < heap_segment_allocated (seg))
36604 size_t s = size (x);
36605 testGCShadowHelper (x);
36606 x = x + Align (s, alignment);
36608 seg = heap_segment_next_rw (seg);
36612 #endif //WRITE_BARRIER_CHECK && !SERVER_GC
36614 #endif // !DACCESS_COMPILE
36616 #ifdef FEATURE_BASICFREEZE
36617 void gc_heap::walk_read_only_segment(heap_segment *seg, void *pvContext, object_callback_func pfnMethodTable, object_callback_func pfnObjRef)
36619 #ifdef DACCESS_COMPILE
36620 UNREFERENCED_PARAMETER(seg);
36621 UNREFERENCED_PARAMETER(pvContext);
36622 UNREFERENCED_PARAMETER(pfnMethodTable);
36623 UNREFERENCED_PARAMETER(pfnObjRef);
36625 uint8_t *o = heap_segment_mem(seg);
36627 // small heap alignment constant
36628 int alignment = get_alignment_constant(TRUE);
36630 while (o < heap_segment_allocated(seg))
36632 pfnMethodTable(pvContext, o);
36634 if (contain_pointers (o))
36636 go_through_object_nostart (method_table (o), o, size(o), oo,
36639 pfnObjRef(pvContext, oo);
36644 o += Align(size(o), alignment);
36646 #endif //!DACCESS_COMPILE
36648 #endif // FEATURE_BASICFREEZE
36650 #ifndef DACCESS_COMPILE
36651 HRESULT GCHeap::WaitUntilConcurrentGCCompleteAsync(int millisecondsTimeout)
36653 #ifdef BACKGROUND_GC
36654 if (recursive_gc_sync::background_running_p())
36656 uint32_t dwRet = pGenGCHeap->background_gc_wait(awr_ignored, millisecondsTimeout);
36657 if (dwRet == WAIT_OBJECT_0)
36659 else if (dwRet == WAIT_TIMEOUT)
36660 return HRESULT_FROM_WIN32(ERROR_TIMEOUT);
36662 return E_FAIL; // It is not clear if what the last error would be if the wait failed,
36663 // as there are too many layers in between. The best we can do is to return E_FAIL;
36669 #endif // !DACCESS_COMPILE
36671 void GCHeap::TemporaryEnableConcurrentGC()
36673 #ifdef BACKGROUND_GC
36674 gc_heap::temp_disable_concurrent_p = false;
36675 #endif //BACKGROUND_GC
36678 void GCHeap::TemporaryDisableConcurrentGC()
36680 #ifdef BACKGROUND_GC
36681 gc_heap::temp_disable_concurrent_p = true;
36682 #endif //BACKGROUND_GC
36685 bool GCHeap::IsConcurrentGCEnabled()
36687 #ifdef BACKGROUND_GC
36688 return (gc_heap::gc_can_use_concurrent && !(gc_heap::temp_disable_concurrent_p));
36691 #endif //BACKGROUND_GC
36694 void GCHeap::SetFinalizeRunOnShutdown(bool value)
36696 g_fFinalizerRunOnShutDown = value;
36699 void PopulateDacVars(GcDacVars *gcDacVars)
36701 #ifndef DACCESS_COMPILE
36702 assert(gcDacVars != nullptr);
36704 gcDacVars->major_version_number = 1;
36705 gcDacVars->minor_version_number = 0;
36706 gcDacVars->built_with_svr = &g_built_with_svr_gc;
36707 gcDacVars->build_variant = &g_build_variant;
36708 gcDacVars->gc_structures_invalid_cnt = const_cast<int32_t*>(&GCScan::m_GcStructuresInvalidCnt);
36709 gcDacVars->generation_size = sizeof(generation);
36710 gcDacVars->max_gen = &g_max_generation;
36711 #ifndef MULTIPLE_HEAPS
36712 gcDacVars->mark_array = &gc_heap::mark_array;
36713 gcDacVars->ephemeral_heap_segment = reinterpret_cast<dac_heap_segment**>(&gc_heap::ephemeral_heap_segment);
36714 gcDacVars->current_c_gc_state = const_cast<c_gc_state*>(&gc_heap::current_c_gc_state);
36715 gcDacVars->saved_sweep_ephemeral_seg = reinterpret_cast<dac_heap_segment**>(&gc_heap::saved_sweep_ephemeral_seg);
36716 gcDacVars->saved_sweep_ephemeral_start = &gc_heap::saved_sweep_ephemeral_start;
36717 gcDacVars->background_saved_lowest_address = &gc_heap::background_saved_lowest_address;
36718 gcDacVars->background_saved_highest_address = &gc_heap::background_saved_highest_address;
36719 gcDacVars->alloc_allocated = &gc_heap::alloc_allocated;
36720 gcDacVars->next_sweep_obj = &gc_heap::next_sweep_obj;
36721 gcDacVars->oom_info = &gc_heap::oom_info;
36722 gcDacVars->finalize_queue = reinterpret_cast<dac_finalize_queue**>(&gc_heap::finalize_queue);
36723 gcDacVars->generation_table = reinterpret_cast<dac_generation**>(&gc_heap::generation_table);
36724 #ifdef GC_CONFIG_DRIVEN
36725 gcDacVars->gc_global_mechanisms = reinterpret_cast<size_t**>(&gc_global_mechanisms);
36726 gcDacVars->interesting_data_per_heap = reinterpret_cast<size_t**>(&gc_heap::interesting_data_per_heap);
36727 gcDacVars->compact_reasons_per_heap = reinterpret_cast<size_t**>(&gc_heap::compact_reasons_per_heap);
36728 gcDacVars->expand_mechanisms_per_heap = reinterpret_cast<size_t**>(&gc_heap::expand_mechanisms_per_heap);
36729 gcDacVars->interesting_mechanism_bits_per_heap = reinterpret_cast<size_t**>(&gc_heap::interesting_mechanism_bits_per_heap);
36730 #endif // GC_CONFIG_DRIVEN
36731 #ifdef HEAP_ANALYZE
36732 gcDacVars->internal_root_array = &gc_heap::internal_root_array;
36733 gcDacVars->internal_root_array_index = &gc_heap::internal_root_array_index;
36734 gcDacVars->heap_analyze_success = &gc_heap::heap_analyze_success;
36735 #endif // HEAP_ANALYZE
36737 gcDacVars->n_heaps = &gc_heap::n_heaps;
36738 gcDacVars->g_heaps = reinterpret_cast<dac_gc_heap***>(&gc_heap::g_heaps);
36739 #endif // MULTIPLE_HEAPS
36740 #endif // DACCESS_COMPILE