1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
3 // See the LICENSE file in the project root for more information.
9 // GC automatically manages memory allocated by managed code.
10 // The design doc for GC can be found at Documentation/botr/garbage-collection.md
12 // This file includes both the code for GC and the allocator. The most common
13 // case for a GC to be triggered is from the allocator code. See
14 // code:#try_allocate_more_space where it calls GarbageCollectGeneration.
16 // Entry points for the allocator are GCHeap::Alloc* which are called by the
17 // allocation helpers in gcscan.cpp
24 // We just needed a simple random number generator for testing.
30 static uint64_t get_rand()
32 x = (314159269*x+278281) & 0x7FFFFFFF;
36 // obtain random number in the range 0 .. r-1
37 static uint64_t get_rand(uint64_t r) {
39 uint64_t x = (uint64_t)((get_rand() * r) >> 31);
44 uint64_t gc_rand::x = 0;
46 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
47 BOOL bgc_heap_walk_for_etw_p = FALSE;
48 #endif //BACKGROUND_GC && FEATURE_EVENT_TRACE
50 #if defined(FEATURE_REDHAWK)
51 #define MAYBE_UNUSED_VAR(v) v = v
53 #define MAYBE_UNUSED_VAR(v)
54 #endif // FEATURE_REDHAWK
56 #define MAX_PTR ((uint8_t*)(~(ptrdiff_t)0))
57 #define commit_min_th (16*OS_PAGE_SIZE)
60 #define partial_size_th 100
61 #define num_partial_refs 64
63 #define partial_size_th 100
64 #define num_partial_refs 32
67 #define demotion_plug_len_th (6*1024*1024)
70 #define MARK_STACK_INITIAL_LENGTH 1024
72 #define MARK_STACK_INITIAL_LENGTH 128
75 #define LOH_PIN_QUEUE_LENGTH 100
76 #define LOH_PIN_DECAY 10
79 // Right now we support maximum 1024 procs - meaning that we will create at most
80 // that many GC threads and GC heaps.
81 #define MAX_SUPPORTED_CPUS 1024
83 #define MAX_SUPPORTED_CPUS 64
86 uint32_t yp_spin_count_unit = 0;
87 size_t loh_size_threshold = LARGE_OBJECT_SIZE;
89 #ifdef GC_CONFIG_DRIVEN
90 int compact_ratio = 0;
91 #endif //GC_CONFIG_DRIVEN
93 // See comments in reset_memory.
94 BOOL reset_mm_p = TRUE;
96 bool g_fFinalizerRunOnShutDown = false;
99 bool g_built_with_svr_gc = true;
101 bool g_built_with_svr_gc = false;
102 #endif // FEATURE_SVR_GC
104 #if defined(BUILDENV_DEBUG)
105 uint8_t g_build_variant = 0;
106 #elif defined(BUILDENV_CHECKED)
107 uint8_t g_build_variant = 1;
109 uint8_t g_build_variant = 2;
110 #endif // defined(BUILDENV_DEBUG)
112 VOLATILE(int32_t) g_no_gc_lock = -1;
114 #if defined (TRACE_GC) && !defined (DACCESS_COMPILE)
115 const char * const allocation_state_str[] = {
124 "try_free_full_seg_in_bgc",
125 "try_free_after_bgc",
128 "acquire_seg_after_cg",
129 "acquire_seg_after_bgc",
130 "check_and_wait_for_bgc",
131 "trigger_full_compact_gc",
132 "trigger_ephemeral_gc",
133 "trigger_2nd_ephemeral_gc",
137 const char * const msl_take_state_str[] = {
153 #endif //TRACE_GC && !DACCESS_COMPILE
156 // Keep this in sync with the definition of gc_reason
157 #if (defined(DT_LOG) || defined(TRACE_GC)) && !defined (DACCESS_COMPILE)
158 static const char* const str_gc_reasons[] =
170 "induced_compacting",
173 "lowmemory_host_blocking"
176 static const char* const str_gc_pause_modes[] =
181 "sustained_low_latency",
184 #endif // defined(DT_LOG) || defined(TRACE_GC)
187 BOOL is_induced (gc_reason reason)
189 return ((reason == reason_induced) ||
190 (reason == reason_induced_noforce) ||
191 (reason == reason_lowmemory) ||
192 (reason == reason_lowmemory_blocking) ||
193 (reason == reason_induced_compacting) ||
194 (reason == reason_lowmemory_host) ||
195 (reason == reason_lowmemory_host_blocking));
199 BOOL is_induced_blocking (gc_reason reason)
201 return ((reason == reason_induced) ||
202 (reason == reason_lowmemory_blocking) ||
203 (reason == reason_induced_compacting) ||
204 (reason == reason_lowmemory_host_blocking));
207 #ifndef DACCESS_COMPILE
210 size_t GetHighPrecisionTimeStamp()
212 int64_t ts = GCToOSInterface::QueryPerformanceCounter();
214 return (size_t)(ts / (qpf / 1000));
219 // There is a current and a prior copy of the statistics. This allows us to display deltas per reporting
220 // interval, as well as running totals. The 'min' and 'max' values require special treatment. They are
221 // Reset (zeroed) in the current statistics when we begin a new interval and they are updated via a
222 // comparison with the global min/max.
223 GCStatistics g_GCStatistics;
224 GCStatistics g_LastGCStatistics;
226 char* GCStatistics::logFileName = NULL;
227 FILE* GCStatistics::logFile = NULL;
229 void GCStatistics::AddGCStats(const gc_mechanisms& settings, size_t timeInMSec)
232 if (settings.concurrent)
234 bgc.Accumulate((uint32_t)timeInMSec*1000);
237 else if (settings.background_p)
239 fgc.Accumulate((uint32_t)timeInMSec*1000);
241 if (settings.compaction)
243 assert(settings.condemned_generation < max_generation);
244 cntFGCGen[settings.condemned_generation]++;
247 #endif // BACKGROUND_GC
249 ngc.Accumulate((uint32_t)timeInMSec*1000);
251 if (settings.compaction)
253 cntNGCGen[settings.condemned_generation]++;
256 if (is_induced (settings.reason))
257 cntReasons[(int)reason_induced]++;
258 else if (settings.stress_induced)
259 cntReasons[(int)reason_gcstress]++;
261 cntReasons[(int)settings.reason]++;
264 if (settings.concurrent || !settings.background_p)
266 #endif // BACKGROUND_GC
270 #endif // BACKGROUND_GC
273 void GCStatistics::Initialize()
275 LIMITED_METHOD_CONTRACT;
276 // for efficiency sake we're taking a dependency on the layout of a C++ object
277 // with a vtable. protect against violations of our premise:
278 static_assert(offsetof(GCStatistics, cntDisplay) == sizeof(void*),
279 "The first field of GCStatistics follows the pointer sized vtable");
281 int podOffs = offsetof(GCStatistics, cntDisplay); // offset of the first POD field
282 memset((uint8_t*)(&g_GCStatistics)+podOffs, 0, sizeof(g_GCStatistics)-podOffs);
283 memset((uint8_t*)(&g_LastGCStatistics)+podOffs, 0, sizeof(g_LastGCStatistics)-podOffs);
286 void GCStatistics::DisplayAndUpdate()
288 LIMITED_METHOD_CONTRACT;
290 if (logFileName == NULL || logFile == NULL)
295 fprintf(logFile, "\nGCMix **** Initialize *****\n\n");
297 fprintf(logFile, "GCMix **** Summary ***** %d\n", cntDisplay);
299 // NGC summary (total, timing info)
300 ngc.DisplayAndUpdate(logFile, "NGC ", &g_LastGCStatistics.ngc, cntNGC, g_LastGCStatistics.cntNGC, msec);
302 // FGC summary (total, timing info)
303 fgc.DisplayAndUpdate(logFile, "FGC ", &g_LastGCStatistics.fgc, cntFGC, g_LastGCStatistics.cntFGC, msec);
306 bgc.DisplayAndUpdate(logFile, "BGC ", &g_LastGCStatistics.bgc, cntBGC, g_LastGCStatistics.cntBGC, msec);
308 // NGC/FGC break out by generation & compacting vs. sweeping
309 fprintf(logFile, "NGC ");
310 for (int i = max_generation; i >= 0; --i)
311 fprintf(logFile, "gen%d %d (%d). ", i, cntNGCGen[i]-g_LastGCStatistics.cntNGCGen[i], cntNGCGen[i]);
312 fprintf(logFile, "\n");
314 fprintf(logFile, "FGC ");
315 for (int i = max_generation-1; i >= 0; --i)
316 fprintf(logFile, "gen%d %d (%d). ", i, cntFGCGen[i]-g_LastGCStatistics.cntFGCGen[i], cntFGCGen[i]);
317 fprintf(logFile, "\n");
319 // Compacting vs. Sweeping break out
320 int _cntSweep = cntNGC-cntCompactNGC;
321 int _cntLastSweep = g_LastGCStatistics.cntNGC-g_LastGCStatistics.cntCompactNGC;
322 fprintf(logFile, "NGC Sweeping %d (%d) Compacting %d (%d)\n",
323 _cntSweep - _cntLastSweep, _cntSweep,
324 cntCompactNGC - g_LastGCStatistics.cntCompactNGC, cntCompactNGC);
326 _cntSweep = cntFGC-cntCompactFGC;
327 _cntLastSweep = g_LastGCStatistics.cntFGC-g_LastGCStatistics.cntCompactFGC;
328 fprintf(logFile, "FGC Sweeping %d (%d) Compacting %d (%d)\n",
329 _cntSweep - _cntLastSweep, _cntSweep,
330 cntCompactFGC - g_LastGCStatistics.cntCompactFGC, cntCompactFGC);
334 for (int reason=(int)reason_alloc_soh; reason <= (int)reason_gcstress; ++reason)
336 if (cntReasons[reason] != 0)
337 fprintf(logFile, "%s %d (%d). ", str_gc_reasons[reason],
338 cntReasons[reason]-g_LastGCStatistics.cntReasons[reason], cntReasons[reason]);
341 fprintf(logFile, "\n\n");
343 // flush the log file...
347 g_LastGCStatistics = *this;
357 size_t round_up_power2 (size_t size)
359 // Get the 0-based index of the most-significant bit in size-1.
360 // If the call failed (because size-1 is zero), size must be 1,
361 // so return 1 (because 1 rounds up to itself).
362 DWORD highest_set_bit_index;
369 &highest_set_bit_index, size - 1)) { return 1; }
371 // The size == 0 case (which would have overflowed to SIZE_MAX when decremented)
372 // is handled below by relying on the fact that highest_set_bit_index is the maximum value
373 // (31 or 63, depending on sizeof(size_t)) and left-shifting a value >= 2 by that
374 // number of bits shifts in zeros from the right, resulting in an output of zero.
375 return static_cast<size_t>(2) << highest_set_bit_index;
379 size_t round_down_power2 (size_t size)
381 // Get the 0-based index of the most-significant bit in size.
382 // If the call failed, size must be zero so return zero.
383 DWORD highest_set_bit_index;
390 &highest_set_bit_index, size)) { return 0; }
392 // Left-shift 1 by highest_set_bit_index to get back a value containing only
393 // the most-significant set bit of size, i.e. size rounded down
394 // to the next power-of-two value.
395 return static_cast<size_t>(1) << highest_set_bit_index;
398 // Get the 0-based index of the most-significant bit in the value.
399 // Returns -1 if the input value is zero (i.e. has no set bits).
401 int index_of_highest_set_bit (size_t value)
403 // Get the 0-based index of the most-significant bit in the value.
404 // If the call failed (because value is zero), return -1.
405 DWORD highest_set_bit_index;
412 &highest_set_bit_index, value)) ? -1 : static_cast<int>(highest_set_bit_index);
416 int relative_index_power2_plug (size_t power2)
418 int index = index_of_highest_set_bit (power2);
419 assert (index <= MAX_INDEX_POWER2);
421 return ((index < MIN_INDEX_POWER2) ? 0 : (index - MIN_INDEX_POWER2));
425 int relative_index_power2_free_space (size_t power2)
427 int index = index_of_highest_set_bit (power2);
428 assert (index <= MAX_INDEX_POWER2);
430 return ((index < MIN_INDEX_POWER2) ? -1 : (index - MIN_INDEX_POWER2));
434 uint32_t bgc_alloc_spin_count = 140;
435 uint32_t bgc_alloc_spin_count_loh = 16;
436 uint32_t bgc_alloc_spin = 2;
440 void c_write (uint32_t& place, uint32_t value)
442 Interlocked::Exchange (&place, value);
446 #ifndef DACCESS_COMPILE
447 // If every heap's gen2 or gen3 size is less than this threshold we will do a blocking GC.
448 const size_t bgc_min_per_heap = 4*1024*1024;
450 int gc_heap::gchist_index = 0;
451 gc_mechanisms_store gc_heap::gchist[max_history_count];
453 #ifndef MULTIPLE_HEAPS
454 size_t gc_heap::total_promoted_bytes = 0;
455 VOLATILE(bgc_state) gc_heap::current_bgc_state = bgc_not_in_process;
456 int gc_heap::gchist_index_per_heap = 0;
457 gc_heap::gc_history gc_heap::gchist_per_heap[max_history_count];
458 #endif //MULTIPLE_HEAPS
460 void gc_heap::add_to_history_per_heap()
463 gc_history* current_hist = &gchist_per_heap[gchist_index_per_heap];
464 current_hist->gc_index = settings.gc_index;
465 current_hist->current_bgc_state = current_bgc_state;
466 size_t elapsed = dd_gc_elapsed_time (dynamic_data_of (0));
467 current_hist->gc_time_ms = (uint32_t)elapsed;
468 current_hist->gc_efficiency = (elapsed ? (total_promoted_bytes / elapsed) : total_promoted_bytes);
469 current_hist->eph_low = generation_allocation_start (generation_of (max_generation-1));
470 current_hist->gen0_start = generation_allocation_start (generation_of (0));
471 current_hist->eph_high = heap_segment_allocated (ephemeral_heap_segment);
473 current_hist->bgc_lowest = background_saved_lowest_address;
474 current_hist->bgc_highest = background_saved_highest_address;
475 #endif //BACKGROUND_GC
476 current_hist->fgc_lowest = lowest_address;
477 current_hist->fgc_highest = highest_address;
478 current_hist->g_lowest = g_gc_lowest_address;
479 current_hist->g_highest = g_gc_highest_address;
481 gchist_index_per_heap++;
482 if (gchist_index_per_heap == max_history_count)
484 gchist_index_per_heap = 0;
489 void gc_heap::add_to_history()
492 gc_mechanisms_store* current_settings = &gchist[gchist_index];
493 current_settings->store (&settings);
496 if (gchist_index == max_history_count)
503 #endif //DACCESS_COMPILE
504 #endif //BACKGROUND_GC
506 #if defined(TRACE_GC) && !defined(DACCESS_COMPILE)
507 BOOL gc_log_on = TRUE;
509 size_t gc_log_file_size = 0;
511 size_t gc_buffer_index = 0;
512 size_t max_gc_buffers = 0;
514 static CLRCriticalSection gc_log_lock;
516 // we keep this much in a buffer and only flush when the buffer is full
517 #define gc_log_buffer_size (1024*1024)
518 uint8_t* gc_log_buffer = 0;
519 size_t gc_log_buffer_offset = 0;
521 void log_va_msg(const char *fmt, va_list args)
525 const int BUFFERSIZE = 512;
526 static char rgchBuffer[BUFFERSIZE];
527 char * pBuffer = &rgchBuffer[0];
530 int buffer_start = 1;
531 int pid_len = sprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, "[%5d]", (uint32_t)GCToOSInterface::GetCurrentThreadIdForLogging());
532 buffer_start += pid_len;
533 memset(&pBuffer[buffer_start], '-', BUFFERSIZE - buffer_start);
534 int msg_len = _vsnprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, _TRUNCATE, fmt, args);
537 msg_len = BUFFERSIZE - buffer_start;
540 msg_len += buffer_start;
542 if ((gc_log_buffer_offset + msg_len) > (gc_log_buffer_size - 12))
545 memset (index_str, '-', 8);
546 sprintf_s (index_str, _countof(index_str), "%d", (int)gc_buffer_index);
547 gc_log_buffer[gc_log_buffer_offset] = '\n';
548 memcpy (gc_log_buffer + (gc_log_buffer_offset + 1), index_str, 8);
551 if (gc_buffer_index > max_gc_buffers)
553 fseek (gc_log, 0, SEEK_SET);
556 fwrite(gc_log_buffer, gc_log_buffer_size, 1, gc_log);
558 memset (gc_log_buffer, '*', gc_log_buffer_size);
559 gc_log_buffer_offset = 0;
562 memcpy (gc_log_buffer + gc_log_buffer_offset, pBuffer, msg_len);
563 gc_log_buffer_offset += msg_len;
568 void GCLog (const char *fmt, ... )
570 if (gc_log_on && (gc_log != NULL))
574 log_va_msg (fmt, args);
578 #endif // TRACE_GC && !DACCESS_COMPILE
580 #if defined(GC_CONFIG_DRIVEN) && !defined(DACCESS_COMPILE)
582 BOOL gc_config_log_on = FALSE;
583 FILE* gc_config_log = NULL;
585 // we keep this much in a buffer and only flush when the buffer is full
586 #define gc_config_log_buffer_size (1*1024) // TEMP
587 uint8_t* gc_config_log_buffer = 0;
588 size_t gc_config_log_buffer_offset = 0;
590 // For config since we log so little we keep the whole history. Also it's only
591 // ever logged by one thread so no need to synchronize.
592 void log_va_msg_config(const char *fmt, va_list args)
594 const int BUFFERSIZE = 256;
595 static char rgchBuffer[BUFFERSIZE];
596 char * pBuffer = &rgchBuffer[0];
599 int buffer_start = 1;
600 int msg_len = _vsnprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, _TRUNCATE, fmt, args );
601 assert (msg_len != -1);
602 msg_len += buffer_start;
604 if ((gc_config_log_buffer_offset + msg_len) > gc_config_log_buffer_size)
606 fwrite(gc_config_log_buffer, gc_config_log_buffer_offset, 1, gc_config_log);
607 fflush(gc_config_log);
608 gc_config_log_buffer_offset = 0;
611 memcpy (gc_config_log_buffer + gc_config_log_buffer_offset, pBuffer, msg_len);
612 gc_config_log_buffer_offset += msg_len;
615 void GCLogConfig (const char *fmt, ... )
617 if (gc_config_log_on && (gc_config_log != NULL))
620 va_start( args, fmt );
621 log_va_msg_config (fmt, args);
624 #endif // GC_CONFIG_DRIVEN && !DACCESS_COMPILE
626 #ifdef SYNCHRONIZATION_STATS
628 // Number of GCs have we done since we last logged.
629 static unsigned int gc_count_during_log;
630 // In ms. This is how often we print out stats.
631 static const unsigned int log_interval = 5000;
632 // Time (in ms) when we start a new log interval.
633 static unsigned int log_start_tick;
634 static unsigned int gc_lock_contended;
635 static int64_t log_start_hires;
636 // Cycles accumulated in SuspendEE during log_interval.
637 static uint64_t suspend_ee_during_log;
638 // Cycles accumulated in RestartEE during log_interval.
639 static uint64_t restart_ee_during_log;
640 static uint64_t gc_during_log;
642 #endif //SYNCHRONIZATION_STATS
645 init_sync_log_stats()
647 #ifdef SYNCHRONIZATION_STATS
648 if (gc_count_during_log == 0)
650 gc_heap::init_sync_stats();
651 suspend_ee_during_log = 0;
652 restart_ee_during_log = 0;
654 gc_lock_contended = 0;
656 log_start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
657 log_start_hires = GCToOSInterface::QueryPerformanceCounter();
659 gc_count_during_log++;
660 #endif //SYNCHRONIZATION_STATS
664 process_sync_log_stats()
666 #ifdef SYNCHRONIZATION_STATS
668 unsigned int log_elapsed = GCToOSInterface::GetLowPrecisionTimeStamp() - log_start_tick;
670 if (log_elapsed > log_interval)
672 uint64_t total = GCToOSInterface::QueryPerformanceCounter() - log_start_hires;
673 // Print out the cycles we spent on average in each suspend and restart.
674 printf("\n_________________________________________________________________________________\n"
675 "Past %d(s): #%3d GCs; Total gc_lock contended: %8u; GC: %12u\n"
676 "SuspendEE: %8u; RestartEE: %8u GC %.3f%%\n",
680 (unsigned int)(gc_during_log / gc_count_during_log),
681 (unsigned int)(suspend_ee_during_log / gc_count_during_log),
682 (unsigned int)(restart_ee_during_log / gc_count_during_log),
683 (double)(100.0f * gc_during_log / total));
684 gc_heap::print_sync_stats(gc_count_during_log);
686 gc_count_during_log = 0;
688 #endif //SYNCHRONIZATION_STATS
691 #ifdef MULTIPLE_HEAPS
695 gc_join_init_cpu_mapping = 0,
697 gc_join_generation_determined = 2,
698 gc_join_begin_mark_phase = 3,
699 gc_join_scan_dependent_handles = 4,
700 gc_join_rescan_dependent_handles = 5,
701 gc_join_scan_sizedref_done = 6,
702 gc_join_null_dead_short_weak = 7,
703 gc_join_scan_finalization = 8,
704 gc_join_null_dead_long_weak = 9,
705 gc_join_null_dead_syncblk = 10,
706 gc_join_decide_on_compaction = 11,
707 gc_join_rearrange_segs_compaction = 12,
708 gc_join_adjust_handle_age_compact = 13,
709 gc_join_adjust_handle_age_sweep = 14,
710 gc_join_begin_relocate_phase = 15,
711 gc_join_relocate_phase_done = 16,
712 gc_join_verify_objects_done = 17,
713 gc_join_start_bgc = 18,
714 gc_join_restart_ee = 19,
715 gc_join_concurrent_overflow = 20,
716 gc_join_suspend_ee = 21,
717 gc_join_bgc_after_ephemeral = 22,
718 gc_join_allow_fgc = 23,
719 gc_join_bgc_sweep = 24,
720 gc_join_suspend_ee_verify = 25,
721 gc_join_restart_ee_verify = 26,
722 gc_join_set_state_free = 27,
723 gc_r_join_update_card_bundle = 28,
724 gc_join_after_absorb = 29,
725 gc_join_verify_copy_table = 30,
726 gc_join_after_reset = 31,
727 gc_join_after_ephemeral_sweep = 32,
728 gc_join_after_profiler_heap_walk = 33,
729 gc_join_minimal_gc = 34,
730 gc_join_after_commit_soh_no_gc = 35,
731 gc_join_expand_loh_no_gc = 36,
732 gc_join_final_no_gc = 37,
733 gc_join_disable_software_write_watch = 38,
739 join_flavor_server_gc = 0,
743 #define first_thread_arrived 2
744 #pragma warning(push)
745 #pragma warning(disable:4324) // don't complain if DECLSPEC_ALIGN actually pads
746 struct DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE) join_structure
748 // Shared non volatile keep on separate line to prevent eviction
751 // Keep polling/wait structures on separate line write once per join
752 DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE)
753 GCEvent joined_event[3]; // the last event in the array is only used for first_thread_arrived.
754 Volatile<int> lock_color;
755 VOLATILE(BOOL) wait_done;
756 VOLATILE(BOOL) joined_p;
758 // Keep volatile counted locks on separate cache line write many per join
759 DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE)
760 VOLATILE(int32_t) join_lock;
761 VOLATILE(int32_t) r_join_lock;
771 type_first_r_join = 3,
783 join_heap_restart = 100,
784 join_heap_r_restart = 200
796 join_structure join_struct;
799 gc_join_flavor flavor;
802 uint64_t start[MAX_SUPPORTED_CPUS], end[MAX_SUPPORTED_CPUS], start_seq;
803 // remember join id and last thread to arrive so restart can use these
805 // we want to print statistics every 10 seconds - this is to remember the start of the 10 sec interval
807 // counters for joins, in 1000's of clock cycles
808 uint64_t elapsed_total[gc_join_max], wake_total[gc_join_max], seq_loss_total[gc_join_max], par_loss_total[gc_join_max], in_join_total[gc_join_max];
812 BOOL init (int n_th, gc_join_flavor f)
814 dprintf (JOIN_LOG, ("Initializing join structure"));
815 join_struct.n_threads = n_th;
816 join_struct.lock_color = 0;
817 for (int i = 0; i < 3; i++)
819 if (!join_struct.joined_event[i].IsValid())
821 join_struct.joined_p = FALSE;
822 dprintf (JOIN_LOG, ("Creating join event %d", i));
823 // TODO - changing this to a non OS event
824 // because this is also used by BGC threads which are
825 // managed threads and WaitEx does not allow you to wait
826 // for an OS event on a managed thread.
827 // But we are not sure if this plays well in the hosting
829 //join_struct.joined_event[i].CreateOSManualEventNoThrow(FALSE);
830 if (!join_struct.joined_event[i].CreateManualEventNoThrow(FALSE))
834 join_struct.join_lock = join_struct.n_threads;
835 join_struct.r_join_lock = join_struct.n_threads;
836 join_struct.wait_done = FALSE;
840 start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
848 dprintf (JOIN_LOG, ("Destroying join structure"));
849 for (int i = 0; i < 3; i++)
851 if (join_struct.joined_event[i].IsValid())
852 join_struct.joined_event[i].CloseEvent();
856 inline void fire_event (int heap, join_time time, join_type type, int join_id)
858 FIRE_EVENT(GCJoin_V2, heap, time, type, join_id);
861 void join (gc_heap* gch, int join_id)
864 // parallel execution ends here
865 end[gch->heap_number] = get_ts();
868 assert (!join_struct.joined_p);
869 int color = join_struct.lock_color.LoadWithoutBarrier();
871 if (Interlocked::Decrement(&join_struct.join_lock) != 0)
873 dprintf (JOIN_LOG, ("join%d(%d): Join() Waiting...join_lock is now %d",
874 flavor, join_id, (int32_t)(join_struct.join_lock)));
876 fire_event (gch->heap_number, time_start, type_join, join_id);
878 //busy wait around the color
879 if (color == join_struct.lock_color.LoadWithoutBarrier())
882 int spin_count = 128 * yp_spin_count_unit;
883 for (int j = 0; j < spin_count; j++)
885 if (color != join_struct.lock_color.LoadWithoutBarrier())
889 YieldProcessor(); // indicate to the processor that we are spinning
892 // we've spun, and if color still hasn't changed, fall into hard wait
893 if (color == join_struct.lock_color.LoadWithoutBarrier())
895 dprintf (JOIN_LOG, ("join%d(%d): Join() hard wait on reset event %d, join_lock is now %d",
896 flavor, join_id, color, (int32_t)(join_struct.join_lock)));
898 //Thread* current_thread = GCToEEInterface::GetThread();
899 //BOOL cooperative_mode = gc_heap::enable_preemptive ();
900 uint32_t dwJoinWait = join_struct.joined_event[color].Wait(INFINITE, FALSE);
901 //gc_heap::disable_preemptive (cooperative_mode);
903 if (dwJoinWait != WAIT_OBJECT_0)
905 STRESS_LOG1 (LF_GC, LL_FATALERROR, "joined event wait failed with code: %Ix", dwJoinWait);
910 // avoid race due to the thread about to reset the event (occasionally) being preempted before ResetEvent()
911 if (color == join_struct.lock_color.LoadWithoutBarrier())
916 dprintf (JOIN_LOG, ("join%d(%d): Join() done, join_lock is %d",
917 flavor, join_id, (int32_t)(join_struct.join_lock)));
920 fire_event (gch->heap_number, time_end, type_join, join_id);
923 // parallel execution starts here
924 start[gch->heap_number] = get_ts();
925 Interlocked::ExchangeAdd(&in_join_total[join_id], (start[gch->heap_number] - end[gch->heap_number]));
930 fire_event (gch->heap_number, time_start, type_last_join, join_id);
932 join_struct.joined_p = TRUE;
933 dprintf (JOIN_LOG, ("join%d(%d): Last thread to complete the join, setting id", flavor, join_id));
934 join_struct.joined_event[!color].Reset();
936 // this one is alone so it can proceed
938 // remember the join id, the last thread arriving, the start of the sequential phase,
939 // and keep track of the cycles spent waiting in the join
940 thd = gch->heap_number;
941 start_seq = get_ts();
942 Interlocked::ExchangeAdd(&in_join_total[join_id], (start_seq - end[gch->heap_number]));
947 // Reverse join - first thread gets here does the work; other threads will only proceed
948 // after the work is done.
949 // Note that you cannot call this twice in a row on the same thread. Plus there's no
950 // need to call it twice in row - you should just merge the work.
951 BOOL r_join (gc_heap* gch, int join_id)
954 if (join_struct.n_threads == 1)
959 if (Interlocked::CompareExchange(&join_struct.r_join_lock, 0, join_struct.n_threads) == 0)
961 if (!join_struct.wait_done)
963 dprintf (JOIN_LOG, ("r_join() Waiting..."));
965 fire_event (gch->heap_number, time_start, type_join, join_id);
967 //busy wait around the color
968 if (!join_struct.wait_done)
971 int spin_count = 256 * yp_spin_count_unit;
972 for (int j = 0; j < spin_count; j++)
974 if (join_struct.wait_done)
978 YieldProcessor(); // indicate to the processor that we are spinning
981 // we've spun, and if color still hasn't changed, fall into hard wait
982 if (!join_struct.wait_done)
984 dprintf (JOIN_LOG, ("Join() hard wait on reset event %d", first_thread_arrived));
985 uint32_t dwJoinWait = join_struct.joined_event[first_thread_arrived].Wait(INFINITE, FALSE);
986 if (dwJoinWait != WAIT_OBJECT_0)
988 STRESS_LOG1 (LF_GC, LL_FATALERROR, "joined event wait failed with code: %Ix", dwJoinWait);
993 // avoid race due to the thread about to reset the event (occasionally) being preempted before ResetEvent()
994 if (!join_struct.wait_done)
999 dprintf (JOIN_LOG, ("r_join() done"));
1002 fire_event (gch->heap_number, time_end, type_join, join_id);
1009 fire_event (gch->heap_number, time_start, type_first_r_join, join_id);
1017 return GCToOSInterface::QueryPerformanceCounter();
1020 void start_ts (gc_heap* gch)
1022 // parallel execution ends here
1023 start[gch->heap_number] = get_ts();
1030 uint64_t elapsed_seq = get_ts() - start_seq;
1031 uint64_t max = 0, sum = 0, wake = 0;
1032 uint64_t min_ts = start[0];
1033 for (int i = 1; i < join_struct.n_threads; i++)
1035 if(min_ts > start[i]) min_ts = start[i];
1038 for (int i = 0; i < join_struct.n_threads; i++)
1040 uint64_t wake_delay = start[i] - min_ts;
1041 uint64_t elapsed = end[i] - start[i];
1047 uint64_t seq_loss = (join_struct.n_threads - 1)*elapsed_seq;
1048 uint64_t par_loss = join_struct.n_threads*max - sum;
1049 double efficiency = 0.0;
1051 efficiency = sum*100.0/(join_struct.n_threads*max);
1053 const double ts_scale = 1e-6;
1055 // enable this printf to get statistics on each individual join as it occurs
1056 // printf("join #%3d seq_loss = %5g par_loss = %5g efficiency = %3.0f%%\n", join_id, ts_scale*seq_loss, ts_scale*par_loss, efficiency);
1058 elapsed_total[id] += sum;
1059 wake_total[id] += wake;
1060 seq_loss_total[id] += seq_loss;
1061 par_loss_total[id] += par_loss;
1063 // every 10 seconds, print a summary of the time spent in each type of join
1064 if (GCToOSInterface::GetLowPrecisionTimeStamp() - start_tick > 10*1000)
1066 printf("**** summary *****\n");
1067 for (int i = 0; i < 16; i++)
1069 printf("join #%3d elapsed_total = %8g wake_loss = %8g seq_loss = %8g par_loss = %8g in_join_total = %8g\n",
1071 ts_scale*elapsed_total[i],
1072 ts_scale*wake_total[i],
1073 ts_scale*seq_loss_total[i],
1074 ts_scale*par_loss_total[i],
1075 ts_scale*in_join_total[i]);
1076 elapsed_total[i] = wake_total[i] = seq_loss_total[i] = par_loss_total[i] = in_join_total[i] = 0;
1078 start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
1082 fire_event (join_heap_restart, time_start, type_restart, -1);
1083 assert (join_struct.joined_p);
1084 join_struct.joined_p = FALSE;
1085 join_struct.join_lock = join_struct.n_threads;
1086 dprintf (JOIN_LOG, ("join%d(%d): Restarting from join: join_lock is %d", flavor, id, (int32_t)(join_struct.join_lock)));
1087 // printf("restart from join #%d at cycle %u from start of gc\n", join_id, GetCycleCount32() - gc_start);
1088 int color = join_struct.lock_color.LoadWithoutBarrier();
1089 join_struct.lock_color = !color;
1090 join_struct.joined_event[color].Set();
1092 // printf("Set joined_event %d\n", !join_struct.lock_color);
1094 fire_event (join_heap_restart, time_end, type_restart, -1);
1097 start[thd] = get_ts();
1103 dprintf (JOIN_LOG, ("join%d(%d): joined, join_lock is %d", flavor, id, (int32_t)(join_struct.join_lock)));
1104 return join_struct.joined_p;
1109 if (join_struct.n_threads != 1)
1111 fire_event (join_heap_r_restart, time_start, type_restart, -1);
1112 join_struct.wait_done = TRUE;
1113 join_struct.joined_event[first_thread_arrived].Set();
1114 fire_event (join_heap_r_restart, time_end, type_restart, -1);
1120 if (join_struct.n_threads != 1)
1122 join_struct.r_join_lock = join_struct.n_threads;
1123 join_struct.wait_done = FALSE;
1124 join_struct.joined_event[first_thread_arrived].Reset();
1131 #ifdef BACKGROUND_GC
1133 #endif //BACKGROUND_GC
1135 #endif //MULTIPLE_HEAPS
1137 #define spin_and_switch(count_to_spin, expr) \
1139 for (int j = 0; j < count_to_spin; j++) \
1149 GCToOSInterface::YieldThread(0); \
1153 #ifndef DACCESS_COMPILE
1154 #ifdef BACKGROUND_GC
1156 #define max_pending_allocs 64
1158 class exclusive_sync
1160 // TODO - verify that this is the right syntax for Volatile.
1161 VOLATILE(uint8_t*) rwp_object;
1162 VOLATILE(int32_t) needs_checking;
1166 uint8_t cache_separator[HS_CACHE_LINE_SIZE - sizeof (int) - sizeof (int32_t)];
1168 // TODO - perhaps each object should be on its own cache line...
1169 VOLATILE(uint8_t*) alloc_objects[max_pending_allocs];
1171 int find_free_index ()
1173 for (int i = 0; i < max_pending_allocs; i++)
1175 if (alloc_objects [i] == (uint8_t*)0)
1187 spin_count = 32 * (g_num_processors - 1);
1190 for (int i = 0; i < max_pending_allocs; i++)
1192 alloc_objects [i] = (uint8_t*)0;
1198 for (int i = 0; i < max_pending_allocs; i++)
1200 if (alloc_objects [i] != (uint8_t*)0)
1202 GCToOSInterface::DebugBreak();
1207 void bgc_mark_set (uint8_t* obj)
1209 dprintf (3, ("cm: probing %Ix", obj));
1211 if (Interlocked::CompareExchange(&needs_checking, 1, 0) == 0)
1213 // If we spend too much time spending all the allocs,
1214 // consider adding a high water mark and scan up
1215 // to that; we'll need to interlock in done when
1216 // we update the high watermark.
1217 for (int i = 0; i < max_pending_allocs; i++)
1219 if (obj == alloc_objects[i])
1222 dprintf (3, ("cm: will spin"));
1223 spin_and_switch (spin_count, (obj != alloc_objects[i]));
1230 dprintf (3, ("cm: set %Ix", obj));
1235 spin_and_switch (spin_count, (needs_checking == 0));
1240 int loh_alloc_set (uint8_t* obj)
1242 if (!gc_heap::cm_in_progress)
1248 dprintf (3, ("loh alloc: probing %Ix", obj));
1250 if (Interlocked::CompareExchange(&needs_checking, 1, 0) == 0)
1252 if (obj == rwp_object)
1255 spin_and_switch (spin_count, (obj != rwp_object));
1260 int cookie = find_free_index();
1264 alloc_objects[cookie] = obj;
1268 // GCToOSInterface::DebugBreak();
1271 dprintf (3, ("loh alloc: set %Ix at %d", obj, cookie));
1277 dprintf (3, ("loh alloc: setting %Ix will spin to acquire a free index", obj));
1278 spin_and_switch (spin_count, (find_free_index () != -1));
1285 dprintf (3, ("loh alloc: will spin on checking %Ix", obj));
1286 spin_and_switch (spin_count, (needs_checking == 0));
1291 void bgc_mark_done ()
1293 dprintf (3, ("cm: release lock on %Ix", (uint8_t *)rwp_object));
1297 void loh_alloc_done_with_index (int index)
1299 dprintf (3, ("loh alloc: release lock on %Ix based on %d", (uint8_t *)alloc_objects[index], index));
1300 assert ((index >= 0) && (index < max_pending_allocs));
1301 alloc_objects[index] = (uint8_t*)0;
1304 void loh_alloc_done (uint8_t* obj)
1306 #ifdef BACKGROUND_GC
1307 if (!gc_heap::cm_in_progress)
1312 for (int i = 0; i < max_pending_allocs; i++)
1314 if (alloc_objects [i] == obj)
1316 dprintf (3, ("loh alloc: release lock on %Ix at %d", (uint8_t *)alloc_objects[i], i));
1317 alloc_objects[i] = (uint8_t*)0;
1321 #endif //BACKGROUND_GC
1325 // Note that this class was written assuming just synchronization between
1326 // one background GC thread and multiple user threads that might request
1327 // an FGC - it does not take into account what kind of locks the multiple
1328 // user threads might be holding at the time (eg, there could only be one
1329 // user thread requesting an FGC because it needs to take gc_lock first)
1330 // so you'll see checks that may not be necessary if you take those conditions
1331 // into consideration.
1333 // With the introduction of Server Background GC we no longer use this
1334 // class to do synchronization between FGCs and BGC.
1335 class recursive_gc_sync
1337 static VOLATILE(int32_t) foreground_request_count;//initial state 0
1338 static VOLATILE(BOOL) gc_background_running; //initial state FALSE
1339 static VOLATILE(int32_t) foreground_count; // initial state 0;
1340 static VOLATILE(uint32_t) foreground_gate; // initial state FALSE;
1341 static GCEvent foreground_complete;//Auto Reset
1342 static GCEvent foreground_allowed;//Auto Reset
1344 static void begin_background();
1345 static void end_background();
1346 static void begin_foreground();
1347 static void end_foreground();
1348 BOOL allow_foreground ();
1350 static void shutdown();
1351 static BOOL background_running_p() {return gc_background_running;}
1354 VOLATILE(int32_t) recursive_gc_sync::foreground_request_count = 0;//initial state 0
1355 VOLATILE(int32_t) recursive_gc_sync::foreground_count = 0; // initial state 0;
1356 VOLATILE(BOOL) recursive_gc_sync::gc_background_running = FALSE; //initial state FALSE
1357 VOLATILE(uint32_t) recursive_gc_sync::foreground_gate = 0;
1358 GCEvent recursive_gc_sync::foreground_complete;//Auto Reset
1359 GCEvent recursive_gc_sync::foreground_allowed;//Manual Reset
1361 BOOL recursive_gc_sync::init ()
1363 foreground_request_count = 0;
1364 foreground_count = 0;
1365 gc_background_running = FALSE;
1366 foreground_gate = 0;
1368 if (!foreground_complete.CreateOSAutoEventNoThrow(FALSE))
1372 if (!foreground_allowed.CreateManualEventNoThrow(FALSE))
1384 void recursive_gc_sync::shutdown()
1386 if (foreground_complete.IsValid())
1387 foreground_complete.CloseEvent();
1388 if (foreground_allowed.IsValid())
1389 foreground_allowed.CloseEvent();
1392 void recursive_gc_sync::begin_background()
1394 dprintf (2, ("begin background"));
1395 foreground_request_count = 1;
1396 foreground_count = 1;
1397 foreground_allowed.Reset();
1398 gc_background_running = TRUE;
1400 void recursive_gc_sync::end_background()
1402 dprintf (2, ("end background"));
1403 gc_background_running = FALSE;
1404 foreground_gate = 1;
1405 foreground_allowed.Set();
1408 void recursive_gc_sync::begin_foreground()
1410 dprintf (2, ("begin_foreground"));
1412 bool cooperative_mode = false;
1413 if (gc_background_running)
1415 gc_heap::fire_alloc_wait_event_begin (awr_fgc_wait_for_bgc);
1416 gc_heap::alloc_wait_event_p = TRUE;
1420 Interlocked::Increment (&foreground_request_count);
1423 dprintf(2, ("Waiting sync gc point"));
1424 assert (foreground_allowed.IsValid());
1425 assert (foreground_complete.IsValid());
1427 cooperative_mode = gc_heap::enable_preemptive ();
1429 foreground_allowed.Wait(INFINITE, FALSE);
1431 dprintf(2, ("Waiting sync gc point is done"));
1433 gc_heap::disable_preemptive (cooperative_mode);
1435 if (foreground_gate)
1437 Interlocked::Increment (&foreground_count);
1438 dprintf (2, ("foreground_count: %d", (int32_t)foreground_count));
1439 if (foreground_gate)
1441 gc_heap::settings.concurrent = FALSE;
1452 goto try_again_no_inc;
1457 void recursive_gc_sync::end_foreground()
1459 dprintf (2, ("end_foreground"));
1460 if (gc_background_running)
1462 Interlocked::Decrement (&foreground_request_count);
1463 dprintf (2, ("foreground_count before decrement: %d", (int32_t)foreground_count));
1464 if (Interlocked::Decrement (&foreground_count) == 0)
1466 //c_write ((BOOL*)&foreground_gate, 0);
1467 // TODO - couldn't make the syntax work with Volatile<T>
1468 foreground_gate = 0;
1469 if (foreground_count == 0)
1471 foreground_allowed.Reset ();
1472 dprintf(2, ("setting foreground complete event"));
1473 foreground_complete.Set();
1480 BOOL recursive_gc_sync::allow_foreground()
1482 assert (gc_heap::settings.concurrent);
1483 dprintf (100, ("enter allow_foreground, f_req_count: %d, f_count: %d",
1484 (int32_t)foreground_request_count, (int32_t)foreground_count));
1486 BOOL did_fgc = FALSE;
1488 //if we have suspended the EE, just return because
1489 //some thread could be waiting on this to proceed.
1490 if (!GCHeap::GcInProgress)
1492 //TODO BACKGROUND_GC This is to stress the concurrency between
1493 //background and foreground
1494 // gc_heap::disallow_new_allocation (0);
1496 //GCToOSInterface::YieldThread(0);
1499 if (foreground_request_count != 0)
1501 //foreground wants to run
1502 //save the important settings
1503 //TODO BACKGROUND_GC be more selective about the important settings.
1504 gc_mechanisms saved_settings = gc_heap::settings;
1508 //c_write ((BOOL*)&foreground_gate, 1);
1509 // TODO - couldn't make the syntax work with Volatile<T>
1510 foreground_gate = 1;
1511 foreground_allowed.Set ();
1512 foreground_complete.Wait (INFINITE, FALSE);
1513 }while (/*foreground_request_count ||*/ foreground_gate);
1515 assert (!foreground_gate);
1517 //restore the important settings
1518 gc_heap::settings = saved_settings;
1519 GCHeap::GcCondemnedGeneration = gc_heap::settings.condemned_generation;
1520 //the background GC shouldn't be using gc_high and gc_low
1521 //gc_low = lowest_address;
1522 //gc_high = highest_address;
1525 //TODO BACKGROUND_GC This is to stress the concurrency between
1526 //background and foreground
1527 // gc_heap::allow_new_allocation (0);
1531 dprintf (100, ("leave allow_foreground"));
1532 assert (gc_heap::settings.concurrent);
1536 #endif //BACKGROUND_GC
1537 #endif //DACCESS_COMPILE
1540 #if defined(COUNT_CYCLES)
1542 #pragma warning(disable:4035)
1546 unsigned GetCycleCount32() // enough for about 40 seconds
1554 #pragma warning(default:4035)
1556 #endif //COUNT_CYCLES
1559 int mark_time, plan_time, sweep_time, reloc_time, compact_time;
1562 #ifndef MULTIPLE_HEAPS
1564 #endif // MULTIPLE_HEAPS
1566 void reset_memory (uint8_t* o, size_t sizeo);
1570 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1571 static bool virtual_alloc_hardware_write_watch = false;
1572 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1574 static bool hardware_write_watch_capability = false;
1576 #ifndef DACCESS_COMPILE
1578 //check if the write watch APIs are supported.
1580 void hardware_write_watch_api_supported()
1582 if (GCToOSInterface::SupportsWriteWatch())
1584 hardware_write_watch_capability = true;
1585 dprintf (2, ("WriteWatch supported"));
1589 dprintf (2,("WriteWatch not supported"));
1593 #endif //!DACCESS_COMPILE
1595 inline bool can_use_hardware_write_watch()
1597 return hardware_write_watch_capability;
1600 inline bool can_use_write_watch_for_gc_heap()
1602 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1604 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1605 return can_use_hardware_write_watch();
1606 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1609 inline bool can_use_write_watch_for_card_table()
1611 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
1614 return can_use_hardware_write_watch();
1619 #define mem_reserve (MEM_RESERVE)
1620 #endif //WRITE_WATCH
1622 //check if the low memory notification is supported
1624 #ifndef DACCESS_COMPILE
1626 void WaitLongerNoInstru (int i)
1628 // every 8th attempt:
1629 bool bToggleGC = GCToEEInterface::EnablePreemptiveGC();
1631 // if we're waiting for gc to finish, we should block immediately
1632 if (g_fSuspensionPending == 0)
1634 if (g_num_processors > 1)
1636 YieldProcessor(); // indicate to the processor that we are spinning
1638 GCToOSInterface::YieldThread (0);
1640 GCToOSInterface::Sleep (5);
1643 GCToOSInterface::Sleep (5);
1646 // If CLR is hosted, a thread may reach here while it is in preemptive GC mode,
1647 // or it has no Thread object, in order to force a task to yield, or to triger a GC.
1648 // It is important that the thread is going to wait for GC. Otherwise the thread
1649 // is in a tight loop. If the thread has high priority, the perf is going to be very BAD.
1653 // In debug builds, all enter_spin_lock operations go through this code. If a GC has
1654 // started, it is important to block until the GC thread calls set_gc_done (since it is
1655 // guaranteed to have cleared g_TrapReturningThreads by this point). This avoids livelock
1656 // conditions which can otherwise occur if threads are allowed to spin in this function
1657 // (and therefore starve the GC thread) between the point when the GC thread sets the
1658 // WaitForGC event and the point when the GC thread clears g_TrapReturningThreads.
1659 if (gc_heap::gc_started)
1661 gc_heap::wait_for_gc_done();
1664 GCToEEInterface::DisablePreemptiveGC();
1666 else if (g_fSuspensionPending > 0)
1668 g_theGCHeap->WaitUntilGCComplete();
1673 static void safe_switch_to_thread()
1675 bool cooperative_mode = gc_heap::enable_preemptive();
1677 GCToOSInterface::YieldThread(0);
1679 gc_heap::disable_preemptive(cooperative_mode);
1683 // We need the following methods to have volatile arguments, so that they can accept
1684 // raw pointers in addition to the results of the & operator on Volatile<T>.
1687 static void enter_spin_lock_noinstru (RAW_KEYWORD(volatile) int32_t* lock)
1691 if (Interlocked::CompareExchange(lock, 0, -1) >= 0)
1694 while (VolatileLoad(lock) >= 0)
1696 if ((++i & 7) && !IsGCInProgress())
1698 if (g_num_processors > 1)
1700 #ifndef MULTIPLE_HEAPS
1701 int spin_count = 32 * yp_spin_count_unit;
1702 #else //!MULTIPLE_HEAPS
1703 int spin_count = yp_spin_count_unit;
1704 #endif //!MULTIPLE_HEAPS
1705 for (int j = 0; j < spin_count; j++)
1707 if (VolatileLoad(lock) < 0 || IsGCInProgress())
1709 YieldProcessor(); // indicate to the processor that we are spinning
1711 if (VolatileLoad(lock) >= 0 && !IsGCInProgress())
1713 safe_switch_to_thread();
1718 safe_switch_to_thread();
1723 WaitLongerNoInstru(i);
1731 static BOOL try_enter_spin_lock_noinstru(RAW_KEYWORD(volatile) int32_t* lock)
1733 return (Interlocked::CompareExchange(&*lock, 0, -1) < 0);
1737 static void leave_spin_lock_noinstru (RAW_KEYWORD(volatile) int32_t* lock)
1739 VolatileStore<int32_t>((int32_t*)lock, -1);
1745 static void enter_spin_lock(GCSpinLock *pSpinLock)
1747 enter_spin_lock_noinstru(&pSpinLock->lock);
1748 assert (pSpinLock->holding_thread == (Thread*)-1);
1749 pSpinLock->holding_thread = GCToEEInterface::GetThread();
1753 static BOOL try_enter_spin_lock(GCSpinLock *pSpinLock)
1755 BOOL ret = try_enter_spin_lock_noinstru(&pSpinLock->lock);
1757 pSpinLock->holding_thread = GCToEEInterface::GetThread();
1762 static void leave_spin_lock(GCSpinLock *pSpinLock)
1764 bool gc_thread_p = GCToEEInterface::WasCurrentThreadCreatedByGC();
1765 // _ASSERTE((pSpinLock->holding_thread == GCToEEInterface::GetThread()) || gc_thread_p || pSpinLock->released_by_gc_p);
1766 pSpinLock->released_by_gc_p = gc_thread_p;
1767 pSpinLock->holding_thread = (Thread*) -1;
1768 if (pSpinLock->lock != -1)
1769 leave_spin_lock_noinstru(&pSpinLock->lock);
1772 #define ASSERT_HOLDING_SPIN_LOCK(pSpinLock) \
1773 _ASSERTE((pSpinLock)->holding_thread == GCToEEInterface::GetThread());
1775 #define ASSERT_NOT_HOLDING_SPIN_LOCK(pSpinLock) \
1776 _ASSERTE((pSpinLock)->holding_thread != GCToEEInterface::GetThread());
1780 //In the concurrent version, the Enable/DisablePreemptiveGC is optional because
1781 //the gc thread call WaitLonger.
1782 void WaitLonger (int i
1783 #ifdef SYNCHRONIZATION_STATS
1784 , GCSpinLock* spin_lock
1785 #endif //SYNCHRONIZATION_STATS
1788 #ifdef SYNCHRONIZATION_STATS
1789 (spin_lock->num_wait_longer)++;
1790 #endif //SYNCHRONIZATION_STATS
1792 // every 8th attempt:
1793 bool bToggleGC = GCToEEInterface::EnablePreemptiveGC();
1796 // if we're waiting for gc to finish, we should block immediately
1797 if (!gc_heap::gc_started)
1799 #ifdef SYNCHRONIZATION_STATS
1800 (spin_lock->num_switch_thread_w)++;
1801 #endif //SYNCHRONIZATION_STATS
1802 if (g_num_processors > 1)
1804 YieldProcessor(); // indicate to the processor that we are spinning
1806 GCToOSInterface::YieldThread (0);
1808 GCToOSInterface::Sleep (5);
1811 GCToOSInterface::Sleep (5);
1814 // If CLR is hosted, a thread may reach here while it is in preemptive GC mode,
1815 // or it has no Thread object, in order to force a task to yield, or to triger a GC.
1816 // It is important that the thread is going to wait for GC. Otherwise the thread
1817 // is in a tight loop. If the thread has high priority, the perf is going to be very BAD.
1818 if (gc_heap::gc_started)
1820 gc_heap::wait_for_gc_done();
1825 #ifdef SYNCHRONIZATION_STATS
1826 (spin_lock->num_disable_preemptive_w)++;
1827 #endif //SYNCHRONIZATION_STATS
1828 GCToEEInterface::DisablePreemptiveGC();
1833 static void enter_spin_lock (GCSpinLock* spin_lock)
1837 if (Interlocked::CompareExchange(&spin_lock->lock, 0, -1) >= 0)
1840 while (spin_lock->lock >= 0)
1842 if ((++i & 7) && !gc_heap::gc_started)
1844 if (g_num_processors > 1)
1846 #ifndef MULTIPLE_HEAPS
1847 int spin_count = 32 * yp_spin_count_unit;
1848 #else //!MULTIPLE_HEAPS
1849 int spin_count = yp_spin_count_unit;
1850 #endif //!MULTIPLE_HEAPS
1851 for (int j = 0; j < spin_count; j++)
1853 if (spin_lock->lock < 0 || gc_heap::gc_started)
1855 YieldProcessor(); // indicate to the processor that we are spinning
1857 if (spin_lock->lock >= 0 && !gc_heap::gc_started)
1859 #ifdef SYNCHRONIZATION_STATS
1860 (spin_lock->num_switch_thread)++;
1861 #endif //SYNCHRONIZATION_STATS
1862 bool cooperative_mode = gc_heap::enable_preemptive ();
1864 GCToOSInterface::YieldThread(0);
1866 gc_heap::disable_preemptive (cooperative_mode);
1870 GCToOSInterface::YieldThread(0);
1875 #ifdef SYNCHRONIZATION_STATS
1877 #endif //SYNCHRONIZATION_STATS
1885 inline BOOL try_enter_spin_lock(GCSpinLock* spin_lock)
1887 return (Interlocked::CompareExchange(&spin_lock->lock, 0, -1) < 0);
1891 static void leave_spin_lock (GCSpinLock * spin_lock)
1893 spin_lock->lock = -1;
1896 #define ASSERT_HOLDING_SPIN_LOCK(pSpinLock)
1900 bool gc_heap::enable_preemptive ()
1902 return GCToEEInterface::EnablePreemptiveGC();
1905 void gc_heap::disable_preemptive (bool restore_cooperative)
1907 if (restore_cooperative)
1909 GCToEEInterface::DisablePreemptiveGC();
1913 #endif // !DACCESS_COMPILE
1915 typedef void ** PTR_PTR;
1916 //This function clears a piece of memory
1917 // size has to be Dword aligned
1920 void memclr ( uint8_t* mem, size_t size)
1922 dprintf (3, ("MEMCLR: %Ix, %d", mem, size));
1923 assert ((size & (sizeof(PTR_PTR)-1)) == 0);
1924 assert (sizeof(PTR_PTR) == DATA_ALIGNMENT);
1927 // The compiler will recognize this pattern and replace it with memset call. We can as well just call
1928 // memset directly to make it obvious what's going on.
1929 PTR_PTR m = (PTR_PTR) mem;
1930 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
1934 memset (mem, 0, size);
1937 void memcopy (uint8_t* dmem, uint8_t* smem, size_t size)
1939 const size_t sz4ptr = sizeof(PTR_PTR)*4;
1940 const size_t sz2ptr = sizeof(PTR_PTR)*2;
1941 const size_t sz1ptr = sizeof(PTR_PTR)*1;
1943 // size must be a multiple of the pointer size
1944 assert ((size & (sizeof (PTR_PTR)-1)) == 0);
1945 assert (sizeof(PTR_PTR) == DATA_ALIGNMENT);
1947 // copy in groups of four pointer sized things at a time
1952 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1953 ((PTR_PTR)dmem)[1] = ((PTR_PTR)smem)[1];
1954 ((PTR_PTR)dmem)[2] = ((PTR_PTR)smem)[2];
1955 ((PTR_PTR)dmem)[3] = ((PTR_PTR)smem)[3];
1959 while ((size -= sz4ptr) >= sz4ptr);
1962 // still two pointer sized things or more left to copy?
1965 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1966 ((PTR_PTR)dmem)[1] = ((PTR_PTR)smem)[1];
1971 // still one pointer sized thing left to copy?
1974 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1982 ptrdiff_t round_down (ptrdiff_t add, int pitch)
1984 return ((add / pitch) * pitch);
1987 #if defined(FEATURE_STRUCTALIGN) && defined(RESPECT_LARGE_ALIGNMENT)
1988 // FEATURE_STRUCTALIGN allows the compiler to dictate the alignment,
1989 // i.e, if a larger alignment matters or is beneficial, the compiler
1990 // generated info tells us so. RESPECT_LARGE_ALIGNMENT is just the
1991 // converse - it's a heuristic for the GC to use a larger alignment.
1992 #error FEATURE_STRUCTALIGN should imply !RESPECT_LARGE_ALIGNMENT
1995 #if defined(FEATURE_STRUCTALIGN) && defined(FEATURE_LOH_COMPACTION)
1996 #error FEATURE_STRUCTALIGN and FEATURE_LOH_COMPACTION are mutually exclusive
1999 #if defined(GROWABLE_SEG_MAPPING_TABLE) && !defined(SEG_MAPPING_TABLE)
2000 #error if GROWABLE_SEG_MAPPING_TABLE is defined, SEG_MAPPING_TABLE must be defined
2004 BOOL same_large_alignment_p (uint8_t* p1, uint8_t* p2)
2006 #ifdef RESPECT_LARGE_ALIGNMENT
2007 return ((((size_t)p1 ^ (size_t)p2) & 7) == 0);
2009 UNREFERENCED_PARAMETER(p1);
2010 UNREFERENCED_PARAMETER(p2);
2012 #endif //RESPECT_LARGE_ALIGNMENT
2016 size_t switch_alignment_size (BOOL already_padded_p)
2018 if (already_padded_p)
2019 return DATA_ALIGNMENT;
2021 return (Align (min_obj_size) +((Align (min_obj_size)&DATA_ALIGNMENT)^DATA_ALIGNMENT));
2025 #ifdef FEATURE_STRUCTALIGN
2026 void set_node_aligninfo (uint8_t *node, int requiredAlignment, ptrdiff_t pad);
2027 void clear_node_aligninfo (uint8_t *node);
2028 #else // FEATURE_STRUCTALIGN
2029 #define node_realigned(node) (((plug_and_reloc*)(node))[-1].reloc & 1)
2030 void set_node_realigned (uint8_t* node);
2031 void clear_node_realigned(uint8_t* node);
2032 #endif // FEATURE_STRUCTALIGN
2035 size_t AlignQword (size_t nbytes)
2037 #ifdef FEATURE_STRUCTALIGN
2038 // This function is used to align everything on the large object
2039 // heap to an 8-byte boundary, to reduce the number of unaligned
2040 // accesses to (say) arrays of doubles. With FEATURE_STRUCTALIGN,
2041 // the compiler dictates the optimal alignment instead of having
2042 // a heuristic in the GC.
2043 return Align (nbytes);
2044 #else // FEATURE_STRUCTALIGN
2045 return (nbytes + 7) & ~7;
2046 #endif // FEATURE_STRUCTALIGN
2050 BOOL Aligned (size_t n)
2052 return (n & ALIGNCONST) == 0;
2055 #define OBJECT_ALIGNMENT_OFFSET (sizeof(MethodTable *))
2057 #ifdef FEATURE_STRUCTALIGN
2058 #define MAX_STRUCTALIGN OS_PAGE_SIZE
2059 #else // FEATURE_STRUCTALIGN
2060 #define MAX_STRUCTALIGN 0
2061 #endif // FEATURE_STRUCTALIGN
2063 #ifdef FEATURE_STRUCTALIGN
2065 ptrdiff_t AdjustmentForMinPadSize(ptrdiff_t pad, int requiredAlignment)
2067 // The resulting alignpad must be either 0 or at least min_obj_size.
2068 // Note that by computing the following difference on unsigned types,
2069 // we can do the range check 0 < alignpad < min_obj_size with a
2070 // single conditional branch.
2071 if ((size_t)(pad - DATA_ALIGNMENT) < Align (min_obj_size) - DATA_ALIGNMENT)
2073 return requiredAlignment;
2079 uint8_t* StructAlign (uint8_t* origPtr, int requiredAlignment, ptrdiff_t alignmentOffset=OBJECT_ALIGNMENT_OFFSET)
2081 // required alignment must be a power of two
2082 _ASSERTE(((size_t)origPtr & ALIGNCONST) == 0);
2083 _ASSERTE(((requiredAlignment - 1) & requiredAlignment) == 0);
2084 _ASSERTE(requiredAlignment >= sizeof(void *));
2085 _ASSERTE(requiredAlignment <= MAX_STRUCTALIGN);
2087 // When this method is invoked for individual objects (i.e., alignmentOffset
2088 // is just the size of the PostHeader), what needs to be aligned when
2089 // we're done is the pointer to the payload of the object (which means
2090 // the actual resulting object pointer is typically not aligned).
2092 uint8_t* result = (uint8_t*)Align ((size_t)origPtr + alignmentOffset, requiredAlignment-1) - alignmentOffset;
2093 ptrdiff_t alignpad = result - origPtr;
2095 return result + AdjustmentForMinPadSize (alignpad, requiredAlignment);
2099 ptrdiff_t ComputeStructAlignPad (uint8_t* plug, int requiredAlignment, size_t alignmentOffset=OBJECT_ALIGNMENT_OFFSET)
2101 return StructAlign (plug, requiredAlignment, alignmentOffset) - plug;
2104 BOOL IsStructAligned (uint8_t *ptr, int requiredAlignment)
2106 return StructAlign (ptr, requiredAlignment) == ptr;
2110 ptrdiff_t ComputeMaxStructAlignPad (int requiredAlignment)
2112 if (requiredAlignment == DATA_ALIGNMENT)
2114 // Since a non-zero alignment padding cannot be less than min_obj_size (so we can fit the
2115 // alignment padding object), the worst-case alignment padding is correspondingly larger
2116 // than the required alignment.
2117 return requiredAlignment + Align (min_obj_size) - DATA_ALIGNMENT;
2121 ptrdiff_t ComputeMaxStructAlignPadLarge (int requiredAlignment)
2123 if (requiredAlignment <= get_alignment_constant (TRUE)+1)
2125 // This is the same as ComputeMaxStructAlignPad, except that in addition to leaving space
2126 // for padding before the actual object, it also leaves space for filling a gap after the
2127 // actual object. This is needed on the large object heap, as the outer allocation functions
2128 // don't operate on an allocation context (which would have left space for the final gap).
2129 return requiredAlignment + Align (min_obj_size) * 2 - DATA_ALIGNMENT;
2132 uint8_t* gc_heap::pad_for_alignment (uint8_t* newAlloc, int requiredAlignment, size_t size, alloc_context* acontext)
2134 uint8_t* alignedPtr = StructAlign (newAlloc, requiredAlignment);
2135 if (alignedPtr != newAlloc) {
2136 make_unused_array (newAlloc, alignedPtr - newAlloc);
2138 acontext->alloc_ptr = alignedPtr + Align (size);
2142 uint8_t* gc_heap::pad_for_alignment_large (uint8_t* newAlloc, int requiredAlignment, size_t size)
2144 uint8_t* alignedPtr = StructAlign (newAlloc, requiredAlignment);
2145 if (alignedPtr != newAlloc) {
2146 make_unused_array (newAlloc, alignedPtr - newAlloc);
2148 if (alignedPtr < newAlloc + ComputeMaxStructAlignPadLarge (requiredAlignment)) {
2149 make_unused_array (alignedPtr + AlignQword (size), newAlloc + ComputeMaxStructAlignPadLarge (requiredAlignment) - alignedPtr);
2153 #else // FEATURE_STRUCTALIGN
2154 #define ComputeMaxStructAlignPad(requiredAlignment) 0
2155 #define ComputeMaxStructAlignPadLarge(requiredAlignment) 0
2156 #endif // FEATURE_STRUCTALIGN
2158 //CLR_SIZE is the max amount of bytes from gen0 that is set to 0 in one chunk
2160 #define CLR_SIZE ((size_t)(8*1024))
2162 #define CLR_SIZE ((size_t)(8*1024))
2165 #define END_SPACE_AFTER_GC (loh_size_threshold + MAX_STRUCTALIGN)
2167 #ifdef BACKGROUND_GC
2168 #define SEGMENT_INITIAL_COMMIT (2*OS_PAGE_SIZE)
2170 #define SEGMENT_INITIAL_COMMIT (OS_PAGE_SIZE)
2171 #endif //BACKGROUND_GC
2173 // This is always power of 2.
2174 const size_t min_segment_size_hard_limit = 1024*1024*16;
2180 #define INITIAL_ALLOC ((size_t)((size_t)4*1024*1024*1024))
2181 #define LHEAP_ALLOC ((size_t)(1024*1024*256))
2185 #define INITIAL_ALLOC ((size_t)(1024*1024*64))
2186 #define LHEAP_ALLOC ((size_t)(1024*1024*32))
2194 #define INITIAL_ALLOC ((size_t)(1024*1024*256))
2195 #define LHEAP_ALLOC ((size_t)(1024*1024*128))
2199 #define INITIAL_ALLOC ((size_t)(1024*1024*16))
2200 #define LHEAP_ALLOC ((size_t)(1024*1024*16))
2206 //amount in bytes of the etw allocation tick
2207 const size_t etw_allocation_tick = 100*1024;
2209 const size_t low_latency_alloc = 256*1024;
2211 const size_t fgn_check_quantum = 2*1024*1024;
2214 const int max_snoop_level = 128;
2219 //threshold of heap size to turn on card bundles.
2220 #define SH_TH_CARD_BUNDLE (40*1024*1024)
2221 #define MH_TH_CARD_BUNDLE (180*1024*1024)
2222 #endif //CARD_BUNDLE
2224 #define GC_EPHEMERAL_DECOMMIT_TIMEOUT 5000
2227 size_t align_on_page (size_t add)
2229 return ((add + OS_PAGE_SIZE - 1) & ~((size_t)OS_PAGE_SIZE - 1));
2233 uint8_t* align_on_page (uint8_t* add)
2235 return (uint8_t*)align_on_page ((size_t) add);
2239 size_t align_lower_page (size_t add)
2241 return (add & ~((size_t)OS_PAGE_SIZE - 1));
2245 uint8_t* align_lower_page (uint8_t* add)
2247 return (uint8_t*)align_lower_page ((size_t)add);
2251 size_t align_write_watch_lower_page (size_t add)
2253 return (add & ~(WRITE_WATCH_UNIT_SIZE - 1));
2257 uint8_t* align_write_watch_lower_page (uint8_t* add)
2259 return (uint8_t*)align_lower_page ((size_t)add);
2264 BOOL power_of_two_p (size_t integer)
2266 return !(integer & (integer-1));
2270 BOOL oddp (size_t integer)
2272 return (integer & 1) != 0;
2275 // we only ever use this for WORDs.
2276 size_t logcount (size_t word)
2278 //counts the number of high bits in a 16 bit word.
2279 assert (word < 0x10000);
2281 count = (word & 0x5555) + ( (word >> 1 ) & 0x5555);
2282 count = (count & 0x3333) + ( (count >> 2) & 0x3333);
2283 count = (count & 0x0F0F) + ( (count >> 4) & 0x0F0F);
2284 count = (count & 0x00FF) + ( (count >> 8) & 0x00FF);
2288 #ifndef DACCESS_COMPILE
2290 void stomp_write_barrier_resize(bool is_runtime_suspended, bool requires_upper_bounds_check)
2292 WriteBarrierParameters args = {};
2293 args.operation = WriteBarrierOp::StompResize;
2294 args.is_runtime_suspended = is_runtime_suspended;
2295 args.requires_upper_bounds_check = requires_upper_bounds_check;
2297 args.card_table = g_gc_card_table;
2298 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
2299 args.card_bundle_table = g_gc_card_bundle_table;
2302 args.lowest_address = g_gc_lowest_address;
2303 args.highest_address = g_gc_highest_address;
2305 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
2306 if (SoftwareWriteWatch::IsEnabledForGCHeap())
2308 args.write_watch_table = g_gc_sw_ww_table;
2310 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
2312 GCToEEInterface::StompWriteBarrier(&args);
2315 void stomp_write_barrier_ephemeral(uint8_t* ephemeral_low, uint8_t* ephemeral_high)
2317 WriteBarrierParameters args = {};
2318 args.operation = WriteBarrierOp::StompEphemeral;
2319 args.is_runtime_suspended = true;
2320 args.ephemeral_low = ephemeral_low;
2321 args.ephemeral_high = ephemeral_high;
2322 GCToEEInterface::StompWriteBarrier(&args);
2325 void stomp_write_barrier_initialize(uint8_t* ephemeral_low, uint8_t* ephemeral_high)
2327 WriteBarrierParameters args = {};
2328 args.operation = WriteBarrierOp::Initialize;
2329 args.is_runtime_suspended = true;
2330 args.requires_upper_bounds_check = false;
2331 args.card_table = g_gc_card_table;
2333 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
2334 args.card_bundle_table = g_gc_card_bundle_table;
2337 args.lowest_address = g_gc_lowest_address;
2338 args.highest_address = g_gc_highest_address;
2339 args.ephemeral_low = ephemeral_low;
2340 args.ephemeral_high = ephemeral_high;
2341 GCToEEInterface::StompWriteBarrier(&args);
2344 #endif // DACCESS_COMPILE
2346 //extract the low bits [0,low[ of a uint32_t
2347 #define lowbits(wrd, bits) ((wrd) & ((1 << (bits))-1))
2348 //extract the high bits [high, 32] of a uint32_t
2349 #define highbits(wrd, bits) ((wrd) & ~((1 << (bits))-1))
2351 // Things we need to manually initialize:
2352 // gen0 min_size - based on cache
2353 // gen0/1 max_size - based on segment size
2354 static static_data static_data_table[latency_level_last - latency_level_first + 1][NUMBERGENERATIONS] =
2356 // latency_level_memory_footprint
2359 {0, 0, 40000, 0.5f, 9.0f, 20.0f, 1000, 1},
2361 {160*1024, 0, 80000, 0.5f, 2.0f, 7.0f, 10000, 10},
2363 {256*1024, SSIZE_T_MAX, 200000, 0.25f, 1.2f, 1.8f, 100000, 100},
2365 {3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0}
2368 // latency_level_balanced
2372 #ifdef MULTIPLE_HEAPS
2376 #endif //MULTIPLE_HEAPS
2379 {256*1024, 0, 80000, 0.5f, 2.0f, 7.0f, 10000, 10},
2381 {256*1024, SSIZE_T_MAX, 200000, 0.25f, 1.2f, 1.8f, 100000, 100},
2383 {3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0}
2390 class CObjectHeader;
2394 class c_synchronize;
2396 #ifdef FEATURE_PREMORTEM_FINALIZATION
2397 #ifndef DACCESS_COMPILE
2399 HRESULT AllocateCFinalize(CFinalize **pCFinalize);
2400 #endif //!DACCESS_COMPILE
2401 #endif // FEATURE_PREMORTEM_FINALIZATION
2403 uint8_t* tree_search (uint8_t* tree, uint8_t* old_address);
2406 #ifdef USE_INTROSORT
2407 #define _sort introsort::sort
2408 #else //USE_INTROSORT
2409 #define _sort qsort1
2410 void qsort1(uint8_t** low, uint8_t** high, unsigned int depth);
2411 #endif //USE_INTROSORT
2413 void* virtual_alloc (size_t size);
2414 void virtual_free (void* add, size_t size);
2416 /* per heap static initialization */
2418 #ifndef MULTIPLE_HEAPS
2419 uint32_t* gc_heap::mark_array;
2420 #endif //MULTIPLE_HEAPS
2424 uint8_t** gc_heap::g_mark_list;
2426 #ifdef PARALLEL_MARK_LIST_SORT
2427 uint8_t** gc_heap::g_mark_list_copy;
2428 #endif //PARALLEL_MARK_LIST_SORT
2430 size_t gc_heap::mark_list_size;
2433 #ifdef SEG_MAPPING_TABLE
2434 seg_mapping* seg_mapping_table;
2435 #endif //SEG_MAPPING_TABLE
2437 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
2438 sorted_table* gc_heap::seg_table;
2439 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
2441 #ifdef MULTIPLE_HEAPS
2442 GCEvent gc_heap::ee_suspend_event;
2443 size_t gc_heap::min_balance_threshold = 0;
2444 #endif //MULTIPLE_HEAPS
2446 VOLATILE(BOOL) gc_heap::gc_started;
2448 #ifdef MULTIPLE_HEAPS
2450 GCEvent gc_heap::gc_start_event;
2451 bool gc_heap::gc_thread_no_affinitize_p = false;
2452 uintptr_t process_mask = 0;
2454 int gc_heap::n_heaps;
2456 gc_heap** gc_heap::g_heaps;
2458 size_t* gc_heap::g_promoted;
2461 int* gc_heap::g_mark_stack_busy;
2465 #ifdef BACKGROUND_GC
2466 size_t* gc_heap::g_bpromoted;
2467 #endif //BACKGROUND_GC
2469 #else //MULTIPLE_HEAPS
2471 size_t gc_heap::g_promoted;
2473 #ifdef BACKGROUND_GC
2474 size_t gc_heap::g_bpromoted;
2475 #endif //BACKGROUND_GC
2477 #endif //MULTIPLE_HEAPS
2479 size_t gc_heap::reserved_memory = 0;
2480 size_t gc_heap::reserved_memory_limit = 0;
2481 BOOL gc_heap::g_low_memory_status;
2483 #ifndef DACCESS_COMPILE
2484 static gc_reason gc_trigger_reason = reason_empty;
2485 #endif //DACCESS_COMPILE
2487 gc_latency_level gc_heap::latency_level = latency_level_default;
2489 gc_mechanisms gc_heap::settings;
2491 gc_history_global gc_heap::gc_data_global;
2493 size_t gc_heap::gc_last_ephemeral_decommit_time = 0;
2495 size_t gc_heap::gc_gen0_desired_high;
2497 CLRCriticalSection gc_heap::check_commit_cs;
2499 size_t gc_heap::current_total_committed = 0;
2501 size_t gc_heap::current_total_committed_bookkeeping = 0;
2504 double gc_heap::short_plugs_pad_ratio = 0;
2505 #endif //SHORT_PLUGS
2508 #define MAX_ALLOWED_MEM_LOAD 85
2510 // consider putting this in dynamic data -
2511 // we may want different values for workstation
2513 #define MIN_YOUNGEST_GEN_DESIRED (16*1024*1024)
2515 size_t gc_heap::youngest_gen_desired_th;
2518 uint32_t gc_heap::last_gc_memory_load = 0;
2520 size_t gc_heap::last_gc_heap_size = 0;
2522 size_t gc_heap::last_gc_fragmentation = 0;
2524 uint64_t gc_heap::mem_one_percent = 0;
2526 uint32_t gc_heap::high_memory_load_th = 0;
2528 uint32_t gc_heap::m_high_memory_load_th;
2530 uint32_t gc_heap::v_high_memory_load_th;
2532 uint64_t gc_heap::total_physical_mem = 0;
2534 uint64_t gc_heap::entry_available_physical_mem = 0;
2536 size_t gc_heap::heap_hard_limit = 0;
2538 #ifdef BACKGROUND_GC
2539 GCEvent gc_heap::bgc_start_event;
2541 gc_mechanisms gc_heap::saved_bgc_settings;
2543 GCEvent gc_heap::background_gc_done_event;
2545 GCEvent gc_heap::ee_proceed_event;
2547 bool gc_heap::gc_can_use_concurrent = false;
2549 bool gc_heap::temp_disable_concurrent_p = false;
2551 uint32_t gc_heap::cm_in_progress = FALSE;
2553 BOOL gc_heap::dont_restart_ee_p = FALSE;
2555 BOOL gc_heap::keep_bgc_threads_p = FALSE;
2557 GCEvent gc_heap::bgc_threads_sync_event;
2559 BOOL gc_heap::do_ephemeral_gc_p = FALSE;
2561 BOOL gc_heap::do_concurrent_p = FALSE;
2563 size_t gc_heap::ephemeral_fgc_counts[max_generation];
2565 BOOL gc_heap::alloc_wait_event_p = FALSE;
2567 VOLATILE(c_gc_state) gc_heap::current_c_gc_state = c_gc_state_free;
2569 #endif //BACKGROUND_GC
2571 #ifndef MULTIPLE_HEAPS
2572 #ifdef SPINLOCK_HISTORY
2573 int gc_heap::spinlock_info_index = 0;
2574 spinlock_info gc_heap::last_spinlock_info[max_saved_spinlock_info + 8];
2575 #endif //SPINLOCK_HISTORY
2577 size_t gc_heap::fgn_last_alloc = 0;
2579 int gc_heap::generation_skip_ratio = 100;
2581 uint64_t gc_heap::loh_alloc_since_cg = 0;
2583 BOOL gc_heap::elevation_requested = FALSE;
2585 BOOL gc_heap::last_gc_before_oom = FALSE;
2587 BOOL gc_heap::sufficient_gen0_space_p = FALSE;
2589 #ifdef BACKGROUND_GC
2590 uint8_t* gc_heap::background_saved_lowest_address = 0;
2591 uint8_t* gc_heap::background_saved_highest_address = 0;
2592 uint8_t* gc_heap::next_sweep_obj = 0;
2593 uint8_t* gc_heap::current_sweep_pos = 0;
2594 exclusive_sync* gc_heap::bgc_alloc_lock;
2595 #endif //BACKGROUND_GC
2597 oom_history gc_heap::oom_info;
2599 fgm_history gc_heap::fgm_result;
2601 size_t gc_heap::allocated_since_last_gc = 0;
2603 BOOL gc_heap::ro_segments_in_range;
2605 size_t gc_heap::gen0_big_free_spaces = 0;
2607 uint8_t* gc_heap::ephemeral_low;
2609 uint8_t* gc_heap::ephemeral_high;
2611 uint8_t* gc_heap::lowest_address;
2613 uint8_t* gc_heap::highest_address;
2615 BOOL gc_heap::ephemeral_promotion;
2617 uint8_t* gc_heap::saved_ephemeral_plan_start[NUMBERGENERATIONS-1];
2618 size_t gc_heap::saved_ephemeral_plan_start_size[NUMBERGENERATIONS-1];
2620 short* gc_heap::brick_table;
2622 uint32_t* gc_heap::card_table;
2625 uint32_t* gc_heap::card_bundle_table;
2626 #endif //CARD_BUNDLE
2628 uint8_t* gc_heap::gc_low;
2630 uint8_t* gc_heap::gc_high;
2632 uint8_t* gc_heap::demotion_low;
2634 uint8_t* gc_heap::demotion_high;
2636 BOOL gc_heap::demote_gen1_p = TRUE;
2638 uint8_t* gc_heap::last_gen1_pin_end;
2640 gen_to_condemn_tuning gc_heap::gen_to_condemn_reasons;
2642 size_t gc_heap::etw_allocation_running_amount[2];
2644 int gc_heap::gc_policy = 0;
2646 size_t gc_heap::allocation_running_time;
2648 size_t gc_heap::allocation_running_amount;
2650 heap_segment* gc_heap::ephemeral_heap_segment = 0;
2652 BOOL gc_heap::blocking_collection = FALSE;
2654 heap_segment* gc_heap::freeable_large_heap_segment = 0;
2656 size_t gc_heap::time_bgc_last = 0;
2658 size_t gc_heap::mark_stack_tos = 0;
2660 size_t gc_heap::mark_stack_bos = 0;
2662 size_t gc_heap::mark_stack_array_length = 0;
2664 mark* gc_heap::mark_stack_array = 0;
2666 #if defined (_DEBUG) && defined (VERIFY_HEAP)
2667 BOOL gc_heap::verify_pinned_queue_p = FALSE;
2668 #endif // defined (_DEBUG) && defined (VERIFY_HEAP)
2670 uint8_t* gc_heap::oldest_pinned_plug = 0;
2672 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
2673 size_t gc_heap::num_pinned_objects = 0;
2674 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
2676 #ifdef FEATURE_LOH_COMPACTION
2677 size_t gc_heap::loh_pinned_queue_tos = 0;
2679 size_t gc_heap::loh_pinned_queue_bos = 0;
2681 size_t gc_heap::loh_pinned_queue_length = 0;
2683 mark* gc_heap::loh_pinned_queue = 0;
2685 BOOL gc_heap::loh_compacted_p = FALSE;
2686 #endif //FEATURE_LOH_COMPACTION
2688 #ifdef BACKGROUND_GC
2690 EEThreadId gc_heap::bgc_thread_id;
2692 uint8_t* gc_heap::background_written_addresses [array_size+2];
2694 heap_segment* gc_heap::freeable_small_heap_segment = 0;
2696 size_t gc_heap::bgc_overflow_count = 0;
2698 size_t gc_heap::bgc_begin_loh_size = 0;
2699 size_t gc_heap::end_loh_size = 0;
2701 uint32_t gc_heap::bgc_alloc_spin_loh = 0;
2703 size_t gc_heap::bgc_loh_size_increased = 0;
2705 size_t gc_heap::bgc_loh_allocated_in_free = 0;
2707 size_t gc_heap::background_soh_alloc_count = 0;
2709 size_t gc_heap::background_loh_alloc_count = 0;
2711 uint8_t** gc_heap::background_mark_stack_tos = 0;
2713 uint8_t** gc_heap::background_mark_stack_array = 0;
2715 size_t gc_heap::background_mark_stack_array_length = 0;
2717 uint8_t* gc_heap::background_min_overflow_address =0;
2719 uint8_t* gc_heap::background_max_overflow_address =0;
2721 BOOL gc_heap::processed_soh_overflow_p = FALSE;
2723 uint8_t* gc_heap::background_min_soh_overflow_address =0;
2725 uint8_t* gc_heap::background_max_soh_overflow_address =0;
2727 heap_segment* gc_heap::saved_sweep_ephemeral_seg = 0;
2729 uint8_t* gc_heap::saved_sweep_ephemeral_start = 0;
2731 heap_segment* gc_heap::saved_overflow_ephemeral_seg = 0;
2733 Thread* gc_heap::bgc_thread = 0;
2735 BOOL gc_heap::expanded_in_fgc = FALSE;
2737 uint8_t** gc_heap::c_mark_list = 0;
2739 size_t gc_heap::c_mark_list_length = 0;
2741 size_t gc_heap::c_mark_list_index = 0;
2743 gc_history_per_heap gc_heap::bgc_data_per_heap;
2745 BOOL gc_heap::bgc_thread_running;
2747 CLRCriticalSection gc_heap::bgc_threads_timeout_cs;
2749 GCEvent gc_heap::gc_lh_block_event;
2751 #endif //BACKGROUND_GC
2754 uint8_t** gc_heap::mark_list;
2755 uint8_t** gc_heap::mark_list_index;
2756 uint8_t** gc_heap::mark_list_end;
2760 snoop_stats_data gc_heap::snoop_stat;
2761 #endif //SNOOP_STATS
2763 uint8_t* gc_heap::min_overflow_address = MAX_PTR;
2765 uint8_t* gc_heap::max_overflow_address = 0;
2767 uint8_t* gc_heap::shigh = 0;
2769 uint8_t* gc_heap::slow = MAX_PTR;
2771 size_t gc_heap::ordered_free_space_indices[MAX_NUM_BUCKETS];
2773 size_t gc_heap::saved_ordered_free_space_indices[MAX_NUM_BUCKETS];
2775 size_t gc_heap::ordered_plug_indices[MAX_NUM_BUCKETS];
2777 size_t gc_heap::saved_ordered_plug_indices[MAX_NUM_BUCKETS];
2779 BOOL gc_heap::ordered_plug_indices_init = FALSE;
2781 BOOL gc_heap::use_bestfit = FALSE;
2783 uint8_t* gc_heap::bestfit_first_pin = 0;
2785 BOOL gc_heap::commit_end_of_seg = FALSE;
2787 size_t gc_heap::max_free_space_items = 0;
2789 size_t gc_heap::free_space_buckets = 0;
2791 size_t gc_heap::free_space_items = 0;
2793 int gc_heap::trimmed_free_space_index = 0;
2795 size_t gc_heap::total_ephemeral_plugs = 0;
2797 seg_free_spaces* gc_heap::bestfit_seg = 0;
2799 size_t gc_heap::total_ephemeral_size = 0;
2803 size_t gc_heap::internal_root_array_length = initial_internal_roots;
2805 uint8_t** gc_heap::internal_root_array = 0;
2807 size_t gc_heap::internal_root_array_index = 0;
2809 BOOL gc_heap::heap_analyze_success = TRUE;
2811 uint8_t* gc_heap::current_obj = 0;
2812 size_t gc_heap::current_obj_size = 0;
2814 #endif //HEAP_ANALYZE
2816 #ifdef GC_CONFIG_DRIVEN
2817 size_t gc_heap::interesting_data_per_gc[max_idp_count];
2818 //size_t gc_heap::interesting_data_per_heap[max_idp_count];
2819 //size_t gc_heap::interesting_mechanisms_per_heap[max_im_count];
2820 #endif //GC_CONFIG_DRIVEN
2821 #endif //MULTIPLE_HEAPS
2823 no_gc_region_info gc_heap::current_no_gc_region_info;
2824 BOOL gc_heap::proceed_with_gc_p = FALSE;
2825 GCSpinLock gc_heap::gc_lock;
2827 size_t gc_heap::eph_gen_starts_size = 0;
2828 heap_segment* gc_heap::segment_standby_list;
2829 size_t gc_heap::last_gc_index = 0;
2830 #ifdef SEG_MAPPING_TABLE
2831 size_t gc_heap::min_segment_size = 0;
2832 size_t gc_heap::min_segment_size_shr = 0;
2833 #endif //SEG_MAPPING_TABLE
2834 size_t gc_heap::soh_segment_size = 0;
2835 size_t gc_heap::min_loh_segment_size = 0;
2836 size_t gc_heap::segment_info_size = 0;
2838 #ifdef GC_CONFIG_DRIVEN
2839 size_t gc_heap::time_init = 0;
2840 size_t gc_heap::time_since_init = 0;
2841 size_t gc_heap::compact_or_sweep_gcs[2];
2842 #endif //GC_CONFIG_DRIVEN
2844 #ifdef FEATURE_LOH_COMPACTION
2845 BOOL gc_heap::loh_compaction_always_p = FALSE;
2846 gc_loh_compaction_mode gc_heap::loh_compaction_mode = loh_compaction_default;
2847 int gc_heap::loh_pinned_queue_decay = LOH_PIN_DECAY;
2849 #endif //FEATURE_LOH_COMPACTION
2851 GCEvent gc_heap::full_gc_approach_event;
2853 GCEvent gc_heap::full_gc_end_event;
2855 uint32_t gc_heap::fgn_maxgen_percent = 0;
2857 uint32_t gc_heap::fgn_loh_percent = 0;
2859 #ifdef BACKGROUND_GC
2860 BOOL gc_heap::fgn_last_gc_was_concurrent = FALSE;
2861 #endif //BACKGROUND_GC
2863 VOLATILE(bool) gc_heap::full_gc_approach_event_set;
2865 size_t gc_heap::full_gc_counts[gc_type_max];
2867 bool gc_heap::maxgen_size_inc_p = false;
2869 BOOL gc_heap::should_expand_in_full_gc = FALSE;
2871 // Provisional mode related stuff.
2872 bool gc_heap::provisional_mode_triggered = false;
2873 bool gc_heap::pm_trigger_full_gc = false;
2874 size_t gc_heap::provisional_triggered_gc_count = 0;
2875 size_t gc_heap::provisional_off_gc_count = 0;
2876 size_t gc_heap::num_provisional_triggered = 0;
2877 bool gc_heap::pm_stress_on = false;
2880 BOOL gc_heap::heap_analyze_enabled = FALSE;
2881 #endif //HEAP_ANALYZE
2883 #ifndef MULTIPLE_HEAPS
2885 alloc_list gc_heap::loh_alloc_list [NUM_LOH_ALIST-1];
2886 alloc_list gc_heap::gen2_alloc_list[NUM_GEN2_ALIST-1];
2888 dynamic_data gc_heap::dynamic_data_table [NUMBERGENERATIONS+1];
2889 gc_history_per_heap gc_heap::gc_data_per_heap;
2890 size_t gc_heap::maxgen_pinned_compact_before_advance = 0;
2892 uint8_t* gc_heap::alloc_allocated = 0;
2894 size_t gc_heap::allocation_quantum = CLR_SIZE;
2896 GCSpinLock gc_heap::more_space_lock_soh;
2897 GCSpinLock gc_heap::more_space_lock_loh;
2898 VOLATILE(int32_t) gc_heap::loh_alloc_thread_count = 0;
2900 #ifdef SYNCHRONIZATION_STATS
2901 unsigned int gc_heap::good_suspension = 0;
2902 unsigned int gc_heap::bad_suspension = 0;
2903 uint64_t gc_heap::total_msl_acquire = 0;
2904 unsigned int gc_heap::num_msl_acquired = 0;
2905 unsigned int gc_heap::num_high_msl_acquire = 0;
2906 unsigned int gc_heap::num_low_msl_acquire = 0;
2907 #endif //SYNCHRONIZATION_STATS
2909 size_t gc_heap::alloc_contexts_used = 0;
2910 size_t gc_heap::soh_allocation_no_gc = 0;
2911 size_t gc_heap::loh_allocation_no_gc = 0;
2912 bool gc_heap::no_gc_oom_p = false;
2913 heap_segment* gc_heap::saved_loh_segment_no_gc = 0;
2915 #endif //MULTIPLE_HEAPS
2917 #ifndef MULTIPLE_HEAPS
2919 BOOL gc_heap::gen0_bricks_cleared = FALSE;
2922 int gc_heap::gen0_must_clear_bricks = 0;
2923 #endif //FFIND_OBJECT
2925 #ifdef FEATURE_PREMORTEM_FINALIZATION
2926 CFinalize* gc_heap::finalize_queue = 0;
2927 #endif // FEATURE_PREMORTEM_FINALIZATION
2929 generation gc_heap::generation_table [NUMBERGENERATIONS + 1];
2931 size_t gc_heap::interesting_data_per_heap[max_idp_count];
2933 size_t gc_heap::compact_reasons_per_heap[max_compact_reasons_count];
2935 size_t gc_heap::expand_mechanisms_per_heap[max_expand_mechanisms_count];
2937 size_t gc_heap::interesting_mechanism_bits_per_heap[max_gc_mechanism_bits_count];
2939 #endif // MULTIPLE_HEAPS
2941 /* end of per heap static initialization */
2943 /* end of static initialization */
2945 #ifndef DACCESS_COMPILE
2947 void gen_to_condemn_tuning::print (int heap_num)
2950 dprintf (DT_LOG_0, ("condemned reasons (%d %d)", condemn_reasons_gen, condemn_reasons_condition));
2951 dprintf (DT_LOG_0, ("%s", record_condemn_reasons_gen_header));
2952 gc_condemn_reason_gen r_gen;
2953 for (int i = 0; i < gcrg_max; i++)
2955 r_gen = (gc_condemn_reason_gen)(i);
2956 str_reasons_gen[i * 2] = get_gen_char (get_gen (r_gen));
2958 dprintf (DT_LOG_0, ("[%2d]%s", heap_num, str_reasons_gen));
2960 dprintf (DT_LOG_0, ("%s", record_condemn_reasons_condition_header));
2961 gc_condemn_reason_condition r_condition;
2962 for (int i = 0; i < gcrc_max; i++)
2964 r_condition = (gc_condemn_reason_condition)(i);
2965 str_reasons_condition[i * 2] = get_condition_char (get_condition (r_condition));
2968 dprintf (DT_LOG_0, ("[%2d]%s", heap_num, str_reasons_condition));
2970 UNREFERENCED_PARAMETER(heap_num);
2974 void gc_generation_data::print (int heap_num, int gen_num)
2976 #if defined(SIMPLE_DPRINTF) && defined(DT_LOG)
2977 dprintf (DT_LOG_0, ("[%2d]gen%d beg %Id fl %Id fo %Id end %Id fl %Id fo %Id in %Id p %Id np %Id alloc %Id",
2980 free_list_space_before, free_obj_space_before,
2982 free_list_space_after, free_obj_space_after,
2983 in, pinned_surv, npinned_surv,
2986 UNREFERENCED_PARAMETER(heap_num);
2987 UNREFERENCED_PARAMETER(gen_num);
2988 #endif //SIMPLE_DPRINTF && DT_LOG
2991 void gc_history_per_heap::set_mechanism (gc_mechanism_per_heap mechanism_per_heap, uint32_t value)
2993 uint32_t* mechanism = &mechanisms[mechanism_per_heap];
2995 *mechanism |= mechanism_mask;
2996 *mechanism |= (1 << value);
2999 gc_mechanism_descr* descr = &gc_mechanisms_descr[mechanism_per_heap];
3000 dprintf (DT_LOG_0, ("setting %s: %s",
3002 (descr->descr)[value]));
3006 void gc_history_per_heap::print()
3008 #if defined(SIMPLE_DPRINTF) && defined(DT_LOG)
3009 for (int i = 0; i < (sizeof (gen_data)/sizeof (gc_generation_data)); i++)
3011 gen_data[i].print (heap_index, i);
3014 dprintf (DT_LOG_0, ("fla %Id flr %Id esa %Id ca %Id pa %Id paa %Id, rfle %d, ec %Id",
3015 maxgen_size_info.free_list_allocated,
3016 maxgen_size_info.free_list_rejected,
3017 maxgen_size_info.end_seg_allocated,
3018 maxgen_size_info.condemned_allocated,
3019 maxgen_size_info.pinned_allocated,
3020 maxgen_size_info.pinned_allocated_advance,
3021 maxgen_size_info.running_free_list_efficiency,
3022 extra_gen0_committed));
3025 gc_mechanism_descr* descr = 0;
3027 for (int i = 0; i < max_mechanism_per_heap; i++)
3029 mechanism = get_mechanism ((gc_mechanism_per_heap)i);
3033 descr = &gc_mechanisms_descr[(gc_mechanism_per_heap)i];
3034 dprintf (DT_LOG_0, ("[%2d]%s%s",
3037 (descr->descr)[mechanism]));
3040 #endif //SIMPLE_DPRINTF && DT_LOG
3043 void gc_history_global::print()
3046 char str_settings[64];
3047 memset (str_settings, '|', sizeof (char) * 64);
3048 str_settings[max_global_mechanisms_count*2] = 0;
3050 for (int i = 0; i < max_global_mechanisms_count; i++)
3052 str_settings[i * 2] = (get_mechanism_p ((gc_global_mechanism_p)i) ? 'Y' : 'N');
3055 dprintf (DT_LOG_0, ("[hp]|c|p|o|d|b|e|"));
3057 dprintf (DT_LOG_0, ("%4d|%s", num_heaps, str_settings));
3058 dprintf (DT_LOG_0, ("Condemned gen%d(reason: %s; mode: %s), youngest budget %Id(%d), memload %d",
3059 condemned_generation,
3060 str_gc_reasons[reason],
3061 str_gc_pause_modes[pause_mode],
3062 final_youngest_desired,
3063 gen0_reduction_count,
3068 void gc_heap::fire_per_heap_hist_event (gc_history_per_heap* current_gc_data_per_heap, int heap_num)
3070 maxgen_size_increase* maxgen_size_info = &(current_gc_data_per_heap->maxgen_size_info);
3071 FIRE_EVENT(GCPerHeapHistory_V3,
3072 (void *)(maxgen_size_info->free_list_allocated),
3073 (void *)(maxgen_size_info->free_list_rejected),
3074 (void *)(maxgen_size_info->end_seg_allocated),
3075 (void *)(maxgen_size_info->condemned_allocated),
3076 (void *)(maxgen_size_info->pinned_allocated),
3077 (void *)(maxgen_size_info->pinned_allocated_advance),
3078 maxgen_size_info->running_free_list_efficiency,
3079 current_gc_data_per_heap->gen_to_condemn_reasons.get_reasons0(),
3080 current_gc_data_per_heap->gen_to_condemn_reasons.get_reasons1(),
3081 current_gc_data_per_heap->mechanisms[gc_heap_compact],
3082 current_gc_data_per_heap->mechanisms[gc_heap_expand],
3083 current_gc_data_per_heap->heap_index,
3084 (void *)(current_gc_data_per_heap->extra_gen0_committed),
3085 (max_generation + 2),
3086 (uint32_t)(sizeof (gc_generation_data)),
3087 (void *)&(current_gc_data_per_heap->gen_data[0]));
3089 current_gc_data_per_heap->print();
3090 current_gc_data_per_heap->gen_to_condemn_reasons.print (heap_num);
3093 void gc_heap::fire_pevents()
3095 settings.record (&gc_data_global);
3096 gc_data_global.print();
3098 FIRE_EVENT(GCGlobalHeapHistory_V2,
3099 gc_data_global.final_youngest_desired,
3100 gc_data_global.num_heaps,
3101 gc_data_global.condemned_generation,
3102 gc_data_global.gen0_reduction_count,
3103 gc_data_global.reason,
3104 gc_data_global.global_mechanims_p,
3105 gc_data_global.pause_mode,
3106 gc_data_global.mem_pressure);
3108 #ifdef MULTIPLE_HEAPS
3109 for (int i = 0; i < gc_heap::n_heaps; i++)
3111 gc_heap* hp = gc_heap::g_heaps[i];
3112 gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap();
3113 fire_per_heap_hist_event (current_gc_data_per_heap, hp->heap_number);
3116 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
3117 fire_per_heap_hist_event (current_gc_data_per_heap, heap_number);
3122 gc_heap::dt_low_ephemeral_space_p (gc_tuning_point tp)
3128 case tuning_deciding_condemned_gen:
3129 case tuning_deciding_compaction:
3130 case tuning_deciding_expansion:
3131 case tuning_deciding_full_gc:
3133 ret = (!ephemeral_gen_fit_p (tp));
3136 case tuning_deciding_promote_ephemeral:
3138 size_t new_gen0size = approximate_new_allocation();
3139 ptrdiff_t plan_ephemeral_size = total_ephemeral_size;
3141 dprintf (GTC_LOG, ("h%d: plan eph size is %Id, new gen0 is %Id",
3142 heap_number, plan_ephemeral_size, new_gen0size));
3143 // If we were in no_gc_region we could have allocated a larger than normal segment,
3144 // and the next seg we allocate will be a normal sized seg so if we can't fit the new
3145 // ephemeral generations there, do an ephemeral promotion.
3146 ret = ((soh_segment_size - segment_info_size) < (plan_ephemeral_size + new_gen0size));
3157 gc_heap::dt_high_frag_p (gc_tuning_point tp,
3165 case tuning_deciding_condemned_gen:
3167 dynamic_data* dd = dynamic_data_of (gen_number);
3168 float fragmentation_burden = 0;
3172 ret = (dd_fragmentation (dynamic_data_of (max_generation)) >= dd_max_size(dd));
3173 dprintf (GTC_LOG, ("h%d: frag is %Id, max size is %Id",
3174 heap_number, dd_fragmentation (dd), dd_max_size(dd)));
3178 #ifndef MULTIPLE_HEAPS
3179 if (gen_number == max_generation)
3181 float frag_ratio = (float)(dd_fragmentation (dynamic_data_of (max_generation))) / (float)generation_size (max_generation);
3182 if (frag_ratio > 0.65)
3184 dprintf (GTC_LOG, ("g2 FR: %d%%", (int)(frag_ratio*100)));
3188 #endif //!MULTIPLE_HEAPS
3189 size_t fr = generation_unusable_fragmentation (generation_of (gen_number));
3190 ret = (fr > dd_fragmentation_limit(dd));
3193 fragmentation_burden = (float)fr / generation_size (gen_number);
3194 ret = (fragmentation_burden > dd_v_fragmentation_burden_limit (dd));
3196 dprintf (GTC_LOG, ("h%d: gen%d, frag is %Id, alloc effi: %d%%, unusable frag is %Id, ratio is %d",
3197 heap_number, gen_number, dd_fragmentation (dd),
3198 (int)(100*generation_allocator_efficiency (generation_of (gen_number))),
3199 fr, (int)(fragmentation_burden*100)));
3211 gc_heap::dt_estimate_reclaim_space_p (gc_tuning_point tp, int gen_number)
3217 case tuning_deciding_condemned_gen:
3219 if (gen_number == max_generation)
3221 size_t est_maxgen_free = estimated_reclaim (gen_number);
3223 uint32_t num_heaps = 1;
3224 #ifdef MULTIPLE_HEAPS
3225 num_heaps = gc_heap::n_heaps;
3226 #endif //MULTIPLE_HEAPS
3228 size_t min_frag_th = min_reclaim_fragmentation_threshold (num_heaps);
3229 dprintf (GTC_LOG, ("h%d, min frag is %Id", heap_number, min_frag_th));
3230 ret = (est_maxgen_free >= min_frag_th);
3246 // DTREVIEW: Right now we only estimate gen2 fragmentation.
3247 // on 64-bit though we should consider gen1 or even gen0 fragmentatioin as
3250 gc_heap::dt_estimate_high_frag_p (gc_tuning_point tp, int gen_number, uint64_t available_mem)
3256 case tuning_deciding_condemned_gen:
3258 if (gen_number == max_generation)
3260 dynamic_data* dd = dynamic_data_of (gen_number);
3261 float est_frag_ratio = 0;
3262 if (dd_current_size (dd) == 0)
3266 else if ((dd_fragmentation (dd) == 0) || (dd_fragmentation (dd) + dd_current_size (dd) == 0))
3272 est_frag_ratio = (float)dd_fragmentation (dd) / (float)(dd_fragmentation (dd) + dd_current_size (dd));
3275 size_t est_frag = (dd_fragmentation (dd) + (size_t)((dd_desired_allocation (dd) - dd_new_allocation (dd)) * est_frag_ratio));
3276 dprintf (GTC_LOG, ("h%d: gen%d: current_size is %Id, frag is %Id, est_frag_ratio is %d%%, estimated frag is %Id",
3279 dd_current_size (dd),
3280 dd_fragmentation (dd),
3281 (int)(est_frag_ratio*100),
3284 uint32_t num_heaps = 1;
3286 #ifdef MULTIPLE_HEAPS
3287 num_heaps = gc_heap::n_heaps;
3288 #endif //MULTIPLE_HEAPS
3289 uint64_t min_frag_th = min_high_fragmentation_threshold(available_mem, num_heaps);
3290 //dprintf (GTC_LOG, ("h%d, min frag is %I64d", heap_number, min_frag_th));
3291 ret = (est_frag >= min_frag_th);
3308 gc_heap::dt_low_card_table_efficiency_p (gc_tuning_point tp)
3314 case tuning_deciding_condemned_gen:
3316 /* promote into max-generation if the card table has too many
3317 * generation faults besides the n -> 0
3319 ret = (generation_skip_ratio < 30);
3331 in_range_for_segment(uint8_t* add, heap_segment* seg)
3333 return ((add >= heap_segment_mem (seg)) && (add < heap_segment_reserved (seg)));
3336 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
3337 // The array we allocate is organized as follows:
3338 // 0th element is the address of the last array we allocated.
3339 // starting from the 1st element are the segment addresses, that's
3340 // what buckets() returns.
3353 bk* buckets() { return (slots + 1); }
3354 uint8_t*& last_slot (bk* arr) { return arr[0].add; }
3357 static sorted_table* make_sorted_table ();
3358 BOOL insert (uint8_t* add, size_t val);;
3359 size_t lookup (uint8_t*& add);
3360 void remove (uint8_t* add);
3362 void delete_sorted_table();
3363 void delete_old_slots();
3364 void enqueue_old_slot(bk* sl);
3365 BOOL ensure_space_for_insert();
3369 sorted_table::make_sorted_table ()
3373 // allocate one more bk to store the older slot address.
3374 sorted_table* res = (sorted_table*)new char [sizeof (sorted_table) + (size + 1) * sizeof (bk)];
3378 res->slots = (bk*)(res + 1);
3385 sorted_table::delete_sorted_table()
3387 if (slots != (bk*)(this+1))
3395 sorted_table::delete_old_slots()
3397 uint8_t* sl = (uint8_t*)old_slots;
3401 sl = last_slot ((bk*)sl);
3407 sorted_table::enqueue_old_slot(bk* sl)
3409 last_slot (sl) = (uint8_t*)old_slots;
3415 sorted_table::lookup (uint8_t*& add)
3417 ptrdiff_t high = (count-1);
3421 bk* buck = buckets();
3424 mid = ((low + high)/2);
3426 if (buck[ti].add > add)
3428 if ((ti > 0) && (buck[ti-1].add <= add))
3430 add = buck[ti-1].add;
3431 return buck[ti - 1].val;
3437 if (buck[ti+1].add > add)
3440 return buck[ti].val;
3450 sorted_table::ensure_space_for_insert()
3454 size = (size * 3)/2;
3455 assert((size * sizeof (bk)) > 0);
3456 bk* res = (bk*)new (nothrow) char [(size + 1) * sizeof (bk)];
3461 last_slot (res) = 0;
3462 memcpy (((bk*)res + 1), buckets(), count * sizeof (bk));
3463 bk* last_old_slots = slots;
3465 if (last_old_slots != (bk*)(this + 1))
3466 enqueue_old_slot (last_old_slots);
3472 sorted_table::insert (uint8_t* add, size_t val)
3474 //grow if no more room
3475 assert (count < size);
3478 ptrdiff_t high = (count-1);
3482 bk* buck = buckets();
3485 mid = ((low + high)/2);
3487 if (buck[ti].add > add)
3489 if ((ti == 0) || (buck[ti-1].add <= add))
3491 // found insertion point
3492 for (ptrdiff_t k = count; k > ti;k--)
3494 buck [k] = buck [k-1];
3505 if (buck[ti+1].add > add)
3507 //found the insertion point
3508 for (ptrdiff_t k = count; k > ti+1;k--)
3510 buck [k] = buck [k-1];
3512 buck[ti+1].add = add;
3513 buck[ti+1].val = val;
3525 sorted_table::remove (uint8_t* add)
3527 ptrdiff_t high = (count-1);
3531 bk* buck = buckets();
3534 mid = ((low + high)/2);
3536 if (buck[ti].add > add)
3538 if (buck[ti-1].add <= add)
3540 // found the guy to remove
3541 for (ptrdiff_t k = ti; k < count; k++)
3542 buck[k-1] = buck[k];
3550 if (buck[ti+1].add > add)
3552 // found the guy to remove
3553 for (ptrdiff_t k = ti+1; k < count; k++)
3554 buck[k-1] = buck[k];
3565 sorted_table::clear()
3568 buckets()[0].add = MAX_PTR;
3570 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
3572 #ifdef SEG_MAPPING_TABLE
3573 #ifdef GROWABLE_SEG_MAPPING_TABLE
3575 uint8_t* align_on_segment (uint8_t* add)
3577 return (uint8_t*)((size_t)(add + (gc_heap::min_segment_size - 1)) & ~(gc_heap::min_segment_size - 1));
3581 uint8_t* align_lower_segment (uint8_t* add)
3583 return (uint8_t*)((size_t)(add) & ~(gc_heap::min_segment_size - 1));
3586 size_t size_seg_mapping_table_of (uint8_t* from, uint8_t* end)
3588 from = align_lower_segment (from);
3589 end = align_on_segment (end);
3590 dprintf (1, ("from: %Ix, end: %Ix, size: %Ix", from, end, sizeof (seg_mapping)*(((size_t)(end - from) >> gc_heap::min_segment_size_shr))));
3591 return sizeof (seg_mapping)*((size_t)(end - from) >> gc_heap::min_segment_size_shr);
3594 // for seg_mapping_table we want it to start from a pointer sized address.
3596 size_t align_for_seg_mapping_table (size_t size)
3598 return ((size + (sizeof (uint8_t*) - 1)) &~ (sizeof (uint8_t*) - 1));
3602 size_t seg_mapping_word_of (uint8_t* add)
3604 return (size_t)add >> gc_heap::min_segment_size_shr;
3606 #else //GROWABLE_SEG_MAPPING_TABLE
3607 BOOL seg_mapping_table_init()
3610 uint64_t total_address_space = (uint64_t)8*1024*1024*1024*1024;
3612 uint64_t total_address_space = (uint64_t)4*1024*1024*1024;
3615 size_t num_entries = (size_t)(total_address_space >> gc_heap::min_segment_size_shr);
3616 seg_mapping_table = new seg_mapping[num_entries];
3618 if (seg_mapping_table)
3620 memset (seg_mapping_table, 0, num_entries * sizeof (seg_mapping));
3621 dprintf (1, ("created %d entries for heap mapping (%Id bytes)",
3622 num_entries, (num_entries * sizeof (seg_mapping))));
3627 dprintf (1, ("failed to create %d entries for heap mapping (%Id bytes)",
3628 num_entries, (num_entries * sizeof (seg_mapping))));
3632 #endif //GROWABLE_SEG_MAPPING_TABLE
3634 #ifdef FEATURE_BASICFREEZE
3636 size_t ro_seg_begin_index (heap_segment* seg)
3638 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3639 begin_index = max (begin_index, (size_t)g_gc_lowest_address >> gc_heap::min_segment_size_shr);
3644 size_t ro_seg_end_index (heap_segment* seg)
3646 size_t end_index = (size_t)(heap_segment_reserved (seg) - 1) >> gc_heap::min_segment_size_shr;
3647 end_index = min (end_index, (size_t)g_gc_highest_address >> gc_heap::min_segment_size_shr);
3651 void seg_mapping_table_add_ro_segment (heap_segment* seg)
3653 #ifdef GROWABLE_SEG_MAPPING_TABLE
3654 if ((heap_segment_reserved (seg) <= g_gc_lowest_address) || (heap_segment_mem (seg) >= g_gc_highest_address))
3656 #endif //GROWABLE_SEG_MAPPING_TABLE
3658 for (size_t entry_index = ro_seg_begin_index (seg); entry_index <= ro_seg_end_index (seg); entry_index++)
3659 seg_mapping_table[entry_index].seg1 = (heap_segment*)((size_t)seg_mapping_table[entry_index].seg1 | ro_in_entry);
3662 void seg_mapping_table_remove_ro_segment (heap_segment* seg)
3664 UNREFERENCED_PARAMETER(seg);
3666 // POSSIBLE PERF TODO: right now we are not doing anything because we can't simply remove the flag. If it proves
3667 // to be a perf problem, we can search in the current ro segs and see if any lands in this range and only
3668 // remove the flag if none lands in this range.
3672 heap_segment* ro_segment_lookup (uint8_t* o)
3674 uint8_t* ro_seg_start = o;
3675 heap_segment* seg = (heap_segment*)gc_heap::seg_table->lookup (ro_seg_start);
3677 if (ro_seg_start && in_range_for_segment (o, seg))
3683 #endif //FEATURE_BASICFREEZE
3685 void gc_heap::seg_mapping_table_add_segment (heap_segment* seg, gc_heap* hp)
3687 size_t seg_end = (size_t)(heap_segment_reserved (seg) - 1);
3688 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3689 seg_mapping* begin_entry = &seg_mapping_table[begin_index];
3690 size_t end_index = seg_end >> gc_heap::min_segment_size_shr;
3691 seg_mapping* end_entry = &seg_mapping_table[end_index];
3693 dprintf (1, ("adding seg %Ix(%d)-%Ix(%d)",
3694 seg, begin_index, heap_segment_reserved (seg), end_index));
3696 dprintf (1, ("before add: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3697 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3698 end_index, (seg_mapping_table[end_index].boundary + 1)));
3700 #ifdef MULTIPLE_HEAPS
3701 #ifdef SIMPLE_DPRINTF
3702 dprintf (1, ("begin %d: h0: %Ix(%d), h1: %Ix(%d); end %d: h0: %Ix(%d), h1: %Ix(%d)",
3703 begin_index, (uint8_t*)(begin_entry->h0), (begin_entry->h0 ? begin_entry->h0->heap_number : -1),
3704 (uint8_t*)(begin_entry->h1), (begin_entry->h1 ? begin_entry->h1->heap_number : -1),
3705 end_index, (uint8_t*)(end_entry->h0), (end_entry->h0 ? end_entry->h0->heap_number : -1),
3706 (uint8_t*)(end_entry->h1), (end_entry->h1 ? end_entry->h1->heap_number : -1)));
3707 #endif //SIMPLE_DPRINTF
3708 assert (end_entry->boundary == 0);
3709 assert (end_entry->h0 == 0);
3711 assert (begin_entry->h1 == 0);
3712 begin_entry->h1 = hp;
3714 UNREFERENCED_PARAMETER(hp);
3715 #endif //MULTIPLE_HEAPS
3717 end_entry->boundary = (uint8_t*)seg_end;
3719 dprintf (1, ("set entry %d seg1 and %d seg0 to %Ix", begin_index, end_index, seg));
3720 assert ((begin_entry->seg1 == 0) || ((size_t)(begin_entry->seg1) == ro_in_entry));
3721 begin_entry->seg1 = (heap_segment*)((size_t)(begin_entry->seg1) | (size_t)seg);
3722 end_entry->seg0 = seg;
3724 // for every entry inbetween we need to set its heap too.
3725 for (size_t entry_index = (begin_index + 1); entry_index <= (end_index - 1); entry_index++)
3727 assert (seg_mapping_table[entry_index].boundary == 0);
3728 #ifdef MULTIPLE_HEAPS
3729 assert (seg_mapping_table[entry_index].h0 == 0);
3730 seg_mapping_table[entry_index].h1 = hp;
3731 #endif //MULTIPLE_HEAPS
3732 seg_mapping_table[entry_index].seg1 = seg;
3735 dprintf (1, ("after add: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3736 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3737 end_index, (seg_mapping_table[end_index].boundary + 1)));
3738 #if defined(MULTIPLE_HEAPS) && defined(SIMPLE_DPRINTF)
3739 dprintf (1, ("begin %d: h0: %Ix(%d), h1: %Ix(%d); end: %d h0: %Ix(%d), h1: %Ix(%d)",
3740 begin_index, (uint8_t*)(begin_entry->h0), (begin_entry->h0 ? begin_entry->h0->heap_number : -1),
3741 (uint8_t*)(begin_entry->h1), (begin_entry->h1 ? begin_entry->h1->heap_number : -1),
3742 end_index, (uint8_t*)(end_entry->h0), (end_entry->h0 ? end_entry->h0->heap_number : -1),
3743 (uint8_t*)(end_entry->h1), (end_entry->h1 ? end_entry->h1->heap_number : -1)));
3744 #endif //MULTIPLE_HEAPS && SIMPLE_DPRINTF
3747 void gc_heap::seg_mapping_table_remove_segment (heap_segment* seg)
3749 size_t seg_end = (size_t)(heap_segment_reserved (seg) - 1);
3750 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3751 seg_mapping* begin_entry = &seg_mapping_table[begin_index];
3752 size_t end_index = seg_end >> gc_heap::min_segment_size_shr;
3753 seg_mapping* end_entry = &seg_mapping_table[end_index];
3754 dprintf (1, ("removing seg %Ix(%d)-%Ix(%d)",
3755 seg, begin_index, heap_segment_reserved (seg), end_index));
3757 assert (end_entry->boundary == (uint8_t*)seg_end);
3758 end_entry->boundary = 0;
3760 #ifdef MULTIPLE_HEAPS
3761 gc_heap* hp = heap_segment_heap (seg);
3762 assert (end_entry->h0 == hp);
3764 assert (begin_entry->h1 == hp);
3765 begin_entry->h1 = 0;
3766 #endif //MULTIPLE_HEAPS
3768 assert (begin_entry->seg1 != 0);
3769 begin_entry->seg1 = (heap_segment*)((size_t)(begin_entry->seg1) & ro_in_entry);
3770 end_entry->seg0 = 0;
3772 // for every entry inbetween we need to reset its heap too.
3773 for (size_t entry_index = (begin_index + 1); entry_index <= (end_index - 1); entry_index++)
3775 assert (seg_mapping_table[entry_index].boundary == 0);
3776 #ifdef MULTIPLE_HEAPS
3777 assert (seg_mapping_table[entry_index].h0 == 0);
3778 assert (seg_mapping_table[entry_index].h1 == hp);
3779 seg_mapping_table[entry_index].h1 = 0;
3780 #endif //MULTIPLE_HEAPS
3781 seg_mapping_table[entry_index].seg1 = 0;
3784 dprintf (1, ("after remove: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3785 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3786 end_index, (seg_mapping_table[end_index].boundary + 1)));
3787 #ifdef MULTIPLE_HEAPS
3788 dprintf (1, ("begin %d: h0: %Ix, h1: %Ix; end: %d h0: %Ix, h1: %Ix",
3789 begin_index, (uint8_t*)(begin_entry->h0), (uint8_t*)(begin_entry->h1),
3790 end_index, (uint8_t*)(end_entry->h0), (uint8_t*)(end_entry->h1)));
3791 #endif //MULTIPLE_HEAPS
3794 #ifdef MULTIPLE_HEAPS
3796 gc_heap* seg_mapping_table_heap_of_worker (uint8_t* o)
3798 size_t index = (size_t)o >> gc_heap::min_segment_size_shr;
3799 seg_mapping* entry = &seg_mapping_table[index];
3801 gc_heap* hp = ((o > entry->boundary) ? entry->h1 : entry->h0);
3803 dprintf (2, ("checking obj %Ix, index is %Id, entry: boundry: %Ix, h0: %Ix, seg0: %Ix, h1: %Ix, seg1: %Ix",
3804 o, index, (entry->boundary + 1),
3805 (uint8_t*)(entry->h0), (uint8_t*)(entry->seg0),
3806 (uint8_t*)(entry->h1), (uint8_t*)(entry->seg1)));
3809 heap_segment* seg = ((o > entry->boundary) ? entry->seg1 : entry->seg0);
3810 #ifdef FEATURE_BASICFREEZE
3811 if ((size_t)seg & ro_in_entry)
3812 seg = (heap_segment*)((size_t)seg & ~ro_in_entry);
3813 #endif //FEATURE_BASICFREEZE
3817 if (in_range_for_segment (o, seg))
3819 dprintf (2, ("obj %Ix belongs to segment %Ix(-%Ix)", o, seg, (uint8_t*)heap_segment_allocated (seg)));
3823 dprintf (2, ("found seg %Ix(-%Ix) for obj %Ix, but it's not on the seg",
3824 seg, (uint8_t*)heap_segment_allocated (seg), o));
3829 dprintf (2, ("could not find obj %Ix in any existing segments", o));
3836 gc_heap* seg_mapping_table_heap_of (uint8_t* o)
3838 #ifdef GROWABLE_SEG_MAPPING_TABLE
3839 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3841 #endif //GROWABLE_SEG_MAPPING_TABLE
3843 return seg_mapping_table_heap_of_worker (o);
3846 gc_heap* seg_mapping_table_heap_of_gc (uint8_t* o)
3848 #if defined(FEATURE_BASICFREEZE) && defined(GROWABLE_SEG_MAPPING_TABLE)
3849 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3851 #endif //FEATURE_BASICFREEZE || GROWABLE_SEG_MAPPING_TABLE
3853 return seg_mapping_table_heap_of_worker (o);
3855 #endif //MULTIPLE_HEAPS
3857 // Only returns a valid seg if we can actually find o on the seg.
3858 heap_segment* seg_mapping_table_segment_of (uint8_t* o)
3860 #if defined(FEATURE_BASICFREEZE) && defined(GROWABLE_SEG_MAPPING_TABLE)
3861 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3862 #ifdef FEATURE_BASICFREEZE
3863 return ro_segment_lookup (o);
3866 #endif //FEATURE_BASICFREEZE
3867 #endif //FEATURE_BASICFREEZE || GROWABLE_SEG_MAPPING_TABLE
3869 size_t index = (size_t)o >> gc_heap::min_segment_size_shr;
3870 seg_mapping* entry = &seg_mapping_table[index];
3872 dprintf (2, ("checking obj %Ix, index is %Id, entry: boundry: %Ix, seg0: %Ix, seg1: %Ix",
3873 o, index, (entry->boundary + 1),
3874 (uint8_t*)(entry->seg0), (uint8_t*)(entry->seg1)));
3876 heap_segment* seg = ((o > entry->boundary) ? entry->seg1 : entry->seg0);
3877 #ifdef FEATURE_BASICFREEZE
3878 if ((size_t)seg & ro_in_entry)
3879 seg = (heap_segment*)((size_t)seg & ~ro_in_entry);
3880 #endif //FEATURE_BASICFREEZE
3884 if (in_range_for_segment (o, seg))
3886 dprintf (2, ("obj %Ix belongs to segment %Ix(-%Ix)", o, (uint8_t*)heap_segment_mem(seg), (uint8_t*)heap_segment_reserved(seg)));
3890 dprintf (2, ("found seg %Ix(-%Ix) for obj %Ix, but it's not on the seg, setting it to 0",
3891 (uint8_t*)heap_segment_mem(seg), (uint8_t*)heap_segment_reserved(seg), o));
3897 dprintf (2, ("could not find obj %Ix in any existing segments", o));
3900 #ifdef FEATURE_BASICFREEZE
3901 // TODO: This was originally written assuming that the seg_mapping_table would always contain entries for ro
3902 // segments whenever the ro segment falls into the [g_gc_lowest_address,g_gc_highest_address) range. I.e., it had an
3903 // extra "&& (size_t)(entry->seg1) & ro_in_entry" expression. However, at the moment, grow_brick_card_table does
3904 // not correctly go through the ro segments and add them back to the seg_mapping_table when the [lowest,highest)
3905 // range changes. We should probably go ahead and modify grow_brick_card_table and put back the
3906 // "&& (size_t)(entry->seg1) & ro_in_entry" here.
3909 seg = ro_segment_lookup (o);
3910 if (seg && !in_range_for_segment (o, seg))
3913 #endif //FEATURE_BASICFREEZE
3917 #endif //SEG_MAPPING_TABLE
3919 size_t gcard_of ( uint8_t*);
3921 #define memref(i) *(uint8_t**)(i)
3924 #define GC_MARKED (size_t)0x1
3925 #define slot(i, j) ((uint8_t**)(i))[j+1]
3927 #define free_object_base_size (plug_skew + sizeof(ArrayBase))
3929 class CObjectHeader : public Object
3933 #if defined(FEATURE_REDHAWK) || defined(BUILD_AS_STANDALONE)
3934 // The GC expects the following methods that are provided by the Object class in the CLR but not provided
3935 // by Redhawk's version of Object.
3936 uint32_t GetNumComponents()
3938 return ((ArrayBase *)this)->GetNumComponents();
3941 void Validate(BOOL bDeep=TRUE, BOOL bVerifyNextHeader = TRUE)
3943 UNREFERENCED_PARAMETER(bVerifyNextHeader);
3948 MethodTable * pMT = GetMethodTable();
3950 _ASSERTE(pMT->SanityCheck());
3952 bool noRangeChecks =
3953 (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_NO_RANGE_CHECKS) == GCConfig::HEAPVERIFY_NO_RANGE_CHECKS;
3955 BOOL fSmallObjectHeapPtr = FALSE, fLargeObjectHeapPtr = FALSE;
3958 fSmallObjectHeapPtr = g_theGCHeap->IsHeapPointer(this, TRUE);
3959 if (!fSmallObjectHeapPtr)
3960 fLargeObjectHeapPtr = g_theGCHeap->IsHeapPointer(this);
3962 _ASSERTE(fSmallObjectHeapPtr || fLargeObjectHeapPtr);
3965 #ifdef FEATURE_STRUCTALIGN
3966 _ASSERTE(IsStructAligned((uint8_t *)this, GetMethodTable()->GetBaseAlignment()));
3967 #endif // FEATURE_STRUCTALIGN
3969 #ifdef FEATURE_64BIT_ALIGNMENT
3970 if (pMT->RequiresAlign8())
3972 _ASSERTE((((size_t)this) & 0x7) == (pMT->IsValueType() ? 4U : 0U));
3974 #endif // FEATURE_64BIT_ALIGNMENT
3977 if (bDeep && (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC))
3978 g_theGCHeap->ValidateObjectMember(this);
3980 if (fSmallObjectHeapPtr)
3982 #ifdef FEATURE_BASICFREEZE
3983 _ASSERTE(!g_theGCHeap->IsLargeObject(pMT) || g_theGCHeap->IsInFrozenSegment(this));
3985 _ASSERTE(!g_theGCHeap->IsLargeObject(pMT));
3990 void ValidatePromote(ScanContext *sc, uint32_t flags)
3992 UNREFERENCED_PARAMETER(sc);
3993 UNREFERENCED_PARAMETER(flags);
3998 void ValidateHeap(Object *from, BOOL bDeep)
4000 UNREFERENCED_PARAMETER(from);
4002 Validate(bDeep, FALSE);
4005 #endif //FEATURE_REDHAWK || BUILD_AS_STANDALONE
4009 // Header Status Information
4012 MethodTable *GetMethodTable() const
4014 return( (MethodTable *) (((size_t) RawGetMethodTable()) & (~(GC_MARKED))));
4019 RawSetMethodTable((MethodTable *) (((size_t) RawGetMethodTable()) | GC_MARKED));
4022 BOOL IsMarked() const
4024 return !!(((size_t)RawGetMethodTable()) & GC_MARKED);
4029 assert (!(gc_heap::settings.concurrent));
4030 GetHeader()->SetGCBit();
4033 BOOL IsPinned() const
4035 return !!((((CObjectHeader*)this)->GetHeader()->GetBits()) & BIT_SBLK_GC_RESERVE);
4040 RawSetMethodTable( GetMethodTable() );
4043 CGCDesc *GetSlotMap ()
4045 assert (GetMethodTable()->ContainsPointers());
4046 return CGCDesc::GetCGCDescFromMT(GetMethodTable());
4049 void SetFree(size_t size)
4051 assert (size >= free_object_base_size);
4053 assert (g_gc_pFreeObjectMethodTable->GetBaseSize() == free_object_base_size);
4054 assert (g_gc_pFreeObjectMethodTable->RawGetComponentSize() == 1);
4056 RawSetMethodTable( g_gc_pFreeObjectMethodTable );
4058 size_t* numComponentsPtr = (size_t*) &((uint8_t*) this)[ArrayBase::GetOffsetOfNumComponents()];
4059 *numComponentsPtr = size - free_object_base_size;
4061 //This introduces a bug in the free list management.
4062 //((void**) this)[-1] = 0; // clear the sync block,
4063 assert (*numComponentsPtr >= 0);
4064 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
4065 memset (((uint8_t*)this)+sizeof(ArrayBase), 0xcc, *numComponentsPtr);
4066 #endif //VERIFY_HEAP
4071 size_t size = free_object_base_size - plug_skew;
4073 // since we only need to clear 2 ptr size, we do it manually
4074 PTR_PTR m = (PTR_PTR) this;
4075 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
4079 BOOL IsFree () const
4081 return (GetMethodTable() == g_gc_pFreeObjectMethodTable);
4084 #ifdef FEATURE_STRUCTALIGN
4085 int GetRequiredAlignment () const
4087 return GetMethodTable()->GetRequiredAlignment();
4089 #endif // FEATURE_STRUCTALIGN
4091 BOOL ContainsPointers() const
4093 return GetMethodTable()->ContainsPointers();
4096 #ifdef COLLECTIBLE_CLASS
4097 BOOL Collectible() const
4099 return GetMethodTable()->Collectible();
4102 FORCEINLINE BOOL ContainsPointersOrCollectible() const
4104 MethodTable *pMethodTable = GetMethodTable();
4105 return (pMethodTable->ContainsPointers() || pMethodTable->Collectible());
4107 #endif //COLLECTIBLE_CLASS
4109 Object* GetObjectBase() const
4111 return (Object*) this;
4115 #define header(i) ((CObjectHeader*)(i))
4117 #define free_list_slot(x) ((uint8_t**)(x))[2]
4118 #define free_list_undo(x) ((uint8_t**)(x))[-1]
4119 #define UNDO_EMPTY ((uint8_t*)1)
4123 void set_plug_padded (uint8_t* node)
4125 header(node)->SetMarked();
4128 void clear_plug_padded (uint8_t* node)
4130 header(node)->ClearMarked();
4133 BOOL is_plug_padded (uint8_t* node)
4135 return header(node)->IsMarked();
4138 inline void set_plug_padded (uint8_t* node){}
4139 inline void clear_plug_padded (uint8_t* node){}
4141 BOOL is_plug_padded (uint8_t* node){return FALSE;}
4142 #endif //SHORT_PLUGS
4145 inline size_t unused_array_size(uint8_t * p)
4147 assert(((CObjectHeader*)p)->IsFree());
4149 size_t* numComponentsPtr = (size_t*)(p + ArrayBase::GetOffsetOfNumComponents());
4150 return free_object_base_size + *numComponentsPtr;
4153 heap_segment* heap_segment_rw (heap_segment* ns)
4155 if ((ns == 0) || !heap_segment_read_only_p (ns))
4163 ns = heap_segment_next (ns);
4164 } while ((ns != 0) && heap_segment_read_only_p (ns));
4169 //returns the next non ro segment.
4170 heap_segment* heap_segment_next_rw (heap_segment* seg)
4172 heap_segment* ns = heap_segment_next (seg);
4173 return heap_segment_rw (ns);
4176 // returns the segment before seg.
4177 heap_segment* heap_segment_prev_rw (heap_segment* begin, heap_segment* seg)
4179 assert (begin != 0);
4180 heap_segment* prev = begin;
4181 heap_segment* current = heap_segment_next_rw (begin);
4183 while (current && current != seg)
4186 current = heap_segment_next_rw (current);
4199 // returns the segment before seg.
4200 heap_segment* heap_segment_prev (heap_segment* begin, heap_segment* seg)
4202 assert (begin != 0);
4203 heap_segment* prev = begin;
4204 heap_segment* current = heap_segment_next (begin);
4206 while (current && current != seg)
4209 current = heap_segment_next (current);
4222 heap_segment* heap_segment_in_range (heap_segment* ns)
4224 if ((ns == 0) || heap_segment_in_range_p (ns))
4232 ns = heap_segment_next (ns);
4233 } while ((ns != 0) && !heap_segment_in_range_p (ns));
4238 heap_segment* heap_segment_next_in_range (heap_segment* seg)
4240 heap_segment* ns = heap_segment_next (seg);
4241 return heap_segment_in_range (ns);
4246 uint8_t* memory_base;
4251 imemory_data *initial_memory;
4252 imemory_data *initial_normal_heap; // points into initial_memory_array
4253 imemory_data *initial_large_heap; // points into initial_memory_array
4255 size_t block_size_normal;
4256 size_t block_size_large;
4258 size_t block_count; // # of blocks in each
4259 size_t current_block_normal;
4260 size_t current_block_large;
4269 size_t allocation_pattern;
4270 } initial_memory_details;
4272 initial_memory_details memory_details;
4274 BOOL reserve_initial_memory (size_t normal_size, size_t large_size, size_t num_heaps)
4276 BOOL reserve_success = FALSE;
4278 // should only be called once
4279 assert (memory_details.initial_memory == 0);
4281 memory_details.initial_memory = new (nothrow) imemory_data[num_heaps*2];
4282 if (memory_details.initial_memory == 0)
4284 dprintf (2, ("failed to reserve %Id bytes for imemory_data", num_heaps*2*sizeof(imemory_data)));
4288 memory_details.initial_normal_heap = memory_details.initial_memory;
4289 memory_details.initial_large_heap = memory_details.initial_memory + num_heaps;
4290 memory_details.block_size_normal = normal_size;
4291 memory_details.block_size_large = large_size;
4292 memory_details.block_count = num_heaps;
4294 memory_details.current_block_normal = 0;
4295 memory_details.current_block_large = 0;
4297 g_gc_lowest_address = MAX_PTR;
4298 g_gc_highest_address = 0;
4300 if (((size_t)MAX_PTR - large_size) < normal_size)
4302 // we are already overflowing with just one heap.
4303 dprintf (2, ("0x%Ix + 0x%Ix already overflow", normal_size, large_size));
4307 if (((size_t)MAX_PTR / memory_details.block_count) < (normal_size + large_size))
4309 dprintf (2, ("(0x%Ix + 0x%Ix)*0x%Ix overflow", normal_size, large_size, memory_details.block_count));
4313 size_t requestedMemory = memory_details.block_count * (normal_size + large_size);
4315 uint8_t* allatonce_block = (uint8_t*)virtual_alloc (requestedMemory);
4316 if (allatonce_block)
4318 g_gc_lowest_address = allatonce_block;
4319 g_gc_highest_address = allatonce_block + (memory_details.block_count * (large_size + normal_size));
4320 memory_details.allocation_pattern = initial_memory_details::ALLATONCE;
4322 for(size_t i = 0; i < memory_details.block_count; i++)
4324 memory_details.initial_normal_heap[i].memory_base = allatonce_block + (i*normal_size);
4325 memory_details.initial_large_heap[i].memory_base = allatonce_block +
4326 (memory_details.block_count*normal_size) + (i*large_size);
4327 reserve_success = TRUE;
4332 // try to allocate 2 blocks
4335 b1 = (uint8_t*)virtual_alloc (memory_details.block_count * normal_size);
4338 b2 = (uint8_t*)virtual_alloc (memory_details.block_count * large_size);
4341 memory_details.allocation_pattern = initial_memory_details::TWO_STAGE;
4342 g_gc_lowest_address = min(b1,b2);
4343 g_gc_highest_address = max(b1 + memory_details.block_count*normal_size,
4344 b2 + memory_details.block_count*large_size);
4345 for(size_t i = 0; i < memory_details.block_count; i++)
4347 memory_details.initial_normal_heap[i].memory_base = b1 + (i*normal_size);
4348 memory_details.initial_large_heap[i].memory_base = b2 + (i*large_size);
4349 reserve_success = TRUE;
4354 // b2 allocation failed, we'll go on to try allocating each block.
4355 // We could preserve the b1 alloc, but code complexity increases
4356 virtual_free (b1, memory_details.block_count * normal_size);
4360 if ((b2==NULL) && ( memory_details.block_count > 1))
4362 memory_details.allocation_pattern = initial_memory_details::EACH_BLOCK;
4364 imemory_data *current_block = memory_details.initial_memory;
4365 for(size_t i = 0; i < (memory_details.block_count*2); i++, current_block++)
4367 size_t block_size = ((i < memory_details.block_count) ?
4368 memory_details.block_size_normal :
4369 memory_details.block_size_large);
4370 current_block->memory_base =
4371 (uint8_t*)virtual_alloc (block_size);
4372 if (current_block->memory_base == 0)
4374 // Free the blocks that we've allocated so far
4375 current_block = memory_details.initial_memory;
4376 for(size_t j = 0; j < i; j++, current_block++){
4377 if (current_block->memory_base != 0){
4378 block_size = ((j < memory_details.block_count) ?
4379 memory_details.block_size_normal :
4380 memory_details.block_size_large);
4381 virtual_free (current_block->memory_base , block_size);
4384 reserve_success = FALSE;
4389 if (current_block->memory_base < g_gc_lowest_address)
4390 g_gc_lowest_address = current_block->memory_base;
4391 if (((uint8_t *) current_block->memory_base + block_size) > g_gc_highest_address)
4392 g_gc_highest_address = (current_block->memory_base + block_size);
4394 reserve_success = TRUE;
4399 return reserve_success;
4402 void destroy_initial_memory()
4404 if (memory_details.initial_memory != NULL)
4406 if (memory_details.allocation_pattern == initial_memory_details::ALLATONCE)
4408 virtual_free(memory_details.initial_memory[0].memory_base,
4409 memory_details.block_count*(memory_details.block_size_normal +
4410 memory_details.block_size_large));
4412 else if (memory_details.allocation_pattern == initial_memory_details::TWO_STAGE)
4414 virtual_free (memory_details.initial_normal_heap[0].memory_base,
4415 memory_details.block_count*memory_details.block_size_normal);
4417 virtual_free (memory_details.initial_large_heap[0].memory_base,
4418 memory_details.block_count*memory_details.block_size_large);
4422 assert (memory_details.allocation_pattern == initial_memory_details::EACH_BLOCK);
4423 imemory_data *current_block = memory_details.initial_memory;
4424 for(size_t i = 0; i < (memory_details.block_count*2); i++, current_block++)
4426 size_t block_size = (i < memory_details.block_count) ? memory_details.block_size_normal :
4427 memory_details.block_size_large;
4428 if (current_block->memory_base != NULL)
4430 virtual_free (current_block->memory_base, block_size);
4435 delete [] memory_details.initial_memory;
4436 memory_details.initial_memory = NULL;
4437 memory_details.initial_normal_heap = NULL;
4438 memory_details.initial_large_heap = NULL;
4442 void* next_initial_memory (size_t size)
4444 assert ((size == memory_details.block_size_normal) || (size == memory_details.block_size_large));
4447 if ((size != memory_details.block_size_normal) ||
4448 ((memory_details.current_block_normal == memory_details.block_count) &&
4449 (memory_details.block_size_normal == memory_details.block_size_large)))
4451 // If the block sizes are the same, flow block requests from normal to large
4452 assert (memory_details.current_block_large < memory_details.block_count);
4453 assert (memory_details.initial_large_heap != 0);
4455 res = memory_details.initial_large_heap[memory_details.current_block_large].memory_base;
4456 memory_details.current_block_large++;
4460 assert (memory_details.current_block_normal < memory_details.block_count);
4461 assert (memory_details.initial_normal_heap != NULL);
4463 res = memory_details.initial_normal_heap[memory_details.current_block_normal].memory_base;
4464 memory_details.current_block_normal++;
4470 heap_segment* get_initial_segment (size_t size, int h_number)
4472 void* mem = next_initial_memory (size);
4473 heap_segment* res = gc_heap::make_heap_segment ((uint8_t*)mem, size , h_number);
4478 void* virtual_alloc (size_t size)
4480 size_t requested_size = size;
4482 if ((gc_heap::reserved_memory_limit - gc_heap::reserved_memory) < requested_size)
4484 gc_heap::reserved_memory_limit =
4485 GCScan::AskForMoreReservedMemory (gc_heap::reserved_memory_limit, requested_size);
4486 if ((gc_heap::reserved_memory_limit - gc_heap::reserved_memory) < requested_size)
4492 uint32_t flags = VirtualReserveFlags::None;
4493 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
4494 if (virtual_alloc_hardware_write_watch)
4496 flags = VirtualReserveFlags::WriteWatch;
4498 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
4499 void* prgmem = GCToOSInterface::VirtualReserve (requested_size, card_size * card_word_width, flags);
4500 void *aligned_mem = prgmem;
4502 // We don't want (prgmem + size) to be right at the end of the address space
4503 // because we'd have to worry about that everytime we do (address + size).
4504 // We also want to make sure that we leave loh_size_threshold at the end
4505 // so we allocate a small object we don't need to worry about overflow there
4506 // when we do alloc_ptr+size.
4509 uint8_t* end_mem = (uint8_t*)prgmem + requested_size;
4511 if ((end_mem == 0) || ((size_t)(MAX_PTR - end_mem) <= END_SPACE_AFTER_GC))
4513 GCToOSInterface::VirtualRelease (prgmem, requested_size);
4514 dprintf (2, ("Virtual Alloc size %Id returned memory right against 4GB [%Ix, %Ix[ - discarding",
4515 requested_size, (size_t)prgmem, (size_t)((uint8_t*)prgmem+requested_size)));
4523 gc_heap::reserved_memory += requested_size;
4526 dprintf (2, ("Virtual Alloc size %Id: [%Ix, %Ix[",
4527 requested_size, (size_t)prgmem, (size_t)((uint8_t*)prgmem+requested_size)));
4532 void virtual_free (void* add, size_t size)
4534 GCToOSInterface::VirtualRelease (add, size);
4535 gc_heap::reserved_memory -= size;
4536 dprintf (2, ("Virtual Free size %Id: [%Ix, %Ix[",
4537 size, (size_t)add, (size_t)((uint8_t*)add+size)));
4540 static size_t get_valid_segment_size (BOOL large_seg=FALSE)
4542 size_t seg_size, initial_seg_size;
4546 initial_seg_size = INITIAL_ALLOC;
4547 seg_size = static_cast<size_t>(GCConfig::GetSegmentSize());
4551 initial_seg_size = LHEAP_ALLOC;
4552 seg_size = static_cast<size_t>(GCConfig::GetSegmentSize()) / 2;
4555 #ifdef MULTIPLE_HEAPS
4560 if (g_num_processors > 4)
4561 initial_seg_size /= 2;
4562 if (g_num_processors > 8)
4563 initial_seg_size /= 2;
4565 #endif //MULTIPLE_HEAPS
4567 // if seg_size is small but not 0 (0 is default if config not set)
4568 // then set the segment to the minimum size
4569 if (!g_theGCHeap->IsValidSegmentSize(seg_size))
4571 // if requested size is between 1 byte and 4MB, use min
4572 if ((seg_size >> 1) && !(seg_size >> 22))
4573 seg_size = 1024*1024*4;
4575 seg_size = initial_seg_size;
4578 #ifdef SEG_MAPPING_TABLE
4580 seg_size = round_up_power2 (seg_size);
4582 seg_size = round_down_power2 (seg_size);
4584 #endif //SEG_MAPPING_TABLE
4590 gc_heap::compute_new_ephemeral_size()
4592 int eph_gen_max = max_generation - 1 - (settings.promotion ? 1 : 0);
4593 size_t padding_size = 0;
4595 for (int i = 0; i <= eph_gen_max; i++)
4597 dynamic_data* dd = dynamic_data_of (i);
4598 total_ephemeral_size += (dd_survived_size (dd) - dd_pinned_survived_size (dd));
4599 #ifdef RESPECT_LARGE_ALIGNMENT
4600 total_ephemeral_size += dd_num_npinned_plugs (dd) * switch_alignment_size (FALSE);
4601 #endif //RESPECT_LARGE_ALIGNMENT
4602 #ifdef FEATURE_STRUCTALIGN
4603 total_ephemeral_size += dd_num_npinned_plugs (dd) * MAX_STRUCTALIGN;
4604 #endif //FEATURE_STRUCTALIGN
4607 padding_size += dd_padding_size (dd);
4608 #endif //SHORT_PLUGS
4611 total_ephemeral_size += eph_gen_starts_size;
4613 #ifdef RESPECT_LARGE_ALIGNMENT
4614 size_t planned_ephemeral_size = heap_segment_plan_allocated (ephemeral_heap_segment) -
4615 generation_plan_allocation_start (generation_of (max_generation-1));
4616 total_ephemeral_size = min (total_ephemeral_size, planned_ephemeral_size);
4617 #endif //RESPECT_LARGE_ALIGNMENT
4620 total_ephemeral_size = Align ((size_t)((double)total_ephemeral_size * short_plugs_pad_ratio) + 1);
4621 total_ephemeral_size += Align (DESIRED_PLUG_LENGTH);
4622 #endif //SHORT_PLUGS
4624 dprintf (3, ("total ephemeral size is %Ix, padding %Ix(%Ix)",
4625 total_ephemeral_size,
4626 padding_size, (total_ephemeral_size - padding_size)));
4630 #pragma warning(disable:4706) // "assignment within conditional expression" is intentional in this function.
4634 gc_heap::soh_get_segment_to_expand()
4636 size_t size = soh_segment_size;
4638 ordered_plug_indices_init = FALSE;
4639 use_bestfit = FALSE;
4641 //compute the size of the new ephemeral heap segment.
4642 compute_new_ephemeral_size();
4644 if ((settings.pause_mode != pause_low_latency) &&
4645 (settings.pause_mode != pause_no_gc)
4646 #ifdef BACKGROUND_GC
4647 && (!recursive_gc_sync::background_running_p())
4648 #endif //BACKGROUND_GC
4651 allocator* gen_alloc = ((settings.condemned_generation == max_generation) ? 0 :
4652 generation_allocator (generation_of (max_generation)));
4653 dprintf (2, ("(gen%d)soh_get_segment_to_expand", settings.condemned_generation));
4655 // try to find one in the gen 2 segment list, search backwards because the first segments
4656 // tend to be more compact than the later ones.
4657 heap_segment* fseg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
4659 PREFIX_ASSUME(fseg != NULL);
4661 #ifdef SEG_REUSE_STATS
4663 #endif //SEG_REUSE_STATS
4665 heap_segment* seg = ephemeral_heap_segment;
4666 while ((seg = heap_segment_prev_rw (fseg, seg)) && (seg != fseg))
4668 #ifdef SEG_REUSE_STATS
4670 #endif //SEG_REUSE_STATS
4672 if (can_expand_into_p (seg, size/3, total_ephemeral_size, gen_alloc))
4674 get_gc_data_per_heap()->set_mechanism (gc_heap_expand,
4675 (use_bestfit ? expand_reuse_bestfit : expand_reuse_normal));
4676 if (settings.condemned_generation == max_generation)
4680 build_ordered_free_spaces (seg);
4681 dprintf (GTC_LOG, ("can use best fit"));
4684 #ifdef SEG_REUSE_STATS
4685 dprintf (SEG_REUSE_LOG_0, ("(gen%d)soh_get_segment_to_expand: found seg #%d to reuse",
4686 settings.condemned_generation, try_reuse));
4687 #endif //SEG_REUSE_STATS
4688 dprintf (GTC_LOG, ("max_gen: Found existing segment to expand into %Ix", (size_t)seg));
4693 #ifdef SEG_REUSE_STATS
4694 dprintf (SEG_REUSE_LOG_0, ("(gen%d)soh_get_segment_to_expand: found seg #%d to reuse - returning",
4695 settings.condemned_generation, try_reuse));
4696 #endif //SEG_REUSE_STATS
4697 dprintf (GTC_LOG, ("max_gen-1: Found existing segment to expand into %Ix", (size_t)seg));
4699 // If we return 0 here, the allocator will think since we are short on end
4700 // of seg we neeed to trigger a full compacting GC. So if sustained low latency
4701 // is set we should acquire a new seg instead, that way we wouldn't be short.
4702 // The real solution, of course, is to actually implement seg reuse in gen1.
4703 if (settings.pause_mode != pause_sustained_low_latency)
4705 dprintf (GTC_LOG, ("max_gen-1: SustainedLowLatency is set, acquire a new seg"));
4706 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, expand_next_full_gc);
4714 heap_segment* result = get_segment (size, FALSE);
4718 #ifdef BACKGROUND_GC
4719 if (current_c_gc_state == c_gc_state_planning)
4721 // When we expand heap during bgc sweep, we set the seg to be swept so
4722 // we'll always look at cards for objects on the new segment.
4723 result->flags |= heap_segment_flags_swept;
4725 #endif //BACKGROUND_GC
4727 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(result),
4728 (size_t)(heap_segment_reserved (result) - heap_segment_mem(result)),
4729 gc_etw_segment_small_object_heap);
4732 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, (result ? expand_new_seg : expand_no_memory));
4736 dprintf (2, ("h%d: failed to allocate a new segment!", heap_number));
4740 #ifdef MULTIPLE_HEAPS
4741 heap_segment_heap (result) = this;
4742 #endif //MULTIPLE_HEAPS
4745 dprintf (GTC_LOG, ("(gen%d)creating new segment %Ix", settings.condemned_generation, result));
4750 #pragma warning(default:4706)
4753 //returns 0 in case of allocation failure
4755 gc_heap::get_segment (size_t size, BOOL loh_p)
4757 if (heap_hard_limit)
4760 heap_segment* result = 0;
4762 if (segment_standby_list != 0)
4764 result = segment_standby_list;
4765 heap_segment* last = 0;
4768 size_t hs = (size_t)(heap_segment_reserved (result) - (uint8_t*)result);
4769 if ((hs >= size) && ((hs / 2) < size))
4771 dprintf (2, ("Hoarded segment %Ix found", (size_t) result));
4774 heap_segment_next (last) = heap_segment_next (result);
4778 segment_standby_list = heap_segment_next (result);
4785 result = heap_segment_next (result);
4792 init_heap_segment (result);
4793 #ifdef BACKGROUND_GC
4794 if (should_commit_mark_array())
4796 dprintf (GC_TABLE_LOG, ("hoarded seg %Ix, mark_array is %Ix", result, mark_array));
4797 if (!commit_mark_array_new_seg (__this, result))
4799 dprintf (GC_TABLE_LOG, ("failed to commit mark array for hoarded seg"));
4800 // If we can't use it we need to thread it back.
4801 if (segment_standby_list != 0)
4803 heap_segment_next (result) = segment_standby_list;
4804 segment_standby_list = result;
4808 segment_standby_list = result;
4814 #endif //BACKGROUND_GC
4816 #ifdef SEG_MAPPING_TABLE
4818 seg_mapping_table_add_segment (result, __this);
4819 #endif //SEG_MAPPING_TABLE
4824 #ifndef SEG_MAPPING_TABLE
4825 if (!seg_table->ensure_space_for_insert ())
4827 #endif //SEG_MAPPING_TABLE
4828 void* mem = virtual_alloc (size);
4831 fgm_result.set_fgm (fgm_reserve_segment, size, loh_p);
4835 result = gc_heap::make_heap_segment ((uint8_t*)mem, size, heap_number);
4841 if (mem < g_gc_lowest_address)
4843 start = (uint8_t*)mem;
4847 start = (uint8_t*)g_gc_lowest_address;
4850 if (((uint8_t*)mem + size) > g_gc_highest_address)
4852 end = (uint8_t*)mem + size;
4856 end = (uint8_t*)g_gc_highest_address;
4859 if (gc_heap::grow_brick_card_tables (start, end, size, result, __this, loh_p) != 0)
4861 virtual_free (mem, size);
4867 fgm_result.set_fgm (fgm_commit_segment_beg, SEGMENT_INITIAL_COMMIT, loh_p);
4868 virtual_free (mem, size);
4873 #ifdef SEG_MAPPING_TABLE
4874 seg_mapping_table_add_segment (result, __this);
4875 #else //SEG_MAPPING_TABLE
4876 gc_heap::seg_table->insert ((uint8_t*)result, delta);
4877 #endif //SEG_MAPPING_TABLE
4881 #ifdef BACKGROUND_GC
4884 ::record_changed_seg ((uint8_t*)result, heap_segment_reserved (result),
4885 settings.gc_index, current_bgc_state,
4887 bgc_verify_mark_array_cleared (result);
4889 #endif //BACKGROUND_GC
4891 dprintf (GC_TABLE_LOG, ("h%d: new seg: %Ix-%Ix (%Id)", heap_number, result, ((uint8_t*)result + size), size));
4895 void release_segment (heap_segment* sg)
4897 ptrdiff_t delta = 0;
4898 FIRE_EVENT(GCFreeSegment_V1, heap_segment_mem(sg));
4899 virtual_free (sg, (uint8_t*)heap_segment_reserved (sg)-(uint8_t*)sg);
4902 heap_segment* gc_heap::get_segment_for_loh (size_t size
4903 #ifdef MULTIPLE_HEAPS
4905 #endif //MULTIPLE_HEAPS
4908 #ifndef MULTIPLE_HEAPS
4910 #endif //MULTIPLE_HEAPS
4911 heap_segment* res = hp->get_segment (size, TRUE);
4914 #ifdef MULTIPLE_HEAPS
4915 heap_segment_heap (res) = hp;
4916 #endif //MULTIPLE_HEAPS
4917 res->flags |= heap_segment_flags_loh;
4919 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(res), (size_t)(heap_segment_reserved (res) - heap_segment_mem(res)), gc_etw_segment_large_object_heap);
4921 GCToEEInterface::DiagUpdateGenerationBounds();
4923 #ifdef MULTIPLE_HEAPS
4924 hp->thread_loh_segment (res);
4926 thread_loh_segment (res);
4927 #endif //MULTIPLE_HEAPS
4933 void gc_heap::thread_loh_segment (heap_segment* new_seg)
4935 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
4937 while (heap_segment_next_rw (seg))
4938 seg = heap_segment_next_rw (seg);
4939 heap_segment_next (seg) = new_seg;
4943 gc_heap::get_large_segment (size_t size, BOOL* did_full_compact_gc)
4945 *did_full_compact_gc = FALSE;
4946 size_t last_full_compact_gc_count = get_full_compact_gc_count();
4948 //access to get_segment needs to be serialized
4949 add_saved_spinlock_info (true, me_release, mt_get_large_seg);
4950 leave_spin_lock (&more_space_lock_loh);
4951 enter_spin_lock (&gc_heap::gc_lock);
4952 dprintf (SPINLOCK_LOG, ("[%d]Seg: Egc", heap_number));
4953 // if a GC happened between here and before we ask for a segment in
4954 // get_large_segment, we need to count that GC.
4955 size_t current_full_compact_gc_count = get_full_compact_gc_count();
4957 if (current_full_compact_gc_count > last_full_compact_gc_count)
4959 *did_full_compact_gc = TRUE;
4962 heap_segment* res = get_segment_for_loh (size
4963 #ifdef MULTIPLE_HEAPS
4965 #endif //MULTIPLE_HEAPS
4968 dprintf (SPINLOCK_LOG, ("[%d]Seg: A Lgc", heap_number));
4969 leave_spin_lock (&gc_heap::gc_lock);
4970 enter_spin_lock (&more_space_lock_loh);
4971 add_saved_spinlock_info (true, me_acquire, mt_get_large_seg);
4977 BOOL gc_heap::unprotect_segment (heap_segment* seg)
4979 uint8_t* start = align_lower_page (heap_segment_mem (seg));
4980 ptrdiff_t region_size = heap_segment_allocated (seg) - start;
4982 if (region_size != 0 )
4984 dprintf (3, ("unprotecting segment %Ix:", (size_t)seg));
4986 BOOL status = GCToOSInterface::VirtualUnprotect (start, region_size);
4994 #ifdef MULTIPLE_HEAPS
4997 #pragma warning(disable:4035)
4998 static ptrdiff_t get_cycle_count()
5002 #pragma warning(default:4035)
5003 #elif defined(__GNUC__)
5004 static ptrdiff_t get_cycle_count()
5008 __asm__ __volatile__
5009 ("rdtsc":"=a" (cycles), "=d" (cyclesHi));
5013 #error Unknown compiler
5015 #elif defined(_TARGET_AMD64_)
5017 extern "C" uint64_t __rdtsc();
5018 #pragma intrinsic(__rdtsc)
5019 static ptrdiff_t get_cycle_count()
5021 return (ptrdiff_t)__rdtsc();
5023 #elif defined(__clang__)
5024 static ptrdiff_t get_cycle_count()
5028 __asm__ __volatile__
5029 ("rdtsc":"=a" (cycles), "=d" (cyclesHi));
5030 return (cyclesHi << 32) | cycles;
5033 extern "C" ptrdiff_t get_cycle_count(void);
5035 #elif defined(_TARGET_ARM_)
5036 static ptrdiff_t get_cycle_count()
5038 // @ARMTODO: cycle counter is not exposed to user mode by CoreARM. For now (until we can show this
5039 // makes a difference on the ARM configurations on which we'll run) just return 0. This will result in
5040 // all buffer access times being reported as equal in access_time().
5043 #elif defined(_TARGET_ARM64_)
5044 static ptrdiff_t get_cycle_count()
5046 // @ARM64TODO: cycle counter is not exposed to user mode by CoreARM. For now (until we can show this
5047 // makes a difference on the ARM configurations on which we'll run) just return 0. This will result in
5048 // all buffer access times being reported as equal in access_time().
5052 #error NYI platform: get_cycle_count
5053 #endif //_TARGET_X86_
5058 static uint8_t* sniff_buffer;
5059 static unsigned n_sniff_buffers;
5060 static unsigned cur_sniff_index;
5062 static uint16_t proc_no_to_heap_no[MAX_SUPPORTED_CPUS];
5063 static uint16_t heap_no_to_proc_no[MAX_SUPPORTED_CPUS];
5064 static uint16_t heap_no_to_numa_node[MAX_SUPPORTED_CPUS];
5065 static uint16_t heap_no_to_cpu_group[MAX_SUPPORTED_CPUS];
5066 static uint16_t heap_no_to_group_proc[MAX_SUPPORTED_CPUS];
5067 static uint16_t numa_node_to_heap_map[MAX_SUPPORTED_CPUS+4];
5069 static int access_time(uint8_t *sniff_buffer, int heap_number, unsigned sniff_index, unsigned n_sniff_buffers)
5071 ptrdiff_t start_cycles = get_cycle_count();
5072 uint8_t sniff = sniff_buffer[(1 + heap_number*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE];
5073 assert (sniff == 0);
5074 ptrdiff_t elapsed_cycles = get_cycle_count() - start_cycles;
5075 // add sniff here just to defeat the optimizer
5076 elapsed_cycles += sniff;
5077 return (int) elapsed_cycles;
5081 static BOOL init(int n_heaps)
5083 assert (sniff_buffer == NULL && n_sniff_buffers == 0);
5084 if (!GCToOSInterface::CanGetCurrentProcessorNumber())
5086 n_sniff_buffers = n_heaps*2+1;
5087 size_t n_cache_lines = 1 + n_heaps * n_sniff_buffers + 1;
5088 size_t sniff_buf_size = n_cache_lines * HS_CACHE_LINE_SIZE;
5089 if (sniff_buf_size / HS_CACHE_LINE_SIZE != n_cache_lines) // check for overlow
5094 sniff_buffer = new (nothrow) uint8_t[sniff_buf_size];
5095 if (sniff_buffer == 0)
5097 memset(sniff_buffer, 0, sniff_buf_size*sizeof(uint8_t));
5100 //can not enable gc numa aware, force all heaps to be in
5101 //one numa node by filling the array with all 0s
5102 if (!GCToOSInterface::CanEnableGCNumaAware())
5103 memset(heap_no_to_numa_node, 0, sizeof (heap_no_to_numa_node));
5108 static void init_cpu_mapping(gc_heap * /*heap*/, int heap_number)
5110 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5112 uint32_t proc_no = GCToOSInterface::GetCurrentProcessorNumber() % gc_heap::n_heaps;
5113 // We can safely cast heap_number to a uint16_t 'cause GetCurrentProcessCpuCount
5114 // only returns up to MAX_SUPPORTED_CPUS procs right now. We only ever create at most
5115 // MAX_SUPPORTED_CPUS GC threads.
5116 proc_no_to_heap_no[proc_no] = (uint16_t)heap_number;
5120 static void mark_heap(int heap_number)
5122 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5125 for (unsigned sniff_index = 0; sniff_index < n_sniff_buffers; sniff_index++)
5126 sniff_buffer[(1 + heap_number*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE] &= 1;
5129 static int select_heap(alloc_context* acontext, int /*hint*/)
5131 UNREFERENCED_PARAMETER(acontext); // only referenced by dprintf
5133 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5134 return proc_no_to_heap_no[GCToOSInterface::GetCurrentProcessorNumber() % gc_heap::n_heaps];
5136 unsigned sniff_index = Interlocked::Increment(&cur_sniff_index);
5137 sniff_index %= n_sniff_buffers;
5140 int best_access_time = 1000*1000*1000;
5141 int second_best_access_time = best_access_time;
5143 uint8_t *l_sniff_buffer = sniff_buffer;
5144 unsigned l_n_sniff_buffers = n_sniff_buffers;
5145 for (int heap_number = 0; heap_number < gc_heap::n_heaps; heap_number++)
5147 int this_access_time = access_time(l_sniff_buffer, heap_number, sniff_index, l_n_sniff_buffers);
5148 if (this_access_time < best_access_time)
5150 second_best_access_time = best_access_time;
5151 best_access_time = this_access_time;
5152 best_heap = heap_number;
5154 else if (this_access_time < second_best_access_time)
5156 second_best_access_time = this_access_time;
5160 if (best_access_time*2 < second_best_access_time)
5162 sniff_buffer[(1 + best_heap*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE] &= 1;
5164 dprintf (3, ("select_heap yields crisp %d for context %p\n", best_heap, (void *)acontext));
5168 dprintf (3, ("select_heap yields vague %d for context %p\n", best_heap, (void *)acontext ));
5174 static bool can_find_heap_fast()
5176 return GCToOSInterface::CanGetCurrentProcessorNumber();
5179 static uint16_t find_proc_no_from_heap_no(int heap_number)
5181 return heap_no_to_proc_no[heap_number];
5184 static void set_proc_no_for_heap(int heap_number, uint16_t proc_no)
5186 heap_no_to_proc_no[heap_number] = proc_no;
5189 static uint16_t find_numa_node_from_heap_no(int heap_number)
5191 return heap_no_to_numa_node[heap_number];
5194 static void set_numa_node_for_heap(int heap_number, uint16_t numa_node)
5196 heap_no_to_numa_node[heap_number] = numa_node;
5199 static uint16_t find_cpu_group_from_heap_no(int heap_number)
5201 return heap_no_to_cpu_group[heap_number];
5204 static void set_cpu_group_for_heap(int heap_number, uint16_t group_number)
5206 heap_no_to_cpu_group[heap_number] = group_number;
5209 static uint16_t find_group_proc_from_heap_no(int heap_number)
5211 return heap_no_to_group_proc[heap_number];
5214 static void set_group_proc_for_heap(int heap_number, uint16_t group_proc)
5216 heap_no_to_group_proc[heap_number] = group_proc;
5219 static void init_numa_node_to_heap_map(int nheaps)
5220 { // called right after GCHeap::Init() for each heap is finished
5221 // when numa is not enabled, heap_no_to_numa_node[] are all filled
5222 // with 0s during initialization, and will be treated as one node
5223 numa_node_to_heap_map[0] = 0;
5226 for (int i=1; i < nheaps; i++)
5228 if (heap_no_to_numa_node[i] != heap_no_to_numa_node[i-1])
5229 numa_node_to_heap_map[node_index++] = (uint16_t)i;
5231 numa_node_to_heap_map[node_index] = (uint16_t)nheaps; //mark the end with nheaps
5234 static void get_heap_range_for_heap(int hn, int* start, int* end)
5235 { // 1-tier/no numa case: heap_no_to_numa_node[] all zeros,
5236 // and treated as in one node. thus: start=0, end=n_heaps
5237 uint16_t numa_node = heap_no_to_numa_node[hn];
5238 *start = (int)numa_node_to_heap_map[numa_node];
5239 *end = (int)(numa_node_to_heap_map[numa_node+1]);
5242 uint8_t* heap_select::sniff_buffer;
5243 unsigned heap_select::n_sniff_buffers;
5244 unsigned heap_select::cur_sniff_index;
5245 uint16_t heap_select::proc_no_to_heap_no[MAX_SUPPORTED_CPUS];
5246 uint16_t heap_select::heap_no_to_proc_no[MAX_SUPPORTED_CPUS];
5247 uint16_t heap_select::heap_no_to_numa_node[MAX_SUPPORTED_CPUS];
5248 uint16_t heap_select::heap_no_to_cpu_group[MAX_SUPPORTED_CPUS];
5249 uint16_t heap_select::heap_no_to_group_proc[MAX_SUPPORTED_CPUS];
5250 uint16_t heap_select::numa_node_to_heap_map[MAX_SUPPORTED_CPUS+4];
5252 BOOL gc_heap::create_thread_support (unsigned number_of_heaps)
5255 if (!gc_start_event.CreateOSManualEventNoThrow (FALSE))
5259 if (!ee_suspend_event.CreateOSAutoEventNoThrow (FALSE))
5263 if (!gc_t_join.init (number_of_heaps, join_flavor_server_gc))
5274 destroy_thread_support();
5280 void gc_heap::destroy_thread_support ()
5282 if (ee_suspend_event.IsValid())
5284 ee_suspend_event.CloseEvent();
5286 if (gc_start_event.IsValid())
5288 gc_start_event.CloseEvent();
5292 void set_thread_group_affinity_for_heap(int heap_number, GCThreadAffinity* affinity)
5294 affinity->Group = GCThreadAffinity::None;
5295 affinity->Processor = GCThreadAffinity::None;
5298 GCToOSInterface::GetGroupForProcessor((uint16_t)heap_number, &gn, &gpn);
5301 for (uintptr_t mask = 1; mask !=0; mask <<=1)
5303 if (bit_number == gpn)
5305 dprintf(3, ("using processor group %d, mask %Ix for heap %d\n", gn, mask, heap_number));
5306 affinity->Processor = gpn;
5307 affinity->Group = gn;
5308 heap_select::set_cpu_group_for_heap(heap_number, gn);
5309 heap_select::set_group_proc_for_heap(heap_number, gpn);
5310 if (GCToOSInterface::CanEnableGCNumaAware())
5312 PROCESSOR_NUMBER proc_no;
5314 proc_no.Number = (uint8_t)gpn;
5315 proc_no.Reserved = 0;
5317 uint16_t node_no = 0;
5318 if (GCToOSInterface::GetNumaProcessorNode(&proc_no, &node_no))
5319 heap_select::set_numa_node_for_heap(heap_number, node_no);
5322 { // no numa setting, each cpu group is treated as a node
5323 heap_select::set_numa_node_for_heap(heap_number, gn);
5331 void set_thread_affinity_mask_for_heap(int heap_number, GCThreadAffinity* affinity)
5333 affinity->Group = GCThreadAffinity::None;
5334 affinity->Processor = GCThreadAffinity::None;
5336 uintptr_t pmask = process_mask;
5338 uint8_t proc_number = 0;
5339 for (uintptr_t mask = 1; mask != 0; mask <<= 1)
5341 if ((mask & pmask) != 0)
5343 if (bit_number == heap_number)
5345 dprintf (3, ("Using processor %d for heap %d", proc_number, heap_number));
5346 affinity->Processor = proc_number;
5347 heap_select::set_proc_no_for_heap(heap_number, proc_number);
5348 if (GCToOSInterface::CanEnableGCNumaAware())
5350 uint16_t node_no = 0;
5351 PROCESSOR_NUMBER proc_no;
5353 proc_no.Number = (uint8_t)proc_number;
5354 proc_no.Reserved = 0;
5355 if (GCToOSInterface::GetNumaProcessorNode(&proc_no, &node_no))
5357 heap_select::set_numa_node_for_heap(heap_number, node_no);
5368 bool gc_heap::create_gc_thread ()
5370 dprintf (3, ("Creating gc thread\n"));
5371 return GCToEEInterface::CreateThread(gc_thread_stub, this, false, ".NET Server GC");
5375 #pragma warning(disable:4715) //IA64 xcompiler recognizes that without the 'break;' the while(1) will never end and therefore not return a value for that code path
5377 void gc_heap::gc_thread_function ()
5379 assert (gc_done_event.IsValid());
5380 assert (gc_start_event.IsValid());
5381 dprintf (3, ("gc thread started"));
5383 heap_select::init_cpu_mapping(this, heap_number);
5387 assert (!gc_t_join.joined());
5389 if (heap_number == 0)
5391 gc_heap::ee_suspend_event.Wait(INFINITE, FALSE);
5393 BEGIN_TIMING(suspend_ee_during_log);
5394 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC);
5395 END_TIMING(suspend_ee_during_log);
5397 proceed_with_gc_p = TRUE;
5399 if (!should_proceed_with_gc())
5401 update_collection_counts_for_no_gc();
5402 proceed_with_gc_p = FALSE;
5406 settings.init_mechanisms();
5407 gc_start_event.Set();
5409 dprintf (3, ("%d gc thread waiting...", heap_number));
5413 gc_start_event.Wait(INFINITE, FALSE);
5414 dprintf (3, ("%d gc thread waiting... Done", heap_number));
5417 assert ((heap_number == 0) || proceed_with_gc_p);
5419 if (proceed_with_gc_p)
5421 garbage_collect (GCHeap::GcCondemnedGeneration);
5423 if (pm_trigger_full_gc)
5425 garbage_collect_pm_full_gc();
5429 if (heap_number == 0)
5431 if (proceed_with_gc_p && (!settings.concurrent))
5436 #ifdef BACKGROUND_GC
5437 recover_bgc_settings();
5438 #endif //BACKGROUND_GC
5440 #ifdef MULTIPLE_HEAPS
5441 for (int i = 0; i < gc_heap::n_heaps; i++)
5443 gc_heap* hp = gc_heap::g_heaps[i];
5444 hp->add_saved_spinlock_info (false, me_release, mt_block_gc);
5445 leave_spin_lock(&hp->more_space_lock_soh);
5447 #endif //MULTIPLE_HEAPS
5449 gc_heap::gc_started = FALSE;
5451 BEGIN_TIMING(restart_ee_during_log);
5452 GCToEEInterface::RestartEE(TRUE);
5453 END_TIMING(restart_ee_during_log);
5454 process_sync_log_stats();
5456 dprintf (SPINLOCK_LOG, ("GC Lgc"));
5457 leave_spin_lock (&gc_heap::gc_lock);
5459 gc_heap::internal_gc_done = true;
5461 if (proceed_with_gc_p)
5465 // If we didn't actually do a GC, it means we didn't wait up the other threads,
5466 // we still need to set the gc_done_event for those threads.
5467 for (int i = 0; i < gc_heap::n_heaps; i++)
5469 gc_heap* hp = gc_heap::g_heaps[i];
5476 int spin_count = 32 * (gc_heap::n_heaps - 1);
5478 // wait until RestartEE has progressed to a stage where we can restart user threads
5479 while (!gc_heap::internal_gc_done && !GCHeap::SafeToRestartManagedThreads())
5481 spin_and_switch (spin_count, (gc_heap::internal_gc_done || GCHeap::SafeToRestartManagedThreads()));
5488 #pragma warning(default:4715) //IA64 xcompiler recognizes that without the 'break;' the while(1) will never end and therefore not return a value for that code path
5491 #endif //MULTIPLE_HEAPS
5493 bool gc_heap::virtual_alloc_commit_for_heap (void* addr, size_t size, int h_number)
5495 #if defined(MULTIPLE_HEAPS) && !defined(FEATURE_REDHAWK)
5496 // Currently there is no way for us to specific the numa node to allocate on via hosting interfaces to
5497 // a host. This will need to be added later.
5498 #if !defined(FEATURE_CORECLR) && !defined(BUILD_AS_STANDALONE)
5499 if (!CLRMemoryHosted())
5502 if (GCToOSInterface::CanEnableGCNumaAware())
5504 uint32_t numa_node = heap_select::find_numa_node_from_heap_no(h_number);
5505 if (GCToOSInterface::VirtualCommit(addr, size, numa_node))
5510 UNREFERENCED_PARAMETER(h_number);
5513 //numa aware not enabled, or call failed --> fallback to VirtualCommit()
5514 return GCToOSInterface::VirtualCommit(addr, size);
5517 bool gc_heap::virtual_commit (void* address, size_t size, int h_number, bool* hard_limit_exceeded_p)
5520 assert (heap_hard_limit == 0);
5523 if (heap_hard_limit)
5525 bool exceeded_p = false;
5527 check_commit_cs.Enter();
5529 if ((current_total_committed + size) > heap_hard_limit)
5531 dprintf (1, ("%Id + %Id = %Id > limit",
5532 current_total_committed, size,
5533 (current_total_committed + size),
5540 current_total_committed += size;
5542 current_total_committed_bookkeeping += size;
5545 check_commit_cs.Leave();
5547 if (hard_limit_exceeded_p)
5548 *hard_limit_exceeded_p = exceeded_p;
5552 dprintf (1, ("can't commit %Ix for %Id bytes > HARD LIMIT %Id", (size_t)address, size, heap_hard_limit));
5557 // If it's a valid heap number it means it's commiting for memory on the GC heap.
5558 bool commit_succeeded_p = ((h_number >= 0) ?
5559 virtual_alloc_commit_for_heap (address, size, h_number) :
5560 GCToOSInterface::VirtualCommit(address, size));
5562 if (!commit_succeeded_p && heap_hard_limit)
5564 check_commit_cs.Enter();
5565 dprintf (1, ("commit failed, updating %Id to %Id",
5566 current_total_committed, (current_total_committed - size)));
5567 current_total_committed -= size;
5569 current_total_committed_bookkeeping -= size;
5571 check_commit_cs.Leave();
5574 return commit_succeeded_p;
5577 bool gc_heap::virtual_decommit (void* address, size_t size, int h_number)
5580 assert (heap_hard_limit == 0);
5583 bool decommit_succeeded_p = GCToOSInterface::VirtualDecommit (address, size);
5585 if (decommit_succeeded_p && heap_hard_limit)
5587 check_commit_cs.Enter();
5588 current_total_committed -= size;
5590 current_total_committed_bookkeeping -= size;
5591 check_commit_cs.Leave();
5594 return decommit_succeeded_p;
5597 #ifndef SEG_MAPPING_TABLE
5599 heap_segment* gc_heap::segment_of (uint8_t* add, ptrdiff_t& delta, BOOL verify_p)
5601 uint8_t* sadd = add;
5602 heap_segment* hs = 0;
5603 heap_segment* hs1 = 0;
5604 if (!((add >= g_gc_lowest_address) && (add < g_gc_highest_address)))
5609 //repeat in case there is a concurrent insertion in the table.
5614 seg_table->lookup (sadd);
5615 hs1 = (heap_segment*)sadd;
5616 } while (hs1 && !in_range_for_segment (add, hs1) && (hs != hs1));
5621 (verify_p && (add > heap_segment_reserved ((heap_segment*)(sadd + delta)))))
5625 #endif //SEG_MAPPING_TABLE
5633 // If we want to save space we can have a pool of plug_and_gap's instead of
5634 // always having 2 allocated for each pinned plug.
5635 gap_reloc_pair saved_pre_plug;
5636 // If we decide to not compact, we need to restore the original values.
5637 gap_reloc_pair saved_pre_plug_reloc;
5639 gap_reloc_pair saved_post_plug;
5641 // Supposedly Pinned objects cannot have references but we are seeing some from pinvoke
5642 // frames. Also if it's an artificially pinned plug created by us, it can certainly
5644 // We know these cases will be rare so we can optimize this to be only allocated on decommand.
5645 gap_reloc_pair saved_post_plug_reloc;
5647 // We need to calculate this after we are done with plan phase and before compact
5648 // phase because compact phase will change the bricks so relocate_address will no
5650 uint8_t* saved_pre_plug_info_reloc_start;
5652 // We need to save this because we will have no way to calculate it, unlike the
5653 // pre plug info start which is right before this plug.
5654 uint8_t* saved_post_plug_info_start;
5657 uint8_t* allocation_context_start_region;
5658 #endif //SHORT_PLUGS
5660 // How the bits in these bytes are organized:
5662 // bit to indicate whether it's a short obj | 3 bits for refs in this short obj | 2 unused bits | bit to indicate if it's collectible | last bit
5663 // last bit indicates if there's pre or post info associated with this plug. If it's not set all other bits will be 0.
5668 // We are seeing this is getting corrupted for a PP with a NP after.
5669 // Save it when we first set it and make sure it doesn't change.
5670 gap_reloc_pair saved_post_plug_debug;
5673 size_t get_max_short_bits()
5675 return (sizeof (gap_reloc_pair) / sizeof (uint8_t*));
5679 size_t get_pre_short_start_bit ()
5681 return (sizeof (saved_pre_p) * 8 - 1 - (sizeof (gap_reloc_pair) / sizeof (uint8_t*)));
5686 return (saved_pre_p & (1 << (sizeof (saved_pre_p) * 8 - 1)));
5689 void set_pre_short()
5691 saved_pre_p |= (1 << (sizeof (saved_pre_p) * 8 - 1));
5694 void set_pre_short_bit (size_t bit)
5696 saved_pre_p |= 1 << (get_pre_short_start_bit() + bit);
5699 BOOL pre_short_bit_p (size_t bit)
5701 return (saved_pre_p & (1 << (get_pre_short_start_bit() + bit)));
5704 #ifdef COLLECTIBLE_CLASS
5705 void set_pre_short_collectible()
5710 BOOL pre_short_collectible_p()
5712 return (saved_pre_p & 2);
5714 #endif //COLLECTIBLE_CLASS
5717 size_t get_post_short_start_bit ()
5719 return (sizeof (saved_post_p) * 8 - 1 - (sizeof (gap_reloc_pair) / sizeof (uint8_t*)));
5724 return (saved_post_p & (1 << (sizeof (saved_post_p) * 8 - 1)));
5727 void set_post_short()
5729 saved_post_p |= (1 << (sizeof (saved_post_p) * 8 - 1));
5732 void set_post_short_bit (size_t bit)
5734 saved_post_p |= 1 << (get_post_short_start_bit() + bit);
5737 BOOL post_short_bit_p (size_t bit)
5739 return (saved_post_p & (1 << (get_post_short_start_bit() + bit)));
5742 #ifdef COLLECTIBLE_CLASS
5743 void set_post_short_collectible()
5748 BOOL post_short_collectible_p()
5750 return (saved_post_p & 2);
5752 #endif //COLLECTIBLE_CLASS
5754 uint8_t* get_plug_address() { return first; }
5756 BOOL has_pre_plug_info() { return saved_pre_p; }
5757 BOOL has_post_plug_info() { return saved_post_p; }
5759 gap_reloc_pair* get_pre_plug_reloc_info() { return &saved_pre_plug_reloc; }
5760 gap_reloc_pair* get_post_plug_reloc_info() { return &saved_post_plug_reloc; }
5761 void set_pre_plug_info_reloc_start (uint8_t* reloc) { saved_pre_plug_info_reloc_start = reloc; }
5762 uint8_t* get_post_plug_info_start() { return saved_post_plug_info_start; }
5764 // We need to temporarily recover the shortened plugs for compact phase so we can
5765 // copy over the whole plug and their related info (mark bits/cards). But we will
5766 // need to set the artificial gap back so compact phase can keep reading the plug info.
5767 // We also need to recover the saved info because we'll need to recover it later.
5769 // So we would call swap_p*_plug_and_saved once to recover the object info; then call
5770 // it again to recover the artificial gap.
5771 void swap_pre_plug_and_saved()
5773 gap_reloc_pair temp;
5774 memcpy (&temp, (first - sizeof (plug_and_gap)), sizeof (temp));
5775 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug_reloc, sizeof (saved_pre_plug_reloc));
5776 saved_pre_plug_reloc = temp;
5779 void swap_post_plug_and_saved()
5781 gap_reloc_pair temp;
5782 memcpy (&temp, saved_post_plug_info_start, sizeof (temp));
5783 memcpy (saved_post_plug_info_start, &saved_post_plug_reloc, sizeof (saved_post_plug_reloc));
5784 saved_post_plug_reloc = temp;
5787 void swap_pre_plug_and_saved_for_profiler()
5789 gap_reloc_pair temp;
5790 memcpy (&temp, (first - sizeof (plug_and_gap)), sizeof (temp));
5791 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug, sizeof (saved_pre_plug));
5792 saved_pre_plug = temp;
5795 void swap_post_plug_and_saved_for_profiler()
5797 gap_reloc_pair temp;
5798 memcpy (&temp, saved_post_plug_info_start, sizeof (temp));
5799 memcpy (saved_post_plug_info_start, &saved_post_plug, sizeof (saved_post_plug));
5800 saved_post_plug = temp;
5803 // We should think about whether it's really necessary to have to copy back the pre plug
5804 // info since it was already copied during compacting plugs. But if a plug doesn't move
5805 // by >= 3 ptr size (the size of gap_reloc_pair), it means we'd have to recover pre plug info.
5806 void recover_plug_info()
5810 if (gc_heap::settings.compaction)
5812 dprintf (3, ("%Ix: REC Pre: %Ix-%Ix",
5814 &saved_pre_plug_reloc,
5815 saved_pre_plug_info_reloc_start));
5816 memcpy (saved_pre_plug_info_reloc_start, &saved_pre_plug_reloc, sizeof (saved_pre_plug_reloc));
5820 dprintf (3, ("%Ix: REC Pre: %Ix-%Ix",
5823 (first - sizeof (plug_and_gap))));
5824 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug, sizeof (saved_pre_plug));
5830 if (gc_heap::settings.compaction)
5832 dprintf (3, ("%Ix: REC Post: %Ix-%Ix",
5834 &saved_post_plug_reloc,
5835 saved_post_plug_info_start));
5836 memcpy (saved_post_plug_info_start, &saved_post_plug_reloc, sizeof (saved_post_plug_reloc));
5840 dprintf (3, ("%Ix: REC Post: %Ix-%Ix",
5843 saved_post_plug_info_start));
5844 memcpy (saved_post_plug_info_start, &saved_post_plug, sizeof (saved_post_plug));
5851 void gc_mechanisms::init_mechanisms()
5853 condemned_generation = 0;
5854 promotion = FALSE;//TRUE;
5856 #ifdef FEATURE_LOH_COMPACTION
5857 loh_compaction = gc_heap::should_compact_loh();
5859 loh_compaction = FALSE;
5860 #endif //FEATURE_LOH_COMPACTION
5861 heap_expansion = FALSE;
5864 elevation_reduced = FALSE;
5865 found_finalizers = FALSE;
5866 #ifdef BACKGROUND_GC
5867 background_p = recursive_gc_sync::background_running_p() != FALSE;
5868 allocations_allowed = TRUE;
5869 #endif //BACKGROUND_GC
5871 entry_memory_load = 0;
5872 exit_memory_load = 0;
5875 stress_induced = FALSE;
5876 #endif // STRESS_HEAP
5879 void gc_mechanisms::first_init()
5882 gen0_reduction_count = 0;
5883 should_lock_elevation = FALSE;
5884 elevation_locked_count = 0;
5885 reason = reason_empty;
5886 #ifdef BACKGROUND_GC
5887 pause_mode = gc_heap::gc_can_use_concurrent ? pause_interactive : pause_batch;
5889 int debug_pause_mode = static_cast<int>(GCConfig::GetLatencyMode());
5890 if (debug_pause_mode >= 0)
5892 assert (debug_pause_mode <= pause_sustained_low_latency);
5893 pause_mode = (gc_pause_mode)debug_pause_mode;
5896 #else //BACKGROUND_GC
5897 pause_mode = pause_batch;
5898 #endif //BACKGROUND_GC
5903 void gc_mechanisms::record (gc_history_global* history)
5905 #ifdef MULTIPLE_HEAPS
5906 history->num_heaps = gc_heap::n_heaps;
5908 history->num_heaps = 1;
5909 #endif //MULTIPLE_HEAPS
5911 history->condemned_generation = condemned_generation;
5912 history->gen0_reduction_count = gen0_reduction_count;
5913 history->reason = reason;
5914 history->pause_mode = (int)pause_mode;
5915 history->mem_pressure = entry_memory_load;
5916 history->global_mechanims_p = 0;
5918 // start setting the boolean values.
5920 history->set_mechanism_p (global_concurrent);
5923 history->set_mechanism_p (global_compaction);
5926 history->set_mechanism_p (global_promotion);
5929 history->set_mechanism_p (global_demotion);
5932 history->set_mechanism_p (global_card_bundles);
5934 if (elevation_reduced)
5935 history->set_mechanism_p (global_elevation);
5938 /**********************************
5939 called at the beginning of GC to fix the allocated size to
5940 what is really allocated, or to turn the free area into an unused object
5941 It needs to be called after all of the other allocation contexts have been
5942 fixed since it relies on alloc_allocated.
5943 ********************************/
5945 //for_gc_p indicates that the work is being done for GC,
5946 //as opposed to concurrent heap verification
5947 void gc_heap::fix_youngest_allocation_area (BOOL for_gc_p)
5949 UNREFERENCED_PARAMETER(for_gc_p);
5951 // The gen 0 alloc context is never used for allocation in the allocator path. It's
5952 // still used in the allocation path during GCs.
5953 assert (generation_allocation_pointer (youngest_generation) == nullptr);
5954 assert (generation_allocation_limit (youngest_generation) == nullptr);
5955 heap_segment_allocated (ephemeral_heap_segment) = alloc_allocated;
5958 void gc_heap::fix_large_allocation_area (BOOL for_gc_p)
5960 UNREFERENCED_PARAMETER(for_gc_p);
5963 alloc_context* acontext =
5965 generation_alloc_context (large_object_generation);
5966 assert (acontext->alloc_ptr == 0);
5967 assert (acontext->alloc_limit == 0);
5969 dprintf (3, ("Large object alloc context: ptr: %Ix, limit %Ix",
5970 (size_t)acontext->alloc_ptr, (size_t)acontext->alloc_limit));
5971 fix_allocation_context (acontext, FALSE, get_alignment_constant (FALSE));
5974 acontext->alloc_ptr = 0;
5975 acontext->alloc_limit = acontext->alloc_ptr;
5980 //for_gc_p indicates that the work is being done for GC,
5981 //as opposed to concurrent heap verification
5982 void gc_heap::fix_allocation_context (alloc_context* acontext, BOOL for_gc_p,
5985 dprintf (3, ("Fixing allocation context %Ix: ptr: %Ix, limit: %Ix",
5987 (size_t)acontext->alloc_ptr, (size_t)acontext->alloc_limit));
5989 if (((size_t)(alloc_allocated - acontext->alloc_limit) > Align (min_obj_size, align_const)) ||
5992 uint8_t* point = acontext->alloc_ptr;
5995 size_t size = (acontext->alloc_limit - acontext->alloc_ptr);
5996 // the allocation area was from the free list
5997 // it was shortened by Align (min_obj_size) to make room for
5998 // at least the shortest unused object
5999 size += Align (min_obj_size, align_const);
6000 assert ((size >= Align (min_obj_size)));
6002 dprintf(3,("Making unused area [%Ix, %Ix[", (size_t)point,
6003 (size_t)point + size ));
6004 make_unused_array (point, size);
6008 generation_free_obj_space (generation_of (0)) += size;
6009 alloc_contexts_used ++;
6015 alloc_allocated = acontext->alloc_ptr;
6016 assert (heap_segment_allocated (ephemeral_heap_segment) <=
6017 heap_segment_committed (ephemeral_heap_segment));
6018 alloc_contexts_used ++;
6023 // We need to update the alloc_bytes to reflect the portion that we have not used
6024 acontext->alloc_bytes -= (acontext->alloc_limit - acontext->alloc_ptr);
6025 acontext->alloc_ptr = 0;
6026 acontext->alloc_limit = acontext->alloc_ptr;
6030 //used by the heap verification for concurrent gc.
6031 //it nulls out the words set by fix_allocation_context for heap_verification
6032 void repair_allocation (gc_alloc_context* acontext, void*)
6034 uint8_t* point = acontext->alloc_ptr;
6038 dprintf (3, ("Clearing [%Ix, %Ix[", (size_t)acontext->alloc_ptr,
6039 (size_t)acontext->alloc_limit+Align(min_obj_size)));
6040 memclr (acontext->alloc_ptr - plug_skew,
6041 (acontext->alloc_limit - acontext->alloc_ptr)+Align (min_obj_size));
6045 void void_allocation (gc_alloc_context* acontext, void*)
6047 uint8_t* point = acontext->alloc_ptr;
6051 dprintf (3, ("Void [%Ix, %Ix[", (size_t)acontext->alloc_ptr,
6052 (size_t)acontext->alloc_limit+Align(min_obj_size)));
6053 acontext->alloc_ptr = 0;
6054 acontext->alloc_limit = acontext->alloc_ptr;
6058 void gc_heap::repair_allocation_contexts (BOOL repair_p)
6060 GCToEEInterface::GcEnumAllocContexts (repair_p ? repair_allocation : void_allocation, NULL);
6063 struct fix_alloc_context_args
6069 void fix_alloc_context (gc_alloc_context* acontext, void* param)
6071 fix_alloc_context_args* args = (fix_alloc_context_args*)param;
6072 g_theGCHeap->FixAllocContext(acontext, (void*)(size_t)(args->for_gc_p), args->heap);
6075 void gc_heap::fix_allocation_contexts (BOOL for_gc_p)
6077 fix_alloc_context_args args;
6078 args.for_gc_p = for_gc_p;
6081 GCToEEInterface::GcEnumAllocContexts(fix_alloc_context, &args);
6082 fix_youngest_allocation_area(for_gc_p);
6083 fix_large_allocation_area(for_gc_p);
6086 void gc_heap::fix_older_allocation_area (generation* older_gen)
6088 heap_segment* older_gen_seg = generation_allocation_segment (older_gen);
6089 if (generation_allocation_limit (older_gen) !=
6090 heap_segment_plan_allocated (older_gen_seg))
6092 uint8_t* point = generation_allocation_pointer (older_gen);
6094 size_t size = (generation_allocation_limit (older_gen) -
6095 generation_allocation_pointer (older_gen));
6098 assert ((size >= Align (min_obj_size)));
6099 dprintf(3,("Making unused area [%Ix, %Ix[", (size_t)point, (size_t)point+size));
6100 make_unused_array (point, size);
6101 if (size >= min_free_list)
6103 generation_allocator (older_gen)->thread_item_front (point, size);
6104 add_gen_free (older_gen->gen_num, size);
6105 generation_free_list_space (older_gen) += size;
6109 generation_free_obj_space (older_gen) += size;
6115 assert (older_gen_seg != ephemeral_heap_segment);
6116 heap_segment_plan_allocated (older_gen_seg) =
6117 generation_allocation_pointer (older_gen);
6118 generation_allocation_limit (older_gen) =
6119 generation_allocation_pointer (older_gen);
6122 generation_allocation_pointer (older_gen) = 0;
6123 generation_allocation_limit (older_gen) = 0;
6126 void gc_heap::set_allocation_heap_segment (generation* gen)
6128 uint8_t* p = generation_allocation_start (gen);
6130 heap_segment* seg = generation_allocation_segment (gen);
6131 if (in_range_for_segment (p, seg))
6134 // try ephemeral heap segment in case of heap expansion
6135 seg = ephemeral_heap_segment;
6136 if (!in_range_for_segment (p, seg))
6138 seg = heap_segment_rw (generation_start_segment (gen));
6140 PREFIX_ASSUME(seg != NULL);
6142 while (!in_range_for_segment (p, seg))
6144 seg = heap_segment_next_rw (seg);
6145 PREFIX_ASSUME(seg != NULL);
6149 generation_allocation_segment (gen) = seg;
6152 void gc_heap::reset_allocation_pointers (generation* gen, uint8_t* start)
6155 assert (Align ((size_t)start) == (size_t)start);
6156 generation_allocation_start (gen) = start;
6157 generation_allocation_pointer (gen) = 0;//start + Align (min_obj_size);
6158 generation_allocation_limit (gen) = 0;//generation_allocation_pointer (gen);
6159 set_allocation_heap_segment (gen);
6162 #ifdef BACKGROUND_GC
6163 //TODO BACKGROUND_GC this is for test only
6165 gc_heap::disallow_new_allocation (int gen_number)
6167 UNREFERENCED_PARAMETER(gen_number);
6168 settings.allocations_allowed = FALSE;
6171 gc_heap::allow_new_allocation (int gen_number)
6173 UNREFERENCED_PARAMETER(gen_number);
6174 settings.allocations_allowed = TRUE;
6177 #endif //BACKGROUND_GC
6179 bool gc_heap::new_allocation_allowed (int gen_number)
6181 #ifdef BACKGROUND_GC
6182 //TODO BACKGROUND_GC this is for test only
6183 if (!settings.allocations_allowed)
6185 dprintf (2, ("new allocation not allowed"));
6188 #endif //BACKGROUND_GC
6190 if (dd_new_allocation (dynamic_data_of (gen_number)) < 0)
6192 if (gen_number != 0)
6194 // For LOH we will give it more budget before we try a GC.
6195 if (settings.concurrent)
6197 dynamic_data* dd2 = dynamic_data_of (max_generation + 1 );
6199 if (dd_new_allocation (dd2) <= (ptrdiff_t)(-2 * dd_desired_allocation (dd2)))
6207 #ifndef MULTIPLE_HEAPS
6208 else if ((settings.pause_mode != pause_no_gc) && (gen_number == 0))
6210 dprintf (3, ("evaluating allocation rate"));
6211 dynamic_data* dd0 = dynamic_data_of (0);
6212 if ((allocation_running_amount - dd_new_allocation (dd0)) >
6215 uint32_t ctime = GCToOSInterface::GetLowPrecisionTimeStamp();
6216 if ((ctime - allocation_running_time) > 1000)
6218 dprintf (2, (">1s since last gen0 gc"));
6223 allocation_running_amount = dd_new_allocation (dd0);
6227 #endif //MULTIPLE_HEAPS
6232 ptrdiff_t gc_heap::get_desired_allocation (int gen_number)
6234 return dd_desired_allocation (dynamic_data_of (gen_number));
6238 ptrdiff_t gc_heap::get_new_allocation (int gen_number)
6240 return dd_new_allocation (dynamic_data_of (gen_number));
6243 //return the amount allocated so far in gen_number
6245 ptrdiff_t gc_heap::get_allocation (int gen_number)
6247 dynamic_data* dd = dynamic_data_of (gen_number);
6249 return dd_desired_allocation (dd) - dd_new_allocation (dd);
6253 BOOL grow_mark_stack (mark*& m, size_t& len, size_t init_len)
6255 size_t new_size = max (init_len, 2*len);
6256 mark* tmp = new (nothrow) mark [new_size];
6259 memcpy (tmp, m, len * sizeof (mark));
6267 dprintf (1, ("Failed to allocate %Id bytes for mark stack", (len * sizeof (mark))));
6273 uint8_t* pinned_plug (mark* m)
6279 size_t& pinned_len (mark* m)
6285 void set_new_pin_info (mark* m, uint8_t* pin_free_space_start)
6287 m->len = pinned_plug (m) - pin_free_space_start;
6289 m->allocation_context_start_region = pin_free_space_start;
6290 #endif //SHORT_PLUGS
6295 uint8_t*& pin_allocation_context_start_region (mark* m)
6297 return m->allocation_context_start_region;
6300 uint8_t* get_plug_start_in_saved (uint8_t* old_loc, mark* pinned_plug_entry)
6302 uint8_t* saved_pre_plug_info = (uint8_t*)(pinned_plug_entry->get_pre_plug_reloc_info());
6303 uint8_t* plug_start_in_saved = saved_pre_plug_info + (old_loc - (pinned_plug (pinned_plug_entry) - sizeof (plug_and_gap)));
6304 //dprintf (1, ("detected a very short plug: %Ix before PP %Ix, pad %Ix",
6305 // old_loc, pinned_plug (pinned_plug_entry), plug_start_in_saved));
6306 dprintf (1, ("EP: %Ix(%Ix), %Ix", old_loc, pinned_plug (pinned_plug_entry), plug_start_in_saved));
6307 return plug_start_in_saved;
6311 void set_padding_in_expand (uint8_t* old_loc,
6312 BOOL set_padding_on_saved_p,
6313 mark* pinned_plug_entry)
6315 if (set_padding_on_saved_p)
6317 set_plug_padded (get_plug_start_in_saved (old_loc, pinned_plug_entry));
6321 set_plug_padded (old_loc);
6326 void clear_padding_in_expand (uint8_t* old_loc,
6327 BOOL set_padding_on_saved_p,
6328 mark* pinned_plug_entry)
6330 if (set_padding_on_saved_p)
6332 clear_plug_padded (get_plug_start_in_saved (old_loc, pinned_plug_entry));
6336 clear_plug_padded (old_loc);
6339 #endif //SHORT_PLUGS
6341 void gc_heap::reset_pinned_queue()
6347 void gc_heap::reset_pinned_queue_bos()
6352 // last_pinned_plug is only for asserting purpose.
6353 void gc_heap::merge_with_last_pinned_plug (uint8_t* last_pinned_plug, size_t plug_size)
6355 if (last_pinned_plug)
6357 mark& last_m = mark_stack_array[mark_stack_tos - 1];
6358 assert (last_pinned_plug == last_m.first);
6359 if (last_m.saved_post_p)
6361 last_m.saved_post_p = FALSE;
6362 dprintf (3, ("setting last plug %Ix post to false", last_m.first));
6363 // We need to recover what the gap has overwritten.
6364 memcpy ((last_m.first + last_m.len - sizeof (plug_and_gap)), &(last_m.saved_post_plug), sizeof (gap_reloc_pair));
6366 last_m.len += plug_size;
6367 dprintf (3, ("recovered the last part of plug %Ix, setting its plug size to %Ix", last_m.first, last_m.len));
6371 void gc_heap::set_allocator_next_pin (uint8_t* alloc_pointer, uint8_t*& alloc_limit)
6373 dprintf (3, ("sanp: ptr: %Ix, limit: %Ix", alloc_pointer, alloc_limit));
6374 dprintf (3, ("oldest %Id: %Ix", mark_stack_bos, pinned_plug (oldest_pin())));
6375 if (!(pinned_plug_que_empty_p()))
6377 mark* oldest_entry = oldest_pin();
6378 uint8_t* plug = pinned_plug (oldest_entry);
6379 if ((plug >= alloc_pointer) && (plug < alloc_limit))
6381 alloc_limit = pinned_plug (oldest_entry);
6382 dprintf (3, ("now setting alloc context: %Ix->%Ix(%Id)",
6383 alloc_pointer, alloc_limit, (alloc_limit - alloc_pointer)));
6388 void gc_heap::set_allocator_next_pin (generation* gen)
6390 dprintf (3, ("SANP: gen%d, ptr; %Ix, limit: %Ix", gen->gen_num, generation_allocation_pointer (gen), generation_allocation_limit (gen)));
6391 if (!(pinned_plug_que_empty_p()))
6393 mark* oldest_entry = oldest_pin();
6394 uint8_t* plug = pinned_plug (oldest_entry);
6395 if ((plug >= generation_allocation_pointer (gen)) &&
6396 (plug < generation_allocation_limit (gen)))
6398 generation_allocation_limit (gen) = pinned_plug (oldest_entry);
6399 dprintf (3, ("SANP: get next pin free space in gen%d for alloc: %Ix->%Ix(%Id)",
6401 generation_allocation_pointer (gen), generation_allocation_limit (gen),
6402 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
6405 assert (!((plug < generation_allocation_pointer (gen)) &&
6406 (plug >= heap_segment_mem (generation_allocation_segment (gen)))));
6410 // After we set the info, we increase tos.
6411 void gc_heap::set_pinned_info (uint8_t* last_pinned_plug, size_t plug_len, uint8_t* alloc_pointer, uint8_t*& alloc_limit)
6413 UNREFERENCED_PARAMETER(last_pinned_plug);
6415 mark& m = mark_stack_array[mark_stack_tos];
6416 assert (m.first == last_pinned_plug);
6420 set_allocator_next_pin (alloc_pointer, alloc_limit);
6423 // After we set the info, we increase tos.
6424 void gc_heap::set_pinned_info (uint8_t* last_pinned_plug, size_t plug_len, generation* gen)
6426 UNREFERENCED_PARAMETER(last_pinned_plug);
6428 mark& m = mark_stack_array[mark_stack_tos];
6429 assert (m.first == last_pinned_plug);
6434 // Why are we checking here? gen is never 0.
6437 set_allocator_next_pin (gen);
6441 size_t gc_heap::deque_pinned_plug ()
6443 dprintf (3, ("dequed: %Id", mark_stack_bos));
6444 size_t m = mark_stack_bos;
6450 mark* gc_heap::pinned_plug_of (size_t bos)
6452 return &mark_stack_array [ bos ];
6456 mark* gc_heap::oldest_pin ()
6458 return pinned_plug_of (mark_stack_bos);
6462 BOOL gc_heap::pinned_plug_que_empty_p ()
6464 return (mark_stack_bos == mark_stack_tos);
6468 mark* gc_heap::before_oldest_pin()
6470 if (mark_stack_bos >= 1)
6471 return pinned_plug_of (mark_stack_bos-1);
6477 BOOL gc_heap::ephemeral_pointer_p (uint8_t* o)
6479 return ((o >= ephemeral_low) && (o < ephemeral_high));
6484 int& gc_heap::mark_stack_busy()
6486 return g_mark_stack_busy [(heap_number+2)*HS_CACHE_LINE_SIZE/sizeof(int)];
6490 void gc_heap::make_mark_stack (mark* arr)
6492 reset_pinned_queue();
6493 mark_stack_array = arr;
6494 mark_stack_array_length = MARK_STACK_INITIAL_LENGTH;
6496 mark_stack_busy() = 0;
6500 #ifdef BACKGROUND_GC
6502 size_t& gc_heap::bpromoted_bytes(int thread)
6504 #ifdef MULTIPLE_HEAPS
6505 return g_bpromoted [thread*16];
6506 #else //MULTIPLE_HEAPS
6507 UNREFERENCED_PARAMETER(thread);
6509 #endif //MULTIPLE_HEAPS
6512 void gc_heap::make_background_mark_stack (uint8_t** arr)
6514 background_mark_stack_array = arr;
6515 background_mark_stack_array_length = MARK_STACK_INITIAL_LENGTH;
6516 background_mark_stack_tos = arr;
6519 void gc_heap::make_c_mark_list (uint8_t** arr)
6522 c_mark_list_index = 0;
6523 c_mark_list_length = 1 + (OS_PAGE_SIZE / MIN_OBJECT_SIZE);
6525 #endif //BACKGROUND_GC
6530 // The card bundle keeps track of groups of card words.
6531 static const size_t card_bundle_word_width = 32;
6533 // How do we express the fact that 32 bits (card_word_width) is one uint32_t?
6534 static const size_t card_bundle_size = (size_t)(GC_PAGE_SIZE / (sizeof(uint32_t)*card_bundle_word_width));
6537 size_t card_bundle_word (size_t cardb)
6539 return cardb / card_bundle_word_width;
6543 uint32_t card_bundle_bit (size_t cardb)
6545 return (uint32_t)(cardb % card_bundle_word_width);
6548 size_t align_cardw_on_bundle (size_t cardw)
6550 return ((size_t)(cardw + card_bundle_size - 1) & ~(card_bundle_size - 1 ));
6553 // Get the card bundle representing a card word
6554 size_t cardw_card_bundle (size_t cardw)
6556 return cardw / card_bundle_size;
6559 // Get the first card word in a card bundle
6560 size_t card_bundle_cardw (size_t cardb)
6562 return cardb * card_bundle_size;
6565 // Clear the specified card bundle
6566 void gc_heap::card_bundle_clear (size_t cardb)
6568 card_bundle_table [card_bundle_word (cardb)] &= ~(1 << card_bundle_bit (cardb));
6569 dprintf (2, ("Cleared card bundle %Ix [%Ix, %Ix[", cardb, (size_t)card_bundle_cardw (cardb),
6570 (size_t)card_bundle_cardw (cardb+1)));
6573 void gc_heap::card_bundle_set (size_t cardb)
6575 if (!card_bundle_set_p (cardb))
6577 card_bundle_table [card_bundle_word (cardb)] |= (1 << card_bundle_bit (cardb));
6581 // Set the card bundle bits between start_cardb and end_cardb
6582 void gc_heap::card_bundles_set (size_t start_cardb, size_t end_cardb)
6584 if (start_cardb == end_cardb)
6586 card_bundle_set(start_cardb);
6590 size_t start_word = card_bundle_word (start_cardb);
6591 size_t end_word = card_bundle_word (end_cardb);
6593 if (start_word < end_word)
6595 // Set the partial words
6596 card_bundle_table [start_word] |= highbits (~0u, card_bundle_bit (start_cardb));
6598 if (card_bundle_bit (end_cardb))
6599 card_bundle_table [end_word] |= lowbits (~0u, card_bundle_bit (end_cardb));
6601 // Set the full words
6602 for (size_t i = start_word + 1; i < end_word; i++)
6603 card_bundle_table [i] = ~0u;
6607 card_bundle_table [start_word] |= (highbits (~0u, card_bundle_bit (start_cardb)) &
6608 lowbits (~0u, card_bundle_bit (end_cardb)));
6612 // Indicates whether the specified bundle is set.
6613 BOOL gc_heap::card_bundle_set_p (size_t cardb)
6615 return (card_bundle_table[card_bundle_word(cardb)] & (1 << card_bundle_bit (cardb)));
6618 // Returns the size (in bytes) of a card bundle representing the region from 'from' to 'end'
6619 size_t size_card_bundle_of (uint8_t* from, uint8_t* end)
6621 // Number of heap bytes represented by a card bundle word
6622 size_t cbw_span = card_size * card_word_width * card_bundle_size * card_bundle_word_width;
6624 // Align the start of the region down
6625 from = (uint8_t*)((size_t)from & ~(cbw_span - 1));
6627 // Align the end of the region up
6628 end = (uint8_t*)((size_t)(end + (cbw_span - 1)) & ~(cbw_span - 1));
6630 // Make sure they're really aligned
6631 assert (((size_t)from & (cbw_span - 1)) == 0);
6632 assert (((size_t)end & (cbw_span - 1)) == 0);
6634 return ((end - from) / cbw_span) * sizeof (uint32_t);
6637 // Takes a pointer to a card bundle table and an address, and returns a pointer that represents
6638 // where a theoretical card bundle table that represents every address (starting from 0) would
6639 // start if the bundle word representing the address were to be located at the pointer passed in.
6640 // The returned 'translated' pointer makes it convenient/fast to calculate where the card bundle
6641 // for a given address is using a simple shift operation on the address.
6642 uint32_t* translate_card_bundle_table (uint32_t* cb, uint8_t* lowest_address)
6644 // The number of bytes of heap memory represented by a card bundle word
6645 const size_t heap_bytes_for_bundle_word = card_size * card_word_width * card_bundle_size * card_bundle_word_width;
6647 // Each card bundle word is 32 bits
6648 return (uint32_t*)((uint8_t*)cb - (((size_t)lowest_address / heap_bytes_for_bundle_word) * sizeof (uint32_t)));
6651 void gc_heap::enable_card_bundles ()
6653 if (can_use_write_watch_for_card_table() && (!card_bundles_enabled()))
6655 dprintf (1, ("Enabling card bundles"));
6657 // We initially set all of the card bundles
6658 card_bundles_set (cardw_card_bundle (card_word (card_of (lowest_address))),
6659 cardw_card_bundle (align_cardw_on_bundle (card_word (card_of (highest_address)))));
6660 settings.card_bundles = TRUE;
6664 BOOL gc_heap::card_bundles_enabled ()
6666 return settings.card_bundles;
6669 #endif // CARD_BUNDLE
6671 #if defined (_TARGET_AMD64_)
6672 #define brick_size ((size_t)4096)
6674 #define brick_size ((size_t)2048)
6675 #endif //_TARGET_AMD64_
6678 size_t gc_heap::brick_of (uint8_t* add)
6680 return (size_t)(add - lowest_address) / brick_size;
6684 uint8_t* gc_heap::brick_address (size_t brick)
6686 return lowest_address + (brick_size * brick);
6690 void gc_heap::clear_brick_table (uint8_t* from, uint8_t* end)
6692 for (size_t i = brick_of (from);i < brick_of (end); i++)
6696 //codes for the brick entries:
6697 //entry == 0 -> not assigned
6698 //entry >0 offset is entry-1
6699 //entry <0 jump back entry bricks
6703 void gc_heap::set_brick (size_t index, ptrdiff_t val)
6709 assert (val < 32767);
6711 brick_table [index] = (short)val+1;
6713 brick_table [index] = (short)val;
6717 int gc_heap::get_brick_entry (size_t index)
6719 #ifdef MULTIPLE_HEAPS
6720 return VolatileLoadWithoutBarrier(&brick_table [index]);
6722 return brick_table[index];
6728 uint8_t* align_on_brick (uint8_t* add)
6730 return (uint8_t*)((size_t)(add + brick_size - 1) & ~(brick_size - 1));
6734 uint8_t* align_lower_brick (uint8_t* add)
6736 return (uint8_t*)(((size_t)add) & ~(brick_size - 1));
6739 size_t size_brick_of (uint8_t* from, uint8_t* end)
6741 assert (((size_t)from & (brick_size-1)) == 0);
6742 assert (((size_t)end & (brick_size-1)) == 0);
6744 return ((end - from) / brick_size) * sizeof (short);
6748 uint8_t* gc_heap::card_address (size_t card)
6750 return (uint8_t*) (card_size * card);
6754 size_t gc_heap::card_of ( uint8_t* object)
6756 return (size_t)(object) / card_size;
6760 size_t gc_heap::card_to_brick (size_t card)
6762 return brick_of (card_address (card));
6766 uint8_t* align_on_card (uint8_t* add)
6768 return (uint8_t*)((size_t)(add + card_size - 1) & ~(card_size - 1 ));
6771 uint8_t* align_on_card_word (uint8_t* add)
6773 return (uint8_t*) ((size_t)(add + (card_size*card_word_width)-1) & ~(card_size*card_word_width - 1));
6777 uint8_t* align_lower_card (uint8_t* add)
6779 return (uint8_t*)((size_t)add & ~(card_size-1));
6783 void gc_heap::clear_card (size_t card)
6785 card_table [card_word (card)] =
6786 (card_table [card_word (card)] & ~(1 << card_bit (card)));
6787 dprintf (3,("Cleared card %Ix [%Ix, %Ix[", card, (size_t)card_address (card),
6788 (size_t)card_address (card+1)));
6792 void gc_heap::set_card (size_t card)
6794 size_t word = card_word (card);
6795 card_table[word] = (card_table [word] | (1 << card_bit (card)));
6797 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
6798 // Also set the card bundle that corresponds to the card
6799 size_t bundle_to_set = cardw_card_bundle(word);
6801 card_bundle_set(bundle_to_set);
6803 dprintf (3,("Set card %Ix [%Ix, %Ix[ and bundle %Ix", card, (size_t)card_address (card), (size_t)card_address (card+1), bundle_to_set));
6804 assert(card_bundle_set_p(bundle_to_set) != 0);
6809 BOOL gc_heap::card_set_p (size_t card)
6811 return ( card_table [ card_word (card) ] & (1 << card_bit (card)));
6814 // Returns the number of DWORDs in the card table that cover the
6815 // range of addresses [from, end[.
6816 size_t count_card_of (uint8_t* from, uint8_t* end)
6818 return card_word (gcard_of (end - 1)) - card_word (gcard_of (from)) + 1;
6821 // Returns the number of bytes to allocate for a card table
6822 // that covers the range of addresses [from, end[.
6823 size_t size_card_of (uint8_t* from, uint8_t* end)
6825 return count_card_of (from, end) * sizeof(uint32_t);
6828 // We don't store seg_mapping_table in card_table_info because there's only always one view.
6829 class card_table_info
6833 uint8_t* lowest_address;
6834 uint8_t* highest_address;
6838 uint32_t* card_bundle_table;
6839 #endif //CARD_BUNDLE
6841 // mark_array is always at the end of the data structure because we
6842 // want to be able to make one commit call for everything before it.
6844 uint32_t* mark_array;
6848 uint32_t* next_card_table;
6851 //These are accessors on untranslated cardtable
6853 unsigned& card_table_refcount (uint32_t* c_table)
6855 return *(unsigned*)((char*)c_table - sizeof (card_table_info));
6859 uint8_t*& card_table_lowest_address (uint32_t* c_table)
6861 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->lowest_address;
6864 uint32_t* translate_card_table (uint32_t* ct)
6866 return (uint32_t*)((uint8_t*)ct - card_word (gcard_of (card_table_lowest_address (ct))) * sizeof(uint32_t));
6870 uint8_t*& card_table_highest_address (uint32_t* c_table)
6872 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->highest_address;
6876 short*& card_table_brick_table (uint32_t* c_table)
6878 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->brick_table;
6883 uint32_t*& card_table_card_bundle_table (uint32_t* c_table)
6885 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->card_bundle_table;
6887 #endif //CARD_BUNDLE
6890 /* Support for mark_array */
6893 uint32_t*& card_table_mark_array (uint32_t* c_table)
6895 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->mark_array;
6899 #define mark_bit_pitch ((size_t)16)
6901 #define mark_bit_pitch ((size_t)8)
6903 #define mark_word_width ((size_t)32)
6904 #define mark_word_size (mark_word_width * mark_bit_pitch)
6907 uint8_t* align_on_mark_bit (uint8_t* add)
6909 return (uint8_t*)((size_t)(add + (mark_bit_pitch - 1)) & ~(mark_bit_pitch - 1));
6913 uint8_t* align_lower_mark_bit (uint8_t* add)
6915 return (uint8_t*)((size_t)(add) & ~(mark_bit_pitch - 1));
6919 BOOL is_aligned_on_mark_word (uint8_t* add)
6921 return ((size_t)add == ((size_t)(add) & ~(mark_word_size - 1)));
6925 uint8_t* align_on_mark_word (uint8_t* add)
6927 return (uint8_t*)((size_t)(add + mark_word_size - 1) & ~(mark_word_size - 1));
6931 uint8_t* align_lower_mark_word (uint8_t* add)
6933 return (uint8_t*)((size_t)(add) & ~(mark_word_size - 1));
6937 size_t mark_bit_of (uint8_t* add)
6939 return ((size_t)add / mark_bit_pitch);
6943 unsigned int mark_bit_bit (size_t mark_bit)
6945 return (unsigned int)(mark_bit % mark_word_width);
6949 size_t mark_bit_word (size_t mark_bit)
6951 return (mark_bit / mark_word_width);
6955 size_t mark_word_of (uint8_t* add)
6957 return ((size_t)add) / mark_word_size;
6960 uint8_t* mark_word_address (size_t wd)
6962 return (uint8_t*)(wd*mark_word_size);
6965 uint8_t* mark_bit_address (size_t mark_bit)
6967 return (uint8_t*)(mark_bit*mark_bit_pitch);
6971 size_t mark_bit_bit_of (uint8_t* add)
6973 return (((size_t)add / mark_bit_pitch) % mark_word_width);
6977 unsigned int gc_heap::mark_array_marked(uint8_t* add)
6979 return mark_array [mark_word_of (add)] & (1 << mark_bit_bit_of (add));
6983 BOOL gc_heap::is_mark_bit_set (uint8_t* add)
6985 return (mark_array [mark_word_of (add)] & (1 << mark_bit_bit_of (add)));
6989 void gc_heap::mark_array_set_marked (uint8_t* add)
6991 size_t index = mark_word_of (add);
6992 uint32_t val = (1 << mark_bit_bit_of (add));
6993 #ifdef MULTIPLE_HEAPS
6994 Interlocked::Or (&(mark_array [index]), val);
6996 mark_array [index] |= val;
7001 void gc_heap::mark_array_clear_marked (uint8_t* add)
7003 mark_array [mark_word_of (add)] &= ~(1 << mark_bit_bit_of (add));
7006 size_t size_mark_array_of (uint8_t* from, uint8_t* end)
7008 assert (((size_t)from & ((mark_word_size)-1)) == 0);
7009 assert (((size_t)end & ((mark_word_size)-1)) == 0);
7010 return sizeof (uint32_t)*(((end - from) / mark_word_size));
7013 //In order to eliminate the lowest_address in the mark array
7014 //computations (mark_word_of, etc) mark_array is offset
7015 // according to the lowest_address.
7016 uint32_t* translate_mark_array (uint32_t* ma)
7018 return (uint32_t*)((uint8_t*)ma - size_mark_array_of (0, g_gc_lowest_address));
7021 // from and end must be page aligned addresses.
7022 void gc_heap::clear_mark_array (uint8_t* from, uint8_t* end, BOOL check_only/*=TRUE*/
7023 #ifdef FEATURE_BASICFREEZE
7024 , BOOL read_only/*=FALSE*/
7025 #endif // FEATURE_BASICFREEZE
7028 if(!gc_can_use_concurrent)
7031 #ifdef FEATURE_BASICFREEZE
7033 #endif // FEATURE_BASICFREEZE
7035 assert (from == align_on_mark_word (from));
7037 assert (end == align_on_mark_word (end));
7039 #ifdef BACKGROUND_GC
7040 uint8_t* current_lowest_address = background_saved_lowest_address;
7041 uint8_t* current_highest_address = background_saved_highest_address;
7043 uint8_t* current_lowest_address = lowest_address;
7044 uint8_t* current_highest_address = highest_address;
7045 #endif //BACKGROUND_GC
7047 //there is a possibility of the addresses to be
7048 //outside of the covered range because of a newly allocated
7049 //large object segment
7050 if ((end <= current_highest_address) && (from >= current_lowest_address))
7052 size_t beg_word = mark_word_of (align_on_mark_word (from));
7053 MAYBE_UNUSED_VAR(beg_word);
7054 //align end word to make sure to cover the address
7055 size_t end_word = mark_word_of (align_on_mark_word (end));
7056 MAYBE_UNUSED_VAR(end_word);
7057 dprintf (3, ("Calling clearing mark array [%Ix, %Ix[ for addresses [%Ix, %Ix[(%s)",
7058 (size_t)mark_word_address (beg_word),
7059 (size_t)mark_word_address (end_word),
7060 (size_t)from, (size_t)end,
7061 (check_only ? "check_only" : "clear")));
7065 while (op < mark_word_address (beg_word))
7067 mark_array_clear_marked (op);
7068 op += mark_bit_pitch;
7071 memset (&mark_array[beg_word], 0, (end_word - beg_word)*sizeof (uint32_t));
7076 //Beware, it is assumed that the mark array word straddling
7077 //start has been cleared before
7078 //verify that the array is empty.
7079 size_t markw = mark_word_of (align_on_mark_word (from));
7080 size_t markw_end = mark_word_of (align_on_mark_word (end));
7081 while (markw < markw_end)
7083 assert (!(mark_array [markw]));
7086 uint8_t* p = mark_word_address (markw_end);
7089 assert (!(mark_array_marked (p)));
7098 //These work on untranslated card tables
7100 uint32_t*& card_table_next (uint32_t* c_table)
7102 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->next_card_table;
7106 size_t& card_table_size (uint32_t* c_table)
7108 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->size;
7111 void own_card_table (uint32_t* c_table)
7113 card_table_refcount (c_table) += 1;
7116 void destroy_card_table (uint32_t* c_table);
7118 void delete_next_card_table (uint32_t* c_table)
7120 uint32_t* n_table = card_table_next (c_table);
7123 if (card_table_next (n_table))
7125 delete_next_card_table (n_table);
7127 if (card_table_refcount (n_table) == 0)
7129 destroy_card_table (n_table);
7130 card_table_next (c_table) = 0;
7135 void release_card_table (uint32_t* c_table)
7137 assert (card_table_refcount (c_table) >0);
7138 card_table_refcount (c_table) -= 1;
7139 if (card_table_refcount (c_table) == 0)
7141 delete_next_card_table (c_table);
7142 if (card_table_next (c_table) == 0)
7144 destroy_card_table (c_table);
7145 // sever the link from the parent
7146 if (&g_gc_card_table[card_word (gcard_of(g_gc_lowest_address))] == c_table)
7148 g_gc_card_table = 0;
7150 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7151 g_gc_card_bundle_table = 0;
7153 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7154 SoftwareWriteWatch::StaticClose();
7155 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7159 uint32_t* p_table = &g_gc_card_table[card_word (gcard_of(g_gc_lowest_address))];
7162 while (p_table && (card_table_next (p_table) != c_table))
7163 p_table = card_table_next (p_table);
7164 card_table_next (p_table) = 0;
7171 void destroy_card_table (uint32_t* c_table)
7173 // delete (uint32_t*)&card_table_refcount(c_table);
7175 GCToOSInterface::VirtualRelease (&card_table_refcount(c_table), card_table_size(c_table));
7176 dprintf (2, ("Table Virtual Free : %Ix", (size_t)&card_table_refcount(c_table)));
7179 uint32_t* gc_heap::make_card_table (uint8_t* start, uint8_t* end)
7181 assert (g_gc_lowest_address == start);
7182 assert (g_gc_highest_address == end);
7184 uint32_t virtual_reserve_flags = VirtualReserveFlags::None;
7186 size_t bs = size_brick_of (start, end);
7187 size_t cs = size_card_of (start, end);
7189 size_t ms = (gc_can_use_concurrent ?
7190 size_mark_array_of (start, end) :
7199 if (can_use_write_watch_for_card_table())
7201 cb = size_card_bundle_of (g_gc_lowest_address, g_gc_highest_address);
7202 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7203 // If we're not manually managing the card bundles, we will need to use OS write
7204 // watch APIs over this region to track changes.
7205 virtual_reserve_flags |= VirtualReserveFlags::WriteWatch;
7208 #endif //CARD_BUNDLE
7211 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7212 size_t sw_ww_table_offset = 0;
7213 if (gc_can_use_concurrent)
7215 size_t sw_ww_size_before_table = sizeof(card_table_info) + cs + bs + cb;
7216 sw_ww_table_offset = SoftwareWriteWatch::GetTableStartByteOffset(sw_ww_size_before_table);
7217 wws = sw_ww_table_offset - sw_ww_size_before_table + SoftwareWriteWatch::GetTableByteSize(start, end);
7219 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7221 #ifdef GROWABLE_SEG_MAPPING_TABLE
7222 size_t st = size_seg_mapping_table_of (g_gc_lowest_address, g_gc_highest_address);
7223 size_t st_table_offset = sizeof(card_table_info) + cs + bs + cb + wws;
7224 size_t st_table_offset_aligned = align_for_seg_mapping_table (st_table_offset);
7226 st += (st_table_offset_aligned - st_table_offset);
7227 #else //GROWABLE_SEG_MAPPING_TABLE
7229 #endif //GROWABLE_SEG_MAPPING_TABLE
7231 // it is impossible for alloc_size to overflow due bounds on each of
7233 size_t alloc_size = sizeof (uint8_t)*(sizeof(card_table_info) + cs + bs + cb + wws + st + ms);
7234 uint8_t* mem = (uint8_t*)GCToOSInterface::VirtualReserve (alloc_size, 0, virtual_reserve_flags);
7239 dprintf (2, ("Init - Card table alloc for %Id bytes: [%Ix, %Ix[",
7240 alloc_size, (size_t)mem, (size_t)(mem+alloc_size)));
7242 // mark array will be committed separately (per segment).
7243 size_t commit_size = alloc_size - ms;
7245 if (!virtual_commit (mem, commit_size))
7247 dprintf (1, ("Card table commit failed"));
7248 GCToOSInterface::VirtualRelease (mem, alloc_size);
7252 // initialize the ref count
7253 uint32_t* ct = (uint32_t*)(mem+sizeof (card_table_info));
7254 card_table_refcount (ct) = 0;
7255 card_table_lowest_address (ct) = start;
7256 card_table_highest_address (ct) = end;
7257 card_table_brick_table (ct) = (short*)((uint8_t*)ct + cs);
7258 card_table_size (ct) = alloc_size;
7259 card_table_next (ct) = 0;
7262 card_table_card_bundle_table (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs);
7264 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7265 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), g_gc_lowest_address);
7268 #endif //CARD_BUNDLE
7270 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7271 if (gc_can_use_concurrent)
7273 SoftwareWriteWatch::InitializeUntranslatedTable(mem + sw_ww_table_offset, start);
7275 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7277 #ifdef GROWABLE_SEG_MAPPING_TABLE
7278 seg_mapping_table = (seg_mapping*)(mem + st_table_offset_aligned);
7279 seg_mapping_table = (seg_mapping*)((uint8_t*)seg_mapping_table -
7280 size_seg_mapping_table_of (0, (align_lower_segment (g_gc_lowest_address))));
7281 #endif //GROWABLE_SEG_MAPPING_TABLE
7284 if (gc_can_use_concurrent)
7285 card_table_mark_array (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs + cb + wws + st);
7287 card_table_mark_array (ct) = NULL;
7290 return translate_card_table(ct);
7293 void gc_heap::set_fgm_result (failure_get_memory f, size_t s, BOOL loh_p)
7295 #ifdef MULTIPLE_HEAPS
7296 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
7298 gc_heap* hp = gc_heap::g_heaps [hn];
7299 hp->fgm_result.set_fgm (f, s, loh_p);
7301 #else //MULTIPLE_HEAPS
7302 fgm_result.set_fgm (f, s, loh_p);
7303 #endif //MULTIPLE_HEAPS
7306 //returns 0 for success, -1 otherwise
7307 // We are doing all the decommitting here because we want to make sure we have
7308 // enough memory to do so - if we do this during copy_brick_card_table and
7309 // and fail to decommit it would make the failure case very complicated to
7310 // handle. This way we can waste some decommit if we call this multiple
7311 // times before the next FGC but it's easier to handle the failure case.
7312 int gc_heap::grow_brick_card_tables (uint8_t* start,
7315 heap_segment* new_seg,
7319 uint8_t* la = g_gc_lowest_address;
7320 uint8_t* ha = g_gc_highest_address;
7321 uint8_t* saved_g_lowest_address = min (start, g_gc_lowest_address);
7322 uint8_t* saved_g_highest_address = max (end, g_gc_highest_address);
7323 seg_mapping* new_seg_mapping_table = nullptr;
7324 #ifdef BACKGROUND_GC
7325 // This value is only for logging purpose - it's not necessarily exactly what we
7326 // would commit for mark array but close enough for diagnostics purpose.
7327 size_t logging_ma_commit_size = size_mark_array_of (0, (uint8_t*)size);
7328 #endif //BACKGROUND_GC
7330 // See if the address is already covered
7331 if ((la != saved_g_lowest_address ) || (ha != saved_g_highest_address))
7334 //modify the higest address so the span covered
7335 //is twice the previous one.
7336 uint8_t* top = (uint8_t*)0 + Align (GCToOSInterface::GetVirtualMemoryLimit());
7337 // On non-Windows systems, we get only an approximate value that can possibly be
7338 // slightly lower than the saved_g_highest_address.
7339 // In such case, we set the top to the saved_g_highest_address so that the
7340 // card and brick tables always cover the whole new range.
7341 if (top < saved_g_highest_address)
7343 top = saved_g_highest_address;
7347 if (ps > (uint64_t)200*1024*1024*1024)
7348 ps += (uint64_t)100*1024*1024*1024;
7353 if (saved_g_lowest_address < g_gc_lowest_address)
7355 if (ps > (size_t)g_gc_lowest_address)
7356 saved_g_lowest_address = (uint8_t*)(size_t)OS_PAGE_SIZE;
7359 assert (((size_t)g_gc_lowest_address - ps) >= OS_PAGE_SIZE);
7360 saved_g_lowest_address = min (saved_g_lowest_address, (g_gc_lowest_address - ps));
7364 if (saved_g_highest_address > g_gc_highest_address)
7366 saved_g_highest_address = max ((saved_g_lowest_address + ps), saved_g_highest_address);
7367 if (saved_g_highest_address > top)
7368 saved_g_highest_address = top;
7371 dprintf (GC_TABLE_LOG, ("Growing card table [%Ix, %Ix[",
7372 (size_t)saved_g_lowest_address,
7373 (size_t)saved_g_highest_address));
7375 bool write_barrier_updated = false;
7376 uint32_t virtual_reserve_flags = VirtualReserveFlags::None;
7377 uint32_t* saved_g_card_table = g_gc_card_table;
7379 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7380 uint32_t* saved_g_card_bundle_table = g_gc_card_bundle_table;
7384 uint32_t* translated_ct = 0;
7387 size_t cs = size_card_of (saved_g_lowest_address, saved_g_highest_address);
7388 size_t bs = size_brick_of (saved_g_lowest_address, saved_g_highest_address);
7391 size_t ms = (gc_heap::gc_can_use_concurrent ?
7392 size_mark_array_of (saved_g_lowest_address, saved_g_highest_address) :
7401 if (can_use_write_watch_for_card_table())
7403 cb = size_card_bundle_of (saved_g_lowest_address, saved_g_highest_address);
7405 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7406 // If we're not manually managing the card bundles, we will need to use OS write
7407 // watch APIs over this region to track changes.
7408 virtual_reserve_flags |= VirtualReserveFlags::WriteWatch;
7411 #endif //CARD_BUNDLE
7414 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7415 size_t sw_ww_table_offset = 0;
7416 if (gc_can_use_concurrent)
7418 size_t sw_ww_size_before_table = sizeof(card_table_info) + cs + bs + cb;
7419 sw_ww_table_offset = SoftwareWriteWatch::GetTableStartByteOffset(sw_ww_size_before_table);
7421 sw_ww_table_offset -
7422 sw_ww_size_before_table +
7423 SoftwareWriteWatch::GetTableByteSize(saved_g_lowest_address, saved_g_highest_address);
7425 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7427 #ifdef GROWABLE_SEG_MAPPING_TABLE
7428 size_t st = size_seg_mapping_table_of (saved_g_lowest_address, saved_g_highest_address);
7429 size_t st_table_offset = sizeof(card_table_info) + cs + bs + cb + wws;
7430 size_t st_table_offset_aligned = align_for_seg_mapping_table (st_table_offset);
7431 st += (st_table_offset_aligned - st_table_offset);
7432 #else //GROWABLE_SEG_MAPPING_TABLE
7434 #endif //GROWABLE_SEG_MAPPING_TABLE
7436 // it is impossible for alloc_size to overflow due bounds on each of
7438 size_t alloc_size = sizeof (uint8_t)*(sizeof(card_table_info) + cs + bs + cb + wws + st + ms);
7439 dprintf (GC_TABLE_LOG, ("card table: %Id; brick table: %Id; card bundle: %Id; sw ww table: %Id; seg table: %Id; mark array: %Id",
7440 cs, bs, cb, wws, st, ms));
7442 uint8_t* mem = (uint8_t*)GCToOSInterface::VirtualReserve (alloc_size, 0, virtual_reserve_flags);
7446 set_fgm_result (fgm_grow_table, alloc_size, loh_p);
7450 dprintf (GC_TABLE_LOG, ("Table alloc for %Id bytes: [%Ix, %Ix[",
7451 alloc_size, (size_t)mem, (size_t)((uint8_t*)mem+alloc_size)));
7454 // mark array will be committed separately (per segment).
7455 size_t commit_size = alloc_size - ms;
7457 if (!virtual_commit (mem, commit_size))
7459 dprintf (GC_TABLE_LOG, ("Table commit failed"));
7460 set_fgm_result (fgm_commit_table, commit_size, loh_p);
7465 ct = (uint32_t*)(mem + sizeof (card_table_info));
7466 card_table_refcount (ct) = 0;
7467 card_table_lowest_address (ct) = saved_g_lowest_address;
7468 card_table_highest_address (ct) = saved_g_highest_address;
7469 card_table_next (ct) = &g_gc_card_table[card_word (gcard_of (la))];
7471 //clear the card table
7473 memclr ((uint8_t*)ct,
7474 (((saved_g_highest_address - saved_g_lowest_address)*sizeof (uint32_t) /
7475 (card_size * card_word_width))
7476 + sizeof (uint32_t)));
7479 bt = (short*)((uint8_t*)ct + cs);
7481 // No initialization needed, will be done in copy_brick_card
7483 card_table_brick_table (ct) = bt;
7486 card_table_card_bundle_table (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs);
7487 //set all bundle to look at all of the cards
7488 memset(card_table_card_bundle_table (ct), 0xFF, cb);
7489 #endif //CARD_BUNDLE
7491 #ifdef GROWABLE_SEG_MAPPING_TABLE
7493 new_seg_mapping_table = (seg_mapping*)(mem + st_table_offset_aligned);
7494 new_seg_mapping_table = (seg_mapping*)((uint8_t*)new_seg_mapping_table -
7495 size_seg_mapping_table_of (0, (align_lower_segment (saved_g_lowest_address))));
7496 memcpy(&new_seg_mapping_table[seg_mapping_word_of(g_gc_lowest_address)],
7497 &seg_mapping_table[seg_mapping_word_of(g_gc_lowest_address)],
7498 size_seg_mapping_table_of(g_gc_lowest_address, g_gc_highest_address));
7500 // new_seg_mapping_table gets assigned to seg_mapping_table at the bottom of this function,
7501 // not here. The reason for this is that, if we fail at mark array committing (OOM) and we've
7502 // already switched seg_mapping_table to point to the new mapping table, we'll decommit it and
7503 // run into trouble. By not assigning here, we're making sure that we will not change seg_mapping_table
7504 // if an OOM occurs.
7506 #endif //GROWABLE_SEG_MAPPING_TABLE
7509 if(gc_can_use_concurrent)
7510 card_table_mark_array (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs + cb + wws + st);
7512 card_table_mark_array (ct) = NULL;
7515 translated_ct = translate_card_table (ct);
7517 dprintf (GC_TABLE_LOG, ("card table: %Ix(translated: %Ix), seg map: %Ix, mark array: %Ix",
7518 (size_t)ct, (size_t)translated_ct, (size_t)new_seg_mapping_table, (size_t)card_table_mark_array (ct)));
7520 #ifdef BACKGROUND_GC
7521 if (hp->should_commit_mark_array())
7523 dprintf (GC_TABLE_LOG, ("new low: %Ix, new high: %Ix, latest mark array is %Ix(translate: %Ix)",
7524 saved_g_lowest_address, saved_g_highest_address,
7525 card_table_mark_array (ct),
7526 translate_mark_array (card_table_mark_array (ct))));
7527 uint32_t* new_mark_array = (uint32_t*)((uint8_t*)card_table_mark_array (ct) - size_mark_array_of (0, saved_g_lowest_address));
7528 if (!commit_new_mark_array_global (new_mark_array))
7530 dprintf (GC_TABLE_LOG, ("failed to commit portions in the mark array for existing segments"));
7531 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7535 if (!commit_mark_array_new_seg (hp, new_seg, translated_ct, saved_g_lowest_address))
7537 dprintf (GC_TABLE_LOG, ("failed to commit mark array for the new seg"));
7538 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7544 clear_commit_flag_global();
7546 #endif //BACKGROUND_GC
7548 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7549 if (gc_can_use_concurrent)
7551 // The current design of software write watch requires that the runtime is suspended during resize. Suspending
7552 // on resize is preferred because it is a far less frequent operation than GetWriteWatch() / ResetWriteWatch().
7553 // Suspending here allows copying dirty state from the old table into the new table, and not have to merge old
7554 // table info lazily as done for card tables.
7556 // Either this thread was the thread that did the suspension which means we are suspended; or this is called
7557 // from a GC thread which means we are in a blocking GC and also suspended.
7558 bool is_runtime_suspended = GCToEEInterface::IsGCThread();
7559 if (!is_runtime_suspended)
7561 // Note on points where the runtime is suspended anywhere in this function. Upon an attempt to suspend the
7562 // runtime, a different thread may suspend first, causing this thread to block at the point of the suspend call.
7563 // So, at any suspend point, externally visible state needs to be consistent, as code that depends on that state
7564 // may run while this thread is blocked. This includes updates to g_gc_card_table, g_gc_lowest_address, and
7565 // g_gc_highest_address.
7569 g_gc_card_table = translated_ct;
7571 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7572 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), saved_g_lowest_address);
7575 SoftwareWriteWatch::SetResizedUntranslatedTable(
7576 mem + sw_ww_table_offset,
7577 saved_g_lowest_address,
7578 saved_g_highest_address);
7580 seg_mapping_table = new_seg_mapping_table;
7582 // Since the runtime is already suspended, update the write barrier here as well.
7583 // This passes a bool telling whether we need to switch to the post
7584 // grow version of the write barrier. This test tells us if the new
7585 // segment was allocated at a lower address than the old, requiring
7586 // that we start doing an upper bounds check in the write barrier.
7587 g_gc_lowest_address = saved_g_lowest_address;
7588 g_gc_highest_address = saved_g_highest_address;
7589 stomp_write_barrier_resize(true, la != saved_g_lowest_address);
7590 write_barrier_updated = true;
7592 if (!is_runtime_suspended)
7598 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7600 g_gc_card_table = translated_ct;
7602 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7603 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), saved_g_lowest_address);
7607 if (!write_barrier_updated)
7609 seg_mapping_table = new_seg_mapping_table;
7610 GCToOSInterface::FlushProcessWriteBuffers();
7611 g_gc_lowest_address = saved_g_lowest_address;
7612 g_gc_highest_address = saved_g_highest_address;
7614 // This passes a bool telling whether we need to switch to the post
7615 // grow version of the write barrier. This test tells us if the new
7616 // segment was allocated at a lower address than the old, requiring
7617 // that we start doing an upper bounds check in the write barrier.
7618 // This will also suspend the runtime if the write barrier type needs
7619 // to be changed, so we are doing this after all global state has
7620 // been updated. See the comment above suspend_EE() above for more
7622 stomp_write_barrier_resize(GCToEEInterface::IsGCThread(), la != saved_g_lowest_address);
7628 //cleanup mess and return -1;
7632 assert(g_gc_card_table == saved_g_card_table);
7634 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7635 assert(g_gc_card_bundle_table == saved_g_card_bundle_table);
7638 //delete (uint32_t*)((uint8_t*)ct - sizeof(card_table_info));
7639 if (!GCToOSInterface::VirtualRelease (mem, alloc_size))
7641 dprintf (GC_TABLE_LOG, ("GCToOSInterface::VirtualRelease failed"));
7642 assert (!"release failed");
7650 #ifdef BACKGROUND_GC
7651 if (hp->should_commit_mark_array())
7653 dprintf (GC_TABLE_LOG, ("in range new seg %Ix, mark_array is %Ix", new_seg, hp->mark_array));
7654 if (!commit_mark_array_new_seg (hp, new_seg))
7656 dprintf (GC_TABLE_LOG, ("failed to commit mark array for the new seg in range"));
7657 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7661 #endif //BACKGROUND_GC
7667 //copy all of the arrays managed by the card table for a page aligned range
7668 void gc_heap::copy_brick_card_range (uint8_t* la, uint32_t* old_card_table,
7669 short* old_brick_table,
7671 uint8_t* start, uint8_t* end)
7673 ptrdiff_t brick_offset = brick_of (start) - brick_of (la);
7676 dprintf (2, ("copying tables for range [%Ix %Ix[", (size_t)start, (size_t)end));
7679 short* brick_start = &brick_table [brick_of (start)];
7680 if (old_brick_table)
7682 // segments are always on page boundaries
7683 memcpy (brick_start, &old_brick_table[brick_offset],
7684 size_brick_of (start, end));
7689 // This is a large heap, just clear the brick table
7692 uint32_t* old_ct = &old_card_table[card_word (card_of (la))];
7694 #ifdef BACKGROUND_GC
7695 UNREFERENCED_PARAMETER(seg);
7696 if (recursive_gc_sync::background_running_p())
7698 uint32_t* old_mark_array = card_table_mark_array (old_ct);
7700 // We don't need to go through all the card tables here because
7701 // we only need to copy from the GC version of the mark array - when we
7702 // mark (even in allocate_large_object) we always use that mark array.
7703 if ((card_table_highest_address (old_ct) >= start) &&
7704 (card_table_lowest_address (old_ct) <= end))
7706 if ((background_saved_highest_address >= start) &&
7707 (background_saved_lowest_address <= end))
7709 //copy the mark bits
7710 // segments are always on page boundaries
7711 uint8_t* m_start = max (background_saved_lowest_address, start);
7712 uint8_t* m_end = min (background_saved_highest_address, end);
7713 memcpy (&mark_array[mark_word_of (m_start)],
7714 &old_mark_array[mark_word_of (m_start) - mark_word_of (la)],
7715 size_mark_array_of (m_start, m_end));
7720 //only large segments can be out of range
7721 assert (old_brick_table == 0);
7724 #else //BACKGROUND_GC
7726 clear_mark_array (start, heap_segment_committed(seg));
7727 #endif //BACKGROUND_GC
7730 // n way merge with all of the card table ever used in between
7731 uint32_t* ct = card_table_next (&card_table[card_word (card_of(lowest_address))]);
7734 while (card_table_next (old_ct) != ct)
7736 //copy if old card table contained [start, end[
7737 if ((card_table_highest_address (ct) >= end) &&
7738 (card_table_lowest_address (ct) <= start))
7740 // or the card_tables
7742 size_t start_word = card_word (card_of (start));
7744 uint32_t* dest = &card_table[start_word];
7745 uint32_t* src = &((translate_card_table (ct))[start_word]);
7746 ptrdiff_t count = count_card_of (start, end);
7747 for (int x = 0; x < count; x++)
7751 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7754 card_bundle_set(cardw_card_bundle(start_word+x));
7762 ct = card_table_next (ct);
7766 //initialize all of the arrays managed by the card table for a page aligned range when an existing ro segment becomes in range
7767 void gc_heap::init_brick_card_range (heap_segment* seg)
7769 dprintf (2, ("initialising tables for range [%Ix %Ix[",
7770 (size_t)heap_segment_mem (seg),
7771 (size_t)heap_segment_allocated (seg)));
7773 // initialize the brick table
7774 for (size_t b = brick_of (heap_segment_mem (seg));
7775 b < brick_of (align_on_brick (heap_segment_allocated (seg)));
7782 if (recursive_gc_sync::background_running_p() && (seg->flags & heap_segment_flags_ma_committed))
7785 clear_mark_array (heap_segment_mem (seg), heap_segment_committed(seg));
7789 clear_card_for_addresses (heap_segment_mem (seg),
7790 heap_segment_allocated (seg));
7793 void gc_heap::copy_brick_card_table()
7795 uint8_t* la = lowest_address;
7796 uint8_t* ha = highest_address;
7797 MAYBE_UNUSED_VAR(ha);
7798 uint32_t* old_card_table = card_table;
7799 short* old_brick_table = brick_table;
7801 assert (la == card_table_lowest_address (&old_card_table[card_word (card_of (la))]));
7802 assert (ha == card_table_highest_address (&old_card_table[card_word (card_of (la))]));
7804 /* todo: Need a global lock for this */
7805 uint32_t* ct = &g_gc_card_table[card_word (gcard_of (g_gc_lowest_address))];
7806 own_card_table (ct);
7807 card_table = translate_card_table (ct);
7808 /* End of global lock */
7809 highest_address = card_table_highest_address (ct);
7810 lowest_address = card_table_lowest_address (ct);
7812 brick_table = card_table_brick_table (ct);
7815 if (gc_can_use_concurrent)
7817 mark_array = translate_mark_array (card_table_mark_array (ct));
7818 assert (mark_word_of (g_gc_highest_address) ==
7819 mark_word_of (align_on_mark_word (g_gc_highest_address)));
7826 #if defined(MARK_ARRAY) && defined(_DEBUG)
7827 size_t cb_end = (size_t)((uint8_t*)card_table_card_bundle_table (ct) + size_card_bundle_of (g_gc_lowest_address, g_gc_highest_address));
7828 #ifdef GROWABLE_SEG_MAPPING_TABLE
7829 size_t st = size_seg_mapping_table_of (g_gc_lowest_address, g_gc_highest_address);
7830 size_t cb_end_aligned = align_for_seg_mapping_table (cb_end);
7831 st += (cb_end_aligned - cb_end);
7832 #else //GROWABLE_SEG_MAPPING_TABLE
7834 #endif //GROWABLE_SEG_MAPPING_TABLE
7835 #endif //MARK_ARRAY && _DEBUG
7836 card_bundle_table = translate_card_bundle_table (card_table_card_bundle_table (ct), g_gc_lowest_address);
7838 // Ensure that the word that represents g_gc_lowest_address in the translated table is located at the
7839 // start of the untranslated table.
7840 assert (&card_bundle_table [card_bundle_word (cardw_card_bundle (card_word (card_of (g_gc_lowest_address))))] ==
7841 card_table_card_bundle_table (ct));
7843 //set the card table if we are in a heap growth scenario
7844 if (card_bundles_enabled())
7846 card_bundles_set (cardw_card_bundle (card_word (card_of (lowest_address))),
7847 cardw_card_bundle (align_cardw_on_bundle (card_word (card_of (highest_address)))));
7849 //check if we need to turn on card_bundles.
7850 #ifdef MULTIPLE_HEAPS
7851 // use INT64 arithmetic here because of possible overflow on 32p
7852 uint64_t th = (uint64_t)MH_TH_CARD_BUNDLE*gc_heap::n_heaps;
7854 // use INT64 arithmetic here because of possible overflow on 32p
7855 uint64_t th = (uint64_t)SH_TH_CARD_BUNDLE;
7856 #endif //MULTIPLE_HEAPS
7857 if (reserved_memory >= th)
7859 enable_card_bundles();
7862 #endif //CARD_BUNDLE
7864 // for each of the segments and heaps, copy the brick table and
7865 // or the card table
7866 heap_segment* seg = generation_start_segment (generation_of (max_generation));
7869 if (heap_segment_read_only_p (seg) && !heap_segment_in_range_p (seg))
7871 //check if it became in range
7872 if ((heap_segment_reserved (seg) > lowest_address) &&
7873 (heap_segment_mem (seg) < highest_address))
7875 set_ro_segment_in_range (seg);
7881 uint8_t* end = align_on_page (heap_segment_allocated (seg));
7882 copy_brick_card_range (la, old_card_table,
7885 align_lower_page (heap_segment_mem (seg)),
7888 seg = heap_segment_next (seg);
7891 seg = generation_start_segment (large_object_generation);
7894 if (heap_segment_read_only_p (seg) && !heap_segment_in_range_p (seg))
7896 //check if it became in range
7897 if ((heap_segment_reserved (seg) > lowest_address) &&
7898 (heap_segment_mem (seg) < highest_address))
7900 set_ro_segment_in_range (seg);
7905 uint8_t* end = align_on_page (heap_segment_allocated (seg));
7906 copy_brick_card_range (la, old_card_table,
7909 align_lower_page (heap_segment_mem (seg)),
7912 seg = heap_segment_next (seg);
7915 release_card_table (&old_card_table[card_word (card_of(la))]);
7918 #ifdef FEATURE_BASICFREEZE
7919 BOOL gc_heap::insert_ro_segment (heap_segment* seg)
7921 enter_spin_lock (&gc_heap::gc_lock);
7923 if (!gc_heap::seg_table->ensure_space_for_insert ()
7924 || (should_commit_mark_array() && !commit_mark_array_new_seg(__this, seg)))
7926 leave_spin_lock(&gc_heap::gc_lock);
7930 //insert at the head of the segment list
7931 generation* gen2 = generation_of (max_generation);
7932 heap_segment* oldhead = generation_start_segment (gen2);
7933 heap_segment_next (seg) = oldhead;
7934 generation_start_segment (gen2) = seg;
7936 seg_table->insert (heap_segment_mem(seg), (size_t)seg);
7938 #ifdef SEG_MAPPING_TABLE
7939 seg_mapping_table_add_ro_segment (seg);
7940 #endif //SEG_MAPPING_TABLE
7943 if ((heap_segment_reserved (seg) > lowest_address) &&
7944 (heap_segment_mem (seg) < highest_address))
7946 set_ro_segment_in_range (seg);
7949 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(seg), (size_t)(heap_segment_reserved (seg) - heap_segment_mem(seg)), gc_etw_segment_read_only_heap);
7951 leave_spin_lock (&gc_heap::gc_lock);
7955 // No one is calling this function right now. If this is getting called we need
7956 // to take care of decommitting the mark array for it - we will need to remember
7957 // which portion of the mark array was committed and only decommit that.
7958 void gc_heap::remove_ro_segment (heap_segment* seg)
7960 //clear the mark bits so a new segment allocated in its place will have a clear mark bits
7962 if (gc_can_use_concurrent)
7964 clear_mark_array (align_lower_mark_word (max (heap_segment_mem (seg), lowest_address)),
7965 align_on_card_word (min (heap_segment_allocated (seg), highest_address)),
7966 false); // read_only segments need the mark clear
7970 enter_spin_lock (&gc_heap::gc_lock);
7972 seg_table->remove ((uint8_t*)seg);
7974 #ifdef SEG_MAPPING_TABLE
7975 seg_mapping_table_remove_ro_segment (seg);
7976 #endif //SEG_MAPPING_TABLE
7978 // Locate segment (and previous segment) in the list.
7979 generation* gen2 = generation_of (max_generation);
7980 heap_segment* curr_seg = generation_start_segment (gen2);
7981 heap_segment* prev_seg = NULL;
7983 while (curr_seg && curr_seg != seg)
7985 prev_seg = curr_seg;
7986 curr_seg = heap_segment_next (curr_seg);
7988 assert (curr_seg == seg);
7990 // Patch previous segment (or list head if there is none) to skip the removed segment.
7992 heap_segment_next (prev_seg) = heap_segment_next (curr_seg);
7994 generation_start_segment (gen2) = heap_segment_next (curr_seg);
7996 leave_spin_lock (&gc_heap::gc_lock);
7998 #endif //FEATURE_BASICFREEZE
8000 BOOL gc_heap::set_ro_segment_in_range (heap_segment* seg)
8003 seg->flags |= heap_segment_flags_inrange;
8004 // init_brick_card_range (seg);
8005 ro_segments_in_range = TRUE;
8006 //right now, segments aren't protected
8007 //unprotect_segment (seg);
8013 uint8_t** make_mark_list (size_t size)
8015 uint8_t** mark_list = new (nothrow) uint8_t* [size];
8019 #define swap(a,b){uint8_t* t; t = a; a = b; b = t;}
8021 void verify_qsort_array (uint8_t* *low, uint8_t* *high)
8025 for (i = low+1; i <= high; i++)
8034 #ifndef USE_INTROSORT
8035 void qsort1( uint8_t* *low, uint8_t* *high, unsigned int depth)
8037 if (((low + 16) >= high) || (depth > 100))
8041 for (i = low+1; i <= high; i++)
8044 for (j=i;j >low && val<*(j-1);j--)
8053 uint8_t *pivot, **left, **right;
8055 //sort low middle and high
8056 if (*(low+((high-low)/2)) < *low)
8057 swap (*(low+((high-low)/2)), *low);
8060 if (*high < *(low+((high-low)/2)))
8061 swap (*(low+((high-low)/2)), *high);
8063 swap (*(low+((high-low)/2)), *(high-1));
8065 left = low; right = high-1;
8067 while (*(--right) > pivot);
8068 while (*(++left) < pivot);
8071 swap(*left, *right);
8076 swap (*left, *(high-1));
8077 qsort1(low, left-1, depth+1);
8078 qsort1(left+1, high, depth+1);
8081 #endif //USE_INTROSORT
8082 void rqsort1( uint8_t* *low, uint8_t* *high)
8084 if ((low + 16) >= high)
8088 for (i = low+1; i <= high; i++)
8091 for (j=i;j >low && val>*(j-1);j--)
8100 uint8_t *pivot, **left, **right;
8102 //sort low middle and high
8103 if (*(low+((high-low)/2)) > *low)
8104 swap (*(low+((high-low)/2)), *low);
8107 if (*high > *(low+((high-low)/2)))
8108 swap (*(low+((high-low)/2)), *high);
8110 swap (*(low+((high-low)/2)), *(high-1));
8112 left = low; right = high-1;
8114 while (*(--right) < pivot);
8115 while (*(++left) > pivot);
8118 swap(*left, *right);
8123 swap (*left, *(high-1));
8124 rqsort1(low, left-1);
8125 rqsort1(left+1, high);
8129 #ifdef USE_INTROSORT
8134 static const int size_threshold = 64;
8135 static const int max_depth = 100;
8138 inline static void swap_elements(uint8_t** i,uint8_t** j)
8146 static void sort (uint8_t** begin, uint8_t** end, int ignored)
8149 introsort_loop (begin, end, max_depth);
8150 insertionsort (begin, end);
8155 static void introsort_loop (uint8_t** lo, uint8_t** hi, int depth_limit)
8157 while (hi-lo >= size_threshold)
8159 if (depth_limit == 0)
8164 uint8_t** p=median_partition (lo, hi);
8165 depth_limit=depth_limit-1;
8166 introsort_loop (p, hi, depth_limit);
8171 static uint8_t** median_partition (uint8_t** low, uint8_t** high)
8173 uint8_t *pivot, **left, **right;
8175 //sort low middle and high
8176 if (*(low+((high-low)/2)) < *low)
8177 swap_elements ((low+((high-low)/2)), low);
8179 swap_elements (low, high);
8180 if (*high < *(low+((high-low)/2)))
8181 swap_elements ((low+((high-low)/2)), high);
8183 swap_elements ((low+((high-low)/2)), (high-1));
8185 left = low; right = high-1;
8187 while (*(--right) > pivot);
8188 while (*(++left) < pivot);
8191 swap_elements(left, right);
8196 swap_elements (left, (high-1));
8201 static void insertionsort (uint8_t** lo, uint8_t** hi)
8203 for (uint8_t** i=lo+1; i <= hi; i++)
8207 while((j > lo) && (t <*(j-1)))
8216 static void heapsort (uint8_t** lo, uint8_t** hi)
8218 size_t n = hi - lo + 1;
8219 for (size_t i=n / 2; i >= 1; i--)
8223 for (size_t i = n; i > 1; i--)
8225 swap_elements (lo, lo + i - 1);
8226 downheap(1, i - 1, lo);
8230 static void downheap (size_t i, size_t n, uint8_t** lo)
8232 uint8_t* d = *(lo + i - 1);
8237 if (child < n && *(lo + child - 1)<(*(lo + child)))
8241 if (!(d<*(lo + child - 1)))
8245 *(lo + i - 1) = *(lo + child - 1);
8253 #endif //USE_INTROSORT
8255 #ifdef MULTIPLE_HEAPS
8256 #ifdef PARALLEL_MARK_LIST_SORT
8257 void gc_heap::sort_mark_list()
8259 // if this heap had a mark list overflow, we don't do anything
8260 if (mark_list_index > mark_list_end)
8262 // printf("sort_mark_list: overflow on heap %d\n", heap_number);
8266 // if any other heap had a mark list overflow, we fake one too,
8267 // so we don't use an incomplete mark list by mistake
8268 for (int i = 0; i < n_heaps; i++)
8270 if (g_heaps[i]->mark_list_index > g_heaps[i]->mark_list_end)
8272 mark_list_index = mark_list_end + 1;
8273 // printf("sort_mark_list: overflow on heap %d\n", i);
8278 // unsigned long start = GetCycleCount32();
8280 dprintf (3, ("Sorting mark lists"));
8281 if (mark_list_index > mark_list)
8282 _sort (mark_list, mark_list_index - 1, 0);
8284 // printf("first phase of sort_mark_list for heap %d took %u cycles to sort %u entries\n", this->heap_number, GetCycleCount32() - start, mark_list_index - mark_list);
8285 // start = GetCycleCount32();
8287 // first set the pieces for all heaps to empty
8289 for (heap_num = 0; heap_num < n_heaps; heap_num++)
8291 mark_list_piece_start[heap_num] = NULL;
8292 mark_list_piece_end[heap_num] = NULL;
8295 uint8_t** x = mark_list;
8297 // predicate means: x is still within the mark list, and within the bounds of this heap
8298 #define predicate(x) (((x) < mark_list_index) && (*(x) < heap->ephemeral_high))
8301 while (x < mark_list_index)
8304 // find the heap x points into - searching cyclically from the last heap,
8305 // because in many cases the right heap is the next one or comes soon after
8306 int last_heap_num = heap_num;
8307 MAYBE_UNUSED_VAR(last_heap_num);
8311 if (heap_num >= n_heaps)
8313 assert(heap_num != last_heap_num); // we should always find the heap - infinite loop if not!
8314 heap = g_heaps[heap_num];
8316 while (!(*x >= heap->ephemeral_low && *x < heap->ephemeral_high));
8318 // x is the start of the mark list piece for this heap
8319 mark_list_piece_start[heap_num] = x;
8321 // to find the end of the mark list piece for this heap, find the first x
8322 // that has !predicate(x), i.e. that is either not in this heap, or beyond the end of the list
8325 // let's see if we get lucky and the whole rest belongs to this piece
8326 if (predicate(mark_list_index-1))
8328 x = mark_list_index;
8329 mark_list_piece_end[heap_num] = x;
8333 // we play a variant of binary search to find the point sooner.
8334 // the first loop advances by increasing steps until the predicate turns false.
8335 // then we retreat the last step, and the second loop advances by decreasing steps, keeping the predicate true.
8340 uint8_t** temp_x = x;
8347 while (predicate(x));
8348 // we know that only the last step was wrong, so we undo it
8352 // loop invariant - predicate holds at x, but not x + inc
8353 assert (predicate(x) && !(((x + inc) > x) && predicate(x + inc)));
8355 if (((x + inc) > x) && predicate(x + inc))
8361 // the termination condition and the loop invariant together imply this:
8362 assert(predicate(x) && !predicate(x + inc) && (inc == 1));
8363 // so the spot we're looking for is one further
8366 mark_list_piece_end[heap_num] = x;
8371 // printf("second phase of sort_mark_list for heap %d took %u cycles\n", this->heap_number, GetCycleCount32() - start);
8374 void gc_heap::append_to_mark_list(uint8_t **start, uint8_t **end)
8376 size_t slots_needed = end - start;
8377 size_t slots_available = mark_list_end + 1 - mark_list_index;
8378 size_t slots_to_copy = min(slots_needed, slots_available);
8379 memcpy(mark_list_index, start, slots_to_copy*sizeof(*start));
8380 mark_list_index += slots_to_copy;
8381 // printf("heap %d: appended %Id slots to mark_list\n", heap_number, slots_to_copy);
8384 void gc_heap::merge_mark_lists()
8386 uint8_t** source[MAX_SUPPORTED_CPUS];
8387 uint8_t** source_end[MAX_SUPPORTED_CPUS];
8388 int source_heap[MAX_SUPPORTED_CPUS];
8389 int source_count = 0;
8391 // in case of mark list overflow, don't bother
8392 if (mark_list_index > mark_list_end)
8394 // printf("merge_mark_lists: overflow\n");
8398 dprintf(3, ("merge_mark_lists: heap_number = %d starts out with %Id entries", heap_number, mark_list_index - mark_list));
8399 // unsigned long start = GetCycleCount32();
8400 for (int i = 0; i < n_heaps; i++)
8402 gc_heap* heap = g_heaps[i];
8403 if (heap->mark_list_piece_start[heap_number] < heap->mark_list_piece_end[heap_number])
8405 source[source_count] = heap->mark_list_piece_start[heap_number];
8406 source_end[source_count] = heap->mark_list_piece_end[heap_number];
8407 source_heap[source_count] = i;
8408 if (source_count < MAX_SUPPORTED_CPUS)
8412 // printf("first phase of merge_mark_lists for heap %d took %u cycles\n", heap_number, GetCycleCount32() - start);
8414 dprintf(3, ("heap_number = %d has %d sources\n", heap_number, source_count));
8415 #if defined(_DEBUG) || defined(TRACE_GC)
8416 for (int j = 0; j < source_count; j++)
8418 dprintf(3, ("heap_number = %d ", heap_number));
8419 dprintf(3, (" source from heap %d = %Ix .. %Ix (%Id entries)",
8420 (size_t)(source_heap[j]), (size_t)(source[j][0]), (size_t)(source_end[j][-1]), (size_t)(source_end[j] - source[j])));
8421 // the sources should all be sorted
8422 for (uint8_t **x = source[j]; x < source_end[j] - 1; x++)
8426 dprintf(3, ("oops, mark_list from source %d for heap %d isn't sorted\n", j, heap_number));
8431 #endif //_DEBUG || TRACE_GC
8433 // start = GetCycleCount32();
8435 mark_list = &g_mark_list_copy [heap_number*mark_list_size];
8436 mark_list_index = mark_list;
8437 mark_list_end = &mark_list [mark_list_size-1];
8438 int piece_count = 0;
8439 if (source_count == 0)
8443 else if (source_count == 1)
8445 mark_list = source[0];
8446 mark_list_index = source_end[0];
8447 mark_list_end = mark_list_index;
8452 while (source_count > 1)
8454 // find the lowest and second lowest value in the sources we're merging from
8455 int lowest_source = 0;
8456 uint8_t *lowest = *source[0];
8457 uint8_t *second_lowest = *source[1];
8458 for (int i = 1; i < source_count; i++)
8460 if (lowest > *source[i])
8462 second_lowest = lowest;
8463 lowest = *source[i];
8466 else if (second_lowest > *source[i])
8468 second_lowest = *source[i];
8472 // find the point in the lowest source where it either runs out or is not <= second_lowest anymore
8474 // let's first try to get lucky and see if the whole source is <= second_lowest -- this is actually quite common
8476 if (source_end[lowest_source][-1] <= second_lowest)
8477 x = source_end[lowest_source];
8480 // use linear search to find the end -- could also use binary search as in sort_mark_list,
8481 // but saw no improvement doing that
8482 for (x = source[lowest_source]; x < source_end[lowest_source] && *x <= second_lowest; x++)
8486 // blast this piece to the mark list
8487 append_to_mark_list(source[lowest_source], x);
8490 source[lowest_source] = x;
8492 // check whether this source is now exhausted
8493 if (x >= source_end[lowest_source])
8495 // if it's not the source with the highest index, copy the source with the highest index
8496 // over it so the non-empty sources are always at the beginning
8497 if (lowest_source < source_count-1)
8499 source[lowest_source] = source[source_count-1];
8500 source_end[lowest_source] = source_end[source_count-1];
8505 // we're left with just one source that we copy
8506 append_to_mark_list(source[0], source_end[0]);
8510 // printf("second phase of merge_mark_lists for heap %d took %u cycles to merge %d pieces\n", heap_number, GetCycleCount32() - start, piece_count);
8512 #if defined(_DEBUG) || defined(TRACE_GC)
8513 // the final mark list must be sorted
8514 for (uint8_t **x = mark_list; x < mark_list_index - 1; x++)
8518 dprintf(3, ("oops, mark_list for heap %d isn't sorted at the end of merge_mark_lists", heap_number));
8522 #endif //defined(_DEBUG) || defined(TRACE_GC)
8524 #else //PARALLEL_MARK_LIST_SORT
8525 void gc_heap::combine_mark_lists()
8527 dprintf (3, ("Combining mark lists"));
8528 //verify if a heap has overflowed its mark list
8529 BOOL use_mark_list = TRUE;
8530 for (int i = 0; i < n_heaps; i++)
8532 if (g_heaps [i]->mark_list_index > g_heaps [i]->mark_list_end)
8534 use_mark_list = FALSE;
8541 dprintf (3, ("Using mark list"));
8542 //compact the gaps out of the mark list
8544 uint8_t** current_gap = g_heaps [gn]->mark_list_index;
8545 uint8_t** current_gap_end = g_heaps[gn]->mark_list_end + 1;
8546 uint8_t** dst_last = current_gap-1;
8548 int srcn = n_heaps-1;
8549 gc_heap* srch = g_heaps [srcn];
8550 uint8_t** src = srch->mark_list_index - 1;
8551 uint8_t** src_beg = srch->mark_list;
8553 while (current_gap <= src)
8555 while ((gn < n_heaps-1) && (current_gap >= current_gap_end))
8557 //go to the next gap
8559 dprintf (3, ("Going to the next gap %d", gn));
8560 assert (gn < n_heaps);
8561 current_gap = g_heaps [gn]->mark_list_index;
8562 current_gap_end = g_heaps[gn]->mark_list_end + 1;
8563 assert ((gn == (n_heaps-1)) || (current_gap_end == g_heaps[gn+1]->mark_list));
8565 while ((srcn > 0) && (src < src_beg))
8567 //go to the previous source
8569 dprintf (3, ("going to the previous source %d", srcn));
8571 gc_heap* srch = g_heaps [srcn];
8572 src = srch->mark_list_index - 1;
8573 src_beg = srch->mark_list;
8575 if (current_gap < src)
8577 dst_last = current_gap;
8578 *current_gap++ = *src--;
8581 dprintf (3, ("src: %Ix dst_last: %Ix", (size_t)src, (size_t)dst_last));
8583 uint8_t** end_of_list = max (src, dst_last);
8585 //sort the resulting compacted list
8586 assert (end_of_list < &g_mark_list [n_heaps*mark_list_size]);
8587 if (end_of_list > &g_mark_list[0])
8588 _sort (&g_mark_list[0], end_of_list, 0);
8589 //adjust the mark_list to the begining of the resulting mark list.
8590 for (int i = 0; i < n_heaps; i++)
8592 g_heaps [i]->mark_list = g_mark_list;
8593 g_heaps [i]->mark_list_index = end_of_list + 1;
8594 g_heaps [i]->mark_list_end = end_of_list + 1;
8599 uint8_t** end_of_list = g_mark_list;
8600 //adjust the mark_list to the begining of the resulting mark list.
8601 //put the index beyond the end to turn off mark list processing
8602 for (int i = 0; i < n_heaps; i++)
8604 g_heaps [i]->mark_list = g_mark_list;
8605 g_heaps [i]->mark_list_index = end_of_list + 1;
8606 g_heaps [i]->mark_list_end = end_of_list;
8610 #endif // PARALLEL_MARK_LIST_SORT
8611 #endif //MULTIPLE_HEAPS
8614 class seg_free_spaces
8616 struct seg_free_space
8622 struct free_space_bucket
8624 seg_free_space* free_space;
8625 ptrdiff_t count_add; // Assigned when we first contruct the array.
8626 ptrdiff_t count_fit; // How many items left when we are fitting plugs.
8629 void move_bucket (int old_power2, int new_power2)
8631 // PREFAST warning 22015: old_power2 could be negative
8632 assert (old_power2 >= 0);
8633 assert (old_power2 >= new_power2);
8635 if (old_power2 == new_power2)
8640 seg_free_space* src_index = free_space_buckets[old_power2].free_space;
8641 for (int i = old_power2; i > new_power2; i--)
8643 seg_free_space** dest = &(free_space_buckets[i].free_space);
8646 seg_free_space* dest_index = free_space_buckets[i - 1].free_space;
8647 if (i > (new_power2 + 1))
8649 seg_free_space temp = *src_index;
8650 *src_index = *dest_index;
8653 src_index = dest_index;
8656 free_space_buckets[old_power2].count_fit--;
8657 free_space_buckets[new_power2].count_fit++;
8662 void dump_free_space (seg_free_space* item)
8669 mark* m = (mark*)(item->start);
8670 len = pinned_len (m);
8671 addr = pinned_plug (m) - len;
8675 heap_segment* seg = (heap_segment*)(item->start);
8676 addr = heap_segment_plan_allocated (seg);
8677 len = heap_segment_committed (seg) - addr;
8680 dprintf (SEG_REUSE_LOG_1, ("[%d]0x%Ix %Id", heap_num, addr, len));
8685 seg_free_space* item = NULL;
8688 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------\nnow the free spaces look like:", heap_num));
8689 for (i = 0; i < (free_space_bucket_count - 1); i++)
8691 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces for 2^%d bucket:", heap_num, (base_power2 + i)));
8692 dprintf (SEG_REUSE_LOG_1, ("[%d]%s %s", heap_num, "start", "len"));
8693 item = free_space_buckets[i].free_space;
8694 while (item < free_space_buckets[i + 1].free_space)
8696 dump_free_space (item);
8699 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------", heap_num));
8702 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces for 2^%d bucket:", heap_num, (base_power2 + i)));
8703 dprintf (SEG_REUSE_LOG_1, ("[%d]%s %s", heap_num, "start", "len"));
8704 item = free_space_buckets[i].free_space;
8706 while (item <= &seg_free_space_array[free_space_item_count - 1])
8708 dump_free_space (item);
8711 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------", heap_num));
8716 free_space_bucket* free_space_buckets;
8717 seg_free_space* seg_free_space_array;
8718 ptrdiff_t free_space_bucket_count;
8719 ptrdiff_t free_space_item_count;
8723 BOOL has_end_of_seg;
8728 seg_free_spaces (int h_number)
8730 heap_num = h_number;
8735 size_t total_prealloc_size =
8736 MAX_NUM_BUCKETS * sizeof (free_space_bucket) +
8737 MAX_NUM_FREE_SPACES * sizeof (seg_free_space);
8739 free_space_buckets = (free_space_bucket*) new (nothrow) uint8_t[total_prealloc_size];
8741 return (!!free_space_buckets);
8744 // We take the ordered free space array we got from the 1st pass,
8745 // and feed the portion that we decided to use to this method, ie,
8746 // the largest item_count free spaces.
8747 void add_buckets (int base, size_t* ordered_free_spaces, int bucket_count, size_t item_count)
8749 assert (free_space_buckets);
8750 assert (item_count <= (size_t)MAX_PTR);
8752 free_space_bucket_count = bucket_count;
8753 free_space_item_count = item_count;
8756 has_end_of_seg = FALSE;
8759 ptrdiff_t total_item_count = 0;
8762 seg_free_space_array = (seg_free_space*)(free_space_buckets + free_space_bucket_count);
8764 for (i = 0; i < (ptrdiff_t)item_count; i++)
8766 seg_free_space_array[i].start = 0;
8767 seg_free_space_array[i].is_plug = FALSE;
8770 for (i = 0; i < bucket_count; i++)
8772 free_space_buckets[i].count_add = ordered_free_spaces[i];
8773 free_space_buckets[i].count_fit = ordered_free_spaces[i];
8774 free_space_buckets[i].free_space = &seg_free_space_array[total_item_count];
8775 total_item_count += free_space_buckets[i].count_add;
8778 assert (total_item_count == (ptrdiff_t)item_count);
8781 // If we are adding a free space before a plug we pass the
8782 // mark stack position so we can update the length; we could
8783 // also be adding the free space after the last plug in which
8784 // case start is the segment which we'll need to update the
8785 // heap_segment_plan_allocated.
8786 void add (void* start, BOOL plug_p, BOOL first_p)
8788 size_t size = (plug_p ?
8789 pinned_len ((mark*)start) :
8790 (heap_segment_committed ((heap_segment*)start) -
8791 heap_segment_plan_allocated ((heap_segment*)start)));
8795 dprintf (SEG_REUSE_LOG_1, ("[%d]Adding a free space before plug: %Id", heap_num, size));
8799 dprintf (SEG_REUSE_LOG_1, ("[%d]Adding a free space at end of seg: %Id", heap_num, size));
8801 has_end_of_seg = TRUE;
8807 size_t eph_gen_starts = gc_heap::eph_gen_starts_size;
8808 size -= eph_gen_starts;
8811 mark* m = (mark*)(start);
8812 pinned_len (m) -= eph_gen_starts;
8816 heap_segment* seg = (heap_segment*)start;
8817 heap_segment_plan_allocated (seg) += eph_gen_starts;
8821 int bucket_power2 = index_of_highest_set_bit (size);
8822 if (bucket_power2 < base_power2)
8827 free_space_bucket* bucket = &free_space_buckets[bucket_power2 - base_power2];
8829 seg_free_space* bucket_free_space = bucket->free_space;
8830 assert (plug_p || (!plug_p && bucket->count_add));
8832 if (bucket->count_add == 0)
8834 dprintf (SEG_REUSE_LOG_1, ("[%d]Already have enough of 2^%d", heap_num, bucket_power2));
8838 ptrdiff_t index = bucket->count_add - 1;
8840 dprintf (SEG_REUSE_LOG_1, ("[%d]Building free spaces: adding %Ix; len: %Id (2^%d)",
8843 (pinned_plug ((mark*)start) - pinned_len ((mark*)start)) :
8844 heap_segment_plan_allocated ((heap_segment*)start)),
8850 bucket_free_space[index].is_plug = TRUE;
8853 bucket_free_space[index].start = start;
8854 bucket->count_add--;
8859 // Do a consistency check after all free spaces are added.
8863 int end_of_seg_count = 0;
8865 for (i = 0; i < free_space_item_count; i++)
8867 assert (seg_free_space_array[i].start);
8868 if (!(seg_free_space_array[i].is_plug))
8876 assert (end_of_seg_count == 1);
8880 assert (end_of_seg_count == 0);
8883 for (i = 0; i < free_space_bucket_count; i++)
8885 assert (free_space_buckets[i].count_add == 0);
8891 uint8_t* fit (uint8_t* old_loc,
8893 BOOL set_padding_on_saved_p,
8894 mark* pinned_plug_entry,
8895 #endif //SHORT_PLUGS
8897 REQD_ALIGN_AND_OFFSET_DCL)
8902 assert (!is_plug_padded (old_loc));
8903 #endif //SHORT_PLUGS
8904 assert (!node_realigned (old_loc));
8907 size_t saved_plug_size = plug_size;
8909 #ifdef FEATURE_STRUCTALIGN
8910 // BARTOKTODO (4841): this code path is disabled (see can_fit_all_blocks_p) until we take alignment requirements into account
8911 _ASSERTE(requiredAlignment == DATA_ALIGNMENT && false);
8912 #endif // FEATURE_STRUCTALIGN
8913 // TODO: this is also not large alignment ready. We would need to consider alignment when chosing the
8916 size_t plug_size_to_fit = plug_size;
8918 // best fit is only done for gen1 to gen2 and we do not pad in gen2.
8919 int pad_in_front = 0;
8922 plug_size_to_fit += (pad_in_front ? Align(min_obj_size) : 0);
8923 #endif //SHORT_PLUGS
8925 int plug_power2 = index_of_highest_set_bit (round_up_power2 (plug_size_to_fit + Align(min_obj_size)));
8927 uint8_t* new_address = 0;
8929 if (plug_power2 < base_power2)
8931 plug_power2 = base_power2;
8934 int chosen_power2 = plug_power2 - base_power2;
8936 for (i = chosen_power2; i < free_space_bucket_count; i++)
8938 if (free_space_buckets[i].count_fit != 0)
8945 dprintf (SEG_REUSE_LOG_1, ("[%d]Fitting plug len %Id (2^%d) using 2^%d free space",
8949 (chosen_power2 + base_power2)));
8951 assert (i < free_space_bucket_count);
8953 seg_free_space* bucket_free_space = free_space_buckets[chosen_power2].free_space;
8954 ptrdiff_t free_space_count = free_space_buckets[chosen_power2].count_fit;
8955 size_t new_free_space_size = 0;
8956 BOOL can_fit = FALSE;
8959 for (i = 0; i < free_space_count; i++)
8961 size_t free_space_size = 0;
8964 BOOL short_plugs_padding_p = FALSE;
8965 #endif //SHORT_PLUGS
8966 BOOL realign_padding_p = FALSE;
8968 if (bucket_free_space[i].is_plug)
8970 mark* m = (mark*)(bucket_free_space[i].start);
8971 uint8_t* plug_free_space_start = pinned_plug (m) - pinned_len (m);
8974 if ((pad_in_front & USE_PADDING_FRONT) &&
8975 (((plug_free_space_start - pin_allocation_context_start_region (m))==0) ||
8976 ((plug_free_space_start - pin_allocation_context_start_region (m))>=DESIRED_PLUG_LENGTH)))
8978 pad = Align (min_obj_size);
8979 short_plugs_padding_p = TRUE;
8981 #endif //SHORT_PLUGS
8983 if (!((old_loc == 0) || same_large_alignment_p (old_loc, plug_free_space_start+pad)))
8985 pad += switch_alignment_size (pad != 0);
8986 realign_padding_p = TRUE;
8989 plug_size = saved_plug_size + pad;
8991 free_space_size = pinned_len (m);
8992 new_address = pinned_plug (m) - pinned_len (m);
8994 if (free_space_size >= (plug_size + Align (min_obj_size)) ||
8995 free_space_size == plug_size)
8997 new_free_space_size = free_space_size - plug_size;
8998 pinned_len (m) = new_free_space_size;
8999 #ifdef SIMPLE_DPRINTF
9000 dprintf (SEG_REUSE_LOG_0, ("[%d]FP: 0x%Ix->0x%Ix(%Ix)(%Ix), [0x%Ix (2^%d) -> [0x%Ix (2^%d)",
9007 index_of_highest_set_bit (free_space_size),
9008 (pinned_plug (m) - pinned_len (m)),
9009 index_of_highest_set_bit (new_free_space_size)));
9010 #endif //SIMPLE_DPRINTF
9013 if (short_plugs_padding_p)
9015 pin_allocation_context_start_region (m) = plug_free_space_start;
9016 set_padding_in_expand (old_loc, set_padding_on_saved_p, pinned_plug_entry);
9018 #endif //SHORT_PLUGS
9020 if (realign_padding_p)
9022 set_node_realigned (old_loc);
9030 heap_segment* seg = (heap_segment*)(bucket_free_space[i].start);
9031 free_space_size = heap_segment_committed (seg) - heap_segment_plan_allocated (seg);
9033 if (!((old_loc == 0) || same_large_alignment_p (old_loc, heap_segment_plan_allocated (seg))))
9035 pad = switch_alignment_size (FALSE);
9036 realign_padding_p = TRUE;
9039 plug_size = saved_plug_size + pad;
9041 if (free_space_size >= (plug_size + Align (min_obj_size)) ||
9042 free_space_size == plug_size)
9044 new_address = heap_segment_plan_allocated (seg);
9045 new_free_space_size = free_space_size - plug_size;
9046 heap_segment_plan_allocated (seg) = new_address + plug_size;
9047 #ifdef SIMPLE_DPRINTF
9048 dprintf (SEG_REUSE_LOG_0, ("[%d]FS: 0x%Ix-> 0x%Ix(%Ix) (2^%d) -> 0x%Ix (2^%d)",
9053 index_of_highest_set_bit (free_space_size),
9054 heap_segment_plan_allocated (seg),
9055 index_of_highest_set_bit (new_free_space_size)));
9056 #endif //SIMPLE_DPRINTF
9058 if (realign_padding_p)
9059 set_node_realigned (old_loc);
9073 assert (chosen_power2 == 0);
9083 assert ((chosen_power2 && (i == 0)) ||
9084 (!chosen_power2) && (i < free_space_count));
9087 int new_bucket_power2 = index_of_highest_set_bit (new_free_space_size);
9089 if (new_bucket_power2 < base_power2)
9091 new_bucket_power2 = base_power2;
9094 move_bucket (chosen_power2, new_bucket_power2 - base_power2);
9103 if (free_space_buckets)
9105 delete [] free_space_buckets;
9107 if (seg_free_space_array)
9109 delete [] seg_free_space_array;
9115 #define marked(i) header(i)->IsMarked()
9116 #define set_marked(i) header(i)->SetMarked()
9117 #define clear_marked(i) header(i)->ClearMarked()
9118 #define pinned(i) header(i)->IsPinned()
9119 #define set_pinned(i) header(i)->SetPinned()
9120 #define clear_pinned(i) header(i)->GetHeader()->ClrGCBit();
9122 inline size_t my_get_size (Object* ob)
9124 MethodTable* mT = header(ob)->GetMethodTable();
9125 return (mT->GetBaseSize() +
9126 (mT->HasComponentSize() ?
9127 ((size_t)((CObjectHeader*)ob)->GetNumComponents() * mT->RawGetComponentSize()) : 0));
9130 //#define size(i) header(i)->GetSize()
9131 #define size(i) my_get_size (header(i))
9133 #define contain_pointers(i) header(i)->ContainsPointers()
9134 #ifdef COLLECTIBLE_CLASS
9135 #define contain_pointers_or_collectible(i) header(i)->ContainsPointersOrCollectible()
9137 #define get_class_object(i) GCToEEInterface::GetLoaderAllocatorObjectForGC((Object *)i)
9138 #define is_collectible(i) method_table(i)->Collectible()
9139 #else //COLLECTIBLE_CLASS
9140 #define contain_pointers_or_collectible(i) header(i)->ContainsPointers()
9141 #endif //COLLECTIBLE_CLASS
9143 #if defined (MARK_ARRAY) && defined (BACKGROUND_GC)
9145 void gc_heap::seg_clear_mark_array_bits_soh (heap_segment* seg)
9147 uint8_t* range_beg = 0;
9148 uint8_t* range_end = 0;
9149 if (bgc_mark_array_range (seg, FALSE, &range_beg, &range_end))
9151 clear_mark_array (range_beg, align_on_mark_word (range_end), FALSE
9152 #ifdef FEATURE_BASICFREEZE
9154 #endif // FEATURE_BASICFREEZE
9159 void gc_heap::clear_batch_mark_array_bits (uint8_t* start, uint8_t* end)
9161 if ((start < background_saved_highest_address) &&
9162 (end > background_saved_lowest_address))
9164 start = max (start, background_saved_lowest_address);
9165 end = min (end, background_saved_highest_address);
9167 size_t start_mark_bit = mark_bit_of (start);
9168 size_t end_mark_bit = mark_bit_of (end);
9169 unsigned int startbit = mark_bit_bit (start_mark_bit);
9170 unsigned int endbit = mark_bit_bit (end_mark_bit);
9171 size_t startwrd = mark_bit_word (start_mark_bit);
9172 size_t endwrd = mark_bit_word (end_mark_bit);
9174 dprintf (3, ("Clearing all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
9175 (size_t)start, (size_t)start_mark_bit,
9176 (size_t)end, (size_t)end_mark_bit));
9178 unsigned int firstwrd = lowbits (~0, startbit);
9179 unsigned int lastwrd = highbits (~0, endbit);
9181 if (startwrd == endwrd)
9183 unsigned int wrd = firstwrd | lastwrd;
9184 mark_array[startwrd] &= wrd;
9188 // clear the first mark word.
9191 mark_array[startwrd] &= firstwrd;
9195 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
9197 mark_array[wrdtmp] = 0;
9200 // clear the last mark word.
9203 mark_array[endwrd] &= lastwrd;
9208 void gc_heap::bgc_clear_batch_mark_array_bits (uint8_t* start, uint8_t* end)
9210 if ((start < background_saved_highest_address) &&
9211 (end > background_saved_lowest_address))
9213 start = max (start, background_saved_lowest_address);
9214 end = min (end, background_saved_highest_address);
9216 clear_batch_mark_array_bits (start, end);
9220 void gc_heap::clear_mark_array_by_objects (uint8_t* from, uint8_t* end, BOOL loh_p)
9222 dprintf (3, ("clearing mark array bits by objects for addr [%Ix,[%Ix",
9224 int align_const = get_alignment_constant (!loh_p);
9230 uint8_t* next_o = o + Align (size (o), align_const);
9232 if (background_object_marked (o, TRUE))
9234 dprintf (3, ("%Ix was marked by bgc, is now cleared", o));
9240 #endif //MARK_ARRAY && BACKGROUND_GC
9243 BOOL gc_heap::is_mark_set (uint8_t* o)
9248 #if defined (_MSC_VER) && defined (_TARGET_X86_)
9249 #pragma optimize("y", on) // Small critical routines, don't put in EBP frame
9250 #endif //_MSC_VER && _TARGET_X86_
9252 // return the generation number of an object.
9253 // It is assumed that the object is valid.
9254 //Note that this will return max_generation for a LOH object
9255 int gc_heap::object_gennum (uint8_t* o)
9257 if (in_range_for_segment (o, ephemeral_heap_segment) &&
9258 (o >= generation_allocation_start (generation_of (max_generation-1))))
9260 // in an ephemeral generation.
9261 for ( int i = 0; i < max_generation-1; i++)
9263 if ((o >= generation_allocation_start (generation_of (i))))
9266 return max_generation-1;
9270 return max_generation;
9274 int gc_heap::object_gennum_plan (uint8_t* o)
9276 if (in_range_for_segment (o, ephemeral_heap_segment))
9278 for (int i = 0; i <= max_generation-1; i++)
9280 uint8_t* plan_start = generation_plan_allocation_start (generation_of (i));
9281 if (plan_start && (o >= plan_start))
9287 return max_generation;
9290 #if defined(_MSC_VER) && defined(_TARGET_X86_)
9291 #pragma optimize("", on) // Go back to command line default optimizations
9292 #endif //_MSC_VER && _TARGET_X86_
9294 heap_segment* gc_heap::make_heap_segment (uint8_t* new_pages, size_t size, int h_number)
9296 size_t initial_commit = SEGMENT_INITIAL_COMMIT;
9298 //Commit the first page
9299 if (!virtual_commit (new_pages, initial_commit, h_number))
9304 //overlay the heap_segment
9305 heap_segment* new_segment = (heap_segment*)new_pages;
9307 uint8_t* start = new_pages + segment_info_size;
9308 heap_segment_mem (new_segment) = start;
9309 heap_segment_used (new_segment) = start;
9310 heap_segment_reserved (new_segment) = new_pages + size;
9311 heap_segment_committed (new_segment) = new_pages + initial_commit;
9312 init_heap_segment (new_segment);
9313 dprintf (2, ("Creating heap segment %Ix", (size_t)new_segment));
9317 void gc_heap::init_heap_segment (heap_segment* seg)
9320 heap_segment_next (seg) = 0;
9321 heap_segment_plan_allocated (seg) = heap_segment_mem (seg);
9322 heap_segment_allocated (seg) = heap_segment_mem (seg);
9323 #ifdef BACKGROUND_GC
9324 heap_segment_background_allocated (seg) = 0;
9325 heap_segment_saved_bg_allocated (seg) = 0;
9326 #endif //BACKGROUND_GC
9329 //Releases the segment to the OS.
9330 // this is always called on one thread only so calling seg_table->remove is fine.
9331 void gc_heap::delete_heap_segment (heap_segment* seg, BOOL consider_hoarding)
9333 if (!heap_segment_loh_p (seg))
9335 //cleanup the brick table back to the empty value
9336 clear_brick_table (heap_segment_mem (seg), heap_segment_reserved (seg));
9339 if (consider_hoarding)
9341 assert ((heap_segment_mem (seg) - (uint8_t*)seg) <= ptrdiff_t(2*OS_PAGE_SIZE));
9342 size_t ss = (size_t) (heap_segment_reserved (seg) - (uint8_t*)seg);
9343 //Don't keep the big ones.
9344 if (ss <= INITIAL_ALLOC)
9346 dprintf (2, ("Hoarding segment %Ix", (size_t)seg));
9347 #ifdef BACKGROUND_GC
9348 // We don't need to clear the decommitted flag because when this segment is used
9349 // for a new segment the flags will be cleared.
9350 if (!heap_segment_decommitted_p (seg))
9351 #endif //BACKGROUND_GC
9353 decommit_heap_segment (seg);
9356 #ifdef SEG_MAPPING_TABLE
9357 seg_mapping_table_remove_segment (seg);
9358 #endif //SEG_MAPPING_TABLE
9360 heap_segment_next (seg) = segment_standby_list;
9361 segment_standby_list = seg;
9368 dprintf (2, ("h%d: del seg: [%Ix, %Ix[",
9369 heap_number, (size_t)seg,
9370 (size_t)(heap_segment_reserved (seg))));
9372 #ifdef BACKGROUND_GC
9373 ::record_changed_seg ((uint8_t*)seg, heap_segment_reserved (seg),
9374 settings.gc_index, current_bgc_state,
9376 decommit_mark_array_by_seg (seg);
9377 #endif //BACKGROUND_GC
9379 #ifdef SEG_MAPPING_TABLE
9380 seg_mapping_table_remove_segment (seg);
9381 #else //SEG_MAPPING_TABLE
9382 seg_table->remove ((uint8_t*)seg);
9383 #endif //SEG_MAPPING_TABLE
9385 release_segment (seg);
9389 //resets the pages beyond allocates size so they won't be swapped out and back in
9391 void gc_heap::reset_heap_segment_pages (heap_segment* seg)
9393 size_t page_start = align_on_page ((size_t)heap_segment_allocated (seg));
9394 size_t size = (size_t)heap_segment_committed (seg) - page_start;
9396 GCToOSInterface::VirtualReset((void*)page_start, size, false /* unlock */);
9399 void gc_heap::decommit_heap_segment_pages (heap_segment* seg,
9402 uint8_t* page_start = align_on_page (heap_segment_allocated(seg));
9403 size_t size = heap_segment_committed (seg) - page_start;
9404 extra_space = align_on_page (extra_space);
9405 if (size >= max ((extra_space + 2*OS_PAGE_SIZE), 100*OS_PAGE_SIZE))
9407 page_start += max(extra_space, 32*OS_PAGE_SIZE);
9408 size -= max (extra_space, 32*OS_PAGE_SIZE);
9410 virtual_decommit (page_start, size, heap_number);
9411 dprintf (3, ("Decommitting heap segment [%Ix, %Ix[(%d)",
9413 (size_t)(page_start + size),
9415 heap_segment_committed (seg) = page_start;
9416 if (heap_segment_used (seg) > heap_segment_committed (seg))
9418 heap_segment_used (seg) = heap_segment_committed (seg);
9423 //decommit all pages except one or 2
9424 void gc_heap::decommit_heap_segment (heap_segment* seg)
9426 uint8_t* page_start = align_on_page (heap_segment_mem (seg));
9428 dprintf (3, ("Decommitting heap segment %Ix", (size_t)seg));
9430 #ifdef BACKGROUND_GC
9431 page_start += OS_PAGE_SIZE;
9432 #endif //BACKGROUND_GC
9434 size_t size = heap_segment_committed (seg) - page_start;
9435 virtual_decommit (page_start, size, heap_number);
9437 //re-init the segment object
9438 heap_segment_committed (seg) = page_start;
9439 if (heap_segment_used (seg) > heap_segment_committed (seg))
9441 heap_segment_used (seg) = heap_segment_committed (seg);
9445 void gc_heap::clear_gen0_bricks()
9447 if (!gen0_bricks_cleared)
9449 gen0_bricks_cleared = TRUE;
9450 //initialize brick table for gen 0
9451 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
9452 b < brick_of (align_on_brick
9453 (heap_segment_allocated (ephemeral_heap_segment)));
9461 #ifdef BACKGROUND_GC
9462 void gc_heap::rearrange_small_heap_segments()
9464 heap_segment* seg = freeable_small_heap_segment;
9467 heap_segment* next_seg = heap_segment_next (seg);
9468 // TODO: we need to consider hoarding here.
9469 delete_heap_segment (seg, FALSE);
9472 freeable_small_heap_segment = 0;
9474 #endif //BACKGROUND_GC
9476 void gc_heap::rearrange_large_heap_segments()
9478 dprintf (2, ("deleting empty large segments"));
9479 heap_segment* seg = freeable_large_heap_segment;
9482 heap_segment* next_seg = heap_segment_next (seg);
9483 delete_heap_segment (seg, GCConfig::GetRetainVM());
9486 freeable_large_heap_segment = 0;
9489 void gc_heap::rearrange_heap_segments(BOOL compacting)
9492 generation_start_segment (generation_of (max_generation));
9494 heap_segment* prev_seg = 0;
9495 heap_segment* next_seg = 0;
9498 next_seg = heap_segment_next (seg);
9500 //link ephemeral segment when expanding
9501 if ((next_seg == 0) && (seg != ephemeral_heap_segment))
9503 seg->next = ephemeral_heap_segment;
9504 next_seg = heap_segment_next (seg);
9507 //re-used expanded heap segment
9508 if ((seg == ephemeral_heap_segment) && next_seg)
9510 heap_segment_next (prev_seg) = next_seg;
9511 heap_segment_next (seg) = 0;
9515 uint8_t* end_segment = (compacting ?
9516 heap_segment_plan_allocated (seg) :
9517 heap_segment_allocated (seg));
9518 // check if the segment was reached by allocation
9519 if ((end_segment == heap_segment_mem (seg))&&
9520 !heap_segment_read_only_p (seg))
9522 //if not, unthread and delete
9524 assert (seg != ephemeral_heap_segment);
9525 heap_segment_next (prev_seg) = next_seg;
9526 delete_heap_segment (seg, GCConfig::GetRetainVM());
9528 dprintf (2, ("Deleting heap segment %Ix", (size_t)seg));
9532 if (!heap_segment_read_only_p (seg))
9536 heap_segment_allocated (seg) =
9537 heap_segment_plan_allocated (seg);
9540 // reset the pages between allocated and committed.
9541 if (seg != ephemeral_heap_segment)
9543 decommit_heap_segment_pages (seg, 0);
9557 uint8_t* g_addresses [array_size+2]; // to get around the bug in GetWriteWatch
9559 #ifdef TIME_WRITE_WATCH
9560 static unsigned int tot_cycles = 0;
9561 #endif //TIME_WRITE_WATCH
9565 inline void gc_heap::verify_card_bundle_bits_set(size_t first_card_word, size_t last_card_word)
9568 for (size_t x = cardw_card_bundle (first_card_word); x < cardw_card_bundle (last_card_word); x++)
9570 if (!card_bundle_set_p (x))
9572 assert (!"Card bundle not set");
9573 dprintf (3, ("Card bundle %Ix not set", x));
9579 // Verifies that any bundles that are not set represent only cards that are not set.
9580 inline void gc_heap::verify_card_bundles()
9583 size_t lowest_card = card_word (card_of (lowest_address));
9584 size_t highest_card = card_word (card_of (highest_address));
9585 size_t cardb = cardw_card_bundle (lowest_card);
9586 size_t end_cardb = cardw_card_bundle (align_cardw_on_bundle (highest_card));
9588 while (cardb < end_cardb)
9590 uint32_t* card_word = &card_table[max(card_bundle_cardw (cardb), lowest_card)];
9591 uint32_t* card_word_end = &card_table[min(card_bundle_cardw (cardb+1), highest_card)];
9593 if (card_bundle_set_p (cardb) == 0)
9595 // Verify that no card is set
9596 while (card_word < card_word_end)
9598 if (*card_word != 0)
9600 dprintf (3, ("gc: %d, Card word %Ix for address %Ix set, card_bundle %Ix clear",
9601 dd_collection_count (dynamic_data_of (0)),
9602 (size_t)(card_word-&card_table[0]),
9603 (size_t)(card_address ((size_t)(card_word-&card_table[0]) * card_word_width)), cardb));
9606 assert((*card_word)==0);
9616 // If card bundles are enabled, use write watch to find pages in the card table that have
9617 // been dirtied, and set the corresponding card bundle bits.
9618 void gc_heap::update_card_table_bundle()
9620 if (card_bundles_enabled())
9622 // The address of the card word containing the card representing the lowest heap address
9623 uint8_t* base_address = (uint8_t*)(&card_table[card_word (card_of (lowest_address))]);
9625 // The address of the card word containing the card representing the highest heap address
9626 uint8_t* high_address = (uint8_t*)(&card_table[card_word (card_of (highest_address))]);
9628 uint8_t* saved_base_address = base_address;
9629 uintptr_t bcount = array_size;
9630 size_t saved_region_size = align_on_page (high_address) - saved_base_address;
9634 size_t region_size = align_on_page (high_address) - base_address;
9636 dprintf (3,("Probing card table pages [%Ix, %Ix[", (size_t)base_address, (size_t)base_address+region_size));
9637 bool success = GCToOSInterface::GetWriteWatch(false /* resetState */,
9640 (void**)g_addresses,
9642 assert (success && "GetWriteWatch failed!");
9644 dprintf (3,("Found %d pages written", bcount));
9645 for (unsigned i = 0; i < bcount; i++)
9647 // Offset of the dirty page from the start of the card table (clamped to base_address)
9648 size_t bcardw = (uint32_t*)(max(g_addresses[i],base_address)) - &card_table[0];
9650 // Offset of the end of the page from the start of the card table (clamped to high addr)
9651 size_t ecardw = (uint32_t*)(min(g_addresses[i]+OS_PAGE_SIZE, high_address)) - &card_table[0];
9652 assert (bcardw >= card_word (card_of (g_gc_lowest_address)));
9654 // Set the card bundle bits representing the dirty card table page
9655 card_bundles_set (cardw_card_bundle (bcardw), cardw_card_bundle (align_cardw_on_bundle (ecardw)));
9656 dprintf (3,("Set Card bundle [%Ix, %Ix[", cardw_card_bundle (bcardw), cardw_card_bundle (align_cardw_on_bundle (ecardw))));
9658 verify_card_bundle_bits_set(bcardw, ecardw);
9661 if (bcount >= array_size)
9663 base_address = g_addresses [array_size-1] + OS_PAGE_SIZE;
9664 bcount = array_size;
9667 } while ((bcount >= array_size) && (base_address < high_address));
9669 // Now that we've updated the card bundle bits, reset the write-tracking state.
9670 GCToOSInterface::ResetWriteWatch (saved_base_address, saved_region_size);
9673 #endif //CARD_BUNDLE
9676 void gc_heap::reset_write_watch_for_gc_heap(void* base_address, size_t region_size)
9678 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9679 SoftwareWriteWatch::ClearDirty(base_address, region_size);
9680 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9681 GCToOSInterface::ResetWriteWatch(base_address, region_size);
9682 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9686 void gc_heap::get_write_watch_for_gc_heap(bool reset, void *base_address, size_t region_size, void** dirty_pages, uintptr_t* dirty_page_count_ref, bool is_runtime_suspended)
9688 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9689 SoftwareWriteWatch::GetDirty(base_address, region_size, dirty_pages, dirty_page_count_ref, reset, is_runtime_suspended);
9690 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9691 UNREFERENCED_PARAMETER(is_runtime_suspended);
9692 bool success = GCToOSInterface::GetWriteWatch(reset, base_address, region_size, dirty_pages, dirty_page_count_ref);
9694 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9697 const size_t ww_reset_quantum = 128*1024*1024;
9700 void gc_heap::switch_one_quantum()
9702 enable_preemptive ();
9703 GCToOSInterface::Sleep (1);
9704 disable_preemptive (true);
9707 void gc_heap::reset_ww_by_chunk (uint8_t* start_address, size_t total_reset_size)
9709 size_t reset_size = 0;
9710 size_t remaining_reset_size = 0;
9711 size_t next_reset_size = 0;
9713 while (reset_size != total_reset_size)
9715 remaining_reset_size = total_reset_size - reset_size;
9716 next_reset_size = ((remaining_reset_size >= ww_reset_quantum) ? ww_reset_quantum : remaining_reset_size);
9717 if (next_reset_size)
9719 reset_write_watch_for_gc_heap(start_address, next_reset_size);
9720 reset_size += next_reset_size;
9722 switch_one_quantum();
9726 assert (reset_size == total_reset_size);
9729 // This does a Sleep(1) for every reset ww_reset_quantum bytes of reset
9730 // we do concurrently.
9731 void gc_heap::switch_on_reset (BOOL concurrent_p, size_t* current_total_reset_size, size_t last_reset_size)
9735 *current_total_reset_size += last_reset_size;
9737 dprintf (2, ("reset %Id bytes so far", *current_total_reset_size));
9739 if (*current_total_reset_size > ww_reset_quantum)
9741 switch_one_quantum();
9743 *current_total_reset_size = 0;
9748 void gc_heap::reset_write_watch (BOOL concurrent_p)
9750 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9751 // Software write watch currently requires the runtime to be suspended during reset. See SoftwareWriteWatch::ClearDirty().
9752 assert(!concurrent_p);
9753 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9755 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
9757 PREFIX_ASSUME(seg != NULL);
9759 size_t reset_size = 0;
9760 size_t region_size = 0;
9762 dprintf (2, ("bgc lowest: %Ix, bgc highest: %Ix", background_saved_lowest_address, background_saved_highest_address));
9766 uint8_t* base_address = align_lower_page (heap_segment_mem (seg));
9767 base_address = max (base_address, background_saved_lowest_address);
9769 uint8_t* high_address = 0;
9770 high_address = ((seg == ephemeral_heap_segment) ? alloc_allocated : heap_segment_allocated (seg));
9771 high_address = min (high_address, background_saved_highest_address);
9773 if (base_address < high_address)
9775 region_size = high_address - base_address;
9777 #ifdef TIME_WRITE_WATCH
9778 unsigned int time_start = GetCycleCount32();
9779 #endif //TIME_WRITE_WATCH
9780 dprintf (3, ("h%d: soh ww: [%Ix(%Id)", heap_number, (size_t)base_address, region_size));
9781 //reset_ww_by_chunk (base_address, region_size);
9782 reset_write_watch_for_gc_heap(base_address, region_size);
9784 #ifdef TIME_WRITE_WATCH
9785 unsigned int time_stop = GetCycleCount32();
9786 tot_cycles += time_stop - time_start;
9787 printf ("ResetWriteWatch Duration: %d, total: %d\n",
9788 time_stop - time_start, tot_cycles);
9789 #endif //TIME_WRITE_WATCH
9791 switch_on_reset (concurrent_p, &reset_size, region_size);
9794 seg = heap_segment_next_rw (seg);
9796 concurrent_print_time_delta ("CRWW soh");
9799 //concurrent_print_time_delta ("CRW soh");
9801 seg = heap_segment_rw (generation_start_segment (large_object_generation));
9803 PREFIX_ASSUME(seg != NULL);
9807 uint8_t* base_address = align_lower_page (heap_segment_mem (seg));
9808 uint8_t* high_address = heap_segment_allocated (seg);
9810 base_address = max (base_address, background_saved_lowest_address);
9811 high_address = min (high_address, background_saved_highest_address);
9813 if (base_address < high_address)
9815 region_size = high_address - base_address;
9817 #ifdef TIME_WRITE_WATCH
9818 unsigned int time_start = GetCycleCount32();
9819 #endif //TIME_WRITE_WATCH
9820 dprintf (3, ("h%d: loh ww: [%Ix(%Id)", heap_number, (size_t)base_address, region_size));
9821 //reset_ww_by_chunk (base_address, region_size);
9822 reset_write_watch_for_gc_heap(base_address, region_size);
9824 #ifdef TIME_WRITE_WATCH
9825 unsigned int time_stop = GetCycleCount32();
9826 tot_cycles += time_stop - time_start;
9827 printf ("ResetWriteWatch Duration: %d, total: %d\n",
9828 time_stop - time_start, tot_cycles);
9829 #endif //TIME_WRITE_WATCH
9831 switch_on_reset (concurrent_p, &reset_size, region_size);
9834 seg = heap_segment_next_rw (seg);
9836 concurrent_print_time_delta ("CRWW loh");
9839 #ifdef DEBUG_WRITE_WATCH
9840 debug_write_watch = (uint8_t**)~0;
9841 #endif //DEBUG_WRITE_WATCH
9844 #endif //WRITE_WATCH
9846 #ifdef BACKGROUND_GC
9847 void gc_heap::restart_vm()
9849 //assert (generation_allocation_pointer (youngest_generation) == 0);
9850 dprintf (3, ("Restarting EE"));
9851 STRESS_LOG0(LF_GC, LL_INFO10000, "Concurrent GC: Retarting EE\n");
9852 ee_proceed_event.Set();
9856 void fire_alloc_wait_event (alloc_wait_reason awr, BOOL begin_p)
9858 if (awr != awr_ignored)
9862 FIRE_EVENT(BGCAllocWaitBegin, awr);
9866 FIRE_EVENT(BGCAllocWaitEnd, awr);
9872 void gc_heap::fire_alloc_wait_event_begin (alloc_wait_reason awr)
9874 fire_alloc_wait_event (awr, TRUE);
9878 void gc_heap::fire_alloc_wait_event_end (alloc_wait_reason awr)
9880 fire_alloc_wait_event (awr, FALSE);
9882 #endif //BACKGROUND_GC
9883 void gc_heap::make_generation (generation& gen, heap_segment* seg, uint8_t* start, uint8_t* pointer)
9885 gen.allocation_start = start;
9886 gen.allocation_context.alloc_ptr = pointer;
9887 gen.allocation_context.alloc_limit = pointer;
9888 gen.allocation_context.alloc_bytes = 0;
9889 gen.allocation_context.alloc_bytes_loh = 0;
9890 gen.allocation_context_start_region = pointer;
9891 gen.start_segment = seg;
9892 gen.allocation_segment = seg;
9893 gen.plan_allocation_start = 0;
9894 gen.free_list_space = 0;
9895 gen.pinned_allocated = 0;
9896 gen.free_list_allocated = 0;
9897 gen.end_seg_allocated = 0;
9898 gen.condemned_allocated = 0;
9899 gen.free_obj_space = 0;
9900 gen.allocation_size = 0;
9901 gen.pinned_allocation_sweep_size = 0;
9902 gen.pinned_allocation_compact_size = 0;
9903 gen.allocate_end_seg_p = FALSE;
9904 gen.free_list_allocator.clear();
9906 #ifdef FREE_USAGE_STATS
9907 memset (gen.gen_free_spaces, 0, sizeof (gen.gen_free_spaces));
9908 memset (gen.gen_current_pinned_free_spaces, 0, sizeof (gen.gen_current_pinned_free_spaces));
9909 memset (gen.gen_plugs, 0, sizeof (gen.gen_plugs));
9910 #endif //FREE_USAGE_STATS
9913 void gc_heap::adjust_ephemeral_limits ()
9915 ephemeral_low = generation_allocation_start (generation_of (max_generation - 1));
9916 ephemeral_high = heap_segment_reserved (ephemeral_heap_segment);
9918 dprintf (3, ("new ephemeral low: %Ix new ephemeral high: %Ix",
9919 (size_t)ephemeral_low, (size_t)ephemeral_high))
9921 #ifndef MULTIPLE_HEAPS
9922 // This updates the write barrier helpers with the new info.
9923 stomp_write_barrier_ephemeral(ephemeral_low, ephemeral_high);
9924 #endif // MULTIPLE_HEAPS
9927 #if defined(TRACE_GC) || defined(GC_CONFIG_DRIVEN)
9928 FILE* CreateLogFile(const GCConfigStringHolder& temp_logfile_name, bool is_config)
9932 if (!temp_logfile_name.Get())
9937 char logfile_name[MAX_LONGPATH+1];
9938 uint32_t pid = GCToOSInterface::GetCurrentProcessId();
9939 const char* suffix = is_config ? ".config.log" : ".log";
9940 _snprintf_s(logfile_name, MAX_LONGPATH+1, _TRUNCATE, "%s.%d%s", temp_logfile_name.Get(), pid, suffix);
9941 logFile = fopen(logfile_name, "wb");
9944 #endif //TRACE_GC || GC_CONFIG_DRIVEN
9946 size_t gc_heap::get_segment_size_hard_limit (uint32_t* num_heaps, bool should_adjust_num_heaps)
9948 assert (heap_hard_limit);
9949 size_t aligned_hard_limit = ((heap_hard_limit + min_segment_size_hard_limit - 1) & ~(min_segment_size_hard_limit - 1));
9950 if (should_adjust_num_heaps)
9952 uint32_t max_num_heaps = (uint32_t)(aligned_hard_limit / min_segment_size_hard_limit);
9953 if (*num_heaps > max_num_heaps)
9955 *num_heaps = max_num_heaps;
9959 size_t seg_size = aligned_hard_limit / *num_heaps;
9960 size_t aligned_seg_size = round_up_power2 (seg_size);
9962 assert (g_theGCHeap->IsValidSegmentSize (aligned_seg_size));
9964 size_t seg_size_from_config = (size_t)GCConfig::GetSegmentSize();
9965 if (seg_size_from_config)
9967 size_t aligned_seg_size_config = round_up_power2 (seg_size_from_config);
9969 aligned_seg_size = max (aligned_seg_size, aligned_seg_size_config);
9972 //printf ("limit: %Idmb, aligned: %Idmb, %d heaps, seg size from config: %Idmb, seg size %Idmb",
9973 // (heap_hard_limit / 1024 / 1024),
9974 // (aligned_hard_limit / 1024 / 1024),
9976 // (seg_size_from_config / 1024 / 1024),
9977 // (aligned_seg_size / 1024 / 1024));
9978 return aligned_seg_size;
9981 HRESULT gc_heap::initialize_gc (size_t segment_size,
9983 #ifdef MULTIPLE_HEAPS
9984 ,unsigned number_of_heaps
9985 #endif //MULTIPLE_HEAPS
9989 if (GCConfig::GetLogEnabled())
9991 gc_log = CreateLogFile(GCConfig::GetLogFile(), false);
9996 // GCLogFileSize in MBs.
9997 gc_log_file_size = static_cast<size_t>(GCConfig::GetLogFileSize());
9999 if (gc_log_file_size <= 0 || gc_log_file_size > 500)
10005 gc_log_lock.Initialize();
10006 gc_log_buffer = new (nothrow) uint8_t [gc_log_buffer_size];
10007 if (!gc_log_buffer)
10013 memset (gc_log_buffer, '*', gc_log_buffer_size);
10015 max_gc_buffers = gc_log_file_size * 1024 * 1024 / gc_log_buffer_size;
10019 #ifdef GC_CONFIG_DRIVEN
10020 if (GCConfig::GetConfigLogEnabled())
10022 gc_config_log = CreateLogFile(GCConfig::GetConfigLogFile(), true);
10024 if (gc_config_log == NULL)
10027 gc_config_log_buffer = new (nothrow) uint8_t [gc_config_log_buffer_size];
10028 if (!gc_config_log_buffer)
10030 fclose(gc_config_log);
10034 compact_ratio = static_cast<int>(GCConfig::GetCompactRatio());
10036 // h# | GC | gen | C | EX | NF | BF | ML | DM || PreS | PostS | Merge | Conv | Pre | Post | PrPo | PreP | PostP |
10037 cprintf (("%2s | %6s | %1s | %1s | %2s | %2s | %2s | %2s | %2s || %5s | %5s | %5s | %5s | %5s | %5s | %5s | %5s | %5s |",
10038 "h#", // heap index
10041 "C", // compaction (empty means sweeping), 'M' means it was mandatory, 'W' means it was not
10042 "EX", // heap expansion
10043 "NF", // normal fit
10044 "BF", // best fit (if it indicates neither NF nor BF it means it had to acquire a new seg.
10047 "PreS", // short object before pinned plug
10048 "PostS", // short object after pinned plug
10049 "Merge", // merged pinned plugs
10050 "Conv", // converted to pinned plug
10051 "Pre", // plug before pinned plug but not after
10052 "Post", // plug after pinned plug but not before
10053 "PrPo", // plug both before and after pinned plug
10054 "PreP", // pre short object padded
10055 "PostP" // post short object padded
10058 #endif //GC_CONFIG_DRIVEN
10061 GCConfigStringHolder logFileName = GCConfig::GetMixLogFile();
10062 if (logFileName.Get() != nullptr)
10064 GCStatistics::logFileName = _strdup(logFileName.Get());
10065 GCStatistics::logFile = fopen(GCStatistics::logFileName, "a");
10066 if (!GCStatistics::logFile)
10073 HRESULT hres = S_OK;
10076 hardware_write_watch_api_supported();
10077 #ifdef BACKGROUND_GC
10078 if (can_use_write_watch_for_gc_heap() && GCConfig::GetConcurrentGC())
10080 gc_can_use_concurrent = true;
10081 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
10082 virtual_alloc_hardware_write_watch = true;
10083 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
10087 gc_can_use_concurrent = false;
10089 #endif //BACKGROUND_GC
10090 #endif //WRITE_WATCH
10092 #ifdef BACKGROUND_GC
10093 // leave the first page to contain only segment info
10094 // because otherwise we could need to revisit the first page frequently in
10096 segment_info_size = OS_PAGE_SIZE;
10098 segment_info_size = Align (sizeof (heap_segment), get_alignment_constant (FALSE));
10099 #endif //BACKGROUND_GC
10101 reserved_memory = 0;
10102 unsigned block_count;
10103 #ifdef MULTIPLE_HEAPS
10104 reserved_memory_limit = (segment_size + heap_size) * number_of_heaps;
10105 block_count = number_of_heaps;
10106 #else //MULTIPLE_HEAPS
10107 reserved_memory_limit = segment_size + heap_size;
10109 #endif //MULTIPLE_HEAPS
10111 if (heap_hard_limit)
10113 check_commit_cs.Initialize();
10116 if (!reserve_initial_memory(segment_size,heap_size,block_count))
10117 return E_OUTOFMEMORY;
10120 //check if we need to turn on card_bundles.
10121 #ifdef MULTIPLE_HEAPS
10122 // use INT64 arithmetic here because of possible overflow on 32p
10123 uint64_t th = (uint64_t)MH_TH_CARD_BUNDLE*number_of_heaps;
10125 // use INT64 arithmetic here because of possible overflow on 32p
10126 uint64_t th = (uint64_t)SH_TH_CARD_BUNDLE;
10127 #endif //MULTIPLE_HEAPS
10129 if (can_use_write_watch_for_card_table() && reserved_memory >= th)
10131 settings.card_bundles = TRUE;
10135 settings.card_bundles = FALSE;
10137 #endif //CARD_BUNDLE
10139 settings.first_init();
10141 int latency_level_from_config = static_cast<int>(GCConfig::GetLatencyLevel());
10142 if (latency_level_from_config >= latency_level_first && latency_level_from_config <= latency_level_last)
10144 gc_heap::latency_level = static_cast<gc_latency_level>(latency_level_from_config);
10147 init_static_data();
10149 g_gc_card_table = make_card_table (g_gc_lowest_address, g_gc_highest_address);
10151 if (!g_gc_card_table)
10152 return E_OUTOFMEMORY;
10154 gc_started = FALSE;
10156 #ifdef MULTIPLE_HEAPS
10157 g_heaps = new (nothrow) gc_heap* [number_of_heaps];
10159 return E_OUTOFMEMORY;
10162 #pragma warning(push)
10163 #pragma warning(disable:22011) // Suppress PREFast warning about integer underflow/overflow
10164 #endif // _PREFAST_
10165 g_promoted = new (nothrow) size_t [number_of_heaps*16];
10166 g_bpromoted = new (nothrow) size_t [number_of_heaps*16];
10168 g_mark_stack_busy = new (nothrow) int[(number_of_heaps+2)*HS_CACHE_LINE_SIZE/sizeof(int)];
10169 #endif //MH_SC_MARK
10171 #pragma warning(pop)
10172 #endif // _PREFAST_
10173 if (!g_promoted || !g_bpromoted)
10174 return E_OUTOFMEMORY;
10177 if (!g_mark_stack_busy)
10178 return E_OUTOFMEMORY;
10179 #endif //MH_SC_MARK
10181 if (!create_thread_support (number_of_heaps))
10182 return E_OUTOFMEMORY;
10184 if (!heap_select::init (number_of_heaps))
10185 return E_OUTOFMEMORY;
10187 #endif //MULTIPLE_HEAPS
10189 #ifdef MULTIPLE_HEAPS
10190 yp_spin_count_unit = 32 * number_of_heaps;
10192 yp_spin_count_unit = 32 * g_num_processors;
10193 #endif //MULTIPLE_HEAPS
10195 if (!init_semi_shared())
10203 //Initializes PER_HEAP_ISOLATED data members.
10205 gc_heap::init_semi_shared()
10209 // This is used for heap expansion - it's to fix exactly the start for gen 0
10210 // through (max_generation-1). When we expand the heap we allocate all these
10211 // gen starts at the beginning of the new ephemeral seg.
10212 eph_gen_starts_size = (Align (min_obj_size)) * max_generation;
10215 #ifdef MULTIPLE_HEAPS
10216 mark_list_size = min (150*1024, max (8192, soh_segment_size/(2*10*32)));
10217 g_mark_list = make_mark_list (mark_list_size*n_heaps);
10219 min_balance_threshold = alloc_quantum_balance_units * CLR_SIZE * 2;
10220 #ifdef PARALLEL_MARK_LIST_SORT
10221 g_mark_list_copy = make_mark_list (mark_list_size*n_heaps);
10222 if (!g_mark_list_copy)
10226 #endif //PARALLEL_MARK_LIST_SORT
10228 #else //MULTIPLE_HEAPS
10230 mark_list_size = max (8192, soh_segment_size/(64*32));
10231 g_mark_list = make_mark_list (mark_list_size);
10233 #endif //MULTIPLE_HEAPS
10235 dprintf (3, ("mark_list_size: %d", mark_list_size));
10243 #if defined(SEG_MAPPING_TABLE) && !defined(GROWABLE_SEG_MAPPING_TABLE)
10244 if (!seg_mapping_table_init())
10246 #endif //SEG_MAPPING_TABLE && !GROWABLE_SEG_MAPPING_TABLE
10248 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
10249 seg_table = sorted_table::make_sorted_table();
10253 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
10255 segment_standby_list = 0;
10257 if (!full_gc_approach_event.CreateManualEventNoThrow(FALSE))
10261 if (!full_gc_end_event.CreateManualEventNoThrow(FALSE))
10266 fgn_maxgen_percent = 0;
10267 fgn_loh_percent = 0;
10268 full_gc_approach_event_set = false;
10270 memset (full_gc_counts, 0, sizeof (full_gc_counts));
10273 should_expand_in_full_gc = FALSE;
10275 #ifdef FEATURE_LOH_COMPACTION
10276 loh_compaction_always_p = GCConfig::GetLOHCompactionMode() != 0;
10277 loh_compaction_mode = loh_compaction_default;
10278 #endif //FEATURE_LOH_COMPACTION
10280 loh_size_threshold = (size_t)GCConfig::GetLOHThreshold();
10281 assert (loh_size_threshold >= LARGE_OBJECT_SIZE);
10283 #ifdef BACKGROUND_GC
10284 memset (ephemeral_fgc_counts, 0, sizeof (ephemeral_fgc_counts));
10285 bgc_alloc_spin_count = static_cast<uint32_t>(GCConfig::GetBGCSpinCount());
10286 bgc_alloc_spin = static_cast<uint32_t>(GCConfig::GetBGCSpin());
10289 int number_bgc_threads = 1;
10290 #ifdef MULTIPLE_HEAPS
10291 number_bgc_threads = n_heaps;
10292 #endif //MULTIPLE_HEAPS
10293 if (!create_bgc_threads_support (number_bgc_threads))
10298 #endif //BACKGROUND_GC
10300 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
10302 #ifdef GC_CONFIG_DRIVEN
10303 compact_or_sweep_gcs[0] = 0;
10304 compact_or_sweep_gcs[1] = 0;
10305 #endif //GC_CONFIG_DRIVEN
10308 short_plugs_pad_ratio = (double)DESIRED_PLUG_LENGTH / (double)(DESIRED_PLUG_LENGTH - Align (min_obj_size));
10309 #endif //SHORT_PLUGS
10317 if (full_gc_approach_event.IsValid())
10319 full_gc_approach_event.CloseEvent();
10321 if (full_gc_end_event.IsValid())
10323 full_gc_end_event.CloseEvent();
10330 gc_heap* gc_heap::make_gc_heap (
10331 #ifdef MULTIPLE_HEAPS
10334 #endif //MULTIPLE_HEAPS
10339 #ifdef MULTIPLE_HEAPS
10340 res = new (nothrow) gc_heap;
10344 res->vm_heap = vm_hp;
10345 res->alloc_context_count = 0;
10348 #ifdef PARALLEL_MARK_LIST_SORT
10349 res->mark_list_piece_start = new (nothrow) uint8_t**[n_heaps];
10350 if (!res->mark_list_piece_start)
10354 #pragma warning(push)
10355 #pragma warning(disable:22011) // Suppress PREFast warning about integer underflow/overflow
10356 #endif // _PREFAST_
10357 res->mark_list_piece_end = new (nothrow) uint8_t**[n_heaps + 32]; // +32 is padding to reduce false sharing
10359 #pragma warning(pop)
10360 #endif // _PREFAST_
10362 if (!res->mark_list_piece_end)
10364 #endif //PARALLEL_MARK_LIST_SORT
10368 #endif //MULTIPLE_HEAPS
10370 if (res->init_gc_heap (
10371 #ifdef MULTIPLE_HEAPS
10373 #else //MULTIPLE_HEAPS
10375 #endif //MULTIPLE_HEAPS
10381 #ifdef MULTIPLE_HEAPS
10384 return (gc_heap*)1;
10385 #endif //MULTIPLE_HEAPS
10389 gc_heap::wait_for_gc_done(int32_t timeOut)
10391 bool cooperative_mode = enable_preemptive ();
10393 uint32_t dwWaitResult = NOERROR;
10395 gc_heap* wait_heap = NULL;
10396 while (gc_heap::gc_started)
10398 #ifdef MULTIPLE_HEAPS
10399 wait_heap = GCHeap::GetHeap(heap_select::select_heap(NULL, 0))->pGenGCHeap;
10400 dprintf(2, ("waiting for the gc_done_event on heap %d", wait_heap->heap_number));
10401 #endif // MULTIPLE_HEAPS
10404 PREFIX_ASSUME(wait_heap != NULL);
10405 #endif // _PREFAST_
10407 dwWaitResult = wait_heap->gc_done_event.Wait(timeOut, FALSE);
10409 disable_preemptive (cooperative_mode);
10411 return dwWaitResult;
10415 gc_heap::set_gc_done()
10417 enter_gc_done_event_lock();
10418 if (!gc_done_event_set)
10420 gc_done_event_set = true;
10421 dprintf (2, ("heap %d: setting gc_done_event", heap_number));
10422 gc_done_event.Set();
10424 exit_gc_done_event_lock();
10428 gc_heap::reset_gc_done()
10430 enter_gc_done_event_lock();
10431 if (gc_done_event_set)
10433 gc_done_event_set = false;
10434 dprintf (2, ("heap %d: resetting gc_done_event", heap_number));
10435 gc_done_event.Reset();
10437 exit_gc_done_event_lock();
10441 gc_heap::enter_gc_done_event_lock()
10443 uint32_t dwSwitchCount = 0;
10446 if (Interlocked::CompareExchange(&gc_done_event_lock, 0, -1) >= 0)
10448 while (gc_done_event_lock >= 0)
10450 if (g_num_processors > 1)
10452 int spin_count = yp_spin_count_unit;
10453 for (int j = 0; j < spin_count; j++)
10455 if (gc_done_event_lock < 0)
10457 YieldProcessor(); // indicate to the processor that we are spinning
10459 if (gc_done_event_lock >= 0)
10460 GCToOSInterface::YieldThread(++dwSwitchCount);
10463 GCToOSInterface::YieldThread(++dwSwitchCount);
10470 gc_heap::exit_gc_done_event_lock()
10472 gc_done_event_lock = -1;
10475 #ifndef MULTIPLE_HEAPS
10477 #ifdef RECORD_LOH_STATE
10478 int gc_heap::loh_state_index = 0;
10479 gc_heap::loh_state_info gc_heap::last_loh_states[max_saved_loh_states];
10480 #endif //RECORD_LOH_STATE
10482 VOLATILE(int32_t) gc_heap::gc_done_event_lock;
10483 VOLATILE(bool) gc_heap::gc_done_event_set;
10484 GCEvent gc_heap::gc_done_event;
10485 #endif //!MULTIPLE_HEAPS
10486 VOLATILE(bool) gc_heap::internal_gc_done;
10488 void gc_heap::add_saved_spinlock_info (
10490 msl_enter_state enter_state,
10491 msl_take_state take_state)
10494 #ifdef SPINLOCK_HISTORY
10495 spinlock_info* current = &last_spinlock_info[spinlock_info_index];
10497 current->enter_state = enter_state;
10498 current->take_state = take_state;
10499 current->thread_id.SetToCurrentThread();
10500 current->loh_p = loh_p;
10501 dprintf (SPINLOCK_LOG, ("[%d]%s %s %s",
10503 (loh_p ? "loh" : "soh"),
10504 ((enter_state == me_acquire) ? "E" : "L"),
10505 msl_take_state_str[take_state]));
10507 spinlock_info_index++;
10509 assert (spinlock_info_index <= max_saved_spinlock_info);
10511 if (spinlock_info_index >= max_saved_spinlock_info)
10513 spinlock_info_index = 0;
10516 MAYBE_UNUSED_VAR(enter_state);
10517 MAYBE_UNUSED_VAR(take_state);
10518 #endif //SPINLOCK_HISTORY
10522 gc_heap::init_gc_heap (int h_number)
10524 #ifdef MULTIPLE_HEAPS
10528 allocated_since_last_gc = 0;
10530 #ifdef SPINLOCK_HISTORY
10531 spinlock_info_index = 0;
10532 memset (last_spinlock_info, 0, sizeof(last_spinlock_info));
10533 #endif //SPINLOCK_HISTORY
10535 // initialize per heap members.
10536 ephemeral_low = (uint8_t*)1;
10538 ephemeral_high = MAX_PTR;
10540 ephemeral_heap_segment = 0;
10542 freeable_large_heap_segment = 0;
10544 condemned_generation_num = 0;
10546 blocking_collection = FALSE;
10548 generation_skip_ratio = 100;
10550 mark_stack_tos = 0;
10552 mark_stack_bos = 0;
10554 mark_stack_array_length = 0;
10556 mark_stack_array = 0;
10558 #if defined (_DEBUG) && defined (VERIFY_HEAP)
10559 verify_pinned_queue_p = FALSE;
10560 #endif // _DEBUG && VERIFY_HEAP
10562 loh_pinned_queue_tos = 0;
10564 loh_pinned_queue_bos = 0;
10566 loh_pinned_queue_length = 0;
10568 loh_pinned_queue_decay = LOH_PIN_DECAY;
10570 loh_pinned_queue = 0;
10572 min_overflow_address = MAX_PTR;
10574 max_overflow_address = 0;
10576 gen0_bricks_cleared = FALSE;
10578 gen0_must_clear_bricks = 0;
10580 allocation_quantum = CLR_SIZE;
10582 more_space_lock_soh = gc_lock;
10584 more_space_lock_loh = gc_lock;
10586 ro_segments_in_range = FALSE;
10588 loh_alloc_since_cg = 0;
10590 new_heap_segment = NULL;
10592 gen0_allocated_after_gc_p = false;
10594 #ifdef RECORD_LOH_STATE
10595 loh_state_index = 0;
10596 #endif //RECORD_LOH_STATE
10597 #endif //MULTIPLE_HEAPS
10599 #ifdef MULTIPLE_HEAPS
10600 if (h_number > n_heaps)
10602 assert (!"Number of heaps exceeded");
10606 heap_number = h_number;
10607 #endif //MULTIPLE_HEAPS
10609 memset (&oom_info, 0, sizeof (oom_info));
10610 memset (&fgm_result, 0, sizeof (fgm_result));
10611 if (!gc_done_event.CreateManualEventNoThrow(FALSE))
10615 gc_done_event_lock = -1;
10616 gc_done_event_set = false;
10618 #ifndef SEG_MAPPING_TABLE
10619 if (!gc_heap::seg_table->ensure_space_for_insert ())
10623 #endif //!SEG_MAPPING_TABLE
10625 heap_segment* seg = get_initial_segment (soh_segment_size, h_number);
10629 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(seg),
10630 (size_t)(heap_segment_reserved (seg) - heap_segment_mem(seg)),
10631 gc_etw_segment_small_object_heap);
10633 #ifdef SEG_MAPPING_TABLE
10634 seg_mapping_table_add_segment (seg, __this);
10635 #else //SEG_MAPPING_TABLE
10636 seg_table->insert ((uint8_t*)seg, sdelta);
10637 #endif //SEG_MAPPING_TABLE
10639 #ifdef MULTIPLE_HEAPS
10640 heap_segment_heap (seg) = this;
10641 #endif //MULTIPLE_HEAPS
10643 /* todo: Need a global lock for this */
10644 uint32_t* ct = &g_gc_card_table [card_word (card_of (g_gc_lowest_address))];
10645 own_card_table (ct);
10646 card_table = translate_card_table (ct);
10647 /* End of global lock */
10649 brick_table = card_table_brick_table (ct);
10650 highest_address = card_table_highest_address (ct);
10651 lowest_address = card_table_lowest_address (ct);
10654 card_bundle_table = translate_card_bundle_table (card_table_card_bundle_table (ct), g_gc_lowest_address);
10655 assert (&card_bundle_table [card_bundle_word (cardw_card_bundle (card_word (card_of (g_gc_lowest_address))))] ==
10656 card_table_card_bundle_table (ct));
10657 #endif //CARD_BUNDLE
10660 if (gc_can_use_concurrent)
10661 mark_array = translate_mark_array (card_table_mark_array (&g_gc_card_table[card_word (card_of (g_gc_lowest_address))]));
10664 #endif //MARK_ARRAY
10666 uint8_t* start = heap_segment_mem (seg);
10668 for (int i = 0; i < 1 + max_generation; i++)
10670 make_generation (generation_table [ (max_generation - i) ],
10672 generation_table [(max_generation - i)].gen_num = max_generation - i;
10673 start += Align (min_obj_size);
10676 heap_segment_allocated (seg) = start;
10677 alloc_allocated = start;
10678 heap_segment_used (seg) = start - plug_skew;
10680 ephemeral_heap_segment = seg;
10682 #ifndef SEG_MAPPING_TABLE
10683 if (!gc_heap::seg_table->ensure_space_for_insert ())
10687 #endif //!SEG_MAPPING_TABLE
10688 //Create the large segment generation
10689 heap_segment* lseg = get_initial_segment(min_loh_segment_size, h_number);
10692 lseg->flags |= heap_segment_flags_loh;
10694 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(lseg),
10695 (size_t)(heap_segment_reserved (lseg) - heap_segment_mem(lseg)),
10696 gc_etw_segment_large_object_heap);
10698 #ifdef SEG_MAPPING_TABLE
10699 seg_mapping_table_add_segment (lseg, __this);
10700 #else //SEG_MAPPING_TABLE
10701 seg_table->insert ((uint8_t*)lseg, sdelta);
10702 #endif //SEG_MAPPING_TABLE
10704 generation_table [max_generation].free_list_allocator = allocator(NUM_GEN2_ALIST, BASE_GEN2_ALIST, gen2_alloc_list);
10705 //assign the alloc_list for the large generation
10706 generation_table [max_generation+1].free_list_allocator = allocator(NUM_LOH_ALIST, BASE_LOH_ALIST, loh_alloc_list);
10707 generation_table [max_generation+1].gen_num = max_generation+1;
10708 make_generation (generation_table [max_generation+1],lseg, heap_segment_mem (lseg), 0);
10709 heap_segment_allocated (lseg) = heap_segment_mem (lseg) + Align (min_obj_size, get_alignment_constant (FALSE));
10710 heap_segment_used (lseg) = heap_segment_allocated (lseg) - plug_skew;
10712 for (int gen_num = 0; gen_num <= 1 + max_generation; gen_num++)
10714 generation* gen = generation_of (gen_num);
10715 make_unused_array (generation_allocation_start (gen), Align (min_obj_size));
10718 #ifdef MULTIPLE_HEAPS
10719 heap_segment_heap (lseg) = this;
10721 //initialize the alloc context heap
10722 generation_alloc_context (generation_of (0))->set_alloc_heap(vm_heap);
10724 //initialize the alloc context heap
10725 generation_alloc_context (generation_of (max_generation+1))->set_alloc_heap(vm_heap);
10727 #endif //MULTIPLE_HEAPS
10729 //Do this only once
10730 #ifdef MULTIPLE_HEAPS
10732 #endif //MULTIPLE_HEAPS
10734 #ifndef INTERIOR_POINTERS
10735 //set the brick_table for large objects
10736 //but default value is clearded
10737 //clear_brick_table ((uint8_t*)heap_segment_mem (lseg),
10738 // (uint8_t*)heap_segment_reserved (lseg));
10740 #else //INTERIOR_POINTERS
10742 //Because of the interior pointer business, we have to clear
10743 //the whole brick table
10744 //but the default value is cleared
10745 // clear_brick_table (lowest_address, highest_address);
10746 #endif //INTERIOR_POINTERS
10749 if (!init_dynamic_data())
10754 etw_allocation_running_amount[0] = 0;
10755 etw_allocation_running_amount[1] = 0;
10757 //needs to be done after the dynamic data has been initialized
10758 #ifndef MULTIPLE_HEAPS
10759 allocation_running_amount = dd_min_size (dynamic_data_of (0));
10760 #endif //!MULTIPLE_HEAPS
10762 fgn_last_alloc = dd_min_size (dynamic_data_of (0));
10764 mark* arr = new (nothrow) (mark [MARK_STACK_INITIAL_LENGTH]);
10768 make_mark_stack(arr);
10770 #ifdef BACKGROUND_GC
10771 freeable_small_heap_segment = 0;
10772 gchist_index_per_heap = 0;
10773 uint8_t** b_arr = new (nothrow) (uint8_t* [MARK_STACK_INITIAL_LENGTH]);
10777 make_background_mark_stack (b_arr);
10778 #endif //BACKGROUND_GC
10780 ephemeral_low = generation_allocation_start(generation_of(max_generation - 1));
10781 ephemeral_high = heap_segment_reserved(ephemeral_heap_segment);
10782 if (heap_number == 0)
10784 stomp_write_barrier_initialize(
10785 #ifdef MULTIPLE_HEAPS
10786 reinterpret_cast<uint8_t*>(1), reinterpret_cast<uint8_t*>(~0)
10788 ephemeral_low, ephemeral_high
10789 #endif //!MULTIPLE_HEAPS
10794 // why would we clear the mark array for this page? it should be cleared..
10795 // clear the first committed page
10796 //if(gc_can_use_concurrent)
10798 // clear_mark_array (align_lower_page (heap_segment_mem (seg)), heap_segment_committed (seg));
10800 #endif //MARK_ARRAY
10802 #ifdef MULTIPLE_HEAPS
10803 //register the heap in the heaps array
10805 if (!create_gc_thread ())
10808 g_heaps [heap_number] = this;
10810 #endif //MULTIPLE_HEAPS
10812 #ifdef FEATURE_PREMORTEM_FINALIZATION
10813 HRESULT hr = AllocateCFinalize(&finalize_queue);
10816 #endif // FEATURE_PREMORTEM_FINALIZATION
10818 max_free_space_items = MAX_NUM_FREE_SPACES;
10820 bestfit_seg = new (nothrow) seg_free_spaces (heap_number);
10827 if (!bestfit_seg->alloc())
10832 last_gc_before_oom = FALSE;
10834 sufficient_gen0_space_p = FALSE;
10836 #ifdef MULTIPLE_HEAPS
10838 #ifdef HEAP_ANALYZE
10840 heap_analyze_success = TRUE;
10842 internal_root_array = 0;
10844 internal_root_array_index = 0;
10846 internal_root_array_length = initial_internal_roots;
10850 current_obj_size = 0;
10852 #endif //HEAP_ANALYZE
10854 #endif // MULTIPLE_HEAPS
10856 #ifdef BACKGROUND_GC
10857 bgc_thread_id.Clear();
10859 if (!create_bgc_thread_support())
10864 bgc_alloc_lock = new (nothrow) exclusive_sync;
10865 if (!bgc_alloc_lock)
10870 bgc_alloc_lock->init();
10874 if (!recursive_gc_sync::init())
10878 bgc_thread_running = 0;
10880 bgc_threads_timeout_cs.Initialize();
10881 expanded_in_fgc = 0;
10882 current_bgc_state = bgc_not_in_process;
10883 background_soh_alloc_count = 0;
10884 background_loh_alloc_count = 0;
10885 bgc_overflow_count = 0;
10886 end_loh_size = dd_min_size (dynamic_data_of (max_generation + 1));
10887 #endif //BACKGROUND_GC
10889 #ifdef GC_CONFIG_DRIVEN
10890 memset (interesting_data_per_heap, 0, sizeof (interesting_data_per_heap));
10891 memset(compact_reasons_per_heap, 0, sizeof (compact_reasons_per_heap));
10892 memset(expand_mechanisms_per_heap, 0, sizeof (expand_mechanisms_per_heap));
10893 memset(interesting_mechanism_bits_per_heap, 0, sizeof (interesting_mechanism_bits_per_heap));
10894 #endif //GC_CONFIG_DRIVEN
10900 gc_heap::destroy_semi_shared()
10902 //TODO: will need to move this to per heap
10903 //#ifdef BACKGROUND_GC
10904 // if (c_mark_list)
10905 // delete c_mark_list;
10906 //#endif //BACKGROUND_GC
10910 delete g_mark_list;
10913 #if defined(SEG_MAPPING_TABLE) && !defined(GROWABLE_SEG_MAPPING_TABLE)
10914 if (seg_mapping_table)
10915 delete seg_mapping_table;
10916 #endif //SEG_MAPPING_TABLE && !GROWABLE_SEG_MAPPING_TABLE
10918 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
10919 //destroy the segment map
10920 seg_table->delete_sorted_table();
10921 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
10925 gc_heap::self_destroy()
10927 #ifdef BACKGROUND_GC
10929 #endif //BACKGROUND_GC
10931 if (gc_done_event.IsValid())
10933 gc_done_event.CloseEvent();
10936 // destroy every segment.
10937 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
10939 PREFIX_ASSUME(seg != NULL);
10941 heap_segment* next_seg;
10944 next_seg = heap_segment_next_rw (seg);
10945 delete_heap_segment (seg);
10949 seg = heap_segment_rw (generation_start_segment (generation_of (max_generation+1)));
10951 PREFIX_ASSUME(seg != NULL);
10955 next_seg = heap_segment_next_rw (seg);
10956 delete_heap_segment (seg);
10960 // get rid of the card table
10961 release_card_table (card_table);
10963 // destroy the mark stack
10964 delete mark_stack_array;
10966 #ifdef FEATURE_PREMORTEM_FINALIZATION
10967 if (finalize_queue)
10968 delete finalize_queue;
10969 #endif // FEATURE_PREMORTEM_FINALIZATION
10973 gc_heap::destroy_gc_heap(gc_heap* heap)
10975 heap->self_destroy();
10979 // Destroys resources owned by gc. It is assumed that a last GC has been performed and that
10980 // the finalizer queue has been drained.
10981 void gc_heap::shutdown_gc()
10983 destroy_semi_shared();
10985 #ifdef MULTIPLE_HEAPS
10986 //delete the heaps array
10988 destroy_thread_support();
10990 #endif //MULTIPLE_HEAPS
10991 //destroy seg_manager
10993 destroy_initial_memory();
10995 GCToOSInterface::Shutdown();
10999 BOOL gc_heap::size_fit_p (size_t size REQD_ALIGN_AND_OFFSET_DCL, uint8_t* alloc_pointer, uint8_t* alloc_limit,
11000 uint8_t* old_loc, int use_padding)
11002 BOOL already_padded = FALSE;
11004 if ((old_loc != 0) && (use_padding & USE_PADDING_FRONT))
11006 alloc_pointer = alloc_pointer + Align (min_obj_size);
11007 already_padded = TRUE;
11009 #endif //SHORT_PLUGS
11011 if (!((old_loc == 0) || same_large_alignment_p (old_loc, alloc_pointer)))
11012 size = size + switch_alignment_size (already_padded);
11014 #ifdef FEATURE_STRUCTALIGN
11015 alloc_pointer = StructAlign(alloc_pointer, requiredAlignment, alignmentOffset);
11016 #endif // FEATURE_STRUCTALIGN
11018 // in allocate_in_condemned_generation we can have this when we
11019 // set the alloc_limit to plan_allocated which could be less than
11021 if (alloc_limit < alloc_pointer)
11028 return (((size_t)(alloc_limit - alloc_pointer) >= (size + ((use_padding & USE_PADDING_TAIL)? Align(min_obj_size) : 0)))
11030 ||((!(use_padding & USE_PADDING_FRONT)) && ((alloc_pointer + size) == alloc_limit))
11031 #else //SHORT_PLUGS
11032 ||((alloc_pointer + size) == alloc_limit)
11033 #endif //SHORT_PLUGS
11038 assert (size == Align (min_obj_size));
11039 return ((size_t)(alloc_limit - alloc_pointer) >= size);
11044 BOOL gc_heap::a_size_fit_p (size_t size, uint8_t* alloc_pointer, uint8_t* alloc_limit,
11047 // We could have run into cases where this is true when alloc_allocated is the
11048 // the same as the seg committed.
11049 if (alloc_limit < alloc_pointer)
11054 return ((size_t)(alloc_limit - alloc_pointer) >= (size + Align(min_obj_size, align_const)));
11057 // Grow by committing more pages
11058 BOOL gc_heap::grow_heap_segment (heap_segment* seg, uint8_t* high_address, bool* hard_limit_exceeded_p)
11060 assert (high_address <= heap_segment_reserved (seg));
11062 if (hard_limit_exceeded_p)
11063 *hard_limit_exceeded_p = false;
11065 //return 0 if we are at the end of the segment.
11066 if (align_on_page (high_address) > heap_segment_reserved (seg))
11069 if (high_address <= heap_segment_committed (seg))
11072 size_t c_size = align_on_page ((size_t)(high_address - heap_segment_committed (seg)));
11073 c_size = max (c_size, commit_min_th);
11074 c_size = min (c_size, (size_t)(heap_segment_reserved (seg) - heap_segment_committed (seg)));
11079 STRESS_LOG2(LF_GC, LL_INFO10000,
11080 "Growing heap_segment: %Ix high address: %Ix\n",
11081 (size_t)seg, (size_t)high_address);
11083 bool ret = virtual_commit (heap_segment_committed (seg), c_size, heap_number, hard_limit_exceeded_p);
11087 #ifndef BACKGROUND_GC
11088 clear_mark_array (heap_segment_committed (seg),
11089 heap_segment_committed (seg)+c_size, TRUE);
11090 #endif //BACKGROUND_GC
11091 #endif //MARK_ARRAY
11092 heap_segment_committed (seg) += c_size;
11094 STRESS_LOG1(LF_GC, LL_INFO10000, "New commit: %Ix",
11095 (size_t)heap_segment_committed (seg));
11097 assert (heap_segment_committed (seg) <= heap_segment_reserved (seg));
11098 assert (high_address <= heap_segment_committed (seg));
11105 int gc_heap::grow_heap_segment (heap_segment* seg, uint8_t* allocated, uint8_t* old_loc, size_t size, BOOL pad_front_p REQD_ALIGN_AND_OFFSET_DCL)
11108 if ((old_loc != 0) && pad_front_p)
11110 allocated = allocated + Align (min_obj_size);
11112 #endif //SHORT_PLUGS
11114 if (!((old_loc == 0) || same_large_alignment_p (old_loc, allocated)))
11115 size = size + switch_alignment_size (FALSE);
11116 #ifdef FEATURE_STRUCTALIGN
11117 size_t pad = ComputeStructAlignPad(allocated, requiredAlignment, alignmentOffset);
11118 return grow_heap_segment (seg, allocated + pad + size);
11119 #else // FEATURE_STRUCTALIGN
11120 return grow_heap_segment (seg, allocated + size);
11121 #endif // FEATURE_STRUCTALIGN
11124 //used only in older generation allocation (i.e during gc).
11125 void gc_heap::adjust_limit (uint8_t* start, size_t limit_size, generation* gen,
11128 UNREFERENCED_PARAMETER(gennum);
11129 dprintf (3, ("gc Expanding segment allocation"));
11130 heap_segment* seg = generation_allocation_segment (gen);
11131 if ((generation_allocation_limit (gen) != start) || (start != heap_segment_plan_allocated (seg)))
11133 if (generation_allocation_limit (gen) == heap_segment_plan_allocated (seg))
11135 assert (generation_allocation_pointer (gen) >= heap_segment_mem (seg));
11136 assert (generation_allocation_pointer (gen) <= heap_segment_committed (seg));
11137 heap_segment_plan_allocated (generation_allocation_segment (gen)) = generation_allocation_pointer (gen);
11141 uint8_t* hole = generation_allocation_pointer (gen);
11142 size_t size = (generation_allocation_limit (gen) - generation_allocation_pointer (gen));
11146 dprintf (3, ("filling up hole: %Ix, size %Ix", hole, size));
11147 size_t allocated_size = generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen);
11148 if (size >= Align (min_free_list))
11150 if (allocated_size < min_free_list)
11152 if (size >= (Align (min_free_list) + Align (min_obj_size)))
11154 //split hole into min obj + threadable free item
11155 make_unused_array (hole, min_obj_size);
11156 generation_free_obj_space (gen) += Align (min_obj_size);
11157 make_unused_array (hole + Align (min_obj_size), size - Align (min_obj_size));
11158 generation_free_list_space (gen) += size - Align (min_obj_size);
11159 generation_allocator(gen)->thread_item_front (hole + Align (min_obj_size),
11160 size - Align (min_obj_size));
11161 add_gen_free (gen->gen_num, (size - Align (min_obj_size)));
11165 dprintf (3, ("allocated size too small, can't put back rest on free list %Ix", allocated_size));
11166 make_unused_array (hole, size);
11167 generation_free_obj_space (gen) += size;
11172 dprintf (3, ("threading hole in front of free list"));
11173 make_unused_array (hole, size);
11174 generation_free_list_space (gen) += size;
11175 generation_allocator(gen)->thread_item_front (hole, size);
11176 add_gen_free (gen->gen_num, size);
11181 make_unused_array (hole, size);
11182 generation_free_obj_space (gen) += size;
11186 generation_allocation_pointer (gen) = start;
11187 generation_allocation_context_start_region (gen) = start;
11189 generation_allocation_limit (gen) = (start + limit_size);
11192 void verify_mem_cleared (uint8_t* start, size_t size)
11194 if (!Aligned (size))
11199 PTR_PTR curr_ptr = (PTR_PTR) start;
11200 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
11202 if (*(curr_ptr++) != 0)
11209 #if defined (VERIFY_HEAP) && defined (BACKGROUND_GC)
11210 void gc_heap::set_batch_mark_array_bits (uint8_t* start, uint8_t* end)
11212 size_t start_mark_bit = mark_bit_of (start);
11213 size_t end_mark_bit = mark_bit_of (end);
11214 unsigned int startbit = mark_bit_bit (start_mark_bit);
11215 unsigned int endbit = mark_bit_bit (end_mark_bit);
11216 size_t startwrd = mark_bit_word (start_mark_bit);
11217 size_t endwrd = mark_bit_word (end_mark_bit);
11219 dprintf (3, ("Setting all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
11220 (size_t)start, (size_t)start_mark_bit,
11221 (size_t)end, (size_t)end_mark_bit));
11223 unsigned int firstwrd = ~(lowbits (~0, startbit));
11224 unsigned int lastwrd = ~(highbits (~0, endbit));
11226 if (startwrd == endwrd)
11228 unsigned int wrd = firstwrd & lastwrd;
11229 mark_array[startwrd] |= wrd;
11233 // set the first mark word.
11236 mark_array[startwrd] |= firstwrd;
11240 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
11242 mark_array[wrdtmp] = ~(unsigned int)0;
11245 // set the last mark word.
11248 mark_array[endwrd] |= lastwrd;
11252 // makes sure that the mark array bits between start and end are 0.
11253 void gc_heap::check_batch_mark_array_bits (uint8_t* start, uint8_t* end)
11255 size_t start_mark_bit = mark_bit_of (start);
11256 size_t end_mark_bit = mark_bit_of (end);
11257 unsigned int startbit = mark_bit_bit (start_mark_bit);
11258 unsigned int endbit = mark_bit_bit (end_mark_bit);
11259 size_t startwrd = mark_bit_word (start_mark_bit);
11260 size_t endwrd = mark_bit_word (end_mark_bit);
11262 //dprintf (3, ("Setting all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
11263 // (size_t)start, (size_t)start_mark_bit,
11264 // (size_t)end, (size_t)end_mark_bit));
11266 unsigned int firstwrd = ~(lowbits (~0, startbit));
11267 unsigned int lastwrd = ~(highbits (~0, endbit));
11269 if (startwrd == endwrd)
11271 unsigned int wrd = firstwrd & lastwrd;
11272 if (mark_array[startwrd] & wrd)
11274 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11276 mark_array [startwrd], mark_word_address (startwrd)));
11282 // set the first mark word.
11285 if (mark_array[startwrd] & firstwrd)
11287 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11288 firstwrd, startwrd,
11289 mark_array [startwrd], mark_word_address (startwrd)));
11296 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
11298 if (mark_array[wrdtmp])
11300 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11302 mark_array [wrdtmp], mark_word_address (wrdtmp)));
11307 // set the last mark word.
11310 if (mark_array[endwrd] & lastwrd)
11312 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11314 mark_array [lastwrd], mark_word_address (lastwrd)));
11319 #endif //VERIFY_HEAP && BACKGROUND_GC
11321 allocator::allocator (unsigned int num_b, size_t fbs, alloc_list* b)
11323 assert (num_b < MAX_BUCKET_COUNT);
11324 num_buckets = num_b;
11325 frst_bucket_size = fbs;
11329 alloc_list& allocator::alloc_list_of (unsigned int bn)
11331 assert (bn < num_buckets);
11333 return first_bucket;
11335 return buckets [bn-1];
11338 size_t& allocator::alloc_list_damage_count_of (unsigned int bn)
11340 assert (bn < num_buckets);
11342 return first_bucket.alloc_list_damage_count();
11344 return buckets [bn-1].alloc_list_damage_count();
11347 void allocator::unlink_item (unsigned int bn, uint8_t* item, uint8_t* prev_item, BOOL use_undo_p)
11349 //unlink the free_item
11350 alloc_list* al = &alloc_list_of (bn);
11353 if (use_undo_p && (free_list_undo (prev_item) == UNDO_EMPTY))
11355 assert (item == free_list_slot (prev_item));
11356 free_list_undo (prev_item) = item;
11357 alloc_list_damage_count_of (bn)++;
11359 free_list_slot (prev_item) = free_list_slot(item);
11363 al->alloc_list_head() = (uint8_t*)free_list_slot(item);
11365 if (al->alloc_list_tail() == item)
11367 al->alloc_list_tail() = prev_item;
11371 void allocator::clear()
11373 for (unsigned int i = 0; i < num_buckets; i++)
11375 alloc_list_head_of (i) = 0;
11376 alloc_list_tail_of (i) = 0;
11380 //always thread to the end.
11381 void allocator::thread_free_item (uint8_t* item, uint8_t*& head, uint8_t*& tail)
11383 free_list_slot (item) = 0;
11384 free_list_undo (item) = UNDO_EMPTY;
11385 assert (item != head);
11391 //TODO: This shouldn't happen anymore - verify that's the case.
11392 //the following is necessary because the last free element
11393 //may have been truncated, and tail isn't updated.
11394 else if (free_list_slot (head) == 0)
11396 free_list_slot (head) = item;
11400 assert (item != tail);
11401 assert (free_list_slot(tail) == 0);
11402 free_list_slot (tail) = item;
11407 void allocator::thread_item (uint8_t* item, size_t size)
11409 size_t sz = frst_bucket_size;
11410 unsigned int a_l_number = 0;
11412 for (; a_l_number < (num_buckets-1); a_l_number++)
11420 alloc_list* al = &alloc_list_of (a_l_number);
11421 thread_free_item (item,
11422 al->alloc_list_head(),
11423 al->alloc_list_tail());
11426 void allocator::thread_item_front (uint8_t* item, size_t size)
11428 //find right free list
11429 size_t sz = frst_bucket_size;
11430 unsigned int a_l_number = 0;
11431 for (; a_l_number < (num_buckets-1); a_l_number++)
11439 alloc_list* al = &alloc_list_of (a_l_number);
11440 free_list_slot (item) = al->alloc_list_head();
11441 free_list_undo (item) = UNDO_EMPTY;
11443 if (al->alloc_list_tail() == 0)
11445 al->alloc_list_tail() = al->alloc_list_head();
11447 al->alloc_list_head() = item;
11448 if (al->alloc_list_tail() == 0)
11450 al->alloc_list_tail() = item;
11454 void allocator::copy_to_alloc_list (alloc_list* toalist)
11456 for (unsigned int i = 0; i < num_buckets; i++)
11458 toalist [i] = alloc_list_of (i);
11459 #ifdef FL_VERIFICATION
11460 uint8_t* free_item = alloc_list_head_of (i);
11465 free_item = free_list_slot (free_item);
11468 toalist[i].item_count = count;
11469 #endif //FL_VERIFICATION
11473 void allocator::copy_from_alloc_list (alloc_list* fromalist)
11475 BOOL repair_list = !discard_if_no_fit_p ();
11476 for (unsigned int i = 0; i < num_buckets; i++)
11478 size_t count = alloc_list_damage_count_of (i);
11479 alloc_list_of (i) = fromalist [i];
11480 assert (alloc_list_damage_count_of (i) == 0);
11484 //repair the the list
11485 //new items may have been added during the plan phase
11486 //items may have been unlinked.
11487 uint8_t* free_item = alloc_list_head_of (i);
11488 while (free_item && count)
11490 assert (((CObjectHeader*)free_item)->IsFree());
11491 if ((free_list_undo (free_item) != UNDO_EMPTY))
11494 free_list_slot (free_item) = free_list_undo (free_item);
11495 free_list_undo (free_item) = UNDO_EMPTY;
11498 free_item = free_list_slot (free_item);
11501 #ifdef FL_VERIFICATION
11502 free_item = alloc_list_head_of (i);
11503 size_t item_count = 0;
11507 free_item = free_list_slot (free_item);
11510 assert (item_count == alloc_list_of (i).item_count);
11511 #endif //FL_VERIFICATION
11514 uint8_t* tail_item = alloc_list_tail_of (i);
11515 assert ((tail_item == 0) || (free_list_slot (tail_item) == 0));
11520 void allocator::commit_alloc_list_changes()
11522 BOOL repair_list = !discard_if_no_fit_p ();
11525 for (unsigned int i = 0; i < num_buckets; i++)
11527 //remove the undo info from list.
11528 uint8_t* free_item = alloc_list_head_of (i);
11529 size_t count = alloc_list_damage_count_of (i);
11530 while (free_item && count)
11532 assert (((CObjectHeader*)free_item)->IsFree());
11534 if (free_list_undo (free_item) != UNDO_EMPTY)
11536 free_list_undo (free_item) = UNDO_EMPTY;
11540 free_item = free_list_slot (free_item);
11543 alloc_list_damage_count_of (i) = 0;
11548 void gc_heap::adjust_limit_clr (uint8_t* start, size_t limit_size,
11549 alloc_context* acontext, heap_segment* seg,
11550 int align_const, int gen_number)
11552 bool loh_p = (gen_number > 0);
11553 GCSpinLock* msl = loh_p ? &more_space_lock_loh : &more_space_lock_soh;
11555 size_t aligned_min_obj_size = Align(min_obj_size, align_const);
11559 assert (heap_segment_used (seg) <= heap_segment_committed (seg));
11562 #ifdef MULTIPLE_HEAPS
11563 if (gen_number == 0)
11565 if (!gen0_allocated_after_gc_p)
11567 gen0_allocated_after_gc_p = true;
11570 #endif //MULTIPLE_HEAPS
11572 dprintf (3, ("Expanding segment allocation [%Ix, %Ix[", (size_t)start,
11573 (size_t)start + limit_size - aligned_min_obj_size));
11575 if ((acontext->alloc_limit != start) &&
11576 (acontext->alloc_limit + aligned_min_obj_size)!= start)
11578 uint8_t* hole = acontext->alloc_ptr;
11581 size_t size = (acontext->alloc_limit - acontext->alloc_ptr);
11582 dprintf (3, ("filling up hole [%Ix, %Ix[", (size_t)hole, (size_t)hole + size + Align (min_obj_size, align_const)));
11583 // when we are finishing an allocation from a free list
11584 // we know that the free area was Align(min_obj_size) larger
11585 acontext->alloc_bytes -= size;
11586 size_t free_obj_size = size + aligned_min_obj_size;
11587 make_unused_array (hole, free_obj_size);
11588 generation_free_obj_space (generation_of (gen_number)) += free_obj_size;
11590 acontext->alloc_ptr = start;
11594 if (gen_number == 0)
11596 size_t pad_size = Align (min_obj_size, align_const);
11597 make_unused_array (acontext->alloc_ptr, pad_size);
11598 dprintf (3, ("contigous ac: making min obj gap %Ix->%Ix(%Id)",
11599 acontext->alloc_ptr, (acontext->alloc_ptr + pad_size), pad_size));
11600 acontext->alloc_ptr += pad_size;
11603 acontext->alloc_limit = (start + limit_size - aligned_min_obj_size);
11604 acontext->alloc_bytes += limit_size - ((gen_number < max_generation + 1) ? aligned_min_obj_size : 0);
11606 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
11607 if (g_fEnableAppDomainMonitoring)
11609 GCToEEInterface::RecordAllocatedBytesForHeap(limit_size, heap_number);
11611 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
11613 uint8_t* saved_used = 0;
11617 saved_used = heap_segment_used (seg);
11620 if (seg == ephemeral_heap_segment)
11622 //Sometimes the allocated size is advanced without clearing the
11623 //memory. Let's catch up here
11624 if (heap_segment_used (seg) < (alloc_allocated - plug_skew))
11627 #ifndef BACKGROUND_GC
11628 clear_mark_array (heap_segment_used (seg) + plug_skew, alloc_allocated);
11629 #endif //BACKGROUND_GC
11630 #endif //MARK_ARRAY
11631 heap_segment_used (seg) = alloc_allocated - plug_skew;
11634 #ifdef BACKGROUND_GC
11637 uint8_t* old_allocated = heap_segment_allocated (seg) - plug_skew - limit_size;
11638 #ifdef FEATURE_LOH_COMPACTION
11639 old_allocated -= Align (loh_padding_obj_size, align_const);
11640 #endif //FEATURE_LOH_COMPACTION
11642 assert (heap_segment_used (seg) >= old_allocated);
11644 #endif //BACKGROUND_GC
11646 (start - plug_skew + limit_size) <= heap_segment_used (seg))
11648 add_saved_spinlock_info (loh_p, me_release, mt_clr_mem);
11649 leave_spin_lock (msl);
11650 dprintf (3, ("clearing memory at %Ix for %d bytes", (start - plug_skew), limit_size));
11651 memclr (start - plug_skew, limit_size);
11655 uint8_t* used = heap_segment_used (seg);
11656 heap_segment_used (seg) = start + limit_size - plug_skew;
11658 add_saved_spinlock_info (loh_p, me_release, mt_clr_mem);
11659 leave_spin_lock (msl);
11661 if ((start - plug_skew) < used)
11663 if (used != saved_used)
11668 dprintf (2, ("clearing memory before used at %Ix for %Id bytes",
11669 (start - plug_skew), (plug_skew + used - start)));
11670 memclr (start - plug_skew, used - (start - plug_skew));
11674 //this portion can be done after we release the lock
11675 if (seg == ephemeral_heap_segment)
11677 #ifdef FFIND_OBJECT
11678 if (gen0_must_clear_bricks > 0)
11680 //set the brick table to speed up find_object
11681 size_t b = brick_of (acontext->alloc_ptr);
11682 set_brick (b, acontext->alloc_ptr - brick_address (b));
11684 dprintf (3, ("Allocation Clearing bricks [%Ix, %Ix[",
11685 b, brick_of (align_on_brick (start + limit_size))));
11686 volatile short* x = &brick_table [b];
11687 short* end_x = &brick_table [brick_of (align_on_brick (start + limit_size))];
11689 for (;x < end_x;x++)
11693 #endif //FFIND_OBJECT
11695 gen0_bricks_cleared = FALSE;
11699 // verifying the memory is completely cleared.
11700 //verify_mem_cleared (start - plug_skew, limit_size);
11703 size_t gc_heap::new_allocation_limit (size_t size, size_t physical_limit, int gen_number)
11705 dynamic_data* dd = dynamic_data_of (gen_number);
11706 ptrdiff_t new_alloc = dd_new_allocation (dd);
11707 assert (new_alloc == (ptrdiff_t)Align (new_alloc,
11708 get_alignment_constant (!(gen_number == (max_generation+1)))));
11710 ptrdiff_t logical_limit = max (new_alloc, (ptrdiff_t)size);
11711 size_t limit = min (logical_limit, (ptrdiff_t)physical_limit);
11712 assert (limit == Align (limit, get_alignment_constant (!(gen_number == (max_generation+1)))));
11713 dd_new_allocation (dd) = (new_alloc - limit);
11717 size_t gc_heap::limit_from_size (size_t size, size_t physical_limit, int gen_number,
11720 size_t padded_size = size + Align (min_obj_size, align_const);
11721 // for LOH this is not true...we could select a physical_limit that's exactly the same
11723 assert ((gen_number != 0) || (physical_limit >= padded_size));
11724 size_t min_size_to_allocate = ((gen_number == 0) ? allocation_quantum : 0);
11726 // For SOH if the size asked for is very small, we want to allocate more than
11727 // just what's asked for if possible.
11728 size_t desired_size_to_allocate = max (padded_size, min_size_to_allocate);
11729 size_t new_physical_limit = min (physical_limit, desired_size_to_allocate);
11731 size_t new_limit = new_allocation_limit (padded_size,
11732 new_physical_limit,
11734 assert (new_limit >= (size + Align (min_obj_size, align_const)));
11735 dprintf (100, ("requested to allocate %Id bytes, actual size is %Id", size, new_limit));
11739 void gc_heap::handle_oom (int heap_num, oom_reason reason, size_t alloc_size,
11740 uint8_t* allocated, uint8_t* reserved)
11742 UNREFERENCED_PARAMETER(heap_num);
11744 if (reason == oom_budget)
11746 alloc_size = dd_min_size (dynamic_data_of (0)) / 2;
11749 if ((reason == oom_budget) && ((!fgm_result.loh_p) && (fgm_result.fgm != fgm_no_failure)))
11751 // This means during the last GC we needed to reserve and/or commit more memory
11752 // but we couldn't. We proceeded with the GC and ended up not having enough
11753 // memory at the end. This is a legitimate OOM situtation. Otherwise we
11754 // probably made a mistake and didn't expand the heap when we should have.
11755 reason = oom_low_mem;
11758 oom_info.reason = reason;
11759 oom_info.allocated = allocated;
11760 oom_info.reserved = reserved;
11761 oom_info.alloc_size = alloc_size;
11762 oom_info.gc_index = settings.gc_index;
11763 oom_info.fgm = fgm_result.fgm;
11764 oom_info.size = fgm_result.size;
11765 oom_info.available_pagefile_mb = fgm_result.available_pagefile_mb;
11766 oom_info.loh_p = fgm_result.loh_p;
11768 fgm_result.fgm = fgm_no_failure;
11770 // Break early - before the more_space_lock is release so no other threads
11771 // could have allocated on the same heap when OOM happened.
11772 if (GCConfig::GetBreakOnOOM())
11774 GCToOSInterface::DebugBreak();
11778 #ifdef BACKGROUND_GC
11779 BOOL gc_heap::background_allowed_p()
11781 return ( gc_can_use_concurrent && ((settings.pause_mode == pause_interactive) || (settings.pause_mode == pause_sustained_low_latency)) );
11783 #endif //BACKGROUND_GC
11785 void gc_heap::check_for_full_gc (int gen_num, size_t size)
11787 BOOL should_notify = FALSE;
11788 // if we detect full gc because of the allocation budget specified this is TRUE;
11789 // it's FALSE if it's due to other factors.
11790 BOOL alloc_factor = TRUE;
11793 int n_initial = gen_num;
11794 BOOL local_blocking_collection = FALSE;
11795 BOOL local_elevation_requested = FALSE;
11796 int new_alloc_remain_percent = 0;
11798 if (full_gc_approach_event_set)
11803 if (gen_num != (max_generation + 1))
11805 gen_num = max_generation;
11808 dynamic_data* dd_full = dynamic_data_of (gen_num);
11809 ptrdiff_t new_alloc_remain = 0;
11810 uint32_t pct = ((gen_num == (max_generation + 1)) ? fgn_loh_percent : fgn_maxgen_percent);
11812 for (int gen_index = 0; gen_index <= (max_generation + 1); gen_index++)
11814 dprintf (2, ("FGN: h#%d: gen%d: %Id(%Id)",
11815 heap_number, gen_index,
11816 dd_new_allocation (dynamic_data_of (gen_index)),
11817 dd_desired_allocation (dynamic_data_of (gen_index))));
11820 // For small object allocations we only check every fgn_check_quantum bytes.
11821 if (n_initial == 0)
11823 dprintf (2, ("FGN: gen0 last recorded alloc: %Id", fgn_last_alloc));
11824 dynamic_data* dd_0 = dynamic_data_of (n_initial);
11825 if (((fgn_last_alloc - dd_new_allocation (dd_0)) < fgn_check_quantum) &&
11826 (dd_new_allocation (dd_0) >= 0))
11832 fgn_last_alloc = dd_new_allocation (dd_0);
11833 dprintf (2, ("FGN: gen0 last recorded alloc is now: %Id", fgn_last_alloc));
11836 // We don't consider the size that came from soh 'cause it doesn't contribute to the
11841 for (i = n+1; i <= max_generation; i++)
11843 if (get_new_allocation (i) <= 0)
11845 n = min (i, max_generation);
11851 dprintf (2, ("FGN: h#%d: gen%d budget exceeded", heap_number, n));
11852 if (gen_num == max_generation)
11854 // If it's small object heap we should first see if we will even be looking at gen2 budget
11855 // in the next GC or not. If not we should go directly to checking other factors.
11856 if (n < (max_generation - 1))
11858 goto check_other_factors;
11862 new_alloc_remain = dd_new_allocation (dd_full) - size;
11864 new_alloc_remain_percent = (int)(((float)(new_alloc_remain) / (float)dd_desired_allocation (dd_full)) * 100);
11866 dprintf (2, ("FGN: alloc threshold for gen%d is %d%%, current threshold is %d%%",
11867 gen_num, pct, new_alloc_remain_percent));
11869 if (new_alloc_remain_percent <= (int)pct)
11871 #ifdef BACKGROUND_GC
11872 // If background GC is enabled, we still want to check whether this will
11873 // be a blocking GC or not because we only want to notify when it's a
11874 // blocking full GC.
11875 if (background_allowed_p())
11877 goto check_other_factors;
11879 #endif //BACKGROUND_GC
11881 should_notify = TRUE;
11885 check_other_factors:
11887 dprintf (2, ("FGC: checking other factors"));
11888 n = generation_to_condemn (n,
11889 &local_blocking_collection,
11890 &local_elevation_requested,
11893 if (local_elevation_requested && (n == max_generation))
11895 if (settings.should_lock_elevation)
11897 int local_elevation_locked_count = settings.elevation_locked_count + 1;
11898 if (local_elevation_locked_count != 6)
11900 dprintf (2, ("FGN: lock count is %d - Condemning max_generation-1",
11901 local_elevation_locked_count));
11902 n = max_generation - 1;
11907 dprintf (2, ("FGN: we estimate gen%d will be collected", n));
11909 #ifdef BACKGROUND_GC
11910 // When background GC is enabled it decreases the accuracy of our predictability -
11911 // by the time the GC happens, we may not be under BGC anymore. If we try to
11912 // predict often enough it should be ok.
11913 if ((n == max_generation) &&
11914 (recursive_gc_sync::background_running_p()))
11916 n = max_generation - 1;
11917 dprintf (2, ("FGN: bgc - 1 instead of 2"));
11920 if ((n == max_generation) && !local_blocking_collection)
11922 if (!background_allowed_p())
11924 local_blocking_collection = TRUE;
11927 #endif //BACKGROUND_GC
11929 dprintf (2, ("FGN: we estimate gen%d will be collected: %s",
11931 (local_blocking_collection ? "blocking" : "background")));
11933 if ((n == max_generation) && local_blocking_collection)
11935 alloc_factor = FALSE;
11936 should_notify = TRUE;
11944 dprintf (2, ("FGN: gen%d detecting full GC approaching(%s) (GC#%d) (%Id%% left in gen%d)",
11946 (alloc_factor ? "alloc" : "other"),
11947 dd_collection_count (dynamic_data_of (0)),
11948 new_alloc_remain_percent,
11951 send_full_gc_notification (n_initial, alloc_factor);
11955 void gc_heap::send_full_gc_notification (int gen_num, BOOL due_to_alloc_p)
11957 if (!full_gc_approach_event_set)
11959 assert (full_gc_approach_event.IsValid());
11960 FIRE_EVENT(GCFullNotify_V1, gen_num, due_to_alloc_p);
11962 full_gc_end_event.Reset();
11963 full_gc_approach_event.Set();
11964 full_gc_approach_event_set = true;
11968 wait_full_gc_status gc_heap::full_gc_wait (GCEvent *event, int time_out_ms)
11970 if (fgn_maxgen_percent == 0)
11972 return wait_full_gc_na;
11975 uint32_t wait_result = user_thread_wait(event, FALSE, time_out_ms);
11977 if ((wait_result == WAIT_OBJECT_0) || (wait_result == WAIT_TIMEOUT))
11979 if (fgn_maxgen_percent == 0)
11981 return wait_full_gc_cancelled;
11984 if (wait_result == WAIT_OBJECT_0)
11986 #ifdef BACKGROUND_GC
11987 if (fgn_last_gc_was_concurrent)
11989 fgn_last_gc_was_concurrent = FALSE;
11990 return wait_full_gc_na;
11993 #endif //BACKGROUND_GC
11995 return wait_full_gc_success;
12000 return wait_full_gc_timeout;
12005 return wait_full_gc_failed;
12009 size_t gc_heap::get_full_compact_gc_count()
12011 return full_gc_counts[gc_type_compacting];
12014 // DTREVIEW - we should check this in dt_low_ephemeral_space_p
12017 BOOL gc_heap::short_on_end_of_seg (int gen_number,
12021 UNREFERENCED_PARAMETER(gen_number);
12022 uint8_t* allocated = heap_segment_allocated(seg);
12024 BOOL sufficient_p = sufficient_space_end_seg (allocated,
12025 heap_segment_reserved (seg),
12026 end_space_after_gc(),
12027 tuning_deciding_short_on_seg);
12030 if (sufficient_gen0_space_p)
12032 dprintf (GTC_LOG, ("gen0 has enough free space"));
12035 sufficient_p = sufficient_gen0_space_p;
12038 return !sufficient_p;
12042 #pragma warning(disable:4706) // "assignment within conditional expression" is intentional in this function.
12046 BOOL gc_heap::a_fit_free_list_p (int gen_number,
12048 alloc_context* acontext,
12051 BOOL can_fit = FALSE;
12052 generation* gen = generation_of (gen_number);
12053 allocator* gen_allocator = generation_allocator (gen);
12054 size_t sz_list = gen_allocator->first_bucket_size();
12055 for (unsigned int a_l_idx = 0; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
12057 if ((size < sz_list) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
12059 uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
12060 uint8_t* prev_free_item = 0;
12062 while (free_list != 0)
12064 dprintf (3, ("considering free list %Ix", (size_t)free_list));
12065 size_t free_list_size = unused_array_size (free_list);
12066 if ((size + Align (min_obj_size, align_const)) <= free_list_size)
12068 dprintf (3, ("Found adequate unused area: [%Ix, size: %Id",
12069 (size_t)free_list, free_list_size));
12071 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
12072 // We ask for more Align (min_obj_size)
12073 // to make sure that we can insert a free object
12074 // in adjust_limit will set the limit lower
12075 size_t limit = limit_from_size (size, free_list_size, gen_number, align_const);
12077 uint8_t* remain = (free_list + limit);
12078 size_t remain_size = (free_list_size - limit);
12079 if (remain_size >= Align(min_free_list, align_const))
12081 make_unused_array (remain, remain_size);
12082 gen_allocator->thread_item_front (remain, remain_size);
12083 assert (remain_size >= Align (min_obj_size, align_const));
12087 //absorb the entire free list
12088 limit += remain_size;
12090 generation_free_list_space (gen) -= limit;
12092 adjust_limit_clr (free_list, limit, acontext, 0, align_const, gen_number);
12097 else if (gen_allocator->discard_if_no_fit_p())
12099 assert (prev_free_item == 0);
12100 dprintf (3, ("couldn't use this free area, discarding"));
12101 generation_free_obj_space (gen) += free_list_size;
12103 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
12104 generation_free_list_space (gen) -= free_list_size;
12108 prev_free_item = free_list;
12110 free_list = free_list_slot (free_list);
12113 sz_list = sz_list * 2;
12120 #ifdef BACKGROUND_GC
12121 void gc_heap::bgc_loh_alloc_clr (uint8_t* alloc_start,
12123 alloc_context* acontext,
12129 make_unused_array (alloc_start, size);
12131 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
12132 if (g_fEnableAppDomainMonitoring)
12134 GCToEEInterface::RecordAllocatedBytesForHeap(size, heap_number);
12136 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
12138 size_t size_of_array_base = sizeof(ArrayBase);
12140 bgc_alloc_lock->loh_alloc_done_with_index (lock_index);
12142 // clear memory while not holding the lock.
12143 size_t size_to_skip = size_of_array_base;
12144 size_t size_to_clear = size - size_to_skip - plug_skew;
12145 size_t saved_size_to_clear = size_to_clear;
12148 uint8_t* end = alloc_start + size - plug_skew;
12149 uint8_t* used = heap_segment_used (seg);
12152 if ((alloc_start + size_to_skip) < used)
12154 size_to_clear = used - (alloc_start + size_to_skip);
12160 dprintf (2, ("bgc loh: setting used to %Ix", end));
12161 heap_segment_used (seg) = end;
12164 dprintf (2, ("bgc loh: used: %Ix, alloc: %Ix, end of alloc: %Ix, clear %Id bytes",
12165 used, alloc_start, end, size_to_clear));
12169 dprintf (2, ("bgc loh: [%Ix-[%Ix(%Id)", alloc_start, alloc_start+size, size));
12173 // since we filled in 0xcc for free object when we verify heap,
12174 // we need to make sure we clear those bytes.
12175 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
12177 if (size_to_clear < saved_size_to_clear)
12179 size_to_clear = saved_size_to_clear;
12182 #endif //VERIFY_HEAP
12184 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to clear large obj", heap_number));
12185 add_saved_spinlock_info (true, me_release, mt_clr_large_mem);
12186 leave_spin_lock (&more_space_lock_loh);
12187 memclr (alloc_start + size_to_skip, size_to_clear);
12189 bgc_alloc_lock->loh_alloc_set (alloc_start);
12191 acontext->alloc_ptr = alloc_start;
12192 acontext->alloc_limit = (alloc_start + size - Align (min_obj_size, align_const));
12194 // need to clear the rest of the object before we hand it out.
12195 clear_unused_array(alloc_start, size);
12197 #endif //BACKGROUND_GC
12199 BOOL gc_heap::a_fit_free_list_large_p (size_t size,
12200 alloc_context* acontext,
12203 BOOL can_fit = FALSE;
12204 int gen_number = max_generation + 1;
12205 generation* gen = generation_of (gen_number);
12206 allocator* loh_allocator = generation_allocator (gen);
12208 #ifdef FEATURE_LOH_COMPACTION
12209 size_t loh_pad = Align (loh_padding_obj_size, align_const);
12210 #endif //FEATURE_LOH_COMPACTION
12212 #ifdef BACKGROUND_GC
12214 #endif //BACKGROUND_GC
12215 size_t sz_list = loh_allocator->first_bucket_size();
12216 for (unsigned int a_l_idx = 0; a_l_idx < loh_allocator->number_of_buckets(); a_l_idx++)
12218 if ((size < sz_list) || (a_l_idx == (loh_allocator->number_of_buckets()-1)))
12220 uint8_t* free_list = loh_allocator->alloc_list_head_of (a_l_idx);
12221 uint8_t* prev_free_item = 0;
12222 while (free_list != 0)
12224 dprintf (3, ("considering free list %Ix", (size_t)free_list));
12226 size_t free_list_size = unused_array_size(free_list);
12228 #ifdef FEATURE_LOH_COMPACTION
12229 if ((size + loh_pad) <= free_list_size)
12231 if (((size + Align (min_obj_size, align_const)) <= free_list_size)||
12232 (size == free_list_size))
12233 #endif //FEATURE_LOH_COMPACTION
12235 #ifdef BACKGROUND_GC
12236 cookie = bgc_alloc_lock->loh_alloc_set (free_list);
12237 bgc_track_loh_alloc();
12238 #endif //BACKGROUND_GC
12240 //unlink the free_item
12241 loh_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
12243 // Substract min obj size because limit_from_size adds it. Not needed for LOH
12244 size_t limit = limit_from_size (size - Align(min_obj_size, align_const), free_list_size,
12245 gen_number, align_const);
12247 #ifdef FEATURE_LOH_COMPACTION
12248 make_unused_array (free_list, loh_pad);
12250 free_list += loh_pad;
12251 free_list_size -= loh_pad;
12252 #endif //FEATURE_LOH_COMPACTION
12254 uint8_t* remain = (free_list + limit);
12255 size_t remain_size = (free_list_size - limit);
12256 if (remain_size != 0)
12258 assert (remain_size >= Align (min_obj_size, align_const));
12259 make_unused_array (remain, remain_size);
12261 if (remain_size >= Align(min_free_list, align_const))
12263 loh_thread_gap_front (remain, remain_size, gen);
12264 assert (remain_size >= Align (min_obj_size, align_const));
12268 generation_free_obj_space (gen) += remain_size;
12270 generation_free_list_space (gen) -= free_list_size;
12271 dprintf (3, ("found fit on loh at %Ix", free_list));
12272 #ifdef BACKGROUND_GC
12275 bgc_loh_alloc_clr (free_list, limit, acontext, align_const, cookie, FALSE, 0);
12278 #endif //BACKGROUND_GC
12280 adjust_limit_clr (free_list, limit, acontext, 0, align_const, gen_number);
12283 //fix the limit to compensate for adjust_limit_clr making it too short
12284 acontext->alloc_limit += Align (min_obj_size, align_const);
12288 prev_free_item = free_list;
12289 free_list = free_list_slot (free_list);
12292 sz_list = sz_list * 2;
12299 #pragma warning(default:4706)
12302 BOOL gc_heap::a_fit_segment_end_p (int gen_number,
12305 alloc_context* acontext,
12307 BOOL* commit_failed_p)
12309 *commit_failed_p = FALSE;
12311 bool hard_limit_short_seg_end_p = false;
12312 #ifdef BACKGROUND_GC
12314 #endif //BACKGROUND_GC
12316 uint8_t*& allocated = ((gen_number == 0) ?
12318 heap_segment_allocated(seg));
12320 size_t pad = Align (min_obj_size, align_const);
12322 #ifdef FEATURE_LOH_COMPACTION
12323 size_t loh_pad = Align (loh_padding_obj_size, align_const);
12324 if (gen_number == (max_generation + 1))
12328 #endif //FEATURE_LOH_COMPACTION
12330 uint8_t* end = heap_segment_committed (seg) - pad;
12332 if (a_size_fit_p (size, allocated, end, align_const))
12334 limit = limit_from_size (size,
12336 gen_number, align_const);
12340 end = heap_segment_reserved (seg) - pad;
12342 if (a_size_fit_p (size, allocated, end, align_const))
12344 limit = limit_from_size (size,
12346 gen_number, align_const);
12348 if (grow_heap_segment (seg, (allocated + limit), &hard_limit_short_seg_end_p))
12354 if (!hard_limit_short_seg_end_p)
12356 dprintf (2, ("can't grow segment, doing a full gc"));
12357 *commit_failed_p = TRUE;
12361 assert (heap_hard_limit);
12370 #ifdef BACKGROUND_GC
12371 if (gen_number != 0)
12373 cookie = bgc_alloc_lock->loh_alloc_set (allocated);
12374 bgc_track_loh_alloc();
12376 #endif //BACKGROUND_GC
12378 uint8_t* old_alloc;
12379 old_alloc = allocated;
12380 #ifdef FEATURE_LOH_COMPACTION
12381 if (gen_number == (max_generation + 1))
12383 make_unused_array (old_alloc, loh_pad);
12384 old_alloc += loh_pad;
12385 allocated += loh_pad;
12388 #endif //FEATURE_LOH_COMPACTION
12390 #if defined (VERIFY_HEAP) && defined (_DEBUG)
12391 ((void**) allocated)[-1] = 0; //clear the sync block
12392 #endif //VERIFY_HEAP && _DEBUG
12393 allocated += limit;
12395 dprintf (3, ("found fit at end of seg: %Ix", old_alloc));
12397 #ifdef BACKGROUND_GC
12400 bgc_loh_alloc_clr (old_alloc, limit, acontext, align_const, cookie, TRUE, seg);
12403 #endif //BACKGROUND_GC
12405 adjust_limit_clr (old_alloc, limit, acontext, seg, align_const, gen_number);
12415 BOOL gc_heap::loh_a_fit_segment_end_p (int gen_number,
12417 alloc_context* acontext,
12419 BOOL* commit_failed_p,
12422 *commit_failed_p = FALSE;
12423 heap_segment* seg = generation_allocation_segment (generation_of (gen_number));
12424 BOOL can_allocate_p = FALSE;
12428 #ifdef BACKGROUND_GC
12429 if (seg->flags & heap_segment_flags_loh_delete)
12431 dprintf (3, ("h%d skipping seg %Ix to be deleted", heap_number, (size_t)seg));
12434 #endif //BACKGROUND_GC
12436 if (a_fit_segment_end_p (gen_number, seg, (size - Align (min_obj_size, align_const)),
12437 acontext, align_const, commit_failed_p))
12439 acontext->alloc_limit += Align (min_obj_size, align_const);
12440 can_allocate_p = TRUE;
12444 if (*commit_failed_p)
12446 *oom_r = oom_cant_commit;
12451 seg = heap_segment_next_rw (seg);
12454 return can_allocate_p;
12457 #ifdef BACKGROUND_GC
12459 void gc_heap::wait_for_background (alloc_wait_reason awr, bool loh_p)
12461 GCSpinLock* msl = loh_p ? &more_space_lock_loh : &more_space_lock_soh;
12463 dprintf (2, ("BGC is already in progress, waiting for it to finish"));
12464 add_saved_spinlock_info (loh_p, me_release, mt_wait_bgc);
12465 leave_spin_lock (msl);
12466 background_gc_wait (awr);
12467 enter_spin_lock (msl);
12468 add_saved_spinlock_info (loh_p, me_acquire, mt_wait_bgc);
12471 void gc_heap::wait_for_bgc_high_memory (alloc_wait_reason awr, bool loh_p)
12473 if (recursive_gc_sync::background_running_p())
12475 uint32_t memory_load;
12476 get_memory_info (&memory_load);
12477 if (memory_load >= m_high_memory_load_th)
12479 dprintf (GTC_LOG, ("high mem - wait for BGC to finish, wait reason: %d", awr));
12480 wait_for_background (awr, loh_p);
12485 #endif //BACKGROUND_GC
12487 // We request to trigger an ephemeral GC but we may get a full compacting GC.
12488 // return TRUE if that's the case.
12489 BOOL gc_heap::trigger_ephemeral_gc (gc_reason gr)
12491 #ifdef BACKGROUND_GC
12492 wait_for_bgc_high_memory (awr_loh_oos_bgc, false);
12493 #endif //BACKGROUND_GC
12495 BOOL did_full_compact_gc = FALSE;
12497 dprintf (2, ("triggering a gen1 GC"));
12498 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12499 vm_heap->GarbageCollectGeneration(max_generation - 1, gr);
12501 #ifdef MULTIPLE_HEAPS
12502 enter_spin_lock (&more_space_lock_soh);
12503 add_saved_spinlock_info (false, me_acquire, mt_t_eph_gc);
12504 #endif //MULTIPLE_HEAPS
12506 size_t current_full_compact_gc_count = get_full_compact_gc_count();
12508 if (current_full_compact_gc_count > last_full_compact_gc_count)
12510 dprintf (2, ("attempted to trigger an ephemeral GC and got a full compacting GC"));
12511 did_full_compact_gc = TRUE;
12514 return did_full_compact_gc;
12517 BOOL gc_heap::soh_try_fit (int gen_number,
12519 alloc_context* acontext,
12521 BOOL* commit_failed_p,
12522 BOOL* short_seg_end_p)
12524 BOOL can_allocate = TRUE;
12525 if (short_seg_end_p)
12527 *short_seg_end_p = FALSE;
12530 can_allocate = a_fit_free_list_p (gen_number, size, acontext, align_const);
12533 if (short_seg_end_p)
12535 *short_seg_end_p = short_on_end_of_seg (gen_number, ephemeral_heap_segment, align_const);
12537 // If the caller doesn't care, we always try to fit at the end of seg;
12538 // otherwise we would only try if we are actually not short at end of seg.
12539 if (!short_seg_end_p || !(*short_seg_end_p))
12541 can_allocate = a_fit_segment_end_p (gen_number, ephemeral_heap_segment, size,
12542 acontext, align_const, commit_failed_p);
12546 return can_allocate;
12549 allocation_state gc_heap::allocate_small (int gen_number,
12551 alloc_context* acontext,
12554 #if defined (BACKGROUND_GC) && !defined (MULTIPLE_HEAPS)
12555 if (recursive_gc_sync::background_running_p())
12557 background_soh_alloc_count++;
12558 if ((background_soh_alloc_count % bgc_alloc_spin_count) == 0)
12560 add_saved_spinlock_info (false, me_release, mt_alloc_small);
12561 leave_spin_lock (&more_space_lock_soh);
12562 bool cooperative_mode = enable_preemptive();
12563 GCToOSInterface::Sleep (bgc_alloc_spin);
12564 disable_preemptive (cooperative_mode);
12565 enter_spin_lock (&more_space_lock_soh);
12566 add_saved_spinlock_info (false, me_acquire, mt_alloc_small);
12570 //GCToOSInterface::YieldThread (0);
12573 #endif //BACKGROUND_GC && !MULTIPLE_HEAPS
12575 gc_reason gr = reason_oos_soh;
12576 oom_reason oom_r = oom_no_failure;
12578 // No variable values should be "carried over" from one state to the other.
12579 // That's why there are local variable for each state
12581 allocation_state soh_alloc_state = a_state_start;
12583 // If we can get a new seg it means allocation will succeed.
12586 dprintf (3, ("[h%d]soh state is %s", heap_number, allocation_state_str[soh_alloc_state]));
12588 switch (soh_alloc_state)
12590 case a_state_can_allocate:
12591 case a_state_cant_allocate:
12595 case a_state_start:
12597 soh_alloc_state = a_state_try_fit;
12600 case a_state_try_fit:
12602 BOOL commit_failed_p = FALSE;
12603 BOOL can_use_existing_p = FALSE;
12605 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12606 align_const, &commit_failed_p,
12608 soh_alloc_state = (can_use_existing_p ?
12609 a_state_can_allocate :
12611 a_state_trigger_full_compact_gc :
12612 a_state_trigger_ephemeral_gc));
12615 case a_state_try_fit_after_bgc:
12617 BOOL commit_failed_p = FALSE;
12618 BOOL can_use_existing_p = FALSE;
12619 BOOL short_seg_end_p = FALSE;
12621 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12622 align_const, &commit_failed_p,
12624 soh_alloc_state = (can_use_existing_p ?
12625 a_state_can_allocate :
12627 a_state_trigger_2nd_ephemeral_gc :
12628 a_state_trigger_full_compact_gc));
12631 case a_state_try_fit_after_cg:
12633 BOOL commit_failed_p = FALSE;
12634 BOOL can_use_existing_p = FALSE;
12635 BOOL short_seg_end_p = FALSE;
12637 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12638 align_const, &commit_failed_p,
12641 if (can_use_existing_p)
12643 soh_alloc_state = a_state_can_allocate;
12645 #ifdef MULTIPLE_HEAPS
12646 else if (gen0_allocated_after_gc_p)
12648 // some other threads already grabbed the more space lock and allocated
12649 // so we should attempt an ephemeral GC again.
12650 soh_alloc_state = a_state_trigger_ephemeral_gc;
12652 #endif //MULTIPLE_HEAPS
12653 else if (short_seg_end_p)
12655 soh_alloc_state = a_state_cant_allocate;
12656 oom_r = oom_budget;
12660 assert (commit_failed_p);
12661 soh_alloc_state = a_state_cant_allocate;
12662 oom_r = oom_cant_commit;
12666 case a_state_check_and_wait_for_bgc:
12668 BOOL bgc_in_progress_p = FALSE;
12669 BOOL did_full_compacting_gc = FALSE;
12671 bgc_in_progress_p = check_and_wait_for_bgc (awr_gen0_oos_bgc, &did_full_compacting_gc, false);
12672 soh_alloc_state = (did_full_compacting_gc ?
12673 a_state_try_fit_after_cg :
12674 a_state_try_fit_after_bgc);
12677 case a_state_trigger_ephemeral_gc:
12679 BOOL commit_failed_p = FALSE;
12680 BOOL can_use_existing_p = FALSE;
12681 BOOL short_seg_end_p = FALSE;
12682 BOOL bgc_in_progress_p = FALSE;
12683 BOOL did_full_compacting_gc = FALSE;
12685 did_full_compacting_gc = trigger_ephemeral_gc (gr);
12686 if (did_full_compacting_gc)
12688 soh_alloc_state = a_state_try_fit_after_cg;
12692 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12693 align_const, &commit_failed_p,
12695 #ifdef BACKGROUND_GC
12696 bgc_in_progress_p = recursive_gc_sync::background_running_p();
12697 #endif //BACKGROUND_GC
12699 if (can_use_existing_p)
12701 soh_alloc_state = a_state_can_allocate;
12705 if (short_seg_end_p)
12707 if (should_expand_in_full_gc)
12709 dprintf (2, ("gen1 GC wanted to expand!"));
12710 soh_alloc_state = a_state_trigger_full_compact_gc;
12714 soh_alloc_state = (bgc_in_progress_p ?
12715 a_state_check_and_wait_for_bgc :
12716 a_state_trigger_full_compact_gc);
12719 else if (commit_failed_p)
12721 soh_alloc_state = a_state_trigger_full_compact_gc;
12725 #ifdef MULTIPLE_HEAPS
12726 // some other threads already grabbed the more space lock and allocated
12727 // so we should attempt an ephemeral GC again.
12728 assert (gen0_allocated_after_gc_p);
12729 soh_alloc_state = a_state_trigger_ephemeral_gc;
12730 #else //MULTIPLE_HEAPS
12731 assert (!"shouldn't get here");
12732 #endif //MULTIPLE_HEAPS
12738 case a_state_trigger_2nd_ephemeral_gc:
12740 BOOL commit_failed_p = FALSE;
12741 BOOL can_use_existing_p = FALSE;
12742 BOOL short_seg_end_p = FALSE;
12743 BOOL did_full_compacting_gc = FALSE;
12746 did_full_compacting_gc = trigger_ephemeral_gc (gr);
12748 if (did_full_compacting_gc)
12750 soh_alloc_state = a_state_try_fit_after_cg;
12754 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12755 align_const, &commit_failed_p,
12757 if (short_seg_end_p || commit_failed_p)
12759 soh_alloc_state = a_state_trigger_full_compact_gc;
12763 assert (can_use_existing_p);
12764 soh_alloc_state = a_state_can_allocate;
12769 case a_state_trigger_full_compact_gc:
12771 if (fgn_maxgen_percent)
12773 dprintf (2, ("FGN: SOH doing last GC before we throw OOM"));
12774 send_full_gc_notification (max_generation, FALSE);
12777 BOOL got_full_compacting_gc = FALSE;
12779 got_full_compacting_gc = trigger_full_compact_gc (gr, &oom_r, false);
12780 soh_alloc_state = (got_full_compacting_gc ? a_state_try_fit_after_cg : a_state_cant_allocate);
12785 assert (!"Invalid state!");
12792 if (soh_alloc_state == a_state_cant_allocate)
12794 assert (oom_r != oom_no_failure);
12795 handle_oom (heap_number,
12798 heap_segment_allocated (ephemeral_heap_segment),
12799 heap_segment_reserved (ephemeral_heap_segment));
12801 add_saved_spinlock_info (false, me_release, mt_alloc_small_cant);
12802 leave_spin_lock (&more_space_lock_soh);
12805 assert ((soh_alloc_state == a_state_can_allocate) ||
12806 (soh_alloc_state == a_state_cant_allocate) ||
12807 (soh_alloc_state == a_state_retry_allocate));
12809 return soh_alloc_state;
12812 #ifdef BACKGROUND_GC
12814 void gc_heap::bgc_track_loh_alloc()
12816 if (current_c_gc_state == c_gc_state_planning)
12818 Interlocked::Increment (&loh_alloc_thread_count);
12819 dprintf (3, ("h%d: inc lc: %d", heap_number, (int32_t)loh_alloc_thread_count));
12824 void gc_heap::bgc_untrack_loh_alloc()
12826 if (current_c_gc_state == c_gc_state_planning)
12828 Interlocked::Decrement (&loh_alloc_thread_count);
12829 dprintf (3, ("h%d: dec lc: %d", heap_number, (int32_t)loh_alloc_thread_count));
12833 BOOL gc_heap::bgc_loh_should_allocate()
12835 size_t min_gc_size = dd_min_size (dynamic_data_of (max_generation + 1));
12837 if ((bgc_begin_loh_size + bgc_loh_size_increased) < (min_gc_size * 10))
12842 if (((bgc_begin_loh_size / end_loh_size) >= 2) || (bgc_loh_size_increased >= bgc_begin_loh_size))
12844 if ((bgc_begin_loh_size / end_loh_size) > 2)
12846 dprintf (3, ("alloc-ed too much before bgc started"));
12850 dprintf (3, ("alloc-ed too much after bgc started"));
12856 bgc_alloc_spin_loh = (uint32_t)(((float)bgc_loh_size_increased / (float)bgc_begin_loh_size) * 10);
12860 #endif //BACKGROUND_GC
12862 size_t gc_heap::get_large_seg_size (size_t size)
12864 size_t default_seg_size = min_loh_segment_size;
12865 #ifdef SEG_MAPPING_TABLE
12866 size_t align_size = default_seg_size;
12867 #else //SEG_MAPPING_TABLE
12868 size_t align_size = default_seg_size / 2;
12869 #endif //SEG_MAPPING_TABLE
12870 int align_const = get_alignment_constant (FALSE);
12871 size_t large_seg_size = align_on_page (
12872 max (default_seg_size,
12873 ((size + 2 * Align(min_obj_size, align_const) + OS_PAGE_SIZE +
12874 align_size) / align_size * align_size)));
12875 return large_seg_size;
12878 BOOL gc_heap::loh_get_new_seg (generation* gen,
12881 BOOL* did_full_compact_gc,
12884 UNREFERENCED_PARAMETER(gen);
12885 UNREFERENCED_PARAMETER(align_const);
12887 *did_full_compact_gc = FALSE;
12889 size_t seg_size = get_large_seg_size (size);
12891 heap_segment* new_seg = get_large_segment (seg_size, did_full_compact_gc);
12895 loh_alloc_since_cg += seg_size;
12902 return (new_seg != 0);
12905 // PERF TODO: this is too aggressive; and in hard limit we should
12906 // count the actual allocated bytes instead of only updating it during
12907 // getting a new seg.
12908 BOOL gc_heap::retry_full_compact_gc (size_t size)
12910 size_t seg_size = get_large_seg_size (size);
12912 if (loh_alloc_since_cg >= (2 * (uint64_t)seg_size))
12917 #ifdef MULTIPLE_HEAPS
12918 uint64_t total_alloc_size = 0;
12919 for (int i = 0; i < n_heaps; i++)
12921 total_alloc_size += g_heaps[i]->loh_alloc_since_cg;
12924 if (total_alloc_size >= (2 * (uint64_t)seg_size))
12928 #endif //MULTIPLE_HEAPS
12933 BOOL gc_heap::check_and_wait_for_bgc (alloc_wait_reason awr,
12934 BOOL* did_full_compact_gc,
12937 BOOL bgc_in_progress = FALSE;
12938 *did_full_compact_gc = FALSE;
12939 #ifdef BACKGROUND_GC
12940 if (recursive_gc_sync::background_running_p())
12942 bgc_in_progress = TRUE;
12943 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12944 wait_for_background (awr, loh_p);
12945 size_t current_full_compact_gc_count = get_full_compact_gc_count();
12946 if (current_full_compact_gc_count > last_full_compact_gc_count)
12948 *did_full_compact_gc = TRUE;
12951 #endif //BACKGROUND_GC
12953 return bgc_in_progress;
12956 BOOL gc_heap::loh_try_fit (int gen_number,
12958 alloc_context* acontext,
12960 BOOL* commit_failed_p,
12963 BOOL can_allocate = TRUE;
12965 if (!a_fit_free_list_large_p (size, acontext, align_const))
12967 can_allocate = loh_a_fit_segment_end_p (gen_number, size,
12968 acontext, align_const,
12969 commit_failed_p, oom_r);
12971 #ifdef BACKGROUND_GC
12972 if (can_allocate && recursive_gc_sync::background_running_p())
12974 bgc_loh_size_increased += size;
12976 #endif //BACKGROUND_GC
12978 #ifdef BACKGROUND_GC
12981 if (recursive_gc_sync::background_running_p())
12983 bgc_loh_allocated_in_free += size;
12986 #endif //BACKGROUND_GC
12988 return can_allocate;
12991 BOOL gc_heap::trigger_full_compact_gc (gc_reason gr,
12995 BOOL did_full_compact_gc = FALSE;
12997 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12999 // Set this so the next GC will be a full compacting GC.
13000 if (!last_gc_before_oom)
13002 last_gc_before_oom = TRUE;
13005 #ifdef BACKGROUND_GC
13006 if (recursive_gc_sync::background_running_p())
13008 wait_for_background (((gr == reason_oos_soh) ? awr_gen0_oos_bgc : awr_loh_oos_bgc), loh_p);
13009 dprintf (2, ("waited for BGC - done"));
13011 #endif //BACKGROUND_GC
13013 GCSpinLock* msl = loh_p ? &more_space_lock_loh : &more_space_lock_soh;
13014 size_t current_full_compact_gc_count = get_full_compact_gc_count();
13015 if (current_full_compact_gc_count > last_full_compact_gc_count)
13017 dprintf (3, ("a full compacting GC triggered while waiting for BGC (%d->%d)", last_full_compact_gc_count, current_full_compact_gc_count));
13018 assert (current_full_compact_gc_count > last_full_compact_gc_count);
13019 did_full_compact_gc = TRUE;
13023 dprintf (3, ("h%d full GC", heap_number));
13025 trigger_gc_for_alloc (max_generation, gr, msl, loh_p, mt_t_full_gc);
13027 current_full_compact_gc_count = get_full_compact_gc_count();
13029 if (current_full_compact_gc_count == last_full_compact_gc_count)
13031 dprintf (2, ("attempted to trigger a full compacting GC but didn't get it"));
13032 // We requested a full GC but didn't get because of the elevation logic
13033 // which means we should fail.
13034 *oom_r = oom_unproductive_full_gc;
13038 dprintf (3, ("h%d: T full compacting GC (%d->%d)",
13040 last_full_compact_gc_count,
13041 current_full_compact_gc_count));
13043 assert (current_full_compact_gc_count > last_full_compact_gc_count);
13044 did_full_compact_gc = TRUE;
13048 return did_full_compact_gc;
13051 #ifdef RECORD_LOH_STATE
13052 void gc_heap::add_saved_loh_state (allocation_state loh_state_to_save, EEThreadId thread_id)
13054 // When the state is can_allocate we already have released the more
13055 // space lock. So we are not logging states here since this code
13056 // is not thread safe.
13057 if (loh_state_to_save != a_state_can_allocate)
13059 last_loh_states[loh_state_index].alloc_state = loh_state_to_save;
13060 last_loh_states[loh_state_index].thread_id = thread_id;
13063 if (loh_state_index == max_saved_loh_states)
13065 loh_state_index = 0;
13068 assert (loh_state_index < max_saved_loh_states);
13071 #endif //RECORD_LOH_STATE
13073 bool gc_heap::should_retry_other_heap (size_t size)
13075 #ifdef MULTIPLE_HEAPS
13076 if (heap_hard_limit)
13078 size_t total_heap_committed_recorded =
13079 current_total_committed - current_total_committed_bookkeeping;
13080 size_t min_size = dd_min_size (g_heaps[0]->dynamic_data_of (max_generation + 1));
13081 size_t slack_space = max (commit_min_th, min_size);
13082 bool retry_p = ((total_heap_committed_recorded + size) < (heap_hard_limit - slack_space));
13083 dprintf (1, ("%Id - %Id - total committed %Id - size %Id = %Id, %s",
13084 heap_hard_limit, slack_space, total_heap_committed_recorded, size,
13085 (heap_hard_limit - slack_space - total_heap_committed_recorded - size),
13086 (retry_p ? "retry" : "no retry")));
13090 #endif //MULTIPLE_HEAPS
13096 allocation_state gc_heap::allocate_large (int gen_number,
13098 alloc_context* acontext,
13101 #ifdef BACKGROUND_GC
13102 if (recursive_gc_sync::background_running_p())
13104 background_loh_alloc_count++;
13105 //if ((background_loh_alloc_count % bgc_alloc_spin_count_loh) == 0)
13107 if (bgc_loh_should_allocate())
13109 if (!bgc_alloc_spin_loh)
13111 add_saved_spinlock_info (true, me_release, mt_alloc_large);
13112 leave_spin_lock (&more_space_lock_loh);
13113 bool cooperative_mode = enable_preemptive();
13114 GCToOSInterface::YieldThread (bgc_alloc_spin_loh);
13115 disable_preemptive (cooperative_mode);
13116 enter_spin_lock (&more_space_lock_loh);
13117 add_saved_spinlock_info (true, me_acquire, mt_alloc_large);
13118 dprintf (SPINLOCK_LOG, ("[%d]spin Emsl loh", heap_number));
13123 wait_for_background (awr_loh_alloc_during_bgc, true);
13127 #endif //BACKGROUND_GC
13129 gc_reason gr = reason_oos_loh;
13130 generation* gen = generation_of (gen_number);
13131 oom_reason oom_r = oom_no_failure;
13132 size_t current_full_compact_gc_count = 0;
13134 // No variable values should be "carried over" from one state to the other.
13135 // That's why there are local variable for each state
13136 allocation_state loh_alloc_state = a_state_start;
13137 #ifdef RECORD_LOH_STATE
13138 EEThreadId current_thread_id;
13139 current_thread_id.SetToCurrentThread();
13140 #endif //RECORD_LOH_STATE
13142 // If we can get a new seg it means allocation will succeed.
13145 dprintf (3, ("[h%d]loh state is %s", heap_number, allocation_state_str[loh_alloc_state]));
13147 #ifdef RECORD_LOH_STATE
13148 add_saved_loh_state (loh_alloc_state, current_thread_id);
13149 #endif //RECORD_LOH_STATE
13150 switch (loh_alloc_state)
13152 case a_state_can_allocate:
13153 case a_state_cant_allocate:
13157 case a_state_start:
13159 loh_alloc_state = a_state_try_fit;
13162 case a_state_try_fit:
13164 BOOL commit_failed_p = FALSE;
13165 BOOL can_use_existing_p = FALSE;
13167 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
13168 align_const, &commit_failed_p, &oom_r);
13169 loh_alloc_state = (can_use_existing_p ?
13170 a_state_can_allocate :
13172 a_state_trigger_full_compact_gc :
13173 a_state_acquire_seg));
13174 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
13177 case a_state_try_fit_new_seg:
13179 BOOL commit_failed_p = FALSE;
13180 BOOL can_use_existing_p = FALSE;
13182 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
13183 align_const, &commit_failed_p, &oom_r);
13184 // Even after we got a new seg it doesn't necessarily mean we can allocate,
13185 // another LOH allocating thread could have beat us to acquire the msl so
13186 // we need to try again.
13187 loh_alloc_state = (can_use_existing_p ? a_state_can_allocate : a_state_try_fit);
13188 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
13191 case a_state_try_fit_after_cg:
13193 BOOL commit_failed_p = FALSE;
13194 BOOL can_use_existing_p = FALSE;
13196 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
13197 align_const, &commit_failed_p, &oom_r);
13198 // If we failed to commit, we bail right away 'cause we already did a
13199 // full compacting GC.
13200 loh_alloc_state = (can_use_existing_p ?
13201 a_state_can_allocate :
13203 a_state_cant_allocate :
13204 a_state_acquire_seg_after_cg));
13205 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
13208 case a_state_try_fit_after_bgc:
13210 BOOL commit_failed_p = FALSE;
13211 BOOL can_use_existing_p = FALSE;
13213 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
13214 align_const, &commit_failed_p, &oom_r);
13215 loh_alloc_state = (can_use_existing_p ?
13216 a_state_can_allocate :
13218 a_state_trigger_full_compact_gc :
13219 a_state_acquire_seg_after_bgc));
13220 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
13223 case a_state_acquire_seg:
13225 BOOL can_get_new_seg_p = FALSE;
13226 BOOL did_full_compacting_gc = FALSE;
13228 current_full_compact_gc_count = get_full_compact_gc_count();
13230 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
13231 loh_alloc_state = (can_get_new_seg_p ?
13232 a_state_try_fit_new_seg :
13233 (did_full_compacting_gc ?
13234 a_state_check_retry_seg :
13235 a_state_check_and_wait_for_bgc));
13238 case a_state_acquire_seg_after_cg:
13240 BOOL can_get_new_seg_p = FALSE;
13241 BOOL did_full_compacting_gc = FALSE;
13243 current_full_compact_gc_count = get_full_compact_gc_count();
13245 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
13246 // Since we release the msl before we try to allocate a seg, other
13247 // threads could have allocated a bunch of segments before us so
13248 // we might need to retry.
13249 loh_alloc_state = (can_get_new_seg_p ?
13250 a_state_try_fit_after_cg :
13251 a_state_check_retry_seg);
13254 case a_state_acquire_seg_after_bgc:
13256 BOOL can_get_new_seg_p = FALSE;
13257 BOOL did_full_compacting_gc = FALSE;
13259 current_full_compact_gc_count = get_full_compact_gc_count();
13261 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
13262 loh_alloc_state = (can_get_new_seg_p ?
13263 a_state_try_fit_new_seg :
13264 (did_full_compacting_gc ?
13265 a_state_check_retry_seg :
13266 a_state_trigger_full_compact_gc));
13267 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13270 case a_state_check_and_wait_for_bgc:
13272 BOOL bgc_in_progress_p = FALSE;
13273 BOOL did_full_compacting_gc = FALSE;
13275 bgc_in_progress_p = check_and_wait_for_bgc (awr_loh_oos_bgc, &did_full_compacting_gc, true);
13276 loh_alloc_state = (!bgc_in_progress_p ?
13277 a_state_trigger_full_compact_gc :
13278 (did_full_compacting_gc ?
13279 a_state_try_fit_after_cg :
13280 a_state_try_fit_after_bgc));
13283 case a_state_trigger_full_compact_gc:
13285 if (fgn_maxgen_percent)
13287 dprintf (2, ("FGN: LOH doing last GC before we throw OOM"));
13288 send_full_gc_notification (max_generation, FALSE);
13291 BOOL got_full_compacting_gc = FALSE;
13293 got_full_compacting_gc = trigger_full_compact_gc (gr, &oom_r, true);
13294 loh_alloc_state = (got_full_compacting_gc ? a_state_try_fit_after_cg : a_state_cant_allocate);
13295 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13298 case a_state_check_retry_seg:
13300 BOOL should_retry_gc = retry_full_compact_gc (size);
13301 BOOL should_retry_get_seg = FALSE;
13302 if (!should_retry_gc)
13304 size_t last_full_compact_gc_count = current_full_compact_gc_count;
13305 current_full_compact_gc_count = get_full_compact_gc_count();
13306 if (current_full_compact_gc_count > last_full_compact_gc_count)
13308 should_retry_get_seg = TRUE;
13312 loh_alloc_state = (should_retry_gc ?
13313 a_state_trigger_full_compact_gc :
13314 (should_retry_get_seg ?
13315 a_state_try_fit_after_cg :
13316 a_state_cant_allocate));
13317 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13322 assert (!"Invalid state!");
13329 if (loh_alloc_state == a_state_cant_allocate)
13331 assert (oom_r != oom_no_failure);
13332 if (should_retry_other_heap (size))
13334 loh_alloc_state = a_state_retry_allocate;
13338 handle_oom (heap_number,
13344 add_saved_spinlock_info (true, me_release, mt_alloc_large_cant);
13345 leave_spin_lock (&more_space_lock_loh);
13348 assert ((loh_alloc_state == a_state_can_allocate) ||
13349 (loh_alloc_state == a_state_cant_allocate) ||
13350 (loh_alloc_state == a_state_retry_allocate));
13351 return loh_alloc_state;
13354 // BGC's final mark phase will acquire the msl, so release it here and re-acquire.
13355 void gc_heap::trigger_gc_for_alloc (int gen_number, gc_reason gr,
13356 GCSpinLock* msl, bool loh_p,
13357 msl_take_state take_state)
13359 #ifdef BACKGROUND_GC
13362 add_saved_spinlock_info (loh_p, me_release, take_state);
13363 leave_spin_lock (msl);
13365 #endif //BACKGROUND_GC
13367 vm_heap->GarbageCollectGeneration (gen_number, gr);
13369 #ifdef MULTIPLE_HEAPS
13372 enter_spin_lock (msl);
13373 add_saved_spinlock_info (loh_p, me_acquire, take_state);
13375 #endif //MULTIPLE_HEAPS
13377 #ifdef BACKGROUND_GC
13380 enter_spin_lock (msl);
13381 add_saved_spinlock_info (loh_p, me_acquire, take_state);
13383 #endif //BACKGROUND_GC
13386 allocation_state gc_heap::try_allocate_more_space (alloc_context* acontext, size_t size,
13389 if (gc_heap::gc_started)
13391 wait_for_gc_done();
13392 return a_state_retry_allocate;
13395 bool loh_p = (gen_number > 0);
13396 GCSpinLock* msl = loh_p ? &more_space_lock_loh : &more_space_lock_soh;
13398 #ifdef SYNCHRONIZATION_STATS
13399 int64_t msl_acquire_start = GCToOSInterface::QueryPerformanceCounter();
13400 #endif //SYNCHRONIZATION_STATS
13401 enter_spin_lock (msl);
13402 add_saved_spinlock_info (loh_p, me_acquire, mt_try_alloc);
13403 dprintf (SPINLOCK_LOG, ("[%d]Emsl for alloc", heap_number));
13404 #ifdef SYNCHRONIZATION_STATS
13405 int64_t msl_acquire = GCToOSInterface::QueryPerformanceCounter() - msl_acquire_start;
13406 total_msl_acquire += msl_acquire;
13407 num_msl_acquired++;
13408 if (msl_acquire > 200)
13410 num_high_msl_acquire++;
13414 num_low_msl_acquire++;
13416 #endif //SYNCHRONIZATION_STATS
13419 // We are commenting this out 'cause we don't see the point - we already
13420 // have checked gc_started when we were acquiring the msl - no need to check
13421 // again. This complicates the logic in bgc_suspend_EE 'cause that one would
13422 // need to release msl which causes all sorts of trouble.
13423 if (gc_heap::gc_started)
13425 #ifdef SYNCHRONIZATION_STATS
13427 #endif //SYNCHRONIZATION_STATS
13428 BOOL fStress = (g_pConfig->GetGCStressLevel() & GCConfig::GCSTRESS_TRANSITION) != 0;
13431 //Rendez vous early (MP scaling issue)
13432 //dprintf (1, ("[%d]waiting for gc", heap_number));
13433 wait_for_gc_done();
13434 #ifdef MULTIPLE_HEAPS
13436 #endif //MULTIPLE_HEAPS
13441 dprintf (3, ("requested to allocate %d bytes on gen%d", size, gen_number));
13443 int align_const = get_alignment_constant (gen_number != (max_generation+1));
13445 if (fgn_maxgen_percent)
13447 check_for_full_gc (gen_number, size);
13450 if (!(new_allocation_allowed (gen_number)))
13452 if (fgn_maxgen_percent && (gen_number == 0))
13454 // We only check gen0 every so often, so take this opportunity to check again.
13455 check_for_full_gc (gen_number, size);
13458 #ifdef BACKGROUND_GC
13459 wait_for_bgc_high_memory (awr_gen0_alloc, loh_p);
13460 #endif //BACKGROUND_GC
13462 #ifdef SYNCHRONIZATION_STATS
13464 #endif //SYNCHRONIZATION_STATS
13465 dprintf (/*100*/ 2, ("running out of budget on gen%d, gc", gen_number));
13467 if (!settings.concurrent || (gen_number == 0))
13469 trigger_gc_for_alloc (0, ((gen_number == 0) ? reason_alloc_soh : reason_alloc_loh),
13470 msl, loh_p, mt_try_budget);
13474 allocation_state can_allocate = ((gen_number == 0) ?
13475 allocate_small (gen_number, size, acontext, align_const) :
13476 allocate_large (gen_number, size, acontext, align_const));
13478 if (can_allocate == a_state_can_allocate)
13480 size_t alloc_context_bytes = acontext->alloc_limit + Align (min_obj_size, align_const) - acontext->alloc_ptr;
13481 int etw_allocation_index = ((gen_number == 0) ? 0 : 1);
13483 etw_allocation_running_amount[etw_allocation_index] += alloc_context_bytes;
13485 allocated_since_last_gc += alloc_context_bytes;
13487 if (etw_allocation_running_amount[etw_allocation_index] > etw_allocation_tick)
13489 #ifdef FEATURE_REDHAWK
13490 FIRE_EVENT(GCAllocationTick_V1, (uint32_t)etw_allocation_running_amount[etw_allocation_index],
13491 (gen_number == 0) ? gc_etw_alloc_soh : gc_etw_alloc_loh);
13493 // Unfortunately some of the ETW macros do not check whether the ETW feature is enabled.
13494 // The ones that do are much less efficient.
13495 #if defined(FEATURE_EVENT_TRACE)
13496 if (EVENT_ENABLED(GCAllocationTick_V3))
13498 fire_etw_allocation_event (etw_allocation_running_amount[etw_allocation_index], gen_number, acontext->alloc_ptr);
13500 #endif //FEATURE_EVENT_TRACE
13501 #endif //FEATURE_REDHAWK
13502 etw_allocation_running_amount[etw_allocation_index] = 0;
13506 return can_allocate;
13509 #ifdef MULTIPLE_HEAPS
13510 void gc_heap::balance_heaps (alloc_context* acontext)
13512 if (acontext->alloc_count < 4)
13514 if (acontext->alloc_count == 0)
13516 acontext->set_home_heap(GCHeap::GetHeap( heap_select::select_heap(acontext, 0) ));
13517 gc_heap* hp = acontext->get_home_heap()->pGenGCHeap;
13518 dprintf (3, ("First allocation for context %Ix on heap %d\n", (size_t)acontext, (size_t)hp->heap_number));
13519 acontext->set_alloc_heap(acontext->get_home_heap());
13520 hp->alloc_context_count++;
13525 BOOL set_home_heap = FALSE;
13528 if (heap_select::can_find_heap_fast())
13530 if (acontext->get_home_heap() != NULL)
13531 hint = acontext->get_home_heap()->pGenGCHeap->heap_number;
13532 if (acontext->get_home_heap() != GCHeap::GetHeap(hint = heap_select::select_heap(acontext, hint)) || ((acontext->alloc_count & 15) == 0))
13534 set_home_heap = TRUE;
13540 if ((acontext->alloc_count & 3) == 0)
13541 set_home_heap = TRUE;
13547 // Since we are balancing up to MAX_SUPPORTED_CPUS, no need for this.
13548 if (n_heaps > MAX_SUPPORTED_CPUS)
13550 // on machines with many processors cache affinity is really king, so don't even try
13551 // to balance on these.
13552 acontext->home_heap = GCHeap::GetHeap( heap_select::select_heap(acontext, hint) );
13553 acontext->alloc_heap = acontext->home_heap;
13558 gc_heap* org_hp = acontext->get_alloc_heap()->pGenGCHeap;
13560 dynamic_data* dd = org_hp->dynamic_data_of (0);
13561 ptrdiff_t org_size = dd_new_allocation (dd);
13562 int org_alloc_context_count;
13563 int max_alloc_context_count;
13565 ptrdiff_t max_size;
13566 size_t delta = dd_min_size (dd)/4;
13568 int start, end, finish;
13569 heap_select::get_heap_range_for_heap(org_hp->heap_number, &start, &end);
13570 finish = start + n_heaps;
13576 max_size = org_size + delta;
13577 acontext->set_home_heap(GCHeap::GetHeap( heap_select::select_heap(acontext, hint) ));
13579 if (org_hp == acontext->get_home_heap()->pGenGCHeap)
13580 max_size = max_size + delta;
13582 org_alloc_context_count = org_hp->alloc_context_count;
13583 max_alloc_context_count = org_alloc_context_count;
13584 if (max_alloc_context_count > 1)
13585 max_size /= max_alloc_context_count;
13587 for (int i = start; i < end; i++)
13589 gc_heap* hp = GCHeap::GetHeap(i%n_heaps)->pGenGCHeap;
13590 dd = hp->dynamic_data_of (0);
13591 ptrdiff_t size = dd_new_allocation (dd);
13592 if (hp == acontext->get_home_heap()->pGenGCHeap)
13593 size = size + delta;
13594 int hp_alloc_context_count = hp->alloc_context_count;
13595 if (hp_alloc_context_count > 0)
13596 size /= (hp_alloc_context_count + 1);
13597 if (size > max_size)
13601 max_alloc_context_count = hp_alloc_context_count;
13605 while (org_alloc_context_count != org_hp->alloc_context_count ||
13606 max_alloc_context_count != max_hp->alloc_context_count);
13608 if ((max_hp == org_hp) && (end < finish))
13610 start = end; end = finish;
13611 delta = dd_min_size(dd)/2; // Make it twice as hard to balance to remote nodes on NUMA.
13615 if (max_hp != org_hp)
13617 org_hp->alloc_context_count--;
13618 max_hp->alloc_context_count++;
13619 acontext->set_alloc_heap(GCHeap::GetHeap(max_hp->heap_number));
13620 if (!gc_thread_no_affinitize_p)
13622 if (GCToOSInterface::CanEnableGCCPUGroups())
13623 { //only set ideal processor when max_hp and org_hp are in the same cpu
13624 //group. DO NOT MOVE THREADS ACROSS CPU GROUPS
13625 uint16_t org_gn = heap_select::find_cpu_group_from_heap_no(org_hp->heap_number);
13626 uint16_t max_gn = heap_select::find_cpu_group_from_heap_no(max_hp->heap_number);
13627 if (org_gn == max_gn) //only set within CPU group, so SetThreadIdealProcessor is enough
13629 uint16_t group_proc_no = heap_select::find_group_proc_from_heap_no(max_hp->heap_number);
13631 GCThreadAffinity affinity;
13632 affinity.Processor = group_proc_no;
13633 affinity.Group = org_gn;
13634 if (!GCToOSInterface::SetCurrentThreadIdealAffinity(&affinity))
13636 dprintf (3, ("Failed to set the ideal processor and group for heap %d.",
13637 org_hp->heap_number));
13643 uint16_t proc_no = heap_select::find_proc_no_from_heap_no(max_hp->heap_number);
13645 GCThreadAffinity affinity;
13646 affinity.Processor = proc_no;
13647 affinity.Group = GCThreadAffinity::None;
13649 if (!GCToOSInterface::SetCurrentThreadIdealAffinity(&affinity))
13651 dprintf (3, ("Failed to set the ideal processor for heap %d.",
13652 org_hp->heap_number));
13656 dprintf (3, ("Switching context %p (home heap %d) ",
13658 acontext->get_home_heap()->pGenGCHeap->heap_number));
13659 dprintf (3, (" from heap %d (%Id free bytes, %d contexts) ",
13660 org_hp->heap_number,
13662 org_alloc_context_count));
13663 dprintf (3, (" to heap %d (%Id free bytes, %d contexts)\n",
13664 max_hp->heap_number,
13665 dd_new_allocation(max_hp->dynamic_data_of(0)),
13666 max_alloc_context_count));
13671 acontext->alloc_count++;
13674 gc_heap* gc_heap::balance_heaps_loh (alloc_context* acontext, size_t alloc_size)
13676 gc_heap* org_hp = acontext->get_alloc_heap()->pGenGCHeap;
13677 dprintf (3, ("[h%d] LA: %Id", org_hp->heap_number, alloc_size));
13679 //if (size > 128*1024)
13682 dynamic_data* dd = org_hp->dynamic_data_of (max_generation + 1);
13684 ptrdiff_t org_size = dd_new_allocation (dd);
13686 ptrdiff_t max_size;
13687 size_t delta = dd_min_size (dd) * 4;
13689 int start, end, finish;
13690 heap_select::get_heap_range_for_heap(org_hp->heap_number, &start, &end);
13691 finish = start + n_heaps;
13696 max_size = org_size + delta;
13697 dprintf (3, ("orig hp: %d, max size: %d",
13698 org_hp->heap_number,
13701 for (int i = start; i < end; i++)
13703 gc_heap* hp = GCHeap::GetHeap(i%n_heaps)->pGenGCHeap;
13704 dd = hp->dynamic_data_of (max_generation + 1);
13705 ptrdiff_t size = dd_new_allocation (dd);
13706 dprintf (3, ("hp: %d, size: %d",
13709 if (size > max_size)
13713 dprintf (3, ("max hp: %d, max size: %d",
13714 max_hp->heap_number,
13720 if ((max_hp == org_hp) && (end < finish))
13722 start = end; end = finish;
13723 delta = dd_min_size(dd) * 4; // Need to tuning delta
13727 if (max_hp != org_hp)
13729 dprintf (3, ("loh: %d(%Id)->%d(%Id)",
13730 org_hp->heap_number, dd_new_allocation (org_hp->dynamic_data_of (max_generation + 1)),
13731 max_hp->heap_number, dd_new_allocation (max_hp->dynamic_data_of (max_generation + 1))));
13741 #endif //MULTIPLE_HEAPS
13743 BOOL gc_heap::allocate_more_space(alloc_context* acontext, size_t size,
13744 int alloc_generation_number)
13746 allocation_state status;
13749 #ifdef MULTIPLE_HEAPS
13750 if (alloc_generation_number == 0)
13752 balance_heaps (acontext);
13753 status = acontext->get_alloc_heap()->pGenGCHeap->try_allocate_more_space (acontext, size, alloc_generation_number);
13757 gc_heap* alloc_heap = balance_heaps_loh (acontext, size);
13758 status = alloc_heap->try_allocate_more_space (acontext, size, alloc_generation_number);
13759 if (status == a_state_retry_allocate)
13761 dprintf (3, ("LOH h%d alloc retry!", alloc_heap->heap_number));
13765 status = try_allocate_more_space (acontext, size, alloc_generation_number);
13766 #endif //MULTIPLE_HEAPS
13768 while (status == a_state_retry_allocate);
13770 return (status == a_state_can_allocate);
13774 CObjectHeader* gc_heap::allocate (size_t jsize, alloc_context* acontext)
13776 size_t size = Align (jsize);
13777 assert (size >= Align (min_obj_size));
13780 uint8_t* result = acontext->alloc_ptr;
13781 acontext->alloc_ptr+=size;
13782 if (acontext->alloc_ptr <= acontext->alloc_limit)
13784 CObjectHeader* obj = (CObjectHeader*)result;
13790 acontext->alloc_ptr -= size;
13793 #pragma inline_depth(0)
13796 if (! allocate_more_space (acontext, size, 0))
13800 #pragma inline_depth(20)
13808 void gc_heap::leave_allocation_segment (generation* gen)
13810 adjust_limit (0, 0, gen, max_generation);
13813 void gc_heap::init_free_and_plug()
13815 #ifdef FREE_USAGE_STATS
13816 for (int i = 0; i <= settings.condemned_generation; i++)
13818 generation* gen = generation_of (i);
13819 memset (gen->gen_free_spaces, 0, sizeof (gen->gen_free_spaces));
13820 memset (gen->gen_plugs, 0, sizeof (gen->gen_plugs));
13821 memset (gen->gen_current_pinned_free_spaces, 0, sizeof (gen->gen_current_pinned_free_spaces));
13824 if (settings.condemned_generation != max_generation)
13826 for (int i = (settings.condemned_generation + 1); i <= max_generation; i++)
13828 generation* gen = generation_of (i);
13829 memset (gen->gen_plugs, 0, sizeof (gen->gen_plugs));
13832 #endif //FREE_USAGE_STATS
13835 void gc_heap::print_free_and_plug (const char* msg)
13837 #if defined(FREE_USAGE_STATS) && defined(SIMPLE_DPRINTF)
13838 int older_gen = ((settings.condemned_generation == max_generation) ? max_generation : (settings.condemned_generation + 1));
13839 for (int i = 0; i <= older_gen; i++)
13841 generation* gen = generation_of (i);
13842 for (int j = 0; j < NUM_GEN_POWER2; j++)
13844 if ((gen->gen_free_spaces[j] != 0) || (gen->gen_plugs[j] != 0))
13846 dprintf (2, ("[%s][h%d][%s#%d]gen%d: 2^%d: F: %Id, P: %Id",
13849 (settings.concurrent ? "BGC" : "GC"),
13852 (j + 9), gen->gen_free_spaces[j], gen->gen_plugs[j]));
13857 UNREFERENCED_PARAMETER(msg);
13858 #endif //FREE_USAGE_STATS && SIMPLE_DPRINTF
13861 void gc_heap::add_gen_plug (int gen_number, size_t plug_size)
13863 #ifdef FREE_USAGE_STATS
13864 dprintf (3, ("adding plug size %Id to gen%d", plug_size, gen_number));
13865 generation* gen = generation_of (gen_number);
13866 size_t sz = BASE_GEN_SIZE;
13869 for (; i < NUM_GEN_POWER2; i++)
13871 if (plug_size < sz)
13878 (gen->gen_plugs[i])++;
13880 UNREFERENCED_PARAMETER(gen_number);
13881 UNREFERENCED_PARAMETER(plug_size);
13882 #endif //FREE_USAGE_STATS
13885 void gc_heap::add_item_to_current_pinned_free (int gen_number, size_t free_size)
13887 #ifdef FREE_USAGE_STATS
13888 generation* gen = generation_of (gen_number);
13889 size_t sz = BASE_GEN_SIZE;
13892 for (; i < NUM_GEN_POWER2; i++)
13894 if (free_size < sz)
13901 (gen->gen_current_pinned_free_spaces[i])++;
13902 generation_pinned_free_obj_space (gen) += free_size;
13903 dprintf (3, ("left pin free %Id(2^%d) to gen%d, total %Id bytes (%Id)",
13904 free_size, (i + 10), gen_number,
13905 generation_pinned_free_obj_space (gen),
13906 gen->gen_current_pinned_free_spaces[i]));
13908 UNREFERENCED_PARAMETER(gen_number);
13909 UNREFERENCED_PARAMETER(free_size);
13910 #endif //FREE_USAGE_STATS
13913 void gc_heap::add_gen_free (int gen_number, size_t free_size)
13915 #ifdef FREE_USAGE_STATS
13916 dprintf (3, ("adding free size %Id to gen%d", free_size, gen_number));
13917 generation* gen = generation_of (gen_number);
13918 size_t sz = BASE_GEN_SIZE;
13921 for (; i < NUM_GEN_POWER2; i++)
13923 if (free_size < sz)
13930 (gen->gen_free_spaces[i])++;
13932 UNREFERENCED_PARAMETER(gen_number);
13933 UNREFERENCED_PARAMETER(free_size);
13934 #endif //FREE_USAGE_STATS
13937 void gc_heap::remove_gen_free (int gen_number, size_t free_size)
13939 #ifdef FREE_USAGE_STATS
13940 dprintf (3, ("removing free %Id from gen%d", free_size, gen_number));
13941 generation* gen = generation_of (gen_number);
13942 size_t sz = BASE_GEN_SIZE;
13945 for (; i < NUM_GEN_POWER2; i++)
13947 if (free_size < sz)
13954 (gen->gen_free_spaces[i])--;
13956 UNREFERENCED_PARAMETER(gen_number);
13957 UNREFERENCED_PARAMETER(free_size);
13958 #endif //FREE_USAGE_STATS
13961 uint8_t* gc_heap::allocate_in_older_generation (generation* gen, size_t size,
13962 int from_gen_number,
13963 uint8_t* old_loc REQD_ALIGN_AND_OFFSET_DCL)
13965 size = Align (size);
13966 assert (size >= Align (min_obj_size));
13967 assert (from_gen_number < max_generation);
13968 assert (from_gen_number >= 0);
13969 assert (generation_of (from_gen_number + 1) == gen);
13971 allocator* gen_allocator = generation_allocator (gen);
13972 BOOL discard_p = gen_allocator->discard_if_no_fit_p ();
13973 int pad_in_front = ((old_loc != 0) && ((from_gen_number+1) != max_generation)) ? USE_PADDING_FRONT : 0;
13975 size_t real_size = size + Align (min_obj_size);
13977 real_size += Align (min_obj_size);
13979 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
13980 generation_allocation_limit (gen), old_loc, USE_PADDING_TAIL | pad_in_front)))
13982 size_t sz_list = gen_allocator->first_bucket_size();
13983 for (unsigned int a_l_idx = 0; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
13985 if ((real_size < (sz_list / 2)) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
13987 uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
13988 uint8_t* prev_free_item = 0;
13989 while (free_list != 0)
13991 dprintf (3, ("considering free list %Ix", (size_t)free_list));
13993 size_t free_list_size = unused_array_size (free_list);
13995 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, free_list, (free_list + free_list_size),
13996 old_loc, USE_PADDING_TAIL | pad_in_front))
13998 dprintf (4, ("F:%Ix-%Id",
13999 (size_t)free_list, free_list_size));
14001 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, !discard_p);
14002 generation_free_list_space (gen) -= free_list_size;
14003 remove_gen_free (gen->gen_num, free_list_size);
14005 adjust_limit (free_list, free_list_size, gen, from_gen_number+1);
14006 generation_allocate_end_seg_p (gen) = FALSE;
14009 // We do first fit on bucket 0 because we are not guaranteed to find a fit there.
14010 else if (discard_p || (a_l_idx == 0))
14012 dprintf (3, ("couldn't use this free area, discarding"));
14013 generation_free_obj_space (gen) += free_list_size;
14015 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
14016 generation_free_list_space (gen) -= free_list_size;
14017 remove_gen_free (gen->gen_num, free_list_size);
14021 prev_free_item = free_list;
14023 free_list = free_list_slot (free_list);
14026 sz_list = sz_list * 2;
14028 //go back to the beginning of the segment list
14029 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
14030 if (seg != generation_allocation_segment (gen))
14032 leave_allocation_segment (gen);
14033 generation_allocation_segment (gen) = seg;
14035 while (seg != ephemeral_heap_segment)
14037 if (size_fit_p(size REQD_ALIGN_AND_OFFSET_ARG, heap_segment_plan_allocated (seg),
14038 heap_segment_committed (seg), old_loc, USE_PADDING_TAIL | pad_in_front))
14040 dprintf (3, ("using what's left in committed"));
14041 adjust_limit (heap_segment_plan_allocated (seg),
14042 heap_segment_committed (seg) -
14043 heap_segment_plan_allocated (seg),
14044 gen, from_gen_number+1);
14045 generation_allocate_end_seg_p (gen) = TRUE;
14046 // dformat (t, 3, "Expanding segment allocation");
14047 heap_segment_plan_allocated (seg) =
14048 heap_segment_committed (seg);
14053 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, heap_segment_plan_allocated (seg),
14054 heap_segment_reserved (seg), old_loc, USE_PADDING_TAIL | pad_in_front) &&
14055 grow_heap_segment (seg, heap_segment_plan_allocated (seg), old_loc, size, pad_in_front REQD_ALIGN_AND_OFFSET_ARG))
14057 dprintf (3, ("using what's left in reserved"));
14058 adjust_limit (heap_segment_plan_allocated (seg),
14059 heap_segment_committed (seg) -
14060 heap_segment_plan_allocated (seg),
14061 gen, from_gen_number+1);
14062 generation_allocate_end_seg_p (gen) = TRUE;
14063 heap_segment_plan_allocated (seg) =
14064 heap_segment_committed (seg);
14070 leave_allocation_segment (gen);
14071 heap_segment* next_seg = heap_segment_next_rw (seg);
14074 dprintf (3, ("getting next segment"));
14075 generation_allocation_segment (gen) = next_seg;
14076 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
14077 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
14086 seg = generation_allocation_segment (gen);
14088 //No need to fix the last region. Will be done later
14099 uint8_t* result = generation_allocation_pointer (gen);
14103 if ((pad_in_front & USE_PADDING_FRONT) &&
14104 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
14105 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
14107 pad = Align (min_obj_size);
14108 set_plug_padded (old_loc);
14110 #endif //SHORT_PLUGS
14112 #ifdef FEATURE_STRUCTALIGN
14113 _ASSERTE(!old_loc || alignmentOffset != 0);
14114 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
14117 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
14118 set_node_aligninfo (old_loc, requiredAlignment, pad1);
14121 #else // FEATURE_STRUCTALIGN
14122 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
14124 pad += switch_alignment_size (is_plug_padded (old_loc));
14125 set_node_realigned (old_loc);
14126 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
14127 (size_t)old_loc, (size_t)(result+pad)));
14128 assert (same_large_alignment_p (result + pad, old_loc));
14130 #endif // FEATURE_STRUCTALIGN
14131 dprintf (3, ("Allocate %Id bytes", size));
14133 if ((old_loc == 0) || (pad != 0))
14135 //allocating a non plug or a gap, so reset the start region
14136 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14139 generation_allocation_pointer (gen) += size + pad;
14140 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
14141 if (generation_allocate_end_seg_p (gen))
14143 generation_end_seg_allocated (gen) += size;
14147 generation_free_list_allocated (gen) += size;
14149 generation_allocation_size (gen) += size;
14151 dprintf (3, ("aio: ptr: %Ix, limit: %Ix, sr: %Ix",
14152 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14153 generation_allocation_context_start_region (gen)));
14155 return result + pad;;
14159 void gc_heap::repair_allocation_in_expanded_heap (generation* consing_gen)
14161 //make sure that every generation has a planned allocation start
14162 int gen_number = max_generation - 1;
14163 while (gen_number>= 0)
14165 generation* gen = generation_of (gen_number);
14166 if (0 == generation_plan_allocation_start (gen))
14168 realloc_plan_generation_start (gen, consing_gen);
14170 assert (generation_plan_allocation_start (gen));
14175 // now we know the planned allocation size
14176 size_t size = (generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
14177 heap_segment* seg = generation_allocation_segment (consing_gen);
14178 if (generation_allocation_limit (consing_gen) == heap_segment_plan_allocated (seg))
14182 heap_segment_plan_allocated (seg) = generation_allocation_pointer (consing_gen);
14187 assert (settings.condemned_generation == max_generation);
14188 uint8_t* first_address = generation_allocation_limit (consing_gen);
14189 //look through the pinned plugs for relevant ones.
14190 //Look for the right pinned plug to start from.
14193 while (mi != mark_stack_tos)
14195 m = pinned_plug_of (mi);
14196 if ((pinned_plug (m) == first_address))
14201 assert (mi != mark_stack_tos);
14202 pinned_len (m) = size;
14206 //tododefrag optimize for new segment (plan_allocated == mem)
14207 uint8_t* gc_heap::allocate_in_expanded_heap (generation* gen,
14212 BOOL set_padding_on_saved_p,
14213 mark* pinned_plug_entry,
14214 #endif //SHORT_PLUGS
14215 BOOL consider_bestfit,
14216 int active_new_gen_number
14217 REQD_ALIGN_AND_OFFSET_DCL)
14219 UNREFERENCED_PARAMETER(active_new_gen_number);
14220 dprintf (3, ("aie: P: %Ix, size: %Ix", old_loc, size));
14222 size = Align (size);
14223 assert (size >= Align (min_obj_size));
14224 int pad_in_front = ((old_loc != 0) && (active_new_gen_number != max_generation)) ? USE_PADDING_FRONT : 0;
14226 if (consider_bestfit && use_bestfit)
14228 assert (bestfit_seg);
14229 dprintf (SEG_REUSE_LOG_1, ("reallocating 0x%Ix in expanded heap, size: %Id",
14231 return bestfit_seg->fit (old_loc,
14233 set_padding_on_saved_p,
14235 #endif //SHORT_PLUGS
14236 size REQD_ALIGN_AND_OFFSET_ARG);
14239 heap_segment* seg = generation_allocation_segment (gen);
14241 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14242 generation_allocation_limit (gen), old_loc,
14243 ((generation_allocation_limit (gen) !=
14244 heap_segment_plan_allocated (seg))? USE_PADDING_TAIL : 0) | pad_in_front)))
14246 dprintf (3, ("aie: can't fit: ptr: %Ix, limit: %Ix", generation_allocation_pointer (gen),
14247 generation_allocation_limit (gen)));
14250 uint8_t* first_address = (generation_allocation_limit (gen) ?
14251 generation_allocation_limit (gen) :
14252 heap_segment_mem (seg));
14253 assert (in_range_for_segment (first_address, seg));
14255 uint8_t* end_address = heap_segment_reserved (seg);
14257 dprintf (3, ("aie: first_addr: %Ix, gen alloc limit: %Ix, end_address: %Ix",
14258 first_address, generation_allocation_limit (gen), end_address));
14263 if (heap_segment_allocated (seg) != heap_segment_mem (seg))
14265 assert (settings.condemned_generation == max_generation);
14266 //look through the pinned plugs for relevant ones.
14267 //Look for the right pinned plug to start from.
14268 while (mi != mark_stack_tos)
14270 m = pinned_plug_of (mi);
14271 if ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address))
14273 dprintf (3, ("aie: found pin: %Ix", pinned_plug (m)));
14279 if (mi != mark_stack_tos)
14281 //fix old free list.
14282 size_t hsize = (generation_allocation_limit (gen) - generation_allocation_pointer (gen));
14284 dprintf(3,("gc filling up hole"));
14285 ptrdiff_t mi1 = (ptrdiff_t)mi;
14286 while ((mi1 >= 0) &&
14287 (pinned_plug (pinned_plug_of(mi1)) != generation_allocation_limit (gen)))
14289 dprintf (3, ("aie: checking pin %Ix", pinned_plug (pinned_plug_of(mi1))));
14294 size_t saved_pinned_len = pinned_len (pinned_plug_of(mi1));
14295 pinned_len (pinned_plug_of(mi1)) = hsize;
14296 dprintf (3, ("changing %Ix len %Ix->%Ix",
14297 pinned_plug (pinned_plug_of(mi1)),
14298 saved_pinned_len, pinned_len (pinned_plug_of(mi1))));
14305 assert (generation_allocation_limit (gen) ==
14306 generation_allocation_pointer (gen));
14307 mi = mark_stack_tos;
14310 while ((mi != mark_stack_tos) && in_range_for_segment (pinned_plug (m), seg))
14312 size_t len = pinned_len (m);
14313 uint8_t* free_list = (pinned_plug (m) - len);
14314 dprintf (3, ("aie: testing free item: %Ix->%Ix(%Ix)",
14315 free_list, (free_list + len), len));
14316 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, free_list, (free_list + len), old_loc, USE_PADDING_TAIL | pad_in_front))
14318 dprintf (3, ("aie: Found adequate unused area: %Ix, size: %Id",
14319 (size_t)free_list, len));
14321 generation_allocation_pointer (gen) = free_list;
14322 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14323 generation_allocation_limit (gen) = (free_list + len);
14325 goto allocate_in_free;
14328 m = pinned_plug_of (mi);
14331 //switch to the end of the segment.
14332 generation_allocation_pointer (gen) = heap_segment_plan_allocated (seg);
14333 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14334 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14335 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14336 dprintf (3, ("aie: switching to end of seg: %Ix->%Ix(%Ix)",
14337 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14338 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
14340 if (!size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14341 generation_allocation_limit (gen), old_loc, USE_PADDING_TAIL | pad_in_front))
14343 dprintf (3, ("aie: ptr: %Ix, limit: %Ix, can't alloc", generation_allocation_pointer (gen),
14344 generation_allocation_limit (gen)));
14345 assert (!"Can't allocate if no free space");
14356 uint8_t* result = generation_allocation_pointer (gen);
14360 if ((pad_in_front & USE_PADDING_FRONT) &&
14361 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
14362 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
14365 pad = Align (min_obj_size);
14366 set_padding_in_expand (old_loc, set_padding_on_saved_p, pinned_plug_entry);
14368 #endif //SHORT_PLUGS
14370 #ifdef FEATURE_STRUCTALIGN
14371 _ASSERTE(!old_loc || alignmentOffset != 0);
14372 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
14375 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
14376 set_node_aligninfo (old_loc, requiredAlignment, pad1);
14380 #else // FEATURE_STRUCTALIGN
14381 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
14383 pad += switch_alignment_size (is_plug_padded (old_loc));
14384 set_node_realigned (old_loc);
14385 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
14386 (size_t)old_loc, (size_t)(result+pad)));
14387 assert (same_large_alignment_p (result + pad, old_loc));
14390 #endif // FEATURE_STRUCTALIGN
14392 if ((old_loc == 0) || (pad != 0))
14394 //allocating a non plug or a gap, so reset the start region
14395 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14398 generation_allocation_pointer (gen) += size + pad;
14399 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
14400 dprintf (3, ("Allocated in expanded heap %Ix:%Id", (size_t)(result+pad), size));
14402 dprintf (3, ("aie: ptr: %Ix, limit: %Ix, sr: %Ix",
14403 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14404 generation_allocation_context_start_region (gen)));
14406 return result + pad;
14410 generation* gc_heap::ensure_ephemeral_heap_segment (generation* consing_gen)
14412 heap_segment* seg = generation_allocation_segment (consing_gen);
14413 if (seg != ephemeral_heap_segment)
14415 assert (generation_allocation_pointer (consing_gen)>= heap_segment_mem (seg));
14416 assert (generation_allocation_pointer (consing_gen)<= heap_segment_committed (seg));
14418 //fix the allocated size of the segment.
14419 heap_segment_plan_allocated (seg) = generation_allocation_pointer (consing_gen);
14421 generation* new_consing_gen = generation_of (max_generation - 1);
14422 generation_allocation_pointer (new_consing_gen) =
14423 heap_segment_mem (ephemeral_heap_segment);
14424 generation_allocation_limit (new_consing_gen) =
14425 generation_allocation_pointer (new_consing_gen);
14426 generation_allocation_context_start_region (new_consing_gen) =
14427 generation_allocation_pointer (new_consing_gen);
14428 generation_allocation_segment (new_consing_gen) = ephemeral_heap_segment;
14430 return new_consing_gen;
14433 return consing_gen;
14436 uint8_t* gc_heap::allocate_in_condemned_generations (generation* gen,
14438 int from_gen_number,
14440 BOOL* convert_to_pinned_p,
14441 uint8_t* next_pinned_plug,
14442 heap_segment* current_seg,
14443 #endif //SHORT_PLUGS
14445 REQD_ALIGN_AND_OFFSET_DCL)
14447 // Make sure that the youngest generation gap hasn't been allocated
14448 if (settings.promotion)
14450 assert (generation_plan_allocation_start (youngest_generation) == 0);
14453 size = Align (size);
14454 assert (size >= Align (min_obj_size));
14455 int to_gen_number = from_gen_number;
14456 if (from_gen_number != (int)max_generation)
14458 to_gen_number = from_gen_number + (settings.promotion ? 1 : 0);
14461 dprintf (3, ("aic gen%d: s: %Id", gen->gen_num, size));
14463 int pad_in_front = ((old_loc != 0) && (to_gen_number != max_generation)) ? USE_PADDING_FRONT : 0;
14465 if ((from_gen_number != -1) && (from_gen_number != (int)max_generation) && settings.promotion)
14467 generation_condemned_allocated (generation_of (from_gen_number + (settings.promotion ? 1 : 0))) += size;
14468 generation_allocation_size (generation_of (from_gen_number + (settings.promotion ? 1 : 0))) += size;
14472 heap_segment* seg = generation_allocation_segment (gen);
14473 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14474 generation_allocation_limit (gen), old_loc,
14475 ((generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))?USE_PADDING_TAIL:0)|pad_in_front)))
14477 if ((! (pinned_plug_que_empty_p()) &&
14478 (generation_allocation_limit (gen) ==
14479 pinned_plug (oldest_pin()))))
14481 size_t entry = deque_pinned_plug();
14482 mark* pinned_plug_entry = pinned_plug_of (entry);
14483 size_t len = pinned_len (pinned_plug_entry);
14484 uint8_t* plug = pinned_plug (pinned_plug_entry);
14485 set_new_pin_info (pinned_plug_entry, generation_allocation_pointer (gen));
14487 #ifdef FREE_USAGE_STATS
14488 generation_allocated_in_pinned_free (gen) += generation_allocated_since_last_pin (gen);
14489 dprintf (3, ("allocated %Id so far within pin %Ix, total->%Id",
14490 generation_allocated_since_last_pin (gen),
14492 generation_allocated_in_pinned_free (gen)));
14493 generation_allocated_since_last_pin (gen) = 0;
14495 add_item_to_current_pinned_free (gen->gen_num, pinned_len (pinned_plug_of (entry)));
14496 #endif //FREE_USAGE_STATS
14498 dprintf (3, ("mark stack bos: %Id, tos: %Id, aic: p %Ix len: %Ix->%Ix",
14499 mark_stack_bos, mark_stack_tos, plug, len, pinned_len (pinned_plug_of (entry))));
14501 assert(mark_stack_array[entry].len == 0 ||
14502 mark_stack_array[entry].len >= Align(min_obj_size));
14503 generation_allocation_pointer (gen) = plug + len;
14504 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14505 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14506 set_allocator_next_pin (gen);
14508 //Add the size of the pinned plug to the right pinned allocations
14509 //find out which gen this pinned plug came from
14510 int frgn = object_gennum (plug);
14511 if ((frgn != (int)max_generation) && settings.promotion)
14513 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
14514 int togn = object_gennum_plan (plug);
14517 generation_pinned_allocation_compact_size (generation_of (togn)) += len;
14523 if (generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))
14525 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14526 dprintf (3, ("changed limit to plan alloc: %Ix", generation_allocation_limit (gen)));
14530 if (heap_segment_plan_allocated (seg) != heap_segment_committed (seg))
14532 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14533 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14534 dprintf (3, ("changed limit to commit: %Ix", generation_allocation_limit (gen)));
14538 #ifndef RESPECT_LARGE_ALIGNMENT
14539 assert (gen != youngest_generation);
14540 #endif //RESPECT_LARGE_ALIGNMENT
14542 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14543 heap_segment_reserved (seg), old_loc, USE_PADDING_TAIL | pad_in_front) &&
14544 (grow_heap_segment (seg, generation_allocation_pointer (gen), old_loc,
14545 size, pad_in_front REQD_ALIGN_AND_OFFSET_ARG)))
14547 dprintf (3, ("Expanded segment allocation by committing more memory"));
14548 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14549 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14553 heap_segment* next_seg = heap_segment_next (seg);
14554 assert (generation_allocation_pointer (gen)>=
14555 heap_segment_mem (seg));
14556 // Verify that all pinned plugs for this segment are consumed
14557 if (!pinned_plug_que_empty_p() &&
14558 ((pinned_plug (oldest_pin()) <
14559 heap_segment_allocated (seg)) &&
14560 (pinned_plug (oldest_pin()) >=
14561 generation_allocation_pointer (gen))))
14563 LOG((LF_GC, LL_INFO10, "remaining pinned plug %Ix while leaving segment on allocation",
14564 pinned_plug (oldest_pin())));
14567 assert (generation_allocation_pointer (gen)>=
14568 heap_segment_mem (seg));
14569 assert (generation_allocation_pointer (gen)<=
14570 heap_segment_committed (seg));
14571 heap_segment_plan_allocated (seg) = generation_allocation_pointer (gen);
14575 generation_allocation_segment (gen) = next_seg;
14576 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
14577 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
14578 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14582 return 0; //should only happen during allocation of generation 0 gap
14583 // in that case we are going to grow the heap anyway
14588 set_allocator_next_pin (gen);
14595 assert (generation_allocation_pointer (gen)>=
14596 heap_segment_mem (generation_allocation_segment (gen)));
14597 uint8_t* result = generation_allocation_pointer (gen);
14600 if ((pad_in_front & USE_PADDING_FRONT) &&
14601 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
14602 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
14604 ptrdiff_t dist = old_loc - result;
14607 dprintf (3, ("old alloc: %Ix, same as new alloc, not padding", old_loc));
14612 if ((dist > 0) && (dist < (ptrdiff_t)Align (min_obj_size)))
14614 dprintf (3, ("old alloc: %Ix, only %d bytes > new alloc! Shouldn't happen", old_loc, dist));
14618 pad = Align (min_obj_size);
14619 set_plug_padded (old_loc);
14622 #endif //SHORT_PLUGS
14623 #ifdef FEATURE_STRUCTALIGN
14624 _ASSERTE(!old_loc || alignmentOffset != 0);
14625 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
14626 if ((old_loc != 0))
14628 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
14629 set_node_aligninfo (old_loc, requiredAlignment, pad1);
14632 #else // FEATURE_STRUCTALIGN
14633 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
14635 pad += switch_alignment_size (is_plug_padded (old_loc));
14636 set_node_realigned(old_loc);
14637 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
14638 (size_t)old_loc, (size_t)(result+pad)));
14639 assert (same_large_alignment_p (result + pad, old_loc));
14641 #endif // FEATURE_STRUCTALIGN
14644 if ((next_pinned_plug != 0) && (pad != 0) && (generation_allocation_segment (gen) == current_seg))
14646 assert (old_loc != 0);
14647 ptrdiff_t dist_to_next_pin = (ptrdiff_t)(next_pinned_plug - (generation_allocation_pointer (gen) + size + pad));
14648 assert (dist_to_next_pin >= 0);
14650 if ((dist_to_next_pin >= 0) && (dist_to_next_pin < (ptrdiff_t)Align (min_obj_size)))
14652 dprintf (3, ("%Ix->(%Ix,%Ix),%Ix(%Ix)(%Ix),NP->PP",
14654 generation_allocation_pointer (gen),
14655 generation_allocation_limit (gen),
14658 dist_to_next_pin));
14659 clear_plug_padded (old_loc);
14661 *convert_to_pinned_p = TRUE;
14662 record_interesting_data_point (idp_converted_pin);
14667 #endif //SHORT_PLUGS
14669 if ((old_loc == 0) || (pad != 0))
14671 //allocating a non plug or a gap, so reset the start region
14672 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14675 generation_allocation_pointer (gen) += size + pad;
14676 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
14678 #ifdef FREE_USAGE_STATS
14679 generation_allocated_since_last_pin (gen) += size;
14680 #endif //FREE_USAGE_STATS
14682 dprintf (3, ("aic: ptr: %Ix, limit: %Ix, sr: %Ix",
14683 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14684 generation_allocation_context_start_region (gen)));
14686 assert (result + pad);
14687 return result + pad;
14691 inline int power (int x, int y)
14694 for (int i = 0; i < y; i++)
14701 int gc_heap::joined_generation_to_condemn (BOOL should_evaluate_elevation,
14704 BOOL* blocking_collection_p
14705 STRESS_HEAP_ARG(int n_original))
14707 int n = current_gen;
14708 #ifdef MULTIPLE_HEAPS
14709 BOOL joined_last_gc_before_oom = FALSE;
14710 for (int i = 0; i < n_heaps; i++)
14712 if (g_heaps[i]->last_gc_before_oom)
14714 dprintf (GTC_LOG, ("h%d is setting blocking to TRUE", i));
14715 joined_last_gc_before_oom = TRUE;
14720 BOOL joined_last_gc_before_oom = last_gc_before_oom;
14721 #endif //MULTIPLE_HEAPS
14723 if (joined_last_gc_before_oom && settings.pause_mode != pause_low_latency)
14725 assert (*blocking_collection_p);
14728 if (should_evaluate_elevation && (n == max_generation))
14730 dprintf (GTC_LOG, ("lock: %d(%d)",
14731 (settings.should_lock_elevation ? 1 : 0),
14732 settings.elevation_locked_count));
14734 if (settings.should_lock_elevation)
14736 settings.elevation_locked_count++;
14737 if (settings.elevation_locked_count == 6)
14739 settings.elevation_locked_count = 0;
14743 n = max_generation - 1;
14744 settings.elevation_reduced = TRUE;
14749 settings.elevation_locked_count = 0;
14754 settings.should_lock_elevation = FALSE;
14755 settings.elevation_locked_count = 0;
14758 if (provisional_mode_triggered && (n == max_generation))
14760 // There are a few cases where we should not reduce the generation.
14761 if ((initial_gen == max_generation) || (settings.reason == reason_alloc_loh))
14763 // If we are doing a full GC in the provisional mode, we always
14764 // make it blocking because we don't want to get into a situation
14765 // where foreground GCs are asking for a compacting full GC right away
14766 // and not getting it.
14767 dprintf (GTC_LOG, ("full GC induced, not reducing gen"));
14768 *blocking_collection_p = TRUE;
14770 else if (should_expand_in_full_gc || joined_last_gc_before_oom)
14772 dprintf (GTC_LOG, ("need full blocking GCs to expand heap or avoid OOM, not reducing gen"));
14773 assert (*blocking_collection_p);
14777 dprintf (GTC_LOG, ("reducing gen in PM: %d->%d->%d", initial_gen, n, (max_generation - 1)));
14778 n = max_generation - 1;
14782 if (should_expand_in_full_gc)
14784 should_expand_in_full_gc = FALSE;
14787 if (heap_hard_limit)
14789 // If we have already consumed 90% of the limit, we should check to see if we should compact LOH.
14790 // TODO: should unify this with gen2.
14791 dprintf (GTC_LOG, ("committed %Id is %d%% of limit %Id",
14792 current_total_committed, (int)((float)current_total_committed * 100.0 / (float)heap_hard_limit),
14794 if ((current_total_committed * 10) >= (heap_hard_limit * 9))
14796 bool full_compact_gc_p = false;
14798 size_t loh_frag = get_total_gen_fragmentation (max_generation + 1);
14800 // If the LOH frag is >= 1/8 it's worth compacting it
14801 if ((loh_frag * 8) >= heap_hard_limit)
14803 dprintf (GTC_LOG, ("loh frag: %Id > 1/8 of limit %Id", loh_frag, (heap_hard_limit / 8)));
14804 full_compact_gc_p = true;
14808 // If there's not much fragmentation but it looks like it'll be productive to
14809 // collect LOH, do that.
14810 size_t est_loh_reclaim = get_total_gen_estimated_reclaim (max_generation + 1);
14811 full_compact_gc_p = ((est_loh_reclaim * 8) >= heap_hard_limit);
14812 dprintf (GTC_LOG, ("loh est reclaim: %Id, 1/8 of limit %Id", est_loh_reclaim, (heap_hard_limit / 8)));
14815 if (full_compact_gc_p)
14817 n = max_generation;
14818 *blocking_collection_p = TRUE;
14819 settings.loh_compaction = TRUE;
14820 dprintf (GTC_LOG, ("compacting LOH due to hard limit"));
14825 if ((n == max_generation) && (*blocking_collection_p == FALSE))
14827 // If we are doing a gen2 we should reset elevation regardless and let the gen2
14828 // decide if we should lock again or in the bgc case by design we will not retract
14830 settings.should_lock_elevation = FALSE;
14831 settings.elevation_locked_count = 0;
14832 dprintf (1, ("doing bgc, reset elevation"));
14836 #ifdef BACKGROUND_GC
14837 // We can only do Concurrent GC Stress if the caller did not explicitly ask for all
14838 // generations to be collected,
14840 // [LOCALGC TODO] STRESS_HEAP is not defined for a standalone GC so there are multiple
14841 // things that need to be fixed in this code block.
14842 if (n_original != max_generation &&
14843 g_pConfig->GetGCStressLevel() && gc_can_use_concurrent)
14845 #ifndef FEATURE_REDHAWK
14846 // for the GC stress mix mode throttle down gen2 collections
14847 if (g_pConfig->IsGCStressMix())
14849 size_t current_gc_count = 0;
14851 #ifdef MULTIPLE_HEAPS
14852 current_gc_count = (size_t)dd_collection_count (g_heaps[0]->dynamic_data_of (0));
14854 current_gc_count = (size_t)dd_collection_count (dynamic_data_of (0));
14855 #endif //MULTIPLE_HEAPS
14856 // in gc stress, only escalate every 10th non-gen2 collection to a gen2...
14857 if ((current_gc_count % 10) == 0)
14859 n = max_generation;
14862 // for traditional GC stress
14864 #endif // !FEATURE_REDHAWK
14865 if (*blocking_collection_p)
14867 // We call StressHeap() a lot for Concurrent GC Stress. However,
14868 // if we can not do a concurrent collection, no need to stress anymore.
14869 // @TODO: Enable stress when the memory pressure goes down again
14870 GCStressPolicy::GlobalDisable();
14874 n = max_generation;
14877 #endif //BACKGROUND_GC
14878 #endif //STRESS_HEAP
14884 size_t get_survived_size (gc_history_per_heap* hist)
14886 size_t surv_size = 0;
14887 gc_generation_data* gen_data;
14889 for (int gen_number = 0; gen_number <= (max_generation + 1); gen_number++)
14891 gen_data = &(hist->gen_data[gen_number]);
14892 surv_size += (gen_data->size_after -
14893 gen_data->free_list_space_after -
14894 gen_data->free_obj_space_after);
14900 size_t gc_heap::get_total_survived_size()
14902 size_t total_surv_size = 0;
14903 #ifdef MULTIPLE_HEAPS
14904 for (int i = 0; i < gc_heap::n_heaps; i++)
14906 gc_heap* hp = gc_heap::g_heaps[i];
14907 gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap();
14908 total_surv_size += get_survived_size (current_gc_data_per_heap);
14911 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
14912 total_surv_size = get_survived_size (current_gc_data_per_heap);
14913 #endif //MULTIPLE_HEAPS
14914 return total_surv_size;
14917 size_t gc_heap::get_total_allocated_since_last_gc()
14919 size_t total_allocated_size = 0;
14920 #ifdef MULTIPLE_HEAPS
14921 for (int i = 0; i < gc_heap::n_heaps; i++)
14923 gc_heap* hp = gc_heap::g_heaps[i];
14924 total_allocated_size += hp->allocated_since_last_gc;
14925 hp->allocated_since_last_gc = 0;
14928 total_allocated_size = allocated_since_last_gc;
14929 allocated_since_last_gc = 0;
14930 #endif //MULTIPLE_HEAPS
14931 return total_allocated_size;
14934 // Gets what's allocated on both SOH and LOH that hasn't been collected.
14935 size_t gc_heap::get_current_allocated()
14937 dynamic_data* dd = dynamic_data_of (0);
14938 size_t current_alloc = dd_desired_allocation (dd) - dd_new_allocation (dd);
14939 dd = dynamic_data_of (max_generation + 1);
14940 current_alloc += dd_desired_allocation (dd) - dd_new_allocation (dd);
14942 return current_alloc;
14945 size_t gc_heap::get_total_allocated()
14947 size_t total_current_allocated = 0;
14948 #ifdef MULTIPLE_HEAPS
14949 for (int i = 0; i < gc_heap::n_heaps; i++)
14951 gc_heap* hp = gc_heap::g_heaps[i];
14952 total_current_allocated += hp->get_current_allocated();
14955 total_current_allocated = get_current_allocated();
14956 #endif //MULTIPLE_HEAPS
14957 return total_current_allocated;
14960 size_t gc_heap::current_generation_size (int gen_number)
14962 dynamic_data* dd = dynamic_data_of (gen_number);
14963 size_t gen_size = (dd_current_size (dd) + dd_desired_allocation (dd)
14964 - dd_new_allocation (dd));
14970 #pragma warning(push)
14971 #pragma warning(disable:6326) // "Potential comparison of a constant with another constant" is intentional in this function.
14975 This is called by when we are actually doing a GC, or when we are just checking whether
14976 we would do a full blocking GC, in which case check_only_p is TRUE.
14978 The difference between calling this with check_only_p TRUE and FALSE is that when it's
14980 settings.reason is ignored
14981 budgets are not checked (since they are checked before this is called)
14982 it doesn't change anything non local like generation_skip_ratio
14984 int gc_heap::generation_to_condemn (int n_initial,
14985 BOOL* blocking_collection_p,
14986 BOOL* elevation_requested_p,
14989 gc_mechanisms temp_settings = settings;
14990 gen_to_condemn_tuning temp_condemn_reasons;
14991 gc_mechanisms* local_settings = (check_only_p ? &temp_settings : &settings);
14992 gen_to_condemn_tuning* local_condemn_reasons = (check_only_p ? &temp_condemn_reasons : &gen_to_condemn_reasons);
14995 if ((local_settings->reason == reason_oos_soh) || (local_settings->reason == reason_oos_loh))
14997 assert (n_initial >= 1);
15000 assert (settings.reason != reason_empty);
15003 local_condemn_reasons->init();
15007 if (heap_number == 0)
15009 dprintf (GTC_LOG, ("init: %d(%d)", n_initial, settings.reason));
15013 BOOL low_memory_detected = g_low_memory_status;
15014 uint32_t memory_load = 0;
15015 uint64_t available_physical = 0;
15016 uint64_t available_page_file = 0;
15017 BOOL check_memory = FALSE;
15018 BOOL high_fragmentation = FALSE;
15019 BOOL v_high_memory_load = FALSE;
15020 BOOL high_memory_load = FALSE;
15021 BOOL low_ephemeral_space = FALSE;
15022 BOOL evaluate_elevation = TRUE;
15023 *elevation_requested_p = FALSE;
15024 *blocking_collection_p = FALSE;
15026 BOOL check_max_gen_alloc = TRUE;
15030 #endif //STRESS_HEAP
15034 dd_fragmentation (dynamic_data_of (0)) =
15035 generation_free_list_space (youngest_generation) +
15036 generation_free_obj_space (youngest_generation);
15038 dd_fragmentation (dynamic_data_of (max_generation + 1)) =
15039 generation_free_list_space (large_object_generation) +
15040 generation_free_obj_space (large_object_generation);
15042 //save new_allocation
15043 for (i = 0; i <= max_generation+1; i++)
15045 dynamic_data* dd = dynamic_data_of (i);
15046 dprintf (GTC_LOG, ("h%d: g%d: l: %Id (%Id)",
15048 dd_new_allocation (dd),
15049 dd_desired_allocation (dd)));
15050 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
15053 local_condemn_reasons->set_gen (gen_initial, n);
15056 #ifdef BACKGROUND_GC
15057 if (recursive_gc_sync::background_running_p())
15059 dprintf (GTC_LOG, ("bgc in prog, 1"));
15060 check_max_gen_alloc = FALSE;
15062 #endif //BACKGROUND_GC
15064 if (check_max_gen_alloc)
15066 //figure out if large objects need to be collected.
15067 if (get_new_allocation (max_generation+1) <= 0)
15069 n = max_generation;
15070 local_condemn_reasons->set_gen (gen_alloc_budget, n);
15074 //figure out which generation ran out of allocation
15075 for (i = n+1; i <= (check_max_gen_alloc ? max_generation : (max_generation - 1)); i++)
15077 if (get_new_allocation (i) <= 0)
15088 local_condemn_reasons->set_gen (gen_alloc_budget, n);
15091 dprintf (GTC_LOG, ("h%d: g%d budget", heap_number, ((get_new_allocation (max_generation+1) <= 0) ? 3 : n)));
15095 #if defined(BACKGROUND_GC) && !defined(MULTIPLE_HEAPS)
15096 //time based tuning
15097 // if enough time has elapsed since the last gc
15098 // and the number of gc is too low (1/10 of lower gen) then collect
15099 // This should also be enabled if we have memory concerns
15100 int n_time_max = max_generation;
15104 if (recursive_gc_sync::background_running_p())
15106 n_time_max = max_generation - 1;
15110 if ((local_settings->pause_mode == pause_interactive) ||
15111 (local_settings->pause_mode == pause_sustained_low_latency))
15113 dynamic_data* dd0 = dynamic_data_of (0);
15114 size_t now = GetHighPrecisionTimeStamp();
15116 for (i = (temp_gen+1); i <= n_time_max; i++)
15118 dynamic_data* dd = dynamic_data_of (i);
15119 if ((now > dd_time_clock(dd) + dd_time_clock_interval(dd)) &&
15120 (dd_gc_clock (dd0) > (dd_gc_clock (dd) + dd_gc_clock_interval(dd))) &&
15121 ((n < max_generation) || ((dd_current_size (dd) < dd_max_size (dd0)))))
15123 n = min (i, n_time_max);
15124 dprintf (GTC_LOG, ("time %d", n));
15129 local_condemn_reasons->set_gen (gen_time_tuning, n);
15135 dprintf (GTC_LOG, ("Condemning %d based on time tuning and fragmentation", n));
15137 #endif //BACKGROUND_GC && !MULTIPLE_HEAPS
15139 if (n < (max_generation - 1))
15141 if (dt_low_card_table_efficiency_p (tuning_deciding_condemned_gen))
15143 n = max (n, max_generation - 1);
15144 local_settings->promotion = TRUE;
15145 dprintf (GTC_LOG, ("h%d: skip %d, c %d",
15146 heap_number, generation_skip_ratio, n));
15147 local_condemn_reasons->set_condition (gen_low_card_p);
15153 generation_skip_ratio = 100;
15156 if (dt_low_ephemeral_space_p (check_only_p ?
15157 tuning_deciding_full_gc :
15158 tuning_deciding_condemned_gen))
15160 low_ephemeral_space = TRUE;
15162 n = max (n, max_generation - 1);
15163 local_condemn_reasons->set_condition (gen_low_ephemeral_p);
15164 dprintf (GTC_LOG, ("h%d: low eph", heap_number));
15166 if (!provisional_mode_triggered)
15168 #ifdef BACKGROUND_GC
15169 if (!gc_can_use_concurrent || (generation_free_list_space (generation_of (max_generation)) == 0))
15170 #endif //BACKGROUND_GC
15172 //It is better to defragment first if we are running out of space for
15173 //the ephemeral generation but we have enough fragmentation to make up for it
15174 //in the non ephemeral generation. Essentially we are trading a gen2 for
15175 // having to expand heap in ephemeral collections.
15176 if (dt_high_frag_p (tuning_deciding_condemned_gen,
15177 max_generation - 1,
15180 high_fragmentation = TRUE;
15181 local_condemn_reasons->set_condition (gen_max_high_frag_e_p);
15182 dprintf (GTC_LOG, ("heap%d: gen1 frag", heap_number));
15188 //figure out which ephemeral generation is too fragramented
15190 for (i = n+1; i < max_generation; i++)
15192 if (dt_high_frag_p (tuning_deciding_condemned_gen, i))
15194 dprintf (GTC_LOG, ("h%d g%d too frag", heap_number, i));
15201 if (low_ephemeral_space)
15204 local_settings->promotion = TRUE;
15209 local_condemn_reasons->set_condition (gen_eph_high_frag_p);
15214 if (settings.pause_mode == pause_low_latency)
15216 if (!is_induced (settings.reason))
15218 n = min (n, max_generation - 1);
15219 dprintf (GTC_LOG, ("low latency mode is enabled, condemning %d", n));
15220 evaluate_elevation = FALSE;
15226 // It's hard to catch when we get to the point that the memory load is so high
15227 // we get an induced GC from the finalizer thread so we are checking the memory load
15228 // for every gen0 GC.
15229 check_memory = (check_only_p ?
15231 ((n >= 1) || low_memory_detected));
15235 //find out if we are short on memory
15236 get_memory_info (&memory_load, &available_physical, &available_page_file);
15237 if (heap_number == 0)
15239 dprintf (GTC_LOG, ("ml: %d", memory_load));
15242 // Need to get it early enough for all heaps to use.
15243 entry_available_physical_mem = available_physical;
15244 local_settings->entry_memory_load = memory_load;
15246 // @TODO: Force compaction more often under GCSTRESS
15247 if (memory_load >= high_memory_load_th || low_memory_detected)
15249 #ifdef SIMPLE_DPRINTF
15250 // stress log can't handle any parameter that's bigger than a void*.
15251 if (heap_number == 0)
15253 dprintf (GTC_LOG, ("tp: %I64d, ap: %I64d", total_physical_mem, available_physical));
15255 #endif //SIMPLE_DPRINTF
15257 high_memory_load = TRUE;
15259 if (memory_load >= v_high_memory_load_th || low_memory_detected)
15261 // TODO: Perhaps in 64-bit we should be estimating gen1's fragmentation as well since
15262 // gen1/gen0 may take a lot more memory than gen2.
15263 if (!high_fragmentation)
15265 high_fragmentation = dt_estimate_reclaim_space_p (tuning_deciding_condemned_gen, max_generation);
15267 v_high_memory_load = TRUE;
15271 if (!high_fragmentation)
15273 high_fragmentation = dt_estimate_high_frag_p (tuning_deciding_condemned_gen, max_generation, available_physical);
15277 if (high_fragmentation)
15279 if (high_memory_load)
15281 local_condemn_reasons->set_condition (gen_max_high_frag_m_p);
15283 else if (v_high_memory_load)
15285 local_condemn_reasons->set_condition (gen_max_high_frag_vm_p);
15291 dprintf (GTC_LOG, ("h%d: le: %d, hm: %d, vm: %d, f: %d",
15292 heap_number, low_ephemeral_space, high_memory_load, v_high_memory_load,
15293 high_fragmentation));
15295 if (should_expand_in_full_gc)
15297 dprintf (GTC_LOG, ("h%d: expand_in_full - BLOCK", heap_number));
15298 *blocking_collection_p = TRUE;
15299 evaluate_elevation = FALSE;
15300 n = max_generation;
15301 local_condemn_reasons->set_condition (gen_expand_fullgc_p);
15304 if (last_gc_before_oom)
15306 dprintf (GTC_LOG, ("h%d: alloc full - BLOCK", heap_number));
15307 n = max_generation;
15308 *blocking_collection_p = TRUE;
15310 if ((local_settings->reason == reason_oos_loh) ||
15311 (local_settings->reason == reason_alloc_loh))
15313 evaluate_elevation = FALSE;
15316 local_condemn_reasons->set_condition (gen_before_oom);
15321 if (is_induced_blocking (settings.reason) &&
15322 n_initial == max_generation
15323 IN_STRESS_HEAP( && !settings.stress_induced ))
15325 if (heap_number == 0)
15327 dprintf (GTC_LOG, ("induced - BLOCK"));
15330 *blocking_collection_p = TRUE;
15331 local_condemn_reasons->set_condition (gen_induced_fullgc_p);
15332 evaluate_elevation = FALSE;
15335 if (settings.reason == reason_induced_noforce)
15337 local_condemn_reasons->set_condition (gen_induced_noforce_p);
15338 evaluate_elevation = FALSE;
15342 if (!provisional_mode_triggered && evaluate_elevation && (low_ephemeral_space || high_memory_load || v_high_memory_load))
15344 *elevation_requested_p = TRUE;
15346 // if we are in high memory load and have consumed 10% of the gen2 budget, do a gen2 now.
15347 if (high_memory_load || v_high_memory_load)
15349 dynamic_data* dd_max = dynamic_data_of (max_generation);
15350 if (((float)dd_new_allocation (dd_max) / (float)dd_desired_allocation (dd_max)) < 0.9)
15352 dprintf (GTC_LOG, ("%Id left in gen2 alloc (%Id)",
15353 dd_new_allocation (dd_max), dd_desired_allocation (dd_max)));
15354 n = max_generation;
15355 local_condemn_reasons->set_condition (gen_almost_max_alloc);
15359 if (n <= max_generation)
15362 if (high_fragmentation)
15364 //elevate to max_generation
15365 n = max_generation;
15366 dprintf (GTC_LOG, ("h%d: f full", heap_number));
15368 #ifdef BACKGROUND_GC
15369 if (high_memory_load || v_high_memory_load)
15371 // For background GC we want to do blocking collections more eagerly because we don't
15372 // want to get into the situation where the memory load becomes high while we are in
15373 // a background GC and we'd have to wait for the background GC to finish to start
15374 // a blocking collection (right now the implemenation doesn't handle converting
15375 // a background GC to a blocking collection midway.
15376 dprintf (GTC_LOG, ("h%d: bgc - BLOCK", heap_number));
15377 *blocking_collection_p = TRUE;
15380 if (v_high_memory_load)
15382 dprintf (GTC_LOG, ("h%d: - BLOCK", heap_number));
15383 *blocking_collection_p = TRUE;
15385 #endif //BACKGROUND_GC
15389 n = max (n, max_generation - 1);
15390 dprintf (GTC_LOG, ("h%d: nf c %d", heap_number, n));
15397 if (!provisional_mode_triggered && (n == (max_generation - 1)) && (n_alloc < (max_generation -1)))
15399 dprintf (GTC_LOG, ("h%d: budget %d, check 2",
15400 heap_number, n_alloc));
15401 if (get_new_allocation (max_generation) <= 0)
15403 dprintf (GTC_LOG, ("h%d: budget alloc", heap_number));
15404 n = max_generation;
15405 local_condemn_reasons->set_condition (gen_max_gen1);
15409 //figure out if max_generation is too fragmented -> blocking collection
15410 if (!provisional_mode_triggered && (n == max_generation))
15412 if (dt_high_frag_p (tuning_deciding_condemned_gen, n))
15414 dprintf (GTC_LOG, ("h%d: g%d too frag", heap_number, n));
15415 local_condemn_reasons->set_condition (gen_max_high_frag_p);
15416 if (local_settings->pause_mode != pause_sustained_low_latency)
15418 *blocking_collection_p = TRUE;
15423 #ifdef BACKGROUND_GC
15424 if (n == max_generation)
15426 if (heap_number == 0)
15428 BOOL bgc_heap_too_small = TRUE;
15429 size_t gen2size = 0;
15430 size_t gen3size = 0;
15431 #ifdef MULTIPLE_HEAPS
15432 for (int i = 0; i < n_heaps; i++)
15434 if (((g_heaps[i]->current_generation_size (max_generation)) > bgc_min_per_heap) ||
15435 ((g_heaps[i]->current_generation_size (max_generation + 1)) > bgc_min_per_heap))
15437 bgc_heap_too_small = FALSE;
15441 #else //MULTIPLE_HEAPS
15442 if ((current_generation_size (max_generation) > bgc_min_per_heap) ||
15443 (current_generation_size (max_generation + 1) > bgc_min_per_heap))
15445 bgc_heap_too_small = FALSE;
15447 #endif //MULTIPLE_HEAPS
15449 if (bgc_heap_too_small)
15451 dprintf (GTC_LOG, ("gen2 and gen3 too small"));
15454 // do not turn stress-induced collections into blocking GCs
15455 if (!settings.stress_induced)
15456 #endif //STRESS_HEAP
15458 *blocking_collection_p = TRUE;
15461 local_condemn_reasons->set_condition (gen_gen2_too_small);
15465 #endif //BACKGROUND_GC
15471 #ifdef BACKGROUND_GC
15472 // We can only do Concurrent GC Stress if the caller did not explicitly ask for all
15473 // generations to be collected,
15475 if (orig_gen != max_generation &&
15476 g_pConfig->GetGCStressLevel() && gc_can_use_concurrent)
15478 *elevation_requested_p = FALSE;
15480 #endif //BACKGROUND_GC
15481 #endif //STRESS_HEAP
15485 fgm_result.available_pagefile_mb = (size_t)(available_page_file / (1024 * 1024));
15488 local_condemn_reasons->set_gen (gen_final_per_heap, n);
15489 get_gc_data_per_heap()->gen_to_condemn_reasons.init (local_condemn_reasons);
15492 local_condemn_reasons->print (heap_number);
15495 if ((local_settings->reason == reason_oos_soh) ||
15496 (local_settings->reason == reason_oos_loh))
15506 #pragma warning(pop)
15510 size_t gc_heap::min_reclaim_fragmentation_threshold (uint32_t num_heaps)
15512 // if the memory load is higher, the threshold we'd want to collect gets lower.
15513 size_t min_mem_based_on_available =
15514 (500 - (settings.entry_memory_load - high_memory_load_th) * 40) * 1024 * 1024 / num_heaps;
15516 size_t ten_percent_size = (size_t)((float)generation_size (max_generation) * 0.10);
15517 uint64_t three_percent_mem = mem_one_percent * 3 / num_heaps;
15519 #ifdef SIMPLE_DPRINTF
15520 dprintf (GTC_LOG, ("min av: %Id, 10%% gen2: %Id, 3%% mem: %I64d",
15521 min_mem_based_on_available, ten_percent_size, three_percent_mem));
15522 #endif //SIMPLE_DPRINTF
15523 return (size_t)(min (min_mem_based_on_available, min (ten_percent_size, three_percent_mem)));
15527 uint64_t gc_heap::min_high_fragmentation_threshold(uint64_t available_mem, uint32_t num_heaps)
15529 return min (available_mem, (256*1024*1024)) / num_heaps;
15533 CORINFO_EXCEPTION_GC = 0xE0004743 // 'GC'
15537 #ifdef BACKGROUND_GC
15538 void gc_heap::init_background_gc ()
15540 //reset the allocation so foreground gc can allocate into older (max_generation) generation
15541 generation* gen = generation_of (max_generation);
15542 generation_allocation_pointer (gen)= 0;
15543 generation_allocation_limit (gen) = 0;
15544 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
15546 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
15548 //reset the plan allocation for each segment
15549 for (heap_segment* seg = generation_allocation_segment (gen); seg != ephemeral_heap_segment;
15550 seg = heap_segment_next_rw (seg))
15552 heap_segment_plan_allocated (seg) = heap_segment_allocated (seg);
15555 if (heap_number == 0)
15557 dprintf (2, ("heap%d: bgc lowest: %Ix, highest: %Ix",
15559 background_saved_lowest_address,
15560 background_saved_highest_address));
15563 gc_lh_block_event.Reset();
15566 #endif //BACKGROUND_GC
15569 void fire_drain_mark_list_event (size_t mark_list_objects)
15571 FIRE_EVENT(BGCDrainMark, mark_list_objects);
15575 void fire_revisit_event (size_t dirtied_pages,
15576 size_t marked_objects,
15577 BOOL large_objects_p)
15579 FIRE_EVENT(BGCRevisit, dirtied_pages, marked_objects, large_objects_p);
15583 void fire_overflow_event (uint8_t* overflow_min,
15584 uint8_t* overflow_max,
15585 size_t marked_objects,
15586 int large_objects_p)
15588 FIRE_EVENT(BGCOverflow, (uint64_t)overflow_min, (uint64_t)overflow_max, marked_objects, large_objects_p);
15591 void gc_heap::concurrent_print_time_delta (const char* msg)
15594 size_t current_time = GetHighPrecisionTimeStamp();
15595 size_t elapsed_time = current_time - time_bgc_last;
15596 time_bgc_last = current_time;
15598 dprintf (2, ("h%d: %s T %Id ms", heap_number, msg, elapsed_time));
15600 UNREFERENCED_PARAMETER(msg);
15604 void gc_heap::free_list_info (int gen_num, const char* msg)
15606 UNREFERENCED_PARAMETER(gen_num);
15607 #if defined (BACKGROUND_GC) && defined (TRACE_GC)
15608 dprintf (3, ("h%d: %s", heap_number, msg));
15609 for (int i = 0; i <= (max_generation + 1); i++)
15611 generation* gen = generation_of (i);
15612 if ((generation_allocation_size (gen) == 0) &&
15613 (generation_free_list_space (gen) == 0) &&
15614 (generation_free_obj_space (gen) == 0))
15616 // don't print if everything is 0.
15620 dprintf (3, ("h%d: g%d: a-%Id, fl-%Id, fo-%Id",
15622 generation_allocation_size (gen),
15623 generation_free_list_space (gen),
15624 generation_free_obj_space (gen)));
15628 UNREFERENCED_PARAMETER(msg);
15629 #endif // BACKGROUND_GC && TRACE_GC
15632 void gc_heap::update_collection_counts_for_no_gc()
15634 assert (settings.pause_mode == pause_no_gc);
15636 settings.condemned_generation = max_generation;
15637 #ifdef MULTIPLE_HEAPS
15638 for (int i = 0; i < n_heaps; i++)
15639 g_heaps[i]->update_collection_counts();
15640 #else //MULTIPLE_HEAPS
15641 update_collection_counts();
15642 #endif //MULTIPLE_HEAPS
15644 full_gc_counts[gc_type_blocking]++;
15647 BOOL gc_heap::should_proceed_with_gc()
15649 if (gc_heap::settings.pause_mode == pause_no_gc)
15651 if (current_no_gc_region_info.started)
15653 // The no_gc mode was already in progress yet we triggered another GC,
15654 // this effectively exits the no_gc mode.
15655 restore_data_for_no_gc();
15658 return should_proceed_for_no_gc();
15664 //internal part of gc used by the serial and concurrent version
15665 void gc_heap::gc1()
15667 #ifdef BACKGROUND_GC
15668 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15669 #endif //BACKGROUND_GC
15672 mark_time = plan_time = reloc_time = compact_time = sweep_time = 0;
15675 verify_soh_segment_list();
15677 int n = settings.condemned_generation;
15679 if (settings.reason == reason_pm_full_gc)
15681 assert (n == max_generation);
15684 gen_to_condemn_tuning* local_condemn_reasons = &(get_gc_data_per_heap()->gen_to_condemn_reasons);
15685 local_condemn_reasons->init();
15686 local_condemn_reasons->set_gen (gen_initial, n);
15687 local_condemn_reasons->set_gen (gen_final_per_heap, n);
15690 update_collection_counts ();
15692 #ifdef BACKGROUND_GC
15693 bgc_alloc_lock->check();
15694 #endif //BACKGROUND_GC
15696 free_list_info (max_generation, "beginning");
15698 vm_heap->GcCondemnedGeneration = settings.condemned_generation;
15700 assert (g_gc_card_table == card_table);
15702 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
15703 assert (g_gc_card_bundle_table == card_bundle_table);
15707 if (n == max_generation)
15709 gc_low = lowest_address;
15710 gc_high = highest_address;
15714 gc_low = generation_allocation_start (generation_of (n));
15715 gc_high = heap_segment_reserved (ephemeral_heap_segment);
15717 #ifdef BACKGROUND_GC
15718 if (settings.concurrent)
15721 time_bgc_last = GetHighPrecisionTimeStamp();
15724 FIRE_EVENT(BGCBegin);
15726 concurrent_print_time_delta ("BGC");
15728 //#ifdef WRITE_WATCH
15729 //reset_write_watch (FALSE);
15730 //#endif //WRITE_WATCH
15732 concurrent_print_time_delta ("RW");
15733 background_mark_phase();
15734 free_list_info (max_generation, "after mark phase");
15736 background_sweep();
15737 free_list_info (max_generation, "after sweep phase");
15740 #endif //BACKGROUND_GC
15742 mark_phase (n, FALSE);
15744 GCScan::GcRuntimeStructuresValid (FALSE);
15746 GCScan::GcRuntimeStructuresValid (TRUE);
15750 size_t end_gc_time = GetHighPrecisionTimeStamp();
15751 // printf ("generation: %d, elapsed time: %Id\n", n, end_gc_time - dd_time_clock (dynamic_data_of (0)));
15753 //adjust the allocation size from the pinned quantities.
15754 for (int gen_number = 0; gen_number <= min (max_generation,n+1); gen_number++)
15756 generation* gn = generation_of (gen_number);
15757 if (settings.compaction)
15759 generation_pinned_allocated (gn) += generation_pinned_allocation_compact_size (gn);
15760 generation_allocation_size (generation_of (gen_number)) += generation_pinned_allocation_compact_size (gn);
15764 generation_pinned_allocated (gn) += generation_pinned_allocation_sweep_size (gn);
15765 generation_allocation_size (generation_of (gen_number)) += generation_pinned_allocation_sweep_size (gn);
15767 generation_pinned_allocation_sweep_size (gn) = 0;
15768 generation_pinned_allocation_compact_size (gn) = 0;
15771 #ifdef BACKGROUND_GC
15772 if (settings.concurrent)
15774 dynamic_data* dd = dynamic_data_of (n);
15775 dd_gc_elapsed_time (dd) = end_gc_time - dd_time_clock (dd);
15777 free_list_info (max_generation, "after computing new dynamic data");
15779 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
15781 for (int gen_number = 0; gen_number < max_generation; gen_number++)
15783 dprintf (2, ("end of BGC: gen%d new_alloc: %Id",
15784 gen_number, dd_desired_allocation (dynamic_data_of (gen_number))));
15785 current_gc_data_per_heap->gen_data[gen_number].size_after = generation_size (gen_number);
15786 current_gc_data_per_heap->gen_data[gen_number].free_list_space_after = generation_free_list_space (generation_of (gen_number));
15787 current_gc_data_per_heap->gen_data[gen_number].free_obj_space_after = generation_free_obj_space (generation_of (gen_number));
15791 #endif //BACKGROUND_GC
15793 free_list_info (max_generation, "end");
15794 for (int gen_number = 0; gen_number <= n; gen_number++)
15796 dynamic_data* dd = dynamic_data_of (gen_number);
15797 dd_gc_elapsed_time (dd) = end_gc_time - dd_time_clock (dd);
15798 compute_new_dynamic_data (gen_number);
15801 if (n != max_generation)
15803 int gen_num_for_data = ((n < (max_generation - 1)) ? (n + 1) : (max_generation + 1));
15804 for (int gen_number = (n + 1); gen_number <= gen_num_for_data; gen_number++)
15806 get_gc_data_per_heap()->gen_data[gen_number].size_after = generation_size (gen_number);
15807 get_gc_data_per_heap()->gen_data[gen_number].free_list_space_after = generation_free_list_space (generation_of (gen_number));
15808 get_gc_data_per_heap()->gen_data[gen_number].free_obj_space_after = generation_free_obj_space (generation_of (gen_number));
15812 get_gc_data_per_heap()->maxgen_size_info.running_free_list_efficiency = (uint32_t)(generation_allocator_efficiency (generation_of (max_generation)) * 100);
15814 free_list_info (max_generation, "after computing new dynamic data");
15816 if (heap_number == 0)
15818 dprintf (GTC_LOG, ("GC#%d(gen%d) took %Idms",
15819 dd_collection_count (dynamic_data_of (0)),
15820 settings.condemned_generation,
15821 dd_gc_elapsed_time (dynamic_data_of (0))));
15824 for (int gen_number = 0; gen_number <= (max_generation + 1); gen_number++)
15826 dprintf (2, ("end of FGC/NGC: gen%d new_alloc: %Id",
15827 gen_number, dd_desired_allocation (dynamic_data_of (gen_number))));
15831 if (n < max_generation)
15833 compute_promoted_allocation (1 + n);
15835 dynamic_data* dd = dynamic_data_of (1 + n);
15836 size_t new_fragmentation = generation_free_list_space (generation_of (1 + n)) +
15837 generation_free_obj_space (generation_of (1 + n));
15839 #ifdef BACKGROUND_GC
15840 if (current_c_gc_state != c_gc_state_planning)
15841 #endif //BACKGROUND_GC
15843 if (settings.promotion)
15845 dd_fragmentation (dd) = new_fragmentation;
15849 //assert (dd_fragmentation (dd) == new_fragmentation);
15854 #ifdef BACKGROUND_GC
15855 if (!settings.concurrent)
15856 #endif //BACKGROUND_GC
15858 #ifndef FEATURE_REDHAWK
15859 // GCToEEInterface::IsGCThread() always returns false on CoreRT, but this assert is useful in CoreCLR.
15860 assert(GCToEEInterface::IsGCThread());
15861 #endif // FEATURE_REDHAWK
15862 adjust_ephemeral_limits();
15865 #ifdef BACKGROUND_GC
15866 assert (ephemeral_low == generation_allocation_start (generation_of ( max_generation -1)));
15867 assert (ephemeral_high == heap_segment_reserved (ephemeral_heap_segment));
15868 #endif //BACKGROUND_GC
15870 if (fgn_maxgen_percent)
15872 if (settings.condemned_generation == (max_generation - 1))
15874 check_for_full_gc (max_generation - 1, 0);
15876 else if (settings.condemned_generation == max_generation)
15878 if (full_gc_approach_event_set
15879 #ifdef MULTIPLE_HEAPS
15880 && (heap_number == 0)
15881 #endif //MULTIPLE_HEAPS
15884 dprintf (2, ("FGN-GC: setting gen2 end event"));
15886 full_gc_approach_event.Reset();
15887 #ifdef BACKGROUND_GC
15888 // By definition WaitForFullGCComplete only succeeds if it's full, *blocking* GC, otherwise need to return N/A
15889 fgn_last_gc_was_concurrent = settings.concurrent ? TRUE : FALSE;
15890 #endif //BACKGROUND_GC
15891 full_gc_end_event.Set();
15892 full_gc_approach_event_set = false;
15897 #ifdef BACKGROUND_GC
15898 if (!settings.concurrent)
15899 #endif //BACKGROUND_GC
15901 //decide on the next allocation quantum
15902 if (alloc_contexts_used >= 1)
15904 allocation_quantum = Align (min ((size_t)CLR_SIZE,
15905 (size_t)max (1024, get_new_allocation (0) / (2 * alloc_contexts_used))),
15906 get_alignment_constant(FALSE));
15907 dprintf (3, ("New allocation quantum: %d(0x%Ix)", allocation_quantum, allocation_quantum));
15911 descr_generations (FALSE);
15913 verify_soh_segment_list();
15915 #ifdef BACKGROUND_GC
15916 add_to_history_per_heap();
15917 if (heap_number == 0)
15921 #endif // BACKGROUND_GC
15924 if (GCStatistics::Enabled() && heap_number == 0)
15925 g_GCStatistics.AddGCStats(settings,
15926 dd_gc_elapsed_time(dynamic_data_of(settings.condemned_generation)));
15930 fprintf (stdout, "%d,%d,%d,%d,%d,%d\n",
15931 n, mark_time, plan_time, reloc_time, compact_time, sweep_time);
15934 #ifdef BACKGROUND_GC
15935 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15936 #endif //BACKGROUND_GC
15938 #if defined(VERIFY_HEAP) || (defined (FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC))
15941 // Note that right now g_pConfig->GetHeapVerifyLevel always returns the same
15942 // value. If we ever allow randomly adjusting this as the process runs,
15943 // we cannot call it this way as joins need to match - we must have the same
15944 // value for all heaps like we do with bgc_heap_walk_for_etw_p.
15945 || (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
15947 #if defined(FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC)
15948 || (bgc_heap_walk_for_etw_p && settings.concurrent)
15952 #ifdef BACKGROUND_GC
15953 bool cooperative_mode = true;
15955 if (settings.concurrent)
15957 cooperative_mode = enable_preemptive ();
15959 #ifdef MULTIPLE_HEAPS
15960 bgc_t_join.join(this, gc_join_suspend_ee_verify);
15961 if (bgc_t_join.joined())
15963 bgc_threads_sync_event.Reset();
15965 dprintf(2, ("Joining BGC threads to suspend EE for verify heap"));
15966 bgc_t_join.restart();
15968 if (heap_number == 0)
15971 bgc_threads_sync_event.Set();
15975 bgc_threads_sync_event.Wait(INFINITE, FALSE);
15976 dprintf (2, ("bgc_threads_sync_event is signalled"));
15980 #endif //MULTIPLE_HEAPS
15982 //fix the allocation area so verify_heap can proceed.
15983 fix_allocation_contexts (FALSE);
15985 #endif //BACKGROUND_GC
15987 #ifdef BACKGROUND_GC
15988 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15989 #ifdef FEATURE_EVENT_TRACE
15990 if (bgc_heap_walk_for_etw_p && settings.concurrent)
15992 GCToEEInterface::DiagWalkBGCSurvivors(__this);
15994 #ifdef MULTIPLE_HEAPS
15995 bgc_t_join.join(this, gc_join_after_profiler_heap_walk);
15996 if (bgc_t_join.joined())
15998 bgc_t_join.restart();
16000 #endif // MULTIPLE_HEAPS
16002 #endif // FEATURE_EVENT_TRACE
16003 #endif //BACKGROUND_GC
16006 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
16007 verify_heap (FALSE);
16008 #endif // VERIFY_HEAP
16010 #ifdef BACKGROUND_GC
16011 if (settings.concurrent)
16013 repair_allocation_contexts (TRUE);
16015 #ifdef MULTIPLE_HEAPS
16016 bgc_t_join.join(this, gc_join_restart_ee_verify);
16017 if (bgc_t_join.joined())
16019 bgc_threads_sync_event.Reset();
16021 dprintf(2, ("Joining BGC threads to restart EE after verify heap"));
16022 bgc_t_join.restart();
16024 if (heap_number == 0)
16027 bgc_threads_sync_event.Set();
16031 bgc_threads_sync_event.Wait(INFINITE, FALSE);
16032 dprintf (2, ("bgc_threads_sync_event is signalled"));
16036 #endif //MULTIPLE_HEAPS
16038 disable_preemptive (cooperative_mode);
16040 #endif //BACKGROUND_GC
16042 #endif // defined(VERIFY_HEAP) || (defined(FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC))
16044 #ifdef MULTIPLE_HEAPS
16045 if (!settings.concurrent)
16047 gc_t_join.join(this, gc_join_done);
16048 if (gc_t_join.joined ())
16050 gc_heap::internal_gc_done = false;
16052 //equalize the new desired size of the generations
16053 int limit = settings.condemned_generation;
16054 if (limit == max_generation)
16056 limit = max_generation+1;
16058 for (int gen = 0; gen <= limit; gen++)
16060 size_t total_desired = 0;
16062 for (int i = 0; i < gc_heap::n_heaps; i++)
16064 gc_heap* hp = gc_heap::g_heaps[i];
16065 dynamic_data* dd = hp->dynamic_data_of (gen);
16066 size_t temp_total_desired = total_desired + dd_desired_allocation (dd);
16067 if (temp_total_desired < total_desired)
16070 total_desired = (size_t)MAX_PTR;
16073 total_desired = temp_total_desired;
16076 size_t desired_per_heap = Align (total_desired/gc_heap::n_heaps,
16077 get_alignment_constant ((gen != (max_generation+1))));
16081 #if 1 //subsumed by the linear allocation model
16082 // to avoid spikes in mem usage due to short terms fluctuations in survivorship,
16083 // apply some smoothing.
16084 static size_t smoothed_desired_per_heap = 0;
16085 size_t smoothing = 3; // exponential smoothing factor
16086 if (smoothing > VolatileLoad(&settings.gc_index))
16087 smoothing = VolatileLoad(&settings.gc_index);
16088 smoothed_desired_per_heap = desired_per_heap / smoothing + ((smoothed_desired_per_heap / smoothing) * (smoothing-1));
16089 dprintf (1, ("sn = %Id n = %Id", smoothed_desired_per_heap, desired_per_heap));
16090 desired_per_heap = Align(smoothed_desired_per_heap, get_alignment_constant (true));
16093 if (!heap_hard_limit)
16095 // if desired_per_heap is close to min_gc_size, trim it
16096 // down to min_gc_size to stay in the cache
16097 gc_heap* hp = gc_heap::g_heaps[0];
16098 dynamic_data* dd = hp->dynamic_data_of (gen);
16099 size_t min_gc_size = dd_min_size(dd);
16100 // if min GC size larger than true on die cache, then don't bother
16101 // limiting the desired size
16102 if ((min_gc_size <= GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE)) &&
16103 desired_per_heap <= 2*min_gc_size)
16105 desired_per_heap = min_gc_size;
16109 desired_per_heap = joined_youngest_desired (desired_per_heap);
16110 dprintf (2, ("final gen0 new_alloc: %Id", desired_per_heap));
16112 gc_data_global.final_youngest_desired = desired_per_heap;
16114 #if 1 //subsumed by the linear allocation model
16115 if (gen == (max_generation + 1))
16117 // to avoid spikes in mem usage due to short terms fluctuations in survivorship,
16118 // apply some smoothing.
16119 static size_t smoothed_desired_per_heap_loh = 0;
16120 size_t smoothing = 3; // exponential smoothing factor
16121 size_t loh_count = dd_collection_count (dynamic_data_of (max_generation));
16122 if (smoothing > loh_count)
16123 smoothing = loh_count;
16124 smoothed_desired_per_heap_loh = desired_per_heap / smoothing + ((smoothed_desired_per_heap_loh / smoothing) * (smoothing-1));
16125 dprintf (2, ("smoothed_desired_per_heap_loh = %Id desired_per_heap = %Id", smoothed_desired_per_heap_loh, desired_per_heap));
16126 desired_per_heap = Align(smoothed_desired_per_heap_loh, get_alignment_constant (false));
16129 for (int i = 0; i < gc_heap::n_heaps; i++)
16131 gc_heap* hp = gc_heap::g_heaps[i];
16132 dynamic_data* dd = hp->dynamic_data_of (gen);
16133 dd_desired_allocation (dd) = desired_per_heap;
16134 dd_gc_new_allocation (dd) = desired_per_heap;
16135 dd_new_allocation (dd) = desired_per_heap;
16139 hp->fgn_last_alloc = desired_per_heap;
16144 #ifdef FEATURE_LOH_COMPACTION
16145 BOOL all_heaps_compacted_p = TRUE;
16146 #endif //FEATURE_LOH_COMPACTION
16147 for (int i = 0; i < gc_heap::n_heaps; i++)
16149 gc_heap* hp = gc_heap::g_heaps[i];
16150 hp->decommit_ephemeral_segment_pages();
16151 hp->rearrange_large_heap_segments();
16152 #ifdef FEATURE_LOH_COMPACTION
16153 all_heaps_compacted_p &= hp->loh_compacted_p;
16154 #endif //FEATURE_LOH_COMPACTION
16157 #ifdef FEATURE_LOH_COMPACTION
16158 check_loh_compact_mode (all_heaps_compacted_p);
16159 #endif //FEATURE_LOH_COMPACTION
16162 pm_full_gc_init_or_clear();
16164 gc_t_join.restart();
16166 alloc_context_count = 0;
16167 heap_select::mark_heap (heap_number);
16171 gc_data_global.final_youngest_desired =
16172 dd_desired_allocation (dynamic_data_of (0));
16174 check_loh_compact_mode (loh_compacted_p);
16176 decommit_ephemeral_segment_pages();
16179 if (!(settings.concurrent))
16181 rearrange_large_heap_segments();
16185 pm_full_gc_init_or_clear();
16187 #ifdef BACKGROUND_GC
16188 recover_bgc_settings();
16189 #endif //BACKGROUND_GC
16190 #endif //MULTIPLE_HEAPS
16193 void gc_heap::save_data_for_no_gc()
16195 current_no_gc_region_info.saved_pause_mode = settings.pause_mode;
16196 #ifdef MULTIPLE_HEAPS
16197 // This is to affect heap balancing.
16198 for (int i = 0; i < n_heaps; i++)
16200 current_no_gc_region_info.saved_gen0_min_size = dd_min_size (g_heaps[i]->dynamic_data_of (0));
16201 dd_min_size (g_heaps[i]->dynamic_data_of (0)) = min_balance_threshold;
16202 current_no_gc_region_info.saved_gen3_min_size = dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1));
16203 dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1)) = 0;
16205 #endif //MULTIPLE_HEAPS
16208 void gc_heap::restore_data_for_no_gc()
16210 gc_heap::settings.pause_mode = current_no_gc_region_info.saved_pause_mode;
16211 #ifdef MULTIPLE_HEAPS
16212 for (int i = 0; i < n_heaps; i++)
16214 dd_min_size (g_heaps[i]->dynamic_data_of (0)) = current_no_gc_region_info.saved_gen0_min_size;
16215 dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1)) = current_no_gc_region_info.saved_gen3_min_size;
16217 #endif //MULTIPLE_HEAPS
16220 start_no_gc_region_status gc_heap::prepare_for_no_gc_region (uint64_t total_size,
16221 BOOL loh_size_known,
16223 BOOL disallow_full_blocking)
16225 if (current_no_gc_region_info.started)
16227 return start_no_gc_in_progress;
16230 start_no_gc_region_status status = start_no_gc_success;
16232 save_data_for_no_gc();
16233 settings.pause_mode = pause_no_gc;
16234 current_no_gc_region_info.start_status = start_no_gc_success;
16236 uint64_t allocation_no_gc_loh = 0;
16237 uint64_t allocation_no_gc_soh = 0;
16238 assert(total_size != 0);
16239 if (loh_size_known)
16241 assert(loh_size != 0);
16242 assert(loh_size <= total_size);
16243 allocation_no_gc_loh = loh_size;
16244 allocation_no_gc_soh = total_size - loh_size;
16248 allocation_no_gc_soh = total_size;
16249 allocation_no_gc_loh = total_size;
16252 int soh_align_const = get_alignment_constant (TRUE);
16253 size_t max_soh_allocated = soh_segment_size - segment_info_size - eph_gen_starts_size;
16254 size_t size_per_heap = 0;
16255 const double scale_factor = 1.05;
16258 #ifdef MULTIPLE_HEAPS
16259 num_heaps = n_heaps;
16260 #endif // MULTIPLE_HEAPS
16262 uint64_t total_allowed_soh_allocation = max_soh_allocated * num_heaps;
16264 // In theory, the upper limit here is the physical memory of the machine, not
16265 // SIZE_T_MAX. This is not true today because total_physical_mem can be
16266 // larger than SIZE_T_MAX if running in wow64 on a machine with more than
16267 // 4GB of RAM. Once Local GC code divergence is resolved and code is flowing
16268 // more freely between branches, it would be good to clean this up to use
16269 // total_physical_mem instead of SIZE_T_MAX.
16270 assert(total_allowed_soh_allocation <= SIZE_T_MAX);
16271 uint64_t total_allowed_loh_allocation = SIZE_T_MAX;
16272 uint64_t total_allowed_soh_alloc_scaled = allocation_no_gc_soh > 0 ? static_cast<uint64_t>(total_allowed_soh_allocation / scale_factor) : 0;
16273 uint64_t total_allowed_loh_alloc_scaled = allocation_no_gc_loh > 0 ? static_cast<uint64_t>(total_allowed_loh_allocation / scale_factor) : 0;
16275 if (allocation_no_gc_soh > total_allowed_soh_alloc_scaled ||
16276 allocation_no_gc_loh > total_allowed_loh_alloc_scaled)
16278 status = start_no_gc_too_large;
16282 if (allocation_no_gc_soh > 0)
16284 allocation_no_gc_soh = static_cast<uint64_t>(allocation_no_gc_soh * scale_factor);
16285 allocation_no_gc_soh = min (allocation_no_gc_soh, total_allowed_soh_alloc_scaled);
16288 if (allocation_no_gc_loh > 0)
16290 allocation_no_gc_loh = static_cast<uint64_t>(allocation_no_gc_loh * scale_factor);
16291 allocation_no_gc_loh = min (allocation_no_gc_loh, total_allowed_loh_alloc_scaled);
16294 if (disallow_full_blocking)
16295 current_no_gc_region_info.minimal_gc_p = TRUE;
16297 if (allocation_no_gc_soh != 0)
16299 current_no_gc_region_info.soh_allocation_size = static_cast<size_t>(allocation_no_gc_soh);
16300 size_per_heap = current_no_gc_region_info.soh_allocation_size;
16301 #ifdef MULTIPLE_HEAPS
16302 size_per_heap /= n_heaps;
16303 for (int i = 0; i < n_heaps; i++)
16305 // due to heap balancing we need to allow some room before we even look to balance to another heap.
16306 g_heaps[i]->soh_allocation_no_gc = min (Align ((size_per_heap + min_balance_threshold), soh_align_const), max_soh_allocated);
16308 #else //MULTIPLE_HEAPS
16309 soh_allocation_no_gc = min (Align (size_per_heap, soh_align_const), max_soh_allocated);
16310 #endif //MULTIPLE_HEAPS
16313 if (allocation_no_gc_loh != 0)
16315 current_no_gc_region_info.loh_allocation_size = static_cast<size_t>(allocation_no_gc_loh);
16316 size_per_heap = current_no_gc_region_info.loh_allocation_size;
16317 #ifdef MULTIPLE_HEAPS
16318 size_per_heap /= n_heaps;
16319 for (int i = 0; i < n_heaps; i++)
16320 g_heaps[i]->loh_allocation_no_gc = Align (size_per_heap, get_alignment_constant (FALSE));
16321 #else //MULTIPLE_HEAPS
16322 loh_allocation_no_gc = Align (size_per_heap, get_alignment_constant (FALSE));
16323 #endif //MULTIPLE_HEAPS
16327 if (status != start_no_gc_success)
16328 restore_data_for_no_gc();
16332 void gc_heap::handle_failure_for_no_gc()
16334 gc_heap::restore_data_for_no_gc();
16335 // sets current_no_gc_region_info.started to FALSE here.
16336 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
16339 start_no_gc_region_status gc_heap::get_start_no_gc_region_status()
16341 return current_no_gc_region_info.start_status;
16344 void gc_heap::record_gcs_during_no_gc()
16346 if (current_no_gc_region_info.started)
16348 current_no_gc_region_info.num_gcs++;
16349 if (is_induced (settings.reason))
16350 current_no_gc_region_info.num_gcs_induced++;
16354 BOOL gc_heap::find_loh_free_for_no_gc()
16356 allocator* loh_allocator = generation_allocator (generation_of (max_generation + 1));
16357 size_t sz_list = loh_allocator->first_bucket_size();
16358 size_t size = loh_allocation_no_gc;
16359 for (unsigned int a_l_idx = 0; a_l_idx < loh_allocator->number_of_buckets(); a_l_idx++)
16361 if ((size < sz_list) || (a_l_idx == (loh_allocator->number_of_buckets()-1)))
16363 uint8_t* free_list = loh_allocator->alloc_list_head_of (a_l_idx);
16366 size_t free_list_size = unused_array_size(free_list);
16368 if (free_list_size > loh_allocation_no_gc)
16370 dprintf (3, ("free item %Ix(%Id) for no gc", (size_t)free_list, free_list_size));
16374 free_list = free_list_slot (free_list);
16377 sz_list = sz_list * 2;
16383 BOOL gc_heap::find_loh_space_for_no_gc()
16385 saved_loh_segment_no_gc = 0;
16387 if (find_loh_free_for_no_gc())
16390 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16394 size_t remaining = heap_segment_reserved (seg) - heap_segment_allocated (seg);
16395 if (remaining >= loh_allocation_no_gc)
16397 saved_loh_segment_no_gc = seg;
16400 seg = heap_segment_next (seg);
16403 if (!saved_loh_segment_no_gc && current_no_gc_region_info.minimal_gc_p)
16405 // If no full GC is allowed, we try to get a new seg right away.
16406 saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc)
16407 #ifdef MULTIPLE_HEAPS
16409 #endif //MULTIPLE_HEAPS
16413 return (saved_loh_segment_no_gc != 0);
16416 BOOL gc_heap::loh_allocated_for_no_gc()
16418 if (!saved_loh_segment_no_gc)
16421 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16424 if (seg == saved_loh_segment_no_gc)
16428 seg = heap_segment_next (seg);
16434 BOOL gc_heap::commit_loh_for_no_gc (heap_segment* seg)
16436 uint8_t* end_committed = heap_segment_allocated (seg) + loh_allocation_no_gc;
16437 assert (end_committed <= heap_segment_reserved (seg));
16438 return (grow_heap_segment (seg, end_committed));
16441 void gc_heap::thread_no_gc_loh_segments()
16443 #ifdef MULTIPLE_HEAPS
16444 for (int i = 0; i < n_heaps; i++)
16446 gc_heap* hp = g_heaps[i];
16447 if (hp->loh_allocated_for_no_gc())
16449 hp->thread_loh_segment (hp->saved_loh_segment_no_gc);
16450 hp->saved_loh_segment_no_gc = 0;
16453 #else //MULTIPLE_HEAPS
16454 if (loh_allocated_for_no_gc())
16456 thread_loh_segment (saved_loh_segment_no_gc);
16457 saved_loh_segment_no_gc = 0;
16459 #endif //MULTIPLE_HEAPS
16462 void gc_heap::set_loh_allocations_for_no_gc()
16464 if (current_no_gc_region_info.loh_allocation_size != 0)
16466 dynamic_data* dd = dynamic_data_of (max_generation + 1);
16467 dd_new_allocation (dd) = loh_allocation_no_gc;
16468 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
16472 void gc_heap::set_soh_allocations_for_no_gc()
16474 if (current_no_gc_region_info.soh_allocation_size != 0)
16476 dynamic_data* dd = dynamic_data_of (0);
16477 dd_new_allocation (dd) = soh_allocation_no_gc;
16478 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
16479 #ifdef MULTIPLE_HEAPS
16480 alloc_context_count = 0;
16481 #endif //MULTIPLE_HEAPS
16485 void gc_heap::set_allocations_for_no_gc()
16487 #ifdef MULTIPLE_HEAPS
16488 for (int i = 0; i < n_heaps; i++)
16490 gc_heap* hp = g_heaps[i];
16491 hp->set_loh_allocations_for_no_gc();
16492 hp->set_soh_allocations_for_no_gc();
16494 #else //MULTIPLE_HEAPS
16495 set_loh_allocations_for_no_gc();
16496 set_soh_allocations_for_no_gc();
16497 #endif //MULTIPLE_HEAPS
16500 BOOL gc_heap::should_proceed_for_no_gc()
16502 BOOL gc_requested = FALSE;
16503 BOOL loh_full_gc_requested = FALSE;
16504 BOOL soh_full_gc_requested = FALSE;
16505 BOOL no_gc_requested = FALSE;
16506 BOOL get_new_loh_segments = FALSE;
16508 if (current_no_gc_region_info.soh_allocation_size)
16510 #ifdef MULTIPLE_HEAPS
16511 for (int i = 0; i < n_heaps; i++)
16513 gc_heap* hp = g_heaps[i];
16514 if ((size_t)(heap_segment_reserved (hp->ephemeral_heap_segment) - hp->alloc_allocated) < hp->soh_allocation_no_gc)
16516 gc_requested = TRUE;
16520 #else //MULTIPLE_HEAPS
16521 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated) < soh_allocation_no_gc)
16522 gc_requested = TRUE;
16523 #endif //MULTIPLE_HEAPS
16527 #ifdef MULTIPLE_HEAPS
16528 for (int i = 0; i < n_heaps; i++)
16530 gc_heap* hp = g_heaps[i];
16531 if (!(hp->grow_heap_segment (hp->ephemeral_heap_segment, (hp->alloc_allocated + hp->soh_allocation_no_gc))))
16533 soh_full_gc_requested = TRUE;
16537 #else //MULTIPLE_HEAPS
16538 if (!grow_heap_segment (ephemeral_heap_segment, (alloc_allocated + soh_allocation_no_gc)))
16539 soh_full_gc_requested = TRUE;
16540 #endif //MULTIPLE_HEAPS
16544 if (!current_no_gc_region_info.minimal_gc_p && gc_requested)
16546 soh_full_gc_requested = TRUE;
16549 no_gc_requested = !(soh_full_gc_requested || gc_requested);
16551 if (soh_full_gc_requested && current_no_gc_region_info.minimal_gc_p)
16553 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16557 if (!soh_full_gc_requested && current_no_gc_region_info.loh_allocation_size)
16559 // Check to see if we have enough reserved space.
16560 #ifdef MULTIPLE_HEAPS
16561 for (int i = 0; i < n_heaps; i++)
16563 gc_heap* hp = g_heaps[i];
16564 if (!hp->find_loh_space_for_no_gc())
16566 loh_full_gc_requested = TRUE;
16570 #else //MULTIPLE_HEAPS
16571 if (!find_loh_space_for_no_gc())
16572 loh_full_gc_requested = TRUE;
16573 #endif //MULTIPLE_HEAPS
16575 // Check to see if we have committed space.
16576 if (!loh_full_gc_requested)
16578 #ifdef MULTIPLE_HEAPS
16579 for (int i = 0; i < n_heaps; i++)
16581 gc_heap* hp = g_heaps[i];
16582 if (hp->saved_loh_segment_no_gc &&!hp->commit_loh_for_no_gc (hp->saved_loh_segment_no_gc))
16584 loh_full_gc_requested = TRUE;
16588 #else //MULTIPLE_HEAPS
16589 if (saved_loh_segment_no_gc && !commit_loh_for_no_gc (saved_loh_segment_no_gc))
16590 loh_full_gc_requested = TRUE;
16591 #endif //MULTIPLE_HEAPS
16595 if (loh_full_gc_requested || soh_full_gc_requested)
16597 if (current_no_gc_region_info.minimal_gc_p)
16598 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16601 no_gc_requested = !(loh_full_gc_requested || soh_full_gc_requested || gc_requested);
16603 if (current_no_gc_region_info.start_status == start_no_gc_success)
16605 if (no_gc_requested)
16606 set_allocations_for_no_gc();
16611 if ((current_no_gc_region_info.start_status == start_no_gc_success) && !no_gc_requested)
16615 // We are done with starting the no_gc_region.
16616 current_no_gc_region_info.started = TRUE;
16621 end_no_gc_region_status gc_heap::end_no_gc_region()
16623 dprintf (1, ("end no gc called"));
16625 end_no_gc_region_status status = end_no_gc_success;
16627 if (!(current_no_gc_region_info.started))
16628 status = end_no_gc_not_in_progress;
16629 if (current_no_gc_region_info.num_gcs_induced)
16630 status = end_no_gc_induced;
16631 else if (current_no_gc_region_info.num_gcs)
16632 status = end_no_gc_alloc_exceeded;
16634 if (settings.pause_mode == pause_no_gc)
16635 restore_data_for_no_gc();
16637 // sets current_no_gc_region_info.started to FALSE here.
16638 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
16644 void gc_heap::update_collection_counts ()
16646 dynamic_data* dd0 = dynamic_data_of (0);
16647 dd_gc_clock (dd0) += 1;
16649 size_t now = GetHighPrecisionTimeStamp();
16651 for (int i = 0; i <= settings.condemned_generation;i++)
16653 dynamic_data* dd = dynamic_data_of (i);
16654 dd_collection_count (dd)++;
16655 //this is needed by the linear allocation model
16656 if (i == max_generation)
16657 dd_collection_count (dynamic_data_of (max_generation+1))++;
16658 dd_gc_clock (dd) = dd_gc_clock (dd0);
16659 dd_time_clock (dd) = now;
16663 BOOL gc_heap::expand_soh_with_minimal_gc()
16665 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment)) >= soh_allocation_no_gc)
16668 heap_segment* new_seg = soh_get_segment_to_expand();
16671 if (g_gc_card_table != card_table)
16672 copy_brick_card_table();
16674 settings.promotion = TRUE;
16675 settings.demotion = FALSE;
16676 ephemeral_promotion = TRUE;
16677 int condemned_gen_number = max_generation - 1;
16679 generation* gen = 0;
16680 int align_const = get_alignment_constant (TRUE);
16682 for (int i = 0; i <= condemned_gen_number; i++)
16684 gen = generation_of (i);
16685 saved_ephemeral_plan_start[i] = generation_allocation_start (gen);
16686 saved_ephemeral_plan_start_size[i] = Align (size (generation_allocation_start (gen)), align_const);
16689 // We do need to clear the bricks here as we are converting a bunch of ephemeral objects to gen2
16690 // and need to make sure that there are no left over bricks from the previous GCs for the space
16691 // we just used for gen0 allocation. We will need to go through the bricks for these objects for
16692 // ephemeral GCs later.
16693 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
16694 b < brick_of (align_on_brick (heap_segment_allocated (ephemeral_heap_segment)));
16700 size_t ephemeral_size = (heap_segment_allocated (ephemeral_heap_segment) -
16701 generation_allocation_start (generation_of (max_generation - 1)));
16702 heap_segment_next (ephemeral_heap_segment) = new_seg;
16703 ephemeral_heap_segment = new_seg;
16704 uint8_t* start = heap_segment_mem (ephemeral_heap_segment);
16706 for (int i = condemned_gen_number; i >= 0; i--)
16708 gen = generation_of (i);
16709 size_t gen_start_size = Align (min_obj_size);
16710 make_generation (generation_table[i], ephemeral_heap_segment, start, 0);
16711 generation_plan_allocation_start (gen) = start;
16712 generation_plan_allocation_start_size (gen) = gen_start_size;
16713 start += gen_start_size;
16715 heap_segment_used (ephemeral_heap_segment) = start - plug_skew;
16716 heap_segment_plan_allocated (ephemeral_heap_segment) = start;
16718 fix_generation_bounds (condemned_gen_number, generation_of (0));
16720 dd_gc_new_allocation (dynamic_data_of (max_generation)) -= ephemeral_size;
16721 dd_new_allocation (dynamic_data_of (max_generation)) = dd_gc_new_allocation (dynamic_data_of (max_generation));
16723 adjust_ephemeral_limits();
16730 // Only to be done on the thread that calls restart in a join for server GC
16731 // and reset the oom status per heap.
16732 void gc_heap::check_and_set_no_gc_oom()
16734 #ifdef MULTIPLE_HEAPS
16735 for (int i = 0; i < n_heaps; i++)
16737 gc_heap* hp = g_heaps[i];
16738 if (hp->no_gc_oom_p)
16740 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16741 hp->no_gc_oom_p = false;
16747 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16748 no_gc_oom_p = false;
16750 #endif //MULTIPLE_HEAPS
16753 void gc_heap::allocate_for_no_gc_after_gc()
16755 if (current_no_gc_region_info.minimal_gc_p)
16756 repair_allocation_contexts (TRUE);
16758 no_gc_oom_p = false;
16760 if (current_no_gc_region_info.start_status != start_no_gc_no_memory)
16762 if (current_no_gc_region_info.soh_allocation_size != 0)
16764 if (((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment)) < soh_allocation_no_gc) ||
16765 (!grow_heap_segment (ephemeral_heap_segment, (heap_segment_allocated (ephemeral_heap_segment) + soh_allocation_no_gc))))
16767 no_gc_oom_p = true;
16770 #ifdef MULTIPLE_HEAPS
16771 gc_t_join.join(this, gc_join_after_commit_soh_no_gc);
16772 if (gc_t_join.joined())
16774 #endif //MULTIPLE_HEAPS
16776 check_and_set_no_gc_oom();
16778 #ifdef MULTIPLE_HEAPS
16779 gc_t_join.restart();
16781 #endif //MULTIPLE_HEAPS
16784 if ((current_no_gc_region_info.start_status == start_no_gc_success) &&
16785 !(current_no_gc_region_info.minimal_gc_p) &&
16786 (current_no_gc_region_info.loh_allocation_size != 0))
16788 gc_policy = policy_compact;
16789 saved_loh_segment_no_gc = 0;
16791 if (!find_loh_free_for_no_gc())
16793 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16794 BOOL found_seg_p = FALSE;
16797 if ((size_t)(heap_segment_reserved (seg) - heap_segment_allocated (seg)) >= loh_allocation_no_gc)
16799 found_seg_p = TRUE;
16800 if (!commit_loh_for_no_gc (seg))
16802 no_gc_oom_p = true;
16806 seg = heap_segment_next (seg);
16810 gc_policy = policy_expand;
16813 #ifdef MULTIPLE_HEAPS
16814 gc_t_join.join(this, gc_join_expand_loh_no_gc);
16815 if (gc_t_join.joined())
16817 check_and_set_no_gc_oom();
16819 if (current_no_gc_region_info.start_status == start_no_gc_success)
16821 for (int i = 0; i < n_heaps; i++)
16823 gc_heap* hp = g_heaps[i];
16824 if (hp->gc_policy == policy_expand)
16826 hp->saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc), hp);
16827 if (!(hp->saved_loh_segment_no_gc))
16829 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16836 gc_t_join.restart();
16838 #else //MULTIPLE_HEAPS
16839 check_and_set_no_gc_oom();
16841 if ((current_no_gc_region_info.start_status == start_no_gc_success) && (gc_policy == policy_expand))
16843 saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc));
16844 if (!saved_loh_segment_no_gc)
16845 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16847 #endif //MULTIPLE_HEAPS
16849 if ((current_no_gc_region_info.start_status == start_no_gc_success) && saved_loh_segment_no_gc)
16851 if (!commit_loh_for_no_gc (saved_loh_segment_no_gc))
16853 no_gc_oom_p = true;
16859 #ifdef MULTIPLE_HEAPS
16860 gc_t_join.join(this, gc_join_final_no_gc);
16861 if (gc_t_join.joined())
16863 #endif //MULTIPLE_HEAPS
16865 check_and_set_no_gc_oom();
16867 if (current_no_gc_region_info.start_status == start_no_gc_success)
16869 set_allocations_for_no_gc();
16870 current_no_gc_region_info.started = TRUE;
16873 #ifdef MULTIPLE_HEAPS
16874 gc_t_join.restart();
16876 #endif //MULTIPLE_HEAPS
16879 void gc_heap::init_records()
16881 // An option is to move this to be after we figure out which gen to condemn so we don't
16882 // need to clear some generations' data 'cause we know they don't change, but that also means
16883 // we can't simply call memset here.
16884 memset (&gc_data_per_heap, 0, sizeof (gc_data_per_heap));
16885 gc_data_per_heap.heap_index = heap_number;
16886 if (heap_number == 0)
16887 memset (&gc_data_global, 0, sizeof (gc_data_global));
16889 #ifdef GC_CONFIG_DRIVEN
16890 memset (interesting_data_per_gc, 0, sizeof (interesting_data_per_gc));
16891 #endif //GC_CONFIG_DRIVEN
16892 memset (&fgm_result, 0, sizeof (fgm_result));
16894 for (int i = 0; i <= (max_generation + 1); i++)
16896 gc_data_per_heap.gen_data[i].size_before = generation_size (i);
16897 generation* gen = generation_of (i);
16898 gc_data_per_heap.gen_data[i].free_list_space_before = generation_free_list_space (gen);
16899 gc_data_per_heap.gen_data[i].free_obj_space_before = generation_free_obj_space (gen);
16902 sufficient_gen0_space_p = FALSE;
16904 #ifdef MULTIPLE_HEAPS
16905 gen0_allocated_after_gc_p = false;
16906 #endif //MULTIPLE_HEAPS
16908 #if defined (_DEBUG) && defined (VERIFY_HEAP)
16909 verify_pinned_queue_p = FALSE;
16910 #endif // _DEBUG && VERIFY_HEAP
16913 void gc_heap::pm_full_gc_init_or_clear()
16915 // This means the next GC will be a full blocking GC and we need to init.
16916 if (settings.condemned_generation == (max_generation - 1))
16918 if (pm_trigger_full_gc)
16920 #ifdef MULTIPLE_HEAPS
16922 #endif //MULTIPLE_HEAPS
16923 dprintf (GTC_LOG, ("init for PM triggered full GC"));
16924 uint32_t saved_entry_memory_load = settings.entry_memory_load;
16925 settings.init_mechanisms();
16926 settings.reason = reason_pm_full_gc;
16927 settings.condemned_generation = max_generation;
16928 settings.entry_memory_load = saved_entry_memory_load;
16929 // Can't assert this since we only check at the end of gen2 GCs,
16930 // during gen1 the memory load could have already dropped.
16931 // Although arguably we should just turn off PM then...
16932 //assert (settings.entry_memory_load >= high_memory_load_th);
16933 assert (settings.entry_memory_load > 0);
16934 settings.gc_index += 1;
16938 // This means we are in the progress of a full blocking GC triggered by
16940 else if (settings.reason == reason_pm_full_gc)
16942 assert (settings.condemned_generation == max_generation);
16943 assert (pm_trigger_full_gc);
16944 pm_trigger_full_gc = false;
16946 dprintf (GTC_LOG, ("PM triggered full GC done"));
16950 void gc_heap::garbage_collect_pm_full_gc()
16952 assert (settings.condemned_generation == max_generation);
16953 assert (settings.reason == reason_pm_full_gc);
16954 assert (!settings.concurrent);
16958 void gc_heap::garbage_collect (int n)
16960 //reset the number of alloc contexts
16961 alloc_contexts_used = 0;
16963 fix_allocation_contexts (TRUE);
16964 #ifdef MULTIPLE_HEAPS
16966 gc_t_join.start_ts(this);
16967 #endif //JOIN_STATS
16968 clear_gen0_bricks();
16969 #endif //MULTIPLE_HEAPS
16971 if ((settings.pause_mode == pause_no_gc) && current_no_gc_region_info.minimal_gc_p)
16973 #ifdef MULTIPLE_HEAPS
16974 gc_t_join.join(this, gc_join_minimal_gc);
16975 if (gc_t_join.joined())
16977 #endif //MULTIPLE_HEAPS
16979 #ifdef MULTIPLE_HEAPS
16980 // this is serialized because we need to get a segment
16981 for (int i = 0; i < n_heaps; i++)
16983 if (!(g_heaps[i]->expand_soh_with_minimal_gc()))
16984 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16987 if (!expand_soh_with_minimal_gc())
16988 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16989 #endif //MULTIPLE_HEAPS
16991 update_collection_counts_for_no_gc();
16993 #ifdef MULTIPLE_HEAPS
16994 gc_t_join.restart();
16996 #endif //MULTIPLE_HEAPS
17003 settings.reason = gc_trigger_reason;
17004 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
17005 num_pinned_objects = 0;
17006 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
17009 if (settings.reason == reason_gcstress)
17011 settings.reason = reason_induced;
17012 settings.stress_induced = TRUE;
17014 #endif // STRESS_HEAP
17016 #ifdef MULTIPLE_HEAPS
17017 //align all heaps on the max generation to condemn
17018 dprintf (3, ("Joining for max generation to condemn"));
17019 condemned_generation_num = generation_to_condemn (n,
17020 &blocking_collection,
17021 &elevation_requested,
17023 gc_t_join.join(this, gc_join_generation_determined);
17024 if (gc_t_join.joined())
17025 #endif //MULTIPLE_HEAPS
17027 #if !defined(SEG_MAPPING_TABLE) && !defined(FEATURE_BASICFREEZE)
17028 //delete old slots from the segment table
17029 seg_table->delete_old_slots();
17030 #endif //!SEG_MAPPING_TABLE && !FEATURE_BASICFREEZE
17032 #ifdef MULTIPLE_HEAPS
17033 for (int i = 0; i < n_heaps; i++)
17035 gc_heap* hp = g_heaps[i];
17036 // check for card table growth
17037 if (g_gc_card_table != hp->card_table)
17038 hp->copy_brick_card_table();
17040 hp->rearrange_large_heap_segments();
17041 #ifdef BACKGROUND_GC
17042 hp->background_delay_delete_loh_segments();
17043 if (!recursive_gc_sync::background_running_p())
17044 hp->rearrange_small_heap_segments();
17045 #endif //BACKGROUND_GC
17047 #else //MULTIPLE_HEAPS
17048 if (g_gc_card_table != card_table)
17049 copy_brick_card_table();
17051 rearrange_large_heap_segments();
17052 #ifdef BACKGROUND_GC
17053 background_delay_delete_loh_segments();
17054 if (!recursive_gc_sync::background_running_p())
17055 rearrange_small_heap_segments();
17056 #endif //BACKGROUND_GC
17057 #endif //MULTIPLE_HEAPS
17059 BOOL should_evaluate_elevation = TRUE;
17060 BOOL should_do_blocking_collection = FALSE;
17062 #ifdef MULTIPLE_HEAPS
17063 int gen_max = condemned_generation_num;
17064 for (int i = 0; i < n_heaps; i++)
17066 if (gen_max < g_heaps[i]->condemned_generation_num)
17067 gen_max = g_heaps[i]->condemned_generation_num;
17068 if (should_evaluate_elevation && !(g_heaps[i]->elevation_requested))
17069 should_evaluate_elevation = FALSE;
17070 if ((!should_do_blocking_collection) && (g_heaps[i]->blocking_collection))
17071 should_do_blocking_collection = TRUE;
17074 settings.condemned_generation = gen_max;
17075 #else //MULTIPLE_HEAPS
17076 settings.condemned_generation = generation_to_condemn (n,
17077 &blocking_collection,
17078 &elevation_requested,
17080 should_evaluate_elevation = elevation_requested;
17081 should_do_blocking_collection = blocking_collection;
17082 #endif //MULTIPLE_HEAPS
17084 settings.condemned_generation = joined_generation_to_condemn (
17085 should_evaluate_elevation,
17087 settings.condemned_generation,
17088 &should_do_blocking_collection
17092 STRESS_LOG1(LF_GCROOTS|LF_GC|LF_GCALLOC, LL_INFO10,
17093 "condemned generation num: %d\n", settings.condemned_generation);
17095 record_gcs_during_no_gc();
17097 if (settings.condemned_generation > 1)
17098 settings.promotion = TRUE;
17100 #ifdef HEAP_ANALYZE
17101 // At this point we've decided what generation is condemned
17102 // See if we've been requested to analyze survivors after the mark phase
17103 if (GCToEEInterface::AnalyzeSurvivorsRequested(settings.condemned_generation))
17105 heap_analyze_enabled = TRUE;
17107 #endif // HEAP_ANALYZE
17109 GCToEEInterface::DiagGCStart(settings.condemned_generation, settings.reason == reason_induced);
17111 #ifdef BACKGROUND_GC
17112 if ((settings.condemned_generation == max_generation) &&
17113 (recursive_gc_sync::background_running_p()))
17115 //TODO BACKGROUND_GC If we just wait for the end of gc, it won't work
17116 // because we have to collect 0 and 1 properly
17117 // in particular, the allocation contexts are gone.
17118 // For now, it is simpler to collect max_generation-1
17119 settings.condemned_generation = max_generation - 1;
17120 dprintf (GTC_LOG, ("bgc - 1 instead of 2"));
17123 if ((settings.condemned_generation == max_generation) &&
17124 (should_do_blocking_collection == FALSE) &&
17125 gc_can_use_concurrent &&
17126 !temp_disable_concurrent_p &&
17127 ((settings.pause_mode == pause_interactive) || (settings.pause_mode == pause_sustained_low_latency)))
17129 keep_bgc_threads_p = TRUE;
17130 c_write (settings.concurrent, TRUE);
17132 #endif //BACKGROUND_GC
17134 settings.gc_index = (uint32_t)dd_collection_count (dynamic_data_of (0)) + 1;
17136 // Call the EE for start of GC work
17137 // just one thread for MP GC
17138 GCToEEInterface::GcStartWork (settings.condemned_generation,
17141 // TODO: we could fire an ETW event to say this GC as a concurrent GC but later on due to not being able to
17142 // create threads or whatever, this could be a non concurrent GC. Maybe for concurrent GC we should fire
17143 // it in do_background_gc and if it failed to be a CGC we fire it in gc1... in other words, this should be
17147 #ifdef MULTIPLE_HEAPS
17148 gc_start_event.Reset();
17149 //start all threads on the roots.
17150 dprintf(3, ("Starting all gc threads for gc"));
17151 gc_t_join.restart();
17152 #endif //MULTIPLE_HEAPS
17155 descr_generations (TRUE);
17158 if ((GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC) &&
17159 !(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_POST_GC_ONLY))
17161 verify_heap (TRUE);
17163 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_BARRIERCHECK)
17164 checkGCWriteBarrier();
17166 #endif // VERIFY_HEAP
17168 #ifdef BACKGROUND_GC
17169 if (settings.concurrent)
17171 // We need to save the settings because we'll need to restore it after each FGC.
17172 assert (settings.condemned_generation == max_generation);
17173 settings.compaction = FALSE;
17174 saved_bgc_settings = settings;
17176 #ifdef MULTIPLE_HEAPS
17177 if (heap_number == 0)
17179 for (int i = 0; i < n_heaps; i++)
17181 prepare_bgc_thread (g_heaps[i]);
17183 dprintf (2, ("setting bgc_threads_sync_event"));
17184 bgc_threads_sync_event.Set();
17188 bgc_threads_sync_event.Wait(INFINITE, FALSE);
17189 dprintf (2, ("bgc_threads_sync_event is signalled"));
17192 prepare_bgc_thread(0);
17193 #endif //MULTIPLE_HEAPS
17195 #ifdef MULTIPLE_HEAPS
17196 gc_t_join.join(this, gc_join_start_bgc);
17197 if (gc_t_join.joined())
17198 #endif //MULTIPLE_HEAPS
17200 do_concurrent_p = TRUE;
17201 do_ephemeral_gc_p = FALSE;
17202 #ifdef MULTIPLE_HEAPS
17203 dprintf(2, ("Joined to perform a background GC"));
17205 for (int i = 0; i < n_heaps; i++)
17207 gc_heap* hp = g_heaps[i];
17208 if (!(hp->bgc_thread) || !hp->commit_mark_array_bgc_init (hp->mark_array))
17210 do_concurrent_p = FALSE;
17215 hp->background_saved_lowest_address = hp->lowest_address;
17216 hp->background_saved_highest_address = hp->highest_address;
17220 do_concurrent_p = (!!bgc_thread && commit_mark_array_bgc_init (mark_array));
17221 if (do_concurrent_p)
17223 background_saved_lowest_address = lowest_address;
17224 background_saved_highest_address = highest_address;
17226 #endif //MULTIPLE_HEAPS
17228 if (do_concurrent_p)
17230 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
17231 SoftwareWriteWatch::EnableForGCHeap();
17232 #endif //FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
17234 #ifdef MULTIPLE_HEAPS
17235 for (int i = 0; i < n_heaps; i++)
17236 g_heaps[i]->current_bgc_state = bgc_initialized;
17238 current_bgc_state = bgc_initialized;
17239 #endif //MULTIPLE_HEAPS
17241 int gen = check_for_ephemeral_alloc();
17242 // always do a gen1 GC before we start BGC.
17243 // This is temporary for testing purpose.
17244 //int gen = max_generation - 1;
17245 dont_restart_ee_p = TRUE;
17248 // If we decide to not do a GC before the BGC we need to
17249 // restore the gen0 alloc context.
17250 #ifdef MULTIPLE_HEAPS
17251 for (int i = 0; i < n_heaps; i++)
17253 generation_allocation_pointer (g_heaps[i]->generation_of (0)) = 0;
17254 generation_allocation_limit (g_heaps[i]->generation_of (0)) = 0;
17257 generation_allocation_pointer (youngest_generation) = 0;
17258 generation_allocation_limit (youngest_generation) = 0;
17259 #endif //MULTIPLE_HEAPS
17263 do_ephemeral_gc_p = TRUE;
17265 settings.init_mechanisms();
17266 settings.condemned_generation = gen;
17267 settings.gc_index = (size_t)dd_collection_count (dynamic_data_of (0)) + 2;
17270 // TODO BACKGROUND_GC need to add the profiling stuff here.
17271 dprintf (GTC_LOG, ("doing gen%d before doing a bgc", gen));
17274 //clear the cards so they don't bleed in gen 1 during collection
17275 // shouldn't this always be done at the beginning of any GC?
17276 //clear_card_for_addresses (
17277 // generation_allocation_start (generation_of (0)),
17278 // heap_segment_allocated (ephemeral_heap_segment));
17280 if (!do_ephemeral_gc_p)
17282 do_background_gc();
17287 settings.compaction = TRUE;
17288 c_write (settings.concurrent, FALSE);
17291 #ifdef MULTIPLE_HEAPS
17292 gc_t_join.restart();
17293 #endif //MULTIPLE_HEAPS
17296 if (do_concurrent_p)
17298 // At this point we are sure we'll be starting a BGC, so save its per heap data here.
17299 // global data is only calculated at the end of the GC so we don't need to worry about
17300 // FGCs overwriting it.
17301 memset (&bgc_data_per_heap, 0, sizeof (bgc_data_per_heap));
17302 memcpy (&bgc_data_per_heap, &gc_data_per_heap, sizeof(gc_data_per_heap));
17304 if (do_ephemeral_gc_p)
17306 dprintf (2, ("GC threads running, doing gen%d GC", settings.condemned_generation));
17308 gen_to_condemn_reasons.init();
17309 gen_to_condemn_reasons.set_condition (gen_before_bgc);
17310 gc_data_per_heap.gen_to_condemn_reasons.init (&gen_to_condemn_reasons);
17312 #ifdef MULTIPLE_HEAPS
17313 gc_t_join.join(this, gc_join_bgc_after_ephemeral);
17314 if (gc_t_join.joined())
17315 #endif //MULTIPLE_HEAPS
17317 #ifdef MULTIPLE_HEAPS
17319 #endif //MULTIPLE_HEAPS
17320 settings = saved_bgc_settings;
17321 assert (settings.concurrent);
17323 do_background_gc();
17325 #ifdef MULTIPLE_HEAPS
17326 gc_t_join.restart();
17327 #endif //MULTIPLE_HEAPS
17333 dprintf (2, ("couldn't create BGC threads, reverting to doing a blocking GC"));
17338 #endif //BACKGROUND_GC
17342 #ifndef MULTIPLE_HEAPS
17343 allocation_running_time = (size_t)GCToOSInterface::GetLowPrecisionTimeStamp();
17344 allocation_running_amount = dd_new_allocation (dynamic_data_of (0));
17345 fgn_last_alloc = dd_new_allocation (dynamic_data_of (0));
17346 #endif //MULTIPLE_HEAPS
17349 if (settings.pause_mode == pause_no_gc)
17350 allocate_for_no_gc_after_gc();
17354 #define mark_stack_empty_p() (mark_stack_base == mark_stack_tos)
17357 size_t& gc_heap::promoted_bytes(int thread)
17359 #ifdef MULTIPLE_HEAPS
17360 return g_promoted [thread*16];
17361 #else //MULTIPLE_HEAPS
17362 UNREFERENCED_PARAMETER(thread);
17364 #endif //MULTIPLE_HEAPS
17367 #ifdef INTERIOR_POINTERS
17368 heap_segment* gc_heap::find_segment (uint8_t* interior, BOOL small_segment_only_p)
17370 #ifdef SEG_MAPPING_TABLE
17371 heap_segment* seg = seg_mapping_table_segment_of (interior);
17374 if (small_segment_only_p && heap_segment_loh_p (seg))
17378 #else //SEG_MAPPING_TABLE
17379 #ifdef MULTIPLE_HEAPS
17380 for (int i = 0; i < gc_heap::n_heaps; i++)
17382 gc_heap* h = gc_heap::g_heaps [i];
17383 hs = h->find_segment_per_heap (o, small_segment_only_p);
17391 gc_heap* h = pGenGCHeap;
17392 hs = h->find_segment_per_heap (o, small_segment_only_p);
17394 #endif //MULTIPLE_HEAPS
17395 #endif //SEG_MAPPING_TABLE
17398 heap_segment* gc_heap::find_segment_per_heap (uint8_t* interior, BOOL small_segment_only_p)
17400 #ifdef SEG_MAPPING_TABLE
17401 return find_segment (interior, small_segment_only_p);
17402 #else //SEG_MAPPING_TABLE
17403 if (in_range_for_segment (interior, ephemeral_heap_segment))
17405 return ephemeral_heap_segment;
17409 heap_segment* found_seg = 0;
17412 heap_segment* seg = generation_start_segment (generation_of (max_generation));
17415 if (in_range_for_segment (interior, seg))
17418 goto end_find_segment;
17421 } while ((seg = heap_segment_next (seg)) != 0);
17423 if (!small_segment_only_p)
17425 #ifdef BACKGROUND_GC
17427 ptrdiff_t delta = 0;
17428 heap_segment* seg = segment_of (interior, delta);
17429 if (seg && in_range_for_segment (interior, seg))
17433 goto end_find_segment;
17435 #else //BACKGROUND_GC
17436 heap_segment* seg = generation_start_segment (generation_of (max_generation+1));
17439 if (in_range_for_segment(interior, seg))
17442 goto end_find_segment;
17445 } while ((seg = heap_segment_next (seg)) != 0);
17446 #endif //BACKGROUND_GC
17452 #endif //SEG_MAPPING_TABLE
17454 #endif //INTERIOR_POINTERS
17456 #if !defined(_DEBUG) && !defined(__GNUC__)
17457 inline // This causes link errors if global optimization is off
17458 #endif //!_DEBUG && !__GNUC__
17459 gc_heap* gc_heap::heap_of (uint8_t* o)
17461 #ifdef MULTIPLE_HEAPS
17463 return g_heaps [0];
17464 #ifdef SEG_MAPPING_TABLE
17465 gc_heap* hp = seg_mapping_table_heap_of (o);
17466 return (hp ? hp : g_heaps[0]);
17467 #else //SEG_MAPPING_TABLE
17468 ptrdiff_t delta = 0;
17469 heap_segment* seg = segment_of (o, delta);
17470 return (seg ? heap_segment_heap (seg) : g_heaps [0]);
17471 #endif //SEG_MAPPING_TABLE
17472 #else //MULTIPLE_HEAPS
17473 UNREFERENCED_PARAMETER(o);
17475 #endif //MULTIPLE_HEAPS
17479 gc_heap* gc_heap::heap_of_gc (uint8_t* o)
17481 #ifdef MULTIPLE_HEAPS
17483 return g_heaps [0];
17484 #ifdef SEG_MAPPING_TABLE
17485 gc_heap* hp = seg_mapping_table_heap_of_gc (o);
17486 return (hp ? hp : g_heaps[0]);
17487 #else //SEG_MAPPING_TABLE
17488 ptrdiff_t delta = 0;
17489 heap_segment* seg = segment_of (o, delta);
17490 return (seg ? heap_segment_heap (seg) : g_heaps [0]);
17491 #endif //SEG_MAPPING_TABLE
17492 #else //MULTIPLE_HEAPS
17493 UNREFERENCED_PARAMETER(o);
17495 #endif //MULTIPLE_HEAPS
17498 #ifdef INTERIOR_POINTERS
17499 // will find all heap objects (large and small)
17500 uint8_t* gc_heap::find_object (uint8_t* interior, uint8_t* low)
17502 if (!gen0_bricks_cleared)
17504 #ifdef MULTIPLE_HEAPS
17505 assert (!"Should have already been done in server GC");
17506 #endif //MULTIPLE_HEAPS
17507 gen0_bricks_cleared = TRUE;
17508 //initialize brick table for gen 0
17509 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
17510 b < brick_of (align_on_brick
17511 (heap_segment_allocated (ephemeral_heap_segment)));
17517 #ifdef FFIND_OBJECT
17518 //indicate that in the future this needs to be done during allocation
17519 #ifdef MULTIPLE_HEAPS
17520 gen0_must_clear_bricks = FFIND_DECAY*gc_heap::n_heaps;
17522 gen0_must_clear_bricks = FFIND_DECAY;
17523 #endif //MULTIPLE_HEAPS
17524 #endif //FFIND_OBJECT
17526 int brick_entry = get_brick_entry(brick_of (interior));
17527 if (brick_entry == 0)
17529 // this is a pointer to a large object
17530 heap_segment* seg = find_segment_per_heap (interior, FALSE);
17532 #ifdef FEATURE_CONSERVATIVE_GC
17533 && (GCConfig::GetConservativeGC() || interior <= heap_segment_allocated(seg))
17537 // If interior falls within the first free object at the beginning of a generation,
17538 // we don't have brick entry for it, and we may incorrectly treat it as on large object heap.
17539 int align_const = get_alignment_constant (heap_segment_read_only_p (seg)
17540 #ifdef FEATURE_CONSERVATIVE_GC
17541 || (GCConfig::GetConservativeGC() && !heap_segment_loh_p (seg))
17544 //int align_const = get_alignment_constant (heap_segment_read_only_p (seg));
17545 assert (interior < heap_segment_allocated (seg));
17547 uint8_t* o = heap_segment_mem (seg);
17548 while (o < heap_segment_allocated (seg))
17550 uint8_t* next_o = o + Align (size (o), align_const);
17551 assert (next_o > o);
17552 if ((o <= interior) && (interior < next_o))
17563 else if (interior >= low)
17565 heap_segment* seg = find_segment_per_heap (interior, TRUE);
17568 #ifdef FEATURE_CONSERVATIVE_GC
17569 if (interior >= heap_segment_allocated (seg))
17572 assert (interior < heap_segment_allocated (seg));
17574 uint8_t* o = find_first_object (interior, heap_segment_mem (seg));
17585 gc_heap::find_object_for_relocation (uint8_t* interior, uint8_t* low, uint8_t* high)
17587 uint8_t* old_address = interior;
17588 if (!((old_address >= low) && (old_address < high)))
17591 size_t brick = brick_of (old_address);
17592 int brick_entry = brick_table [ brick ];
17593 if (brick_entry != 0)
17597 while (brick_entry < 0)
17599 brick = (brick + brick_entry);
17600 brick_entry = brick_table [ brick ];
17602 uint8_t* old_loc = old_address;
17603 uint8_t* node = tree_search ((brick_address (brick) + brick_entry-1),
17605 if (node <= old_loc)
17610 brick_entry = brick_table [ brick ];
17616 //find the object by going along the plug
17618 while (o <= interior)
17620 uint8_t* next_o = o + Align (size (o));
17621 assert (next_o > o);
17622 if (next_o > interior)
17628 assert ((o <= interior) && ((o + Align (size (o))) > interior));
17633 // this is a pointer to a large object
17634 heap_segment* seg = find_segment_per_heap (interior, FALSE);
17637 assert (interior < heap_segment_allocated (seg));
17639 uint8_t* o = heap_segment_mem (seg);
17640 while (o < heap_segment_allocated (seg))
17642 uint8_t* next_o = o + Align (size (o));
17643 assert (next_o > o);
17644 if ((o < interior) && (interior < next_o))
17656 #else //INTERIOR_POINTERS
17658 uint8_t* gc_heap::find_object (uint8_t* o, uint8_t* low)
17662 #endif //INTERIOR_POINTERS
17664 #ifdef MULTIPLE_HEAPS
17667 #ifdef GC_CONFIG_DRIVEN
17668 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;} else {mark_list_index++;}}
17669 #else //GC_CONFIG_DRIVEN
17670 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;}}
17671 #endif //GC_CONFIG_DRIVEN
17673 #define m_boundary(o) {}
17676 #define m_boundary_fullgc(o) {}
17678 #else //MULTIPLE_HEAPS
17681 #ifdef GC_CONFIG_DRIVEN
17682 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;} else {mark_list_index++;} if (slow > o) slow = o; if (shigh < o) shigh = o;}
17684 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;}if (slow > o) slow = o; if (shigh < o) shigh = o;}
17685 #endif //GC_CONFIG_DRIVEN
17687 #define m_boundary(o) {if (slow > o) slow = o; if (shigh < o) shigh = o;}
17690 #define m_boundary_fullgc(o) {if (slow > o) slow = o; if (shigh < o) shigh = o;}
17692 #endif //MULTIPLE_HEAPS
17694 #define method_table(o) ((CObjectHeader*)(o))->GetMethodTable()
17697 BOOL gc_heap::gc_mark1 (uint8_t* o)
17699 BOOL marked = !marked (o);
17701 dprintf (3, ("*%Ix*, newly marked: %d", (size_t)o, marked));
17706 BOOL gc_heap::gc_mark (uint8_t* o, uint8_t* low, uint8_t* high)
17708 BOOL marked = FALSE;
17709 if ((o >= low) && (o < high))
17710 marked = gc_mark1 (o);
17711 #ifdef MULTIPLE_HEAPS
17715 gc_heap* hp = heap_of_gc (o);
17717 if ((o >= hp->gc_low) && (o < hp->gc_high))
17718 marked = gc_mark1 (o);
17721 snoop_stat.objects_checked_count++;
17725 snoop_stat.objects_marked_count++;
17729 snoop_stat.zero_ref_count++;
17732 #endif //SNOOP_STATS
17733 #endif //MULTIPLE_HEAPS
17737 #ifdef BACKGROUND_GC
17740 BOOL gc_heap::background_marked (uint8_t* o)
17742 return mark_array_marked (o);
17745 BOOL gc_heap::background_mark1 (uint8_t* o)
17747 BOOL to_mark = !mark_array_marked (o);
17749 dprintf (3, ("b*%Ix*b(%d)", (size_t)o, (to_mark ? 1 : 0)));
17752 mark_array_set_marked (o);
17753 dprintf (4, ("n*%Ix*n", (size_t)o));
17760 // TODO: we could consider filtering out NULL's here instead of going to
17761 // look for it on other heaps
17763 BOOL gc_heap::background_mark (uint8_t* o, uint8_t* low, uint8_t* high)
17765 BOOL marked = FALSE;
17766 if ((o >= low) && (o < high))
17767 marked = background_mark1 (o);
17768 #ifdef MULTIPLE_HEAPS
17772 gc_heap* hp = heap_of (o);
17774 if ((o >= hp->background_saved_lowest_address) && (o < hp->background_saved_highest_address))
17775 marked = background_mark1 (o);
17777 #endif //MULTIPLE_HEAPS
17781 #endif //BACKGROUND_GC
17784 uint8_t* gc_heap::next_end (heap_segment* seg, uint8_t* f)
17786 if (seg == ephemeral_heap_segment)
17789 return heap_segment_allocated (seg);
17792 #define new_start() {if (ppstop <= start) {break;} else {parm = start}}
17793 #define ignore_start 0
17794 #define use_start 1
17796 #define go_through_object(mt,o,size,parm,start,start_useful,limit,exp) \
17798 CGCDesc* map = CGCDesc::GetCGCDescFromMT((MethodTable*)(mt)); \
17799 CGCDescSeries* cur = map->GetHighestSeries(); \
17800 ptrdiff_t cnt = (ptrdiff_t) map->GetNumSeries(); \
17804 CGCDescSeries* last = map->GetLowestSeries(); \
17805 uint8_t** parm = 0; \
17808 assert (parm <= (uint8_t**)((o) + cur->GetSeriesOffset())); \
17809 parm = (uint8_t**)((o) + cur->GetSeriesOffset()); \
17810 uint8_t** ppstop = \
17811 (uint8_t**)((uint8_t*)parm + cur->GetSeriesSize() + (size));\
17812 if (!start_useful || (uint8_t*)ppstop > (start)) \
17814 if (start_useful && (uint8_t*)parm < (start)) parm = (uint8_t**)(start);\
17815 while (parm < ppstop) \
17823 } while (cur >= last); \
17827 /* Handle the repeating case - array of valuetypes */ \
17828 uint8_t** parm = (uint8_t**)((o) + cur->startoffset); \
17829 if (start_useful && start > (uint8_t*)parm) \
17831 ptrdiff_t cs = mt->RawGetComponentSize(); \
17832 parm = (uint8_t**)((uint8_t*)parm + (((start) - (uint8_t*)parm)/cs)*cs); \
17834 while ((uint8_t*)parm < ((o)+(size)-plug_skew)) \
17836 for (ptrdiff_t __i = 0; __i > cnt; __i--) \
17838 HALF_SIZE_T skip = cur->val_serie[__i].skip; \
17839 HALF_SIZE_T nptrs = cur->val_serie[__i].nptrs; \
17840 uint8_t** ppstop = parm + nptrs; \
17841 if (!start_useful || (uint8_t*)ppstop > (start)) \
17843 if (start_useful && (uint8_t*)parm < (start)) parm = (uint8_t**)(start); \
17848 } while (parm < ppstop); \
17850 parm = (uint8_t**)((uint8_t*)ppstop + skip); \
17856 #define go_through_object_nostart(mt,o,size,parm,exp) {go_through_object(mt,o,size,parm,o,ignore_start,(o + size),exp); }
17858 // 1 thing to note about this macro:
17859 // 1) you can use *parm safely but in general you don't want to use parm
17860 // because for the collectible types it's not an address on the managed heap.
17861 #ifndef COLLECTIBLE_CLASS
17862 #define go_through_object_cl(mt,o,size,parm,exp) \
17864 if (header(o)->ContainsPointers()) \
17866 go_through_object_nostart(mt,o,size,parm,exp); \
17869 #else //COLLECTIBLE_CLASS
17870 #define go_through_object_cl(mt,o,size,parm,exp) \
17872 if (header(o)->Collectible()) \
17874 uint8_t* class_obj = get_class_object (o); \
17875 uint8_t** parm = &class_obj; \
17876 do {exp} while (false); \
17878 if (header(o)->ContainsPointers()) \
17880 go_through_object_nostart(mt,o,size,parm,exp); \
17883 #endif //COLLECTIBLE_CLASS
17885 // This starts a plug. But mark_stack_tos isn't increased until set_pinned_info is called.
17886 void gc_heap::enque_pinned_plug (uint8_t* plug,
17887 BOOL save_pre_plug_info_p,
17888 uint8_t* last_object_in_last_plug)
17890 if (mark_stack_array_length <= mark_stack_tos)
17892 if (!grow_mark_stack (mark_stack_array, mark_stack_array_length, MARK_STACK_INITIAL_LENGTH))
17894 // we don't want to continue here due to security
17895 // risks. This happens very rarely and fixing it in the
17896 // way so that we can continue is a bit involved and will
17897 // not be done in Dev10.
17898 GCToEEInterface::HandleFatalError(CORINFO_EXCEPTION_GC);
17902 dprintf (3, ("enqueuing P #%Id(%Ix): %Ix. oldest: %Id, LO: %Ix, pre: %d",
17903 mark_stack_tos, &mark_stack_array[mark_stack_tos], plug, mark_stack_bos, last_object_in_last_plug, (save_pre_plug_info_p ? 1 : 0)));
17904 mark& m = mark_stack_array[mark_stack_tos];
17906 // Must be set now because if we have a short object we'll need the value of saved_pre_p.
17907 m.saved_pre_p = save_pre_plug_info_p;
17909 if (save_pre_plug_info_p)
17912 BOOL is_padded = is_plug_padded (last_object_in_last_plug);
17914 clear_plug_padded (last_object_in_last_plug);
17915 #endif //SHORT_PLUGS
17916 memcpy (&(m.saved_pre_plug), &(((plug_and_gap*)plug)[-1]), sizeof (gap_reloc_pair));
17919 set_plug_padded (last_object_in_last_plug);
17920 #endif //SHORT_PLUGS
17922 memcpy (&(m.saved_pre_plug_reloc), &(((plug_and_gap*)plug)[-1]), sizeof (gap_reloc_pair));
17924 // If the last object in the last plug is too short, it requires special handling.
17925 size_t last_obj_size = plug - last_object_in_last_plug;
17926 if (last_obj_size < min_pre_pin_obj_size)
17928 record_interesting_data_point (idp_pre_short);
17931 record_interesting_data_point (idp_pre_short_padded);
17932 #endif //SHORT_PLUGS
17933 dprintf (3, ("encountered a short object %Ix right before pinned plug %Ix!",
17934 last_object_in_last_plug, plug));
17935 // Need to set the short bit regardless of having refs or not because we need to
17936 // indicate that this object is not walkable.
17939 #ifdef COLLECTIBLE_CLASS
17940 if (is_collectible (last_object_in_last_plug))
17942 m.set_pre_short_collectible();
17944 #endif //COLLECTIBLE_CLASS
17946 if (contain_pointers (last_object_in_last_plug))
17948 dprintf (3, ("short object: %Ix(%Ix)", last_object_in_last_plug, last_obj_size));
17950 go_through_object_nostart (method_table(last_object_in_last_plug), last_object_in_last_plug, last_obj_size, pval,
17952 size_t gap_offset = (((size_t)pval - (size_t)(plug - sizeof (gap_reloc_pair) - plug_skew))) / sizeof (uint8_t*);
17953 dprintf (3, ("member: %Ix->%Ix, %Id ptrs from beginning of gap", (uint8_t*)pval, *pval, gap_offset));
17954 m.set_pre_short_bit (gap_offset);
17961 m.saved_post_p = FALSE;
17964 void gc_heap::save_post_plug_info (uint8_t* last_pinned_plug, uint8_t* last_object_in_last_plug, uint8_t* post_plug)
17966 UNREFERENCED_PARAMETER(last_pinned_plug);
17968 mark& m = mark_stack_array[mark_stack_tos - 1];
17969 assert (last_pinned_plug == m.first);
17970 m.saved_post_plug_info_start = (uint8_t*)&(((plug_and_gap*)post_plug)[-1]);
17973 BOOL is_padded = is_plug_padded (last_object_in_last_plug);
17975 clear_plug_padded (last_object_in_last_plug);
17976 #endif //SHORT_PLUGS
17977 memcpy (&(m.saved_post_plug), m.saved_post_plug_info_start, sizeof (gap_reloc_pair));
17980 set_plug_padded (last_object_in_last_plug);
17981 #endif //SHORT_PLUGS
17983 memcpy (&(m.saved_post_plug_reloc), m.saved_post_plug_info_start, sizeof (gap_reloc_pair));
17985 // This is important - we need to clear all bits here except the last one.
17986 m.saved_post_p = TRUE;
17989 m.saved_post_plug_debug.gap = 1;
17992 dprintf (3, ("PP %Ix has NP %Ix right after", last_pinned_plug, post_plug));
17994 size_t last_obj_size = post_plug - last_object_in_last_plug;
17995 if (last_obj_size < min_pre_pin_obj_size)
17997 dprintf (3, ("PP %Ix last obj %Ix is too short", last_pinned_plug, last_object_in_last_plug));
17998 record_interesting_data_point (idp_post_short);
18001 record_interesting_data_point (idp_post_short_padded);
18002 #endif //SHORT_PLUGS
18003 m.set_post_short();
18004 #if defined (_DEBUG) && defined (VERIFY_HEAP)
18005 verify_pinned_queue_p = TRUE;
18006 #endif // _DEBUG && VERIFY_HEAP
18008 #ifdef COLLECTIBLE_CLASS
18009 if (is_collectible (last_object_in_last_plug))
18011 m.set_post_short_collectible();
18013 #endif //COLLECTIBLE_CLASS
18015 if (contain_pointers (last_object_in_last_plug))
18017 dprintf (3, ("short object: %Ix(%Ix)", last_object_in_last_plug, last_obj_size));
18019 // TODO: since we won't be able to walk this object in relocation, we still need to
18020 // take care of collectible assemblies here.
18021 go_through_object_nostart (method_table(last_object_in_last_plug), last_object_in_last_plug, last_obj_size, pval,
18023 size_t gap_offset = (((size_t)pval - (size_t)(post_plug - sizeof (gap_reloc_pair) - plug_skew))) / sizeof (uint8_t*);
18024 dprintf (3, ("member: %Ix->%Ix, %Id ptrs from beginning of gap", (uint8_t*)pval, *pval, gap_offset));
18025 m.set_post_short_bit (gap_offset);
18034 __declspec(naked) void __fastcall Prefetch(void* addr)
18042 inline void Prefetch (void* addr)
18044 UNREFERENCED_PARAMETER(addr);
18049 VOLATILE(uint8_t*)& gc_heap::ref_mark_stack (gc_heap* hp, int index)
18051 return ((VOLATILE(uint8_t*)*)(hp->mark_stack_array))[index];
18054 #endif //MH_SC_MARK
18058 #define partial_object 3
18060 uint8_t* ref_from_slot (uint8_t* r)
18062 return (uint8_t*)((size_t)r & ~(stolen | partial));
18065 BOOL stolen_p (uint8_t* r)
18067 return (((size_t)r&2) && !((size_t)r&1));
18070 BOOL ready_p (uint8_t* r)
18072 return ((size_t)r != 1);
18075 BOOL partial_p (uint8_t* r)
18077 return (((size_t)r&1) && !((size_t)r&2));
18080 BOOL straight_ref_p (uint8_t* r)
18082 return (!stolen_p (r) && !partial_p (r));
18085 BOOL partial_object_p (uint8_t* r)
18087 return (((size_t)r & partial_object) == partial_object);
18090 BOOL ref_p (uint8_t* r)
18092 return (straight_ref_p (r) || partial_object_p (r));
18095 void gc_heap::mark_object_simple1 (uint8_t* oo, uint8_t* start THREAD_NUMBER_DCL)
18097 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_tos = (SERVER_SC_MARK_VOLATILE(uint8_t*)*)mark_stack_array;
18098 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_limit = (SERVER_SC_MARK_VOLATILE(uint8_t*)*)&mark_stack_array[mark_stack_array_length];
18099 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_base = mark_stack_tos;
18100 #ifdef SORT_MARK_STACK
18101 SERVER_SC_MARK_VOLATILE(uint8_t*)* sorted_tos = mark_stack_base;
18102 #endif //SORT_MARK_STACK
18104 // If we are doing a full GC we don't use mark list anyway so use m_boundary_fullgc that doesn't
18105 // update mark list.
18106 BOOL full_p = (settings.condemned_generation == max_generation);
18108 assert ((start >= oo) && (start < oo+size(oo)));
18111 *mark_stack_tos = oo;
18112 #endif //!MH_SC_MARK
18116 #ifdef MULTIPLE_HEAPS
18117 #else //MULTIPLE_HEAPS
18118 const int thread = 0;
18119 #endif //MULTIPLE_HEAPS
18121 if (oo && ((size_t)oo != 4))
18129 else if (!partial_p (oo) && ((s = size (oo)) < (partial_size_th*sizeof (uint8_t*))))
18131 BOOL overflow_p = FALSE;
18133 if (mark_stack_tos + (s) /sizeof (uint8_t*) >= (mark_stack_limit - 1))
18135 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
18136 if (mark_stack_tos + CGCDesc::GetNumPointers(method_table(oo), s, num_components) >= (mark_stack_limit - 1))
18142 if (overflow_p == FALSE)
18144 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18146 go_through_object_cl (method_table(oo), oo, s, ppslot,
18148 uint8_t* o = *ppslot;
18150 if (gc_mark (o, gc_low, gc_high))
18154 m_boundary_fullgc (o);
18160 size_t obj_size = size (o);
18161 promoted_bytes (thread) += obj_size;
18162 if (contain_pointers_or_collectible (o))
18164 *(mark_stack_tos++) = o;
18172 dprintf(3,("mark stack overflow for object %Ix ", (size_t)oo));
18173 min_overflow_address = min (min_overflow_address, oo);
18174 max_overflow_address = max (max_overflow_address, oo);
18179 if (partial_p (oo))
18181 start = ref_from_slot (oo);
18182 oo = ref_from_slot (*(--mark_stack_tos));
18183 dprintf (4, ("oo: %Ix, start: %Ix\n", (size_t)oo, (size_t)start));
18184 assert ((oo < start) && (start < (oo + size (oo))));
18186 #ifdef COLLECTIBLE_CLASS
18189 // If there's a class object, push it now. We are guaranteed to have the slot since
18190 // we just popped one object off.
18191 if (is_collectible (oo))
18193 uint8_t* class_obj = get_class_object (oo);
18194 if (gc_mark (class_obj, gc_low, gc_high))
18198 m_boundary_fullgc (class_obj);
18202 m_boundary (class_obj);
18205 size_t obj_size = size (class_obj);
18206 promoted_bytes (thread) += obj_size;
18207 *(mark_stack_tos++) = class_obj;
18208 // The code below expects that the oo is still stored in the stack slot that was
18209 // just popped and it "pushes" it back just by incrementing the mark_stack_tos.
18210 // But the class_obj has just overwritten that stack slot and so the oo needs to
18211 // be stored to the new slot that's pointed to by the mark_stack_tos.
18212 *mark_stack_tos = oo;
18216 if (!contain_pointers (oo))
18221 #endif //COLLECTIBLE_CLASS
18225 BOOL overflow_p = FALSE;
18227 if (mark_stack_tos + (num_partial_refs + 2) >= mark_stack_limit)
18231 if (overflow_p == FALSE)
18233 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18235 //push the object and its current
18236 SERVER_SC_MARK_VOLATILE(uint8_t*)* place = ++mark_stack_tos;
18240 *(place) = (uint8_t*)partial;
18241 #endif //MH_SC_MARK
18242 int i = num_partial_refs;
18243 uint8_t* ref_to_continue = 0;
18245 go_through_object (method_table(oo), oo, s, ppslot,
18246 start, use_start, (oo + s),
18248 uint8_t* o = *ppslot;
18250 if (gc_mark (o, gc_low, gc_high))
18254 m_boundary_fullgc (o);
18260 size_t obj_size = size (o);
18261 promoted_bytes (thread) += obj_size;
18262 if (contain_pointers_or_collectible (o))
18264 *(mark_stack_tos++) = o;
18267 ref_to_continue = (uint8_t*)((size_t)(ppslot+1) | partial);
18276 //we are finished with this object
18277 assert (ref_to_continue == 0);
18279 assert ((*(place-1)) == (uint8_t*)0);
18282 #endif //MH_SC_MARK
18284 // shouldn't we decrease tos by 2 here??
18287 if (ref_to_continue)
18291 assert ((*(place-1)) == (uint8_t*)0);
18292 *(place-1) = (uint8_t*)((size_t)oo | partial_object);
18293 assert (((*place) == (uint8_t*)1) || ((*place) == (uint8_t*)2));
18294 #endif //MH_SC_MARK
18295 *place = ref_to_continue;
18300 dprintf(3,("mark stack overflow for object %Ix ", (size_t)oo));
18301 min_overflow_address = min (min_overflow_address, oo);
18302 max_overflow_address = max (max_overflow_address, oo);
18305 #ifdef SORT_MARK_STACK
18306 if (mark_stack_tos > sorted_tos + mark_stack_array_length/8)
18308 rqsort1 (sorted_tos, mark_stack_tos-1);
18309 sorted_tos = mark_stack_tos-1;
18311 #endif //SORT_MARK_STACK
18314 if (!(mark_stack_empty_p()))
18316 oo = *(--mark_stack_tos);
18319 #ifdef SORT_MARK_STACK
18320 sorted_tos = min ((size_t)sorted_tos, (size_t)mark_stack_tos);
18321 #endif //SORT_MARK_STACK
18329 BOOL same_numa_node_p (int hn1, int hn2)
18331 return (heap_select::find_numa_node_from_heap_no (hn1) == heap_select::find_numa_node_from_heap_no (hn2));
18334 int find_next_buddy_heap (int this_heap_number, int current_buddy, int n_heaps)
18336 int hn = (current_buddy+1)%n_heaps;
18337 while (hn != current_buddy)
18339 if ((this_heap_number != hn) && (same_numa_node_p (this_heap_number, hn)))
18341 hn = (hn+1)%n_heaps;
18343 return current_buddy;
18347 gc_heap::mark_steal()
18349 mark_stack_busy() = 0;
18350 //clear the mark stack in the snooping range
18351 for (int i = 0; i < max_snoop_level; i++)
18353 ((VOLATILE(uint8_t*)*)(mark_stack_array))[i] = 0;
18356 //pick the next heap as our buddy
18357 int thpn = find_next_buddy_heap (heap_number, heap_number, n_heaps);
18360 dprintf (SNOOP_LOG, ("(GC%d)heap%d: start snooping %d", settings.gc_index, heap_number, (heap_number+1)%n_heaps));
18361 uint32_t begin_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
18362 #endif //SNOOP_STATS
18364 int idle_loop_count = 0;
18365 int first_not_ready_level = 0;
18369 gc_heap* hp = g_heaps [thpn];
18370 int level = first_not_ready_level;
18371 first_not_ready_level = 0;
18373 while (check_next_mark_stack (hp) && (level < (max_snoop_level-1)))
18375 idle_loop_count = 0;
18377 snoop_stat.busy_count++;
18378 dprintf (SNOOP_LOG, ("heap%d: looking at next heap level %d stack contents: %Ix",
18379 heap_number, level, (int)((uint8_t**)(hp->mark_stack_array))[level]));
18380 #endif //SNOOP_STATS
18382 uint8_t* o = ref_mark_stack (hp, level);
18384 uint8_t* start = o;
18387 mark_stack_busy() = 1;
18389 BOOL success = TRUE;
18390 uint8_t* next = (ref_mark_stack (hp, level+1));
18393 if (((size_t)o > 4) && !partial_object_p (o))
18395 //this is a normal object, not a partial mark tuple
18396 //success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level), 0, o)==o);
18397 success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level), (uint8_t*)4, o)==o);
18399 snoop_stat.interlocked_count++;
18401 snoop_stat.normal_count++;
18402 #endif //SNOOP_STATS
18406 //it is a stolen entry, or beginning/ending of a partial mark
18409 snoop_stat.stolen_or_pm_count++;
18410 #endif //SNOOP_STATS
18414 else if (stolen_p (next))
18416 //ignore the stolen guy and go to the next level
18420 snoop_stat.stolen_entry_count++;
18421 #endif //SNOOP_STATS
18425 assert (partial_p (next));
18426 start = ref_from_slot (next);
18427 //re-read the object
18428 o = ref_from_slot (ref_mark_stack (hp, level));
18432 success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level+1), (uint8_t*)stolen, next)==next);
18434 snoop_stat.interlocked_count++;
18437 snoop_stat.partial_mark_parent_count++;
18439 #endif //SNOOP_STATS
18443 // stack is not ready, or o is completely different from the last time we read from this stack level.
18444 // go up 2 levels to steal children or totally unrelated objects.
18446 if (first_not_ready_level == 0)
18448 first_not_ready_level = level;
18452 snoop_stat.pm_not_ready_count++;
18453 #endif //SNOOP_STATS
18460 dprintf (SNOOP_LOG, ("heap%d: marking %Ix from %d [%d] tl:%dms",
18461 heap_number, (size_t)o, (heap_number+1)%n_heaps, level,
18462 (GCToOSInterface::GetLowPrecisionTimeStamp()-begin_tick)));
18463 uint32_t start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
18464 #endif //SNOOP_STATS
18466 mark_object_simple1 (o, start, heap_number);
18469 dprintf (SNOOP_LOG, ("heap%d: done marking %Ix from %d [%d] %dms tl:%dms",
18470 heap_number, (size_t)o, (heap_number+1)%n_heaps, level,
18471 (GCToOSInterface::GetLowPrecisionTimeStamp()-start_tick),(GCToOSInterface::GetLowPrecisionTimeStamp()-begin_tick)));
18472 #endif //SNOOP_STATS
18474 mark_stack_busy() = 0;
18476 //clear the mark stack in snooping range
18477 for (int i = 0; i < max_snoop_level; i++)
18479 if (((uint8_t**)mark_stack_array)[i] != 0)
18481 ((VOLATILE(uint8_t*)*)(mark_stack_array))[i] = 0;
18483 snoop_stat.stack_bottom_clear_count++;
18484 #endif //SNOOP_STATS
18490 mark_stack_busy() = 0;
18494 //slot is either partial or stolen
18498 if ((first_not_ready_level != 0) && hp->mark_stack_busy())
18502 if (!hp->mark_stack_busy())
18504 first_not_ready_level = 0;
18507 if ((idle_loop_count % (6) )==1)
18510 snoop_stat.switch_to_thread_count++;
18511 #endif //SNOOP_STATS
18512 GCToOSInterface::Sleep(1);
18514 int free_count = 1;
18516 snoop_stat.stack_idle_count++;
18517 //dprintf (SNOOP_LOG, ("heap%d: counting idle threads", heap_number));
18518 #endif //SNOOP_STATS
18519 for (int hpn = (heap_number+1)%n_heaps; hpn != heap_number;)
18521 if (!((g_heaps [hpn])->mark_stack_busy()))
18525 dprintf (SNOOP_LOG, ("heap%d: %d idle", heap_number, free_count));
18526 #endif //SNOOP_STATS
18528 else if (same_numa_node_p (hpn, heap_number) || ((idle_loop_count%1000))==999)
18533 hpn = (hpn+1)%n_heaps;
18536 if (free_count == n_heaps)
18545 BOOL gc_heap::check_next_mark_stack (gc_heap* next_heap)
18548 snoop_stat.check_level_count++;
18549 #endif //SNOOP_STATS
18550 return (next_heap->mark_stack_busy()>=1);
18552 #endif //MH_SC_MARK
18555 void gc_heap::print_snoop_stat()
18557 dprintf (1234, ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s",
18558 "heap", "check", "zero", "mark", "stole", "pstack", "nstack", "nonsk"));
18559 dprintf (1234, ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d",
18560 snoop_stat.heap_index,
18561 snoop_stat.objects_checked_count,
18562 snoop_stat.zero_ref_count,
18563 snoop_stat.objects_marked_count,
18564 snoop_stat.stolen_stack_count,
18565 snoop_stat.partial_stack_count,
18566 snoop_stat.normal_stack_count,
18567 snoop_stat.non_stack_count));
18568 dprintf (1234, ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s",
18569 "heap", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "clear"));
18570 dprintf (1234, ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
18571 snoop_stat.heap_index,
18572 snoop_stat.check_level_count,
18573 snoop_stat.busy_count,
18574 snoop_stat.interlocked_count,
18575 snoop_stat.partial_mark_parent_count,
18576 snoop_stat.stolen_or_pm_count,
18577 snoop_stat.stolen_entry_count,
18578 snoop_stat.pm_not_ready_count,
18579 snoop_stat.normal_count,
18580 snoop_stat.stack_bottom_clear_count));
18582 printf ("\n%4s | %8s | %8s | %8s | %8s | %8s\n",
18583 "heap", "check", "zero", "mark", "idle", "switch");
18584 printf ("%4d | %8d | %8d | %8d | %8d | %8d\n",
18585 snoop_stat.heap_index,
18586 snoop_stat.objects_checked_count,
18587 snoop_stat.zero_ref_count,
18588 snoop_stat.objects_marked_count,
18589 snoop_stat.stack_idle_count,
18590 snoop_stat.switch_to_thread_count);
18591 printf ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s\n",
18592 "heap", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "normal", "clear");
18593 printf ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
18594 snoop_stat.heap_index,
18595 snoop_stat.check_level_count,
18596 snoop_stat.busy_count,
18597 snoop_stat.interlocked_count,
18598 snoop_stat.partial_mark_parent_count,
18599 snoop_stat.stolen_or_pm_count,
18600 snoop_stat.stolen_entry_count,
18601 snoop_stat.pm_not_ready_count,
18602 snoop_stat.normal_count,
18603 snoop_stat.stack_bottom_clear_count);
18605 #endif //SNOOP_STATS
18607 #ifdef HEAP_ANALYZE
18609 gc_heap::ha_mark_object_simple (uint8_t** po THREAD_NUMBER_DCL)
18611 if (!internal_root_array)
18613 internal_root_array = new (nothrow) uint8_t* [internal_root_array_length];
18614 if (!internal_root_array)
18616 heap_analyze_success = FALSE;
18620 if (heap_analyze_success && (internal_root_array_length <= internal_root_array_index))
18622 size_t new_size = 2*internal_root_array_length;
18624 uint64_t available_physical = 0;
18625 get_memory_info (NULL, &available_physical);
18626 if (new_size > (size_t)(available_physical / 10))
18628 heap_analyze_success = FALSE;
18632 uint8_t** tmp = new (nothrow) uint8_t* [new_size];
18635 memcpy (tmp, internal_root_array,
18636 internal_root_array_length*sizeof (uint8_t*));
18637 delete[] internal_root_array;
18638 internal_root_array = tmp;
18639 internal_root_array_length = new_size;
18643 heap_analyze_success = FALSE;
18648 if (heap_analyze_success)
18650 PREFIX_ASSUME(internal_root_array_index < internal_root_array_length);
18652 uint8_t* ref = (uint8_t*)po;
18653 if (!current_obj ||
18654 !((ref >= current_obj) && (ref < (current_obj + current_obj_size))))
18656 gc_heap* hp = gc_heap::heap_of (ref);
18657 current_obj = hp->find_object (ref, hp->lowest_address);
18658 current_obj_size = size (current_obj);
18660 internal_root_array[internal_root_array_index] = current_obj;
18661 internal_root_array_index++;
18665 mark_object_simple (po THREAD_NUMBER_ARG);
18667 #endif //HEAP_ANALYZE
18669 //this method assumes that *po is in the [low. high[ range
18671 gc_heap::mark_object_simple (uint8_t** po THREAD_NUMBER_DCL)
18674 #ifdef MULTIPLE_HEAPS
18675 #else //MULTIPLE_HEAPS
18676 const int thread = 0;
18677 #endif //MULTIPLE_HEAPS
18680 snoop_stat.objects_checked_count++;
18681 #endif //SNOOP_STATS
18686 size_t s = size (o);
18687 promoted_bytes (thread) += s;
18689 go_through_object_cl (method_table(o), o, s, poo,
18691 uint8_t* oo = *poo;
18692 if (gc_mark (oo, gc_low, gc_high))
18695 size_t obj_size = size (oo);
18696 promoted_bytes (thread) += obj_size;
18698 if (contain_pointers_or_collectible (oo))
18699 mark_object_simple1 (oo, oo THREAD_NUMBER_ARG);
18709 uint8_t* gc_heap::mark_object (uint8_t* o THREAD_NUMBER_DCL)
18711 if ((o >= gc_low) && (o < gc_high))
18712 mark_object_simple (&o THREAD_NUMBER_ARG);
18713 #ifdef MULTIPLE_HEAPS
18717 gc_heap* hp = heap_of (o);
18719 if ((o >= hp->gc_low) && (o < hp->gc_high))
18720 mark_object_simple (&o THREAD_NUMBER_ARG);
18722 #endif //MULTIPLE_HEAPS
18727 #ifdef BACKGROUND_GC
18729 void gc_heap::background_mark_simple1 (uint8_t* oo THREAD_NUMBER_DCL)
18731 uint8_t** mark_stack_limit = &background_mark_stack_array[background_mark_stack_array_length];
18733 #ifdef SORT_MARK_STACK
18734 uint8_t** sorted_tos = background_mark_stack_array;
18735 #endif //SORT_MARK_STACK
18737 background_mark_stack_tos = background_mark_stack_array;
18741 #ifdef MULTIPLE_HEAPS
18742 #else //MULTIPLE_HEAPS
18743 const int thread = 0;
18744 #endif //MULTIPLE_HEAPS
18748 if ((((size_t)oo & 1) == 0) && ((s = size (oo)) < (partial_size_th*sizeof (uint8_t*))))
18750 BOOL overflow_p = FALSE;
18752 if (background_mark_stack_tos + (s) /sizeof (uint8_t*) >= (mark_stack_limit - 1))
18754 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
18755 size_t num_pointers = CGCDesc::GetNumPointers(method_table(oo), s, num_components);
18756 if (background_mark_stack_tos + num_pointers >= (mark_stack_limit - 1))
18758 dprintf (2, ("h%d: %Id left, obj (mt: %Ix) %Id ptrs",
18760 (size_t)(mark_stack_limit - 1 - background_mark_stack_tos),
18764 bgc_overflow_count++;
18769 if (overflow_p == FALSE)
18771 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18773 go_through_object_cl (method_table(oo), oo, s, ppslot,
18775 uint8_t* o = *ppslot;
18777 if (background_mark (o,
18778 background_saved_lowest_address,
18779 background_saved_highest_address))
18782 size_t obj_size = size (o);
18783 bpromoted_bytes (thread) += obj_size;
18784 if (contain_pointers_or_collectible (o))
18786 *(background_mark_stack_tos++) = o;
18795 dprintf (3,("mark stack overflow for object %Ix ", (size_t)oo));
18796 background_min_overflow_address = min (background_min_overflow_address, oo);
18797 background_max_overflow_address = max (background_max_overflow_address, oo);
18802 uint8_t* start = oo;
18803 if ((size_t)oo & 1)
18805 oo = (uint8_t*)((size_t)oo & ~1);
18806 start = *(--background_mark_stack_tos);
18807 dprintf (4, ("oo: %Ix, start: %Ix\n", (size_t)oo, (size_t)start));
18809 #ifdef COLLECTIBLE_CLASS
18812 // If there's a class object, push it now. We are guaranteed to have the slot since
18813 // we just popped one object off.
18814 if (is_collectible (oo))
18816 uint8_t* class_obj = get_class_object (oo);
18817 if (background_mark (class_obj,
18818 background_saved_lowest_address,
18819 background_saved_highest_address))
18821 size_t obj_size = size (class_obj);
18822 bpromoted_bytes (thread) += obj_size;
18824 *(background_mark_stack_tos++) = class_obj;
18828 if (!contain_pointers (oo))
18833 #endif //COLLECTIBLE_CLASS
18837 BOOL overflow_p = FALSE;
18839 if (background_mark_stack_tos + (num_partial_refs + 2) >= mark_stack_limit)
18841 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
18842 size_t num_pointers = CGCDesc::GetNumPointers(method_table(oo), s, num_components);
18844 dprintf (2, ("h%d: PM: %Id left, obj %Ix (mt: %Ix) start: %Ix, total: %Id",
18846 (size_t)(mark_stack_limit - background_mark_stack_tos),
18852 bgc_overflow_count++;
18855 if (overflow_p == FALSE)
18857 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18859 //push the object and its current
18860 uint8_t** place = background_mark_stack_tos++;
18862 *(background_mark_stack_tos++) = (uint8_t*)((size_t)oo | 1);
18864 int i = num_partial_refs;
18866 go_through_object (method_table(oo), oo, s, ppslot,
18867 start, use_start, (oo + s),
18869 uint8_t* o = *ppslot;
18872 if (background_mark (o,
18873 background_saved_lowest_address,
18874 background_saved_highest_address))
18877 size_t obj_size = size (o);
18878 bpromoted_bytes (thread) += obj_size;
18879 if (contain_pointers_or_collectible (o))
18881 *(background_mark_stack_tos++) = o;
18885 *place = (uint8_t*)(ppslot+1);
18894 //we are finished with this object
18902 dprintf (3,("mark stack overflow for object %Ix ", (size_t)oo));
18903 background_min_overflow_address = min (background_min_overflow_address, oo);
18904 background_max_overflow_address = max (background_max_overflow_address, oo);
18908 #ifdef SORT_MARK_STACK
18909 if (background_mark_stack_tos > sorted_tos + mark_stack_array_length/8)
18911 rqsort1 (sorted_tos, background_mark_stack_tos-1);
18912 sorted_tos = background_mark_stack_tos-1;
18914 #endif //SORT_MARK_STACK
18916 #ifdef COLLECTIBLE_CLASS
18918 #endif // COLLECTIBLE_CLASS
18921 if (!(background_mark_stack_tos == background_mark_stack_array))
18923 oo = *(--background_mark_stack_tos);
18925 #ifdef SORT_MARK_STACK
18926 sorted_tos = (uint8_t**)min ((size_t)sorted_tos, (size_t)background_mark_stack_tos);
18927 #endif //SORT_MARK_STACK
18933 assert (background_mark_stack_tos == background_mark_stack_array);
18938 //this version is different than the foreground GC because
18939 //it can't keep pointers to the inside of an object
18940 //while calling background_mark_simple1. The object could be moved
18941 //by an intervening foreground gc.
18942 //this method assumes that *po is in the [low. high[ range
18944 gc_heap::background_mark_simple (uint8_t* o THREAD_NUMBER_DCL)
18946 #ifdef MULTIPLE_HEAPS
18947 #else //MULTIPLE_HEAPS
18948 const int thread = 0;
18949 #endif //MULTIPLE_HEAPS
18951 dprintf (3, ("bmarking %Ix", o));
18953 if (background_mark1 (o))
18956 size_t s = size (o);
18957 bpromoted_bytes (thread) += s;
18959 if (contain_pointers_or_collectible (o))
18961 background_mark_simple1 (o THREAD_NUMBER_ARG);
18968 uint8_t* gc_heap::background_mark_object (uint8_t* o THREAD_NUMBER_DCL)
18970 if ((o >= background_saved_lowest_address) && (o < background_saved_highest_address))
18972 background_mark_simple (o THREAD_NUMBER_ARG);
18978 dprintf (3, ("or-%Ix", o));
18984 void gc_heap::background_verify_mark (Object*& object, ScanContext* sc, uint32_t flags)
18986 UNREFERENCED_PARAMETER(sc);
18988 assert (settings.concurrent);
18989 uint8_t* o = (uint8_t*)object;
18991 gc_heap* hp = gc_heap::heap_of (o);
18992 #ifdef INTERIOR_POINTERS
18993 if (flags & GC_CALL_INTERIOR)
18995 o = hp->find_object (o, background_saved_lowest_address);
18997 #endif //INTERIOR_POINTERS
18999 if (!background_object_marked (o, FALSE))
19005 void gc_heap::background_promote (Object** ppObject, ScanContext* sc, uint32_t flags)
19007 UNREFERENCED_PARAMETER(sc);
19008 //in order to save space on the array, mark the object,
19009 //knowing that it will be visited later
19010 assert (settings.concurrent);
19012 THREAD_NUMBER_FROM_CONTEXT;
19013 #ifndef MULTIPLE_HEAPS
19014 const int thread = 0;
19015 #endif //!MULTIPLE_HEAPS
19017 uint8_t* o = (uint8_t*)*ppObject;
19022 #ifdef DEBUG_DestroyedHandleValue
19023 // we can race with destroy handle during concurrent scan
19024 if (o == (uint8_t*)DEBUG_DestroyedHandleValue)
19026 #endif //DEBUG_DestroyedHandleValue
19030 gc_heap* hp = gc_heap::heap_of (o);
19032 if ((o < hp->background_saved_lowest_address) || (o >= hp->background_saved_highest_address))
19037 #ifdef INTERIOR_POINTERS
19038 if (flags & GC_CALL_INTERIOR)
19040 o = hp->find_object (o, hp->background_saved_lowest_address);
19044 #endif //INTERIOR_POINTERS
19046 #ifdef FEATURE_CONSERVATIVE_GC
19047 // For conservative GC, a value on stack may point to middle of a free object.
19048 // In this case, we don't need to promote the pointer.
19049 if (GCConfig::GetConservativeGC() && ((CObjectHeader*)o)->IsFree())
19053 #endif //FEATURE_CONSERVATIVE_GC
19056 ((CObjectHeader*)o)->Validate();
19059 dprintf (BGC_LOG, ("Background Promote %Ix", (size_t)o));
19061 //needs to be called before the marking because it is possible for a foreground
19062 //gc to take place during the mark and move the object
19063 STRESS_LOG3(LF_GC|LF_GCROOTS, LL_INFO1000000, " GCHeap::Promote: Promote GC Root *%p = %p MT = %pT", ppObject, o, o ? ((Object*) o)->GetGCSafeMethodTable() : NULL);
19065 hpt->background_mark_simple (o THREAD_NUMBER_ARG);
19068 //used by the ephemeral collection to scan the local background structures
19069 //containing references.
19071 gc_heap::scan_background_roots (promote_func* fn, int hn, ScanContext *pSC)
19077 pSC->thread_number = hn;
19079 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
19080 pSC->pCurrentDomain = 0;
19083 BOOL relocate_p = (fn == &GCHeap::Relocate);
19085 dprintf (3, ("Scanning background mark list"));
19088 size_t mark_list_finger = 0;
19089 while (mark_list_finger < c_mark_list_index)
19091 uint8_t** o = &c_mark_list [mark_list_finger];
19094 // We may not be able to calculate the size during relocate as POPO
19095 // may have written over the object.
19096 size_t s = size (*o);
19097 assert (Align (s) >= Align (min_obj_size));
19098 dprintf(3,("background root %Ix", (size_t)*o));
19100 (*fn) ((Object**)o, pSC, 0);
19101 mark_list_finger++;
19104 //scan the mark stack
19105 dprintf (3, ("Scanning background mark stack"));
19107 uint8_t** finger = background_mark_stack_array;
19108 while (finger < background_mark_stack_tos)
19110 if ((finger + 1) < background_mark_stack_tos)
19112 // We need to check for the partial mark case here.
19113 uint8_t* parent_obj = *(finger + 1);
19114 if ((size_t)parent_obj & 1)
19116 uint8_t* place = *finger;
19117 size_t place_offset = 0;
19118 uint8_t* real_parent_obj = (uint8_t*)((size_t)parent_obj & ~1);
19122 *(finger + 1) = real_parent_obj;
19123 place_offset = place - real_parent_obj;
19124 dprintf(3,("relocating background root %Ix", (size_t)real_parent_obj));
19125 (*fn) ((Object**)(finger + 1), pSC, 0);
19126 real_parent_obj = *(finger + 1);
19127 *finger = real_parent_obj + place_offset;
19128 *(finger + 1) = (uint8_t*)((size_t)real_parent_obj | 1);
19129 dprintf(3,("roots changed to %Ix, %Ix", *finger, *(finger + 1)));
19133 uint8_t** temp = &real_parent_obj;
19134 dprintf(3,("marking background root %Ix", (size_t)real_parent_obj));
19135 (*fn) ((Object**)temp, pSC, 0);
19142 dprintf(3,("background root %Ix", (size_t)*finger));
19143 (*fn) ((Object**)finger, pSC, 0);
19149 void gc_heap::background_mark_through_object (uint8_t* oo THREAD_NUMBER_DCL)
19151 if (contain_pointers (oo))
19153 size_t total_refs = 0;
19154 size_t s = size (oo);
19155 go_through_object_nostart (method_table(oo), oo, s, po,
19159 background_mark_object (o THREAD_NUMBER_ARG);
19163 dprintf (3,("Background marking through %Ix went through %Id refs",
19169 uint8_t* gc_heap::background_seg_end (heap_segment* seg, BOOL concurrent_p)
19171 if (concurrent_p && (seg == saved_overflow_ephemeral_seg))
19173 // for now we stop at where gen1 started when we started processing
19174 return background_min_soh_overflow_address;
19178 return heap_segment_allocated (seg);
19182 uint8_t* gc_heap::background_first_overflow (uint8_t* min_add,
19185 BOOL small_object_p)
19189 if (small_object_p)
19191 if (in_range_for_segment (min_add, seg))
19193 // min_add was the beginning of gen1 when we did the concurrent
19194 // overflow. Now we could be in a situation where min_add is
19195 // actually the same as allocated for that segment (because
19196 // we expanded heap), in which case we can not call
19197 // find first on this address or we will AV.
19198 if (min_add >= heap_segment_allocated (seg))
19204 if (concurrent_p &&
19205 ((seg == saved_overflow_ephemeral_seg) && (min_add >= background_min_soh_overflow_address)))
19207 return background_min_soh_overflow_address;
19211 o = find_first_object (min_add, heap_segment_mem (seg));
19218 o = max (heap_segment_mem (seg), min_add);
19222 void gc_heap::background_process_mark_overflow_internal (int condemned_gen_number,
19223 uint8_t* min_add, uint8_t* max_add,
19228 current_bgc_state = bgc_overflow_soh;
19231 size_t total_marked_objects = 0;
19233 #ifdef MULTIPLE_HEAPS
19234 int thread = heap_number;
19235 #endif //MULTIPLE_HEAPS
19237 exclusive_sync* loh_alloc_lock = 0;
19239 dprintf (2,("Processing Mark overflow [%Ix %Ix]", (size_t)min_add, (size_t)max_add));
19240 #ifdef MULTIPLE_HEAPS
19241 // We don't have each heap scan all heaps concurrently because we are worried about
19242 // multiple threads calling things like find_first_object.
19243 int h_start = (concurrent_p ? heap_number : 0);
19244 int h_end = (concurrent_p ? (heap_number + 1) : n_heaps);
19245 for (int hi = h_start; hi < h_end; hi++)
19247 gc_heap* hp = (concurrent_p ? this : g_heaps [(heap_number + hi) % n_heaps]);
19253 #endif //MULTIPLE_HEAPS
19254 BOOL small_object_segments = TRUE;
19255 int align_const = get_alignment_constant (small_object_segments);
19256 generation* gen = hp->generation_of (condemned_gen_number);
19257 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
19258 PREFIX_ASSUME(seg != NULL);
19259 loh_alloc_lock = hp->bgc_alloc_lock;
19261 uint8_t* o = hp->background_first_overflow (min_add,
19264 small_object_segments);
19268 while ((o < hp->background_seg_end (seg, concurrent_p)) && (o <= max_add))
19270 dprintf (3, ("considering %Ix", (size_t)o));
19274 if (concurrent_p && !small_object_segments)
19276 loh_alloc_lock->bgc_mark_set (o);
19278 if (((CObjectHeader*)o)->IsFree())
19280 s = unused_array_size (o);
19292 if (background_object_marked (o, FALSE) && contain_pointers_or_collectible (o))
19294 total_marked_objects++;
19295 go_through_object_cl (method_table(o), o, s, poo,
19296 uint8_t* oo = *poo;
19297 background_mark_object (oo THREAD_NUMBER_ARG);
19301 if (concurrent_p && !small_object_segments)
19303 loh_alloc_lock->bgc_mark_done ();
19306 o = o + Align (s, align_const);
19314 dprintf (2, ("went through overflow objects in segment %Ix (%d) (so far %Id marked)",
19315 heap_segment_mem (seg), (small_object_segments ? 0 : 1), total_marked_objects));
19317 if ((concurrent_p && (seg == hp->saved_overflow_ephemeral_seg)) ||
19318 (seg = heap_segment_next_in_range (seg)) == 0)
19320 if (small_object_segments)
19324 current_bgc_state = bgc_overflow_loh;
19327 dprintf (2, ("h%d: SOH: ov-mo: %Id", heap_number, total_marked_objects));
19328 fire_overflow_event (min_add, max_add, total_marked_objects, !small_object_segments);
19329 concurrent_print_time_delta (concurrent_p ? "Cov SOH" : "Nov SOH");
19330 total_marked_objects = 0;
19331 small_object_segments = FALSE;
19332 align_const = get_alignment_constant (small_object_segments);
19333 seg = heap_segment_in_range (generation_start_segment (hp->generation_of (max_generation+1)));
19335 PREFIX_ASSUME(seg != NULL);
19337 o = max (heap_segment_mem (seg), min_add);
19342 dprintf (GTC_LOG, ("h%d: LOH: ov-mo: %Id", heap_number, total_marked_objects));
19343 fire_overflow_event (min_add, max_add, total_marked_objects, !small_object_segments);
19349 o = hp->background_first_overflow (min_add,
19352 small_object_segments);
19359 BOOL gc_heap::background_process_mark_overflow (BOOL concurrent_p)
19361 BOOL grow_mark_array_p = TRUE;
19365 assert (!processed_soh_overflow_p);
19367 if ((background_max_overflow_address != 0) &&
19368 (background_min_overflow_address != MAX_PTR))
19370 // We have overflow to process but we know we can't process the ephemeral generations
19371 // now (we actually could process till the current gen1 start but since we are going to
19372 // make overflow per segment, for now I'll just stop at the saved gen1 start.
19373 saved_overflow_ephemeral_seg = ephemeral_heap_segment;
19374 background_max_soh_overflow_address = heap_segment_reserved (saved_overflow_ephemeral_seg);
19375 background_min_soh_overflow_address = generation_allocation_start (generation_of (max_generation-1));
19380 assert ((saved_overflow_ephemeral_seg == 0) ||
19381 ((background_max_soh_overflow_address != 0) &&
19382 (background_min_soh_overflow_address != MAX_PTR)));
19384 if (!processed_soh_overflow_p)
19386 // if there was no more overflow we just need to process what we didn't process
19387 // on the saved ephemeral segment.
19388 if ((background_max_overflow_address == 0) && (background_min_overflow_address == MAX_PTR))
19390 dprintf (2, ("final processing mark overflow - no more overflow since last time"));
19391 grow_mark_array_p = FALSE;
19394 background_min_overflow_address = min (background_min_overflow_address,
19395 background_min_soh_overflow_address);
19396 background_max_overflow_address = max (background_max_overflow_address,
19397 background_max_soh_overflow_address);
19398 processed_soh_overflow_p = TRUE;
19402 BOOL overflow_p = FALSE;
19404 if ((! ((background_max_overflow_address == 0)) ||
19405 ! ((background_min_overflow_address == MAX_PTR))))
19409 if (grow_mark_array_p)
19411 // Try to grow the array.
19412 size_t new_size = max (MARK_STACK_INITIAL_LENGTH, 2*background_mark_stack_array_length);
19414 if ((new_size * sizeof(mark)) > 100*1024)
19416 size_t new_max_size = (get_total_heap_size() / 10) / sizeof(mark);
19418 new_size = min(new_max_size, new_size);
19421 if ((background_mark_stack_array_length < new_size) &&
19422 ((new_size - background_mark_stack_array_length) > (background_mark_stack_array_length / 2)))
19424 dprintf (2, ("h%d: ov grow to %Id", heap_number, new_size));
19426 uint8_t** tmp = new (nothrow) uint8_t* [new_size];
19429 delete background_mark_stack_array;
19430 background_mark_stack_array = tmp;
19431 background_mark_stack_array_length = new_size;
19432 background_mark_stack_tos = background_mark_stack_array;
19438 grow_mark_array_p = TRUE;
19441 uint8_t* min_add = background_min_overflow_address;
19442 uint8_t* max_add = background_max_overflow_address;
19444 background_max_overflow_address = 0;
19445 background_min_overflow_address = MAX_PTR;
19447 background_process_mark_overflow_internal (max_generation, min_add, max_add, concurrent_p);
19457 #endif //BACKGROUND_GC
19460 void gc_heap::mark_through_object (uint8_t* oo, BOOL mark_class_object_p THREAD_NUMBER_DCL)
19462 #ifndef COLLECTIBLE_CLASS
19463 UNREFERENCED_PARAMETER(mark_class_object_p);
19464 BOOL to_mark_class_object = FALSE;
19465 #else //COLLECTIBLE_CLASS
19466 BOOL to_mark_class_object = (mark_class_object_p && (is_collectible(oo)));
19467 #endif //COLLECTIBLE_CLASS
19468 if (contain_pointers (oo) || to_mark_class_object)
19470 dprintf(3,( "Marking through %Ix", (size_t)oo));
19471 size_t s = size (oo);
19473 #ifdef COLLECTIBLE_CLASS
19474 if (to_mark_class_object)
19476 uint8_t* class_obj = get_class_object (oo);
19477 mark_object (class_obj THREAD_NUMBER_ARG);
19479 #endif //COLLECTIBLE_CLASS
19481 if (contain_pointers (oo))
19483 go_through_object_nostart (method_table(oo), oo, s, po,
19485 mark_object (o THREAD_NUMBER_ARG);
19491 size_t gc_heap::get_total_heap_size()
19493 size_t total_heap_size = 0;
19495 #ifdef MULTIPLE_HEAPS
19498 for (hn = 0; hn < gc_heap::n_heaps; hn++)
19500 gc_heap* hp2 = gc_heap::g_heaps [hn];
19501 total_heap_size += hp2->generation_size (max_generation + 1) + hp2->generation_sizes (hp2->generation_of (max_generation));
19504 total_heap_size = generation_size (max_generation + 1) + generation_sizes (generation_of (max_generation));
19505 #endif //MULTIPLE_HEAPS
19507 return total_heap_size;
19510 size_t gc_heap::get_total_fragmentation()
19512 size_t total_fragmentation = 0;
19514 #ifdef MULTIPLE_HEAPS
19515 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
19517 gc_heap* hp = gc_heap::g_heaps[hn];
19518 #else //MULTIPLE_HEAPS
19520 gc_heap* hp = pGenGCHeap;
19521 #endif //MULTIPLE_HEAPS
19522 for (int i = 0; i <= (max_generation + 1); i++)
19524 generation* gen = hp->generation_of (i);
19525 total_fragmentation += (generation_free_list_space (gen) + generation_free_obj_space (gen));
19529 return total_fragmentation;
19532 size_t gc_heap::get_total_gen_fragmentation (int gen_number)
19534 size_t total_fragmentation = 0;
19536 #ifdef MULTIPLE_HEAPS
19537 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
19539 gc_heap* hp = gc_heap::g_heaps[hn];
19540 #else //MULTIPLE_HEAPS
19542 gc_heap* hp = pGenGCHeap;
19543 #endif //MULTIPLE_HEAPS
19544 generation* gen = hp->generation_of (gen_number);
19545 total_fragmentation += (generation_free_list_space (gen) + generation_free_obj_space (gen));
19548 return total_fragmentation;
19551 size_t gc_heap::get_total_gen_estimated_reclaim (int gen_number)
19553 size_t total_estimated_reclaim = 0;
19555 #ifdef MULTIPLE_HEAPS
19556 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
19558 gc_heap* hp = gc_heap::g_heaps[hn];
19559 #else //MULTIPLE_HEAPS
19561 gc_heap* hp = pGenGCHeap;
19562 #endif //MULTIPLE_HEAPS
19563 total_estimated_reclaim += hp->estimated_reclaim (gen_number);
19566 return total_estimated_reclaim;
19569 size_t gc_heap::committed_size()
19571 generation* gen = generation_of (max_generation);
19572 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
19573 size_t total_committed = 0;
19577 total_committed += heap_segment_committed (seg) - (uint8_t*)seg;
19579 seg = heap_segment_next (seg);
19582 if (gen != large_object_generation)
19584 gen = generation_of (max_generation + 1);
19585 seg = generation_start_segment (gen);
19592 return total_committed;
19595 size_t gc_heap::get_total_committed_size()
19597 size_t total_committed = 0;
19599 #ifdef MULTIPLE_HEAPS
19602 for (hn = 0; hn < gc_heap::n_heaps; hn++)
19604 gc_heap* hp = gc_heap::g_heaps [hn];
19605 total_committed += hp->committed_size();
19608 total_committed = committed_size();
19609 #endif //MULTIPLE_HEAPS
19611 return total_committed;
19614 size_t gc_heap::committed_size (bool loh_p, size_t* allocated)
19616 int gen_number = (loh_p ? (max_generation + 1) : max_generation);
19617 generation* gen = generation_of (gen_number);
19618 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
19619 size_t total_committed = 0;
19620 size_t total_allocated = 0;
19624 total_committed += heap_segment_committed (seg) - (uint8_t*)seg;
19625 total_allocated += heap_segment_allocated (seg) - (uint8_t*)seg;
19626 seg = heap_segment_next (seg);
19629 *allocated = total_allocated;
19630 return total_committed;
19633 void gc_heap::get_memory_info (uint32_t* memory_load,
19634 uint64_t* available_physical,
19635 uint64_t* available_page_file)
19637 GCToOSInterface::GetMemoryStatus(memory_load, available_physical, available_page_file);
19640 void fire_mark_event (int heap_num, int root_type, size_t bytes_marked)
19642 dprintf (DT_LOG_0, ("-----------[%d]mark %d: %Id", heap_num, root_type, bytes_marked));
19643 FIRE_EVENT(GCMarkWithType, heap_num, root_type, bytes_marked);
19646 //returns TRUE is an overflow happened.
19647 BOOL gc_heap::process_mark_overflow(int condemned_gen_number)
19649 size_t last_promoted_bytes = promoted_bytes (heap_number);
19650 BOOL overflow_p = FALSE;
19652 if ((! (max_overflow_address == 0) ||
19653 ! (min_overflow_address == MAX_PTR)))
19656 // Try to grow the array.
19658 max (MARK_STACK_INITIAL_LENGTH, 2*mark_stack_array_length);
19660 if ((new_size * sizeof(mark)) > 100*1024)
19662 size_t new_max_size = (get_total_heap_size() / 10) / sizeof(mark);
19664 new_size = min(new_max_size, new_size);
19667 if ((mark_stack_array_length < new_size) &&
19668 ((new_size - mark_stack_array_length) > (mark_stack_array_length / 2)))
19670 mark* tmp = new (nothrow) mark [new_size];
19673 delete mark_stack_array;
19674 mark_stack_array = tmp;
19675 mark_stack_array_length = new_size;
19679 uint8_t* min_add = min_overflow_address;
19680 uint8_t* max_add = max_overflow_address;
19681 max_overflow_address = 0;
19682 min_overflow_address = MAX_PTR;
19683 process_mark_overflow_internal (condemned_gen_number, min_add, max_add);
19687 size_t current_promoted_bytes = promoted_bytes (heap_number);
19689 if (current_promoted_bytes != last_promoted_bytes)
19690 fire_mark_event (heap_number, ETW::GC_ROOT_OVERFLOW, (current_promoted_bytes - last_promoted_bytes));
19694 void gc_heap::process_mark_overflow_internal (int condemned_gen_number,
19695 uint8_t* min_add, uint8_t* max_add)
19697 #ifdef MULTIPLE_HEAPS
19698 int thread = heap_number;
19699 #endif //MULTIPLE_HEAPS
19700 BOOL full_p = (condemned_gen_number == max_generation);
19702 dprintf(3,("Processing Mark overflow [%Ix %Ix]", (size_t)min_add, (size_t)max_add));
19703 #ifdef MULTIPLE_HEAPS
19704 for (int hi = 0; hi < n_heaps; hi++)
19706 gc_heap* hp = g_heaps [(heap_number + hi) % n_heaps];
19712 #endif //MULTIPLE_HEAPS
19713 BOOL small_object_segments = TRUE;
19714 int align_const = get_alignment_constant (small_object_segments);
19715 generation* gen = hp->generation_of (condemned_gen_number);
19716 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
19718 PREFIX_ASSUME(seg != NULL);
19719 uint8_t* o = max (heap_segment_mem (seg), min_add);
19722 uint8_t* end = heap_segment_allocated (seg);
19724 while ((o < end) && (o <= max_add))
19726 assert ((min_add <= o) && (max_add >= o));
19727 dprintf (3, ("considering %Ix", (size_t)o));
19730 mark_through_object (o, TRUE THREAD_NUMBER_ARG);
19733 o = o + Align (size (o), align_const);
19736 if (( seg = heap_segment_next_in_range (seg)) == 0)
19738 if (small_object_segments && full_p)
19740 small_object_segments = FALSE;
19741 align_const = get_alignment_constant (small_object_segments);
19742 seg = heap_segment_in_range (generation_start_segment (hp->generation_of (max_generation+1)));
19744 PREFIX_ASSUME(seg != NULL);
19746 o = max (heap_segment_mem (seg), min_add);
19756 o = max (heap_segment_mem (seg), min_add);
19763 // Scanning for promotion for dependent handles need special handling. Because the primary holds a strong
19764 // reference to the secondary (when the primary itself is reachable) and this can cause a cascading series of
19765 // promotions (the secondary of one handle is or promotes the primary of another) we might need to perform the
19766 // promotion scan multiple times.
19767 // This helper encapsulates the logic to complete all dependent handle promotions when running a server GC. It
19768 // also has the effect of processing any mark stack overflow.
19770 #ifdef MULTIPLE_HEAPS
19771 // When multiple heaps are enabled we have must utilize a more complex algorithm in order to keep all the GC
19772 // worker threads synchronized. The algorithms are sufficiently divergent that we have different
19773 // implementations based on whether MULTIPLE_HEAPS is defined or not.
19775 // Define some static variables used for synchronization in the method below. These should really be defined
19776 // locally but MSVC complains when the VOLATILE macro is expanded into an instantiation of the Volatile class.
19778 // A note about the synchronization used within this method. Communication between the worker threads is
19779 // achieved via two shared booleans (defined below). These both act as latches that are transitioned only from
19780 // false -> true by unsynchronized code. They are only read or reset to false by a single thread under the
19781 // protection of a join.
19782 static VOLATILE(BOOL) s_fUnpromotedHandles = FALSE;
19783 static VOLATILE(BOOL) s_fUnscannedPromotions = FALSE;
19784 static VOLATILE(BOOL) s_fScanRequired;
19785 void gc_heap::scan_dependent_handles (int condemned_gen_number, ScanContext *sc, BOOL initial_scan_p)
19787 // Whenever we call this method there may have been preceding object promotions. So set
19788 // s_fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
19789 // based on the how the scanning proceeded).
19790 s_fUnscannedPromotions = TRUE;
19792 // We don't know how many times we need to loop yet. In particular we can't base the loop condition on
19793 // the state of this thread's portion of the dependent handle table. That's because promotions on other
19794 // threads could cause handle promotions to become necessary here. Even if there are definitely no more
19795 // promotions possible in this thread's handles, we still have to stay in lock-step with those worker
19796 // threads that haven't finished yet (each GC worker thread has to join exactly the same number of times
19797 // as all the others or they'll get out of step).
19800 // The various worker threads are all currently racing in this code. We need to work out if at least
19801 // one of them think they have work to do this cycle. Each thread needs to rescan its portion of the
19802 // dependent handle table when both of the following conditions apply:
19803 // 1) At least one (arbitrary) object might have been promoted since the last scan (because if this
19804 // object happens to correspond to a primary in one of our handles we might potentially have to
19805 // promote the associated secondary).
19806 // 2) The table for this thread has at least one handle with a secondary that isn't promoted yet.
19808 // The first condition is represented by s_fUnscannedPromotions. This is always non-zero for the first
19809 // iteration of this loop (see comment above) and in subsequent cycles each thread updates this
19810 // whenever a mark stack overflow occurs or scanning their dependent handles results in a secondary
19811 // being promoted. This value is cleared back to zero in a synchronized fashion in the join that
19812 // follows below. Note that we can't read this outside of the join since on any iteration apart from
19813 // the first threads will be racing between reading this value and completing their previous
19814 // iteration's table scan.
19816 // The second condition is tracked by the dependent handle code itself on a per worker thread basis
19817 // (and updated by the GcDhReScan() method). We call GcDhUnpromotedHandlesExist() on each thread to
19818 // determine the local value and collect the results into the s_fUnpromotedHandles variable in what is
19819 // effectively an OR operation. As per s_fUnscannedPromotions we can't read the final result until
19820 // we're safely joined.
19821 if (GCScan::GcDhUnpromotedHandlesExist(sc))
19822 s_fUnpromotedHandles = TRUE;
19824 // Synchronize all the threads so we can read our state variables safely. The shared variable
19825 // s_fScanRequired, indicating whether we should scan the tables or terminate the loop, will be set by
19826 // a single thread inside the join.
19827 gc_t_join.join(this, gc_join_scan_dependent_handles);
19828 if (gc_t_join.joined())
19830 // We're synchronized so it's safe to read our shared state variables. We update another shared
19831 // variable to indicate to all threads whether we'll be scanning for another cycle or terminating
19832 // the loop. We scan if there has been at least one object promotion since last time and at least
19833 // one thread has a dependent handle table with a potential handle promotion possible.
19834 s_fScanRequired = s_fUnscannedPromotions && s_fUnpromotedHandles;
19836 // Reset our shared state variables (ready to be set again on this scan or with a good initial
19837 // value for the next call if we're terminating the loop).
19838 s_fUnscannedPromotions = FALSE;
19839 s_fUnpromotedHandles = FALSE;
19841 if (!s_fScanRequired)
19843 // We're terminating the loop. Perform any last operations that require single threaded access.
19844 if (!initial_scan_p)
19846 // On the second invocation we reconcile all mark overflow ranges across the heaps. This can help
19847 // load balance if some of the heaps have an abnormally large workload.
19848 uint8_t* all_heaps_max = 0;
19849 uint8_t* all_heaps_min = MAX_PTR;
19851 for (i = 0; i < n_heaps; i++)
19853 if (all_heaps_max < g_heaps[i]->max_overflow_address)
19854 all_heaps_max = g_heaps[i]->max_overflow_address;
19855 if (all_heaps_min > g_heaps[i]->min_overflow_address)
19856 all_heaps_min = g_heaps[i]->min_overflow_address;
19858 for (i = 0; i < n_heaps; i++)
19860 g_heaps[i]->max_overflow_address = all_heaps_max;
19861 g_heaps[i]->min_overflow_address = all_heaps_min;
19866 // Restart all the workers.
19867 dprintf(3, ("Starting all gc thread mark stack overflow processing"));
19868 gc_t_join.restart();
19871 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
19872 // being visible. If there really was an overflow (process_mark_overflow returns true) then set the
19873 // global flag indicating that at least one object promotion may have occurred (the usual comment
19874 // about races applies). (Note it's OK to set this flag even if we're about to terminate the loop and
19875 // exit the method since we unconditionally set this variable on method entry anyway).
19876 if (process_mark_overflow(condemned_gen_number))
19877 s_fUnscannedPromotions = TRUE;
19879 // If we decided that no scan was required we can terminate the loop now.
19880 if (!s_fScanRequired)
19883 // Otherwise we must join with the other workers to ensure that all mark stack overflows have been
19884 // processed before we start scanning dependent handle tables (if overflows remain while we scan we
19885 // could miss noting the promotion of some primary objects).
19886 gc_t_join.join(this, gc_join_rescan_dependent_handles);
19887 if (gc_t_join.joined())
19889 // Restart all the workers.
19890 dprintf(3, ("Starting all gc thread for dependent handle promotion"));
19891 gc_t_join.restart();
19894 // If the portion of the dependent handle table managed by this worker has handles that could still be
19895 // promoted perform a rescan. If the rescan resulted in at least one promotion note this fact since it
19896 // could require a rescan of handles on this or other workers.
19897 if (GCScan::GcDhUnpromotedHandlesExist(sc))
19898 if (GCScan::GcDhReScan(sc))
19899 s_fUnscannedPromotions = TRUE;
19902 #else //MULTIPLE_HEAPS
19903 // Non-multiple heap version of scan_dependent_handles: much simpler without the need to keep multiple worker
19904 // threads synchronized.
19905 void gc_heap::scan_dependent_handles (int condemned_gen_number, ScanContext *sc, BOOL initial_scan_p)
19907 UNREFERENCED_PARAMETER(initial_scan_p);
19909 // Whenever we call this method there may have been preceding object promotions. So set
19910 // fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
19911 // based on the how the scanning proceeded).
19912 bool fUnscannedPromotions = true;
19914 // Loop until there are either no more dependent handles that can have their secondary promoted or we've
19915 // managed to perform a scan without promoting anything new.
19916 while (GCScan::GcDhUnpromotedHandlesExist(sc) && fUnscannedPromotions)
19918 // On each iteration of the loop start with the assumption that no further objects have been promoted.
19919 fUnscannedPromotions = false;
19921 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
19922 // being visible. If there was an overflow (process_mark_overflow returned true) then additional
19923 // objects now appear to be promoted and we should set the flag.
19924 if (process_mark_overflow(condemned_gen_number))
19925 fUnscannedPromotions = true;
19927 // Perform the scan and set the flag if any promotions resulted.
19928 if (GCScan::GcDhReScan(sc))
19929 fUnscannedPromotions = true;
19932 // Process any mark stack overflow that may have resulted from scanning handles (or if we didn't need to
19933 // scan any handles at all this is the processing of overflows that may have occurred prior to this method
19935 process_mark_overflow(condemned_gen_number);
19937 #endif //MULTIPLE_HEAPS
19939 void gc_heap::mark_phase (int condemned_gen_number, BOOL mark_only_p)
19941 assert (settings.concurrent == FALSE);
19944 sc.thread_number = heap_number;
19945 sc.promotion = TRUE;
19946 sc.concurrent = FALSE;
19948 dprintf(2,("---- Mark Phase condemning %d ----", condemned_gen_number));
19949 BOOL full_p = (condemned_gen_number == max_generation);
19954 start = GetCycleCount32();
19957 int gen_to_init = condemned_gen_number;
19958 if (condemned_gen_number == max_generation)
19960 gen_to_init = max_generation + 1;
19962 for (int gen_idx = 0; gen_idx <= gen_to_init; gen_idx++)
19964 dynamic_data* dd = dynamic_data_of (gen_idx);
19965 dd_begin_data_size (dd) = generation_size (gen_idx) -
19966 dd_fragmentation (dd) -
19967 Align (size (generation_allocation_start (generation_of (gen_idx))));
19968 dprintf (2, ("begin data size for gen%d is %Id", gen_idx, dd_begin_data_size (dd)));
19969 dd_survived_size (dd) = 0;
19970 dd_pinned_survived_size (dd) = 0;
19971 dd_artificial_pinned_survived_size (dd) = 0;
19972 dd_added_pinned_size (dd) = 0;
19974 dd_padding_size (dd) = 0;
19975 #endif //SHORT_PLUGS
19976 #if defined (RESPECT_LARGE_ALIGNMENT) || defined (FEATURE_STRUCTALIGN)
19977 dd_num_npinned_plugs (dd) = 0;
19978 #endif //RESPECT_LARGE_ALIGNMENT || FEATURE_STRUCTALIGN
19981 #ifdef FFIND_OBJECT
19982 if (gen0_must_clear_bricks > 0)
19983 gen0_must_clear_bricks--;
19984 #endif //FFIND_OBJECT
19986 size_t last_promoted_bytes = 0;
19988 promoted_bytes (heap_number) = 0;
19989 reset_mark_stack();
19992 memset (&snoop_stat, 0, sizeof(snoop_stat));
19993 snoop_stat.heap_index = heap_number;
19994 #endif //SNOOP_STATS
19999 //initialize the mark stack
20000 for (int i = 0; i < max_snoop_level; i++)
20002 ((uint8_t**)(mark_stack_array))[i] = 0;
20005 mark_stack_busy() = 1;
20007 #endif //MH_SC_MARK
20009 static uint32_t num_sizedrefs = 0;
20012 static BOOL do_mark_steal_p = FALSE;
20013 #endif //MH_SC_MARK
20015 #ifdef MULTIPLE_HEAPS
20016 gc_t_join.join(this, gc_join_begin_mark_phase);
20017 if (gc_t_join.joined())
20019 #endif //MULTIPLE_HEAPS
20021 maxgen_size_inc_p = false;
20023 num_sizedrefs = GCToEEInterface::GetTotalNumSizedRefHandles();
20025 #ifdef MULTIPLE_HEAPS
20030 size_t total_heap_size = get_total_heap_size();
20032 if (total_heap_size > (100 * 1024 * 1024))
20034 do_mark_steal_p = TRUE;
20038 do_mark_steal_p = FALSE;
20043 do_mark_steal_p = FALSE;
20045 #endif //MH_SC_MARK
20047 gc_t_join.restart();
20049 #endif //MULTIPLE_HEAPS
20054 //set up the mark lists from g_mark_list
20055 assert (g_mark_list);
20056 #ifdef MULTIPLE_HEAPS
20057 mark_list = &g_mark_list [heap_number*mark_list_size];
20059 mark_list = g_mark_list;
20060 #endif //MULTIPLE_HEAPS
20061 //dont use the mark list for full gc
20062 //because multiple segments are more complex to handle and the list
20063 //is likely to overflow
20064 if (condemned_gen_number != max_generation)
20065 mark_list_end = &mark_list [mark_list_size-1];
20067 mark_list_end = &mark_list [0];
20068 mark_list_index = &mark_list [0];
20071 #ifndef MULTIPLE_HEAPS
20072 shigh = (uint8_t*) 0;
20074 #endif //MULTIPLE_HEAPS
20076 //%type% category = quote (mark);
20078 if ((condemned_gen_number == max_generation) && (num_sizedrefs > 0))
20080 GCScan::GcScanSizedRefs(GCHeap::Promote, condemned_gen_number, max_generation, &sc);
20081 fire_mark_event (heap_number, ETW::GC_ROOT_SIZEDREF, (promoted_bytes (heap_number) - last_promoted_bytes));
20082 last_promoted_bytes = promoted_bytes (heap_number);
20084 #ifdef MULTIPLE_HEAPS
20085 gc_t_join.join(this, gc_join_scan_sizedref_done);
20086 if (gc_t_join.joined())
20088 dprintf(3, ("Done with marking all sized refs. Starting all gc thread for marking other strong roots"));
20089 gc_t_join.restart();
20091 #endif //MULTIPLE_HEAPS
20094 dprintf(3,("Marking Roots"));
20096 GCScan::GcScanRoots(GCHeap::Promote,
20097 condemned_gen_number, max_generation,
20100 fire_mark_event (heap_number, ETW::GC_ROOT_STACK, (promoted_bytes (heap_number) - last_promoted_bytes));
20101 last_promoted_bytes = promoted_bytes (heap_number);
20103 #ifdef BACKGROUND_GC
20104 if (recursive_gc_sync::background_running_p())
20106 scan_background_roots (GCHeap::Promote, heap_number, &sc);
20108 #endif //BACKGROUND_GC
20110 #ifdef FEATURE_PREMORTEM_FINALIZATION
20111 dprintf(3, ("Marking finalization data"));
20112 finalize_queue->GcScanRoots(GCHeap::Promote, heap_number, 0);
20113 #endif // FEATURE_PREMORTEM_FINALIZATION
20115 fire_mark_event (heap_number, ETW::GC_ROOT_FQ, (promoted_bytes (heap_number) - last_promoted_bytes));
20116 last_promoted_bytes = promoted_bytes (heap_number);
20121 dprintf(3,("Marking handle table"));
20122 GCScan::GcScanHandles(GCHeap::Promote,
20123 condemned_gen_number, max_generation,
20125 fire_mark_event (heap_number, ETW::GC_ROOT_HANDLES, (promoted_bytes (heap_number) - last_promoted_bytes));
20126 last_promoted_bytes = promoted_bytes (heap_number);
20130 size_t promoted_before_cards = promoted_bytes (heap_number);
20133 dprintf (3, ("before cards: %Id", promoted_before_cards));
20137 #ifdef MULTIPLE_HEAPS
20138 if (gc_t_join.r_join(this, gc_r_join_update_card_bundle))
20140 #endif //MULTIPLE_HEAPS
20142 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
20143 // If we are manually managing card bundles, every write to the card table should already be
20144 // accounted for in the card bundle table so there's nothing to update here.
20145 update_card_table_bundle();
20147 if (card_bundles_enabled())
20149 verify_card_bundles();
20152 #ifdef MULTIPLE_HEAPS
20153 gc_t_join.r_restart();
20155 #endif //MULTIPLE_HEAPS
20156 #endif //CARD_BUNDLE
20158 card_fn mark_object_fn = &gc_heap::mark_object_simple;
20159 #ifdef HEAP_ANALYZE
20160 heap_analyze_success = TRUE;
20161 if (heap_analyze_enabled)
20163 internal_root_array_index = 0;
20165 current_obj_size = 0;
20166 mark_object_fn = &gc_heap::ha_mark_object_simple;
20168 #endif //HEAP_ANALYZE
20170 dprintf(3,("Marking cross generation pointers"));
20171 mark_through_cards_for_segments (mark_object_fn, FALSE);
20173 dprintf(3,("Marking cross generation pointers for large objects"));
20174 mark_through_cards_for_large_objects (mark_object_fn, FALSE);
20176 dprintf (3, ("marked by cards: %Id",
20177 (promoted_bytes (heap_number) - promoted_before_cards)));
20178 fire_mark_event (heap_number, ETW::GC_ROOT_OLDER, (promoted_bytes (heap_number) - last_promoted_bytes));
20179 last_promoted_bytes = promoted_bytes (heap_number);
20184 if (do_mark_steal_p)
20188 #endif //MH_SC_MARK
20190 // Dependent handles need to be scanned with a special algorithm (see the header comment on
20191 // scan_dependent_handles for more detail). We perform an initial scan without synchronizing with other
20192 // worker threads or processing any mark stack overflow. This is not guaranteed to complete the operation
20193 // but in a common case (where there are no dependent handles that are due to be collected) it allows us
20194 // to optimize away further scans. The call to scan_dependent_handles is what will cycle through more
20195 // iterations if required and will also perform processing of any mark stack overflow once the dependent
20196 // handle table has been fully promoted.
20197 GCScan::GcDhInitialScan(GCHeap::Promote, condemned_gen_number, max_generation, &sc);
20198 scan_dependent_handles(condemned_gen_number, &sc, true);
20200 #ifdef MULTIPLE_HEAPS
20201 dprintf(3, ("Joining for short weak handle scan"));
20202 gc_t_join.join(this, gc_join_null_dead_short_weak);
20203 if (gc_t_join.joined())
20204 #endif //MULTIPLE_HEAPS
20206 #ifdef HEAP_ANALYZE
20207 heap_analyze_enabled = FALSE;
20208 GCToEEInterface::AnalyzeSurvivorsFinished(condemned_gen_number);
20209 #endif // HEAP_ANALYZE
20210 GCToEEInterface::AfterGcScanRoots (condemned_gen_number, max_generation, &sc);
20212 #ifdef MULTIPLE_HEAPS
20215 // we used r_join and need to reinitialize states for it here.
20216 gc_t_join.r_init();
20219 //start all threads on the roots.
20220 dprintf(3, ("Starting all gc thread for short weak handle scan"));
20221 gc_t_join.restart();
20222 #endif //MULTIPLE_HEAPS
20226 // null out the target of short weakref that were not promoted.
20227 GCScan::GcShortWeakPtrScan(GCHeap::Promote, condemned_gen_number, max_generation,&sc);
20229 // MTHTS: keep by single thread
20230 #ifdef MULTIPLE_HEAPS
20231 dprintf(3, ("Joining for finalization"));
20232 gc_t_join.join(this, gc_join_scan_finalization);
20233 if (gc_t_join.joined())
20234 #endif //MULTIPLE_HEAPS
20237 #ifdef MULTIPLE_HEAPS
20238 //start all threads on the roots.
20239 dprintf(3, ("Starting all gc thread for Finalization"));
20240 gc_t_join.restart();
20241 #endif //MULTIPLE_HEAPS
20244 //Handle finalization.
20245 size_t promoted_bytes_live = promoted_bytes (heap_number);
20247 #ifdef FEATURE_PREMORTEM_FINALIZATION
20248 dprintf (3, ("Finalize marking"));
20249 finalize_queue->ScanForFinalization (GCHeap::Promote, condemned_gen_number, mark_only_p, __this);
20251 GCToEEInterface::DiagWalkFReachableObjects(__this);
20252 #endif // FEATURE_PREMORTEM_FINALIZATION
20254 // Scan dependent handles again to promote any secondaries associated with primaries that were promoted
20255 // for finalization. As before scan_dependent_handles will also process any mark stack overflow.
20256 scan_dependent_handles(condemned_gen_number, &sc, false);
20258 #ifdef MULTIPLE_HEAPS
20259 dprintf(3, ("Joining for weak pointer deletion"));
20260 gc_t_join.join(this, gc_join_null_dead_long_weak);
20261 if (gc_t_join.joined())
20263 //start all threads on the roots.
20264 dprintf(3, ("Starting all gc thread for weak pointer deletion"));
20265 gc_t_join.restart();
20267 #endif //MULTIPLE_HEAPS
20269 // null out the target of long weakref that were not promoted.
20270 GCScan::GcWeakPtrScan (GCHeap::Promote, condemned_gen_number, max_generation, &sc);
20272 // MTHTS: keep by single thread
20273 #ifdef MULTIPLE_HEAPS
20275 #ifdef PARALLEL_MARK_LIST_SORT
20276 // unsigned long start = GetCycleCount32();
20278 // printf("sort_mark_list took %u cycles\n", GetCycleCount32() - start);
20279 #endif //PARALLEL_MARK_LIST_SORT
20282 dprintf (3, ("Joining for sync block cache entry scanning"));
20283 gc_t_join.join(this, gc_join_null_dead_syncblk);
20284 if (gc_t_join.joined())
20285 #endif //MULTIPLE_HEAPS
20287 // scan for deleted entries in the syncblk cache
20288 GCScan::GcWeakPtrScanBySingleThread (condemned_gen_number, max_generation, &sc);
20290 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
20291 if (g_fEnableAppDomainMonitoring)
20293 size_t promoted_all_heaps = 0;
20294 #ifdef MULTIPLE_HEAPS
20295 for (int i = 0; i < n_heaps; i++)
20297 promoted_all_heaps += promoted_bytes (i);
20300 promoted_all_heaps = promoted_bytes (heap_number);
20301 #endif //MULTIPLE_HEAPS
20302 GCToEEInterface::RecordTotalSurvivedBytes(promoted_all_heaps);
20304 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
20306 #ifdef MULTIPLE_HEAPS
20309 #ifndef PARALLEL_MARK_LIST_SORT
20310 //compact g_mark_list and sort it.
20311 combine_mark_lists();
20312 #endif //PARALLEL_MARK_LIST_SORT
20315 //decide on promotion
20316 if (!settings.promotion)
20319 for (int n = 0; n <= condemned_gen_number;n++)
20321 m += (size_t)(dd_min_size (dynamic_data_of (n))*(n+1)*0.1);
20324 for (int i = 0; i < n_heaps; i++)
20326 dynamic_data* dd = g_heaps[i]->dynamic_data_of (min (condemned_gen_number +1,
20328 size_t older_gen_size = (dd_current_size (dd) +
20329 (dd_desired_allocation (dd) -
20330 dd_new_allocation (dd)));
20332 if ((m > (older_gen_size)) ||
20333 (promoted_bytes (i) > m))
20335 settings.promotion = TRUE;
20341 if (do_mark_steal_p)
20343 size_t objects_checked_count = 0;
20344 size_t zero_ref_count = 0;
20345 size_t objects_marked_count = 0;
20346 size_t check_level_count = 0;
20347 size_t busy_count = 0;
20348 size_t interlocked_count = 0;
20349 size_t partial_mark_parent_count = 0;
20350 size_t stolen_or_pm_count = 0;
20351 size_t stolen_entry_count = 0;
20352 size_t pm_not_ready_count = 0;
20353 size_t normal_count = 0;
20354 size_t stack_bottom_clear_count = 0;
20356 for (int i = 0; i < n_heaps; i++)
20358 gc_heap* hp = g_heaps[i];
20359 hp->print_snoop_stat();
20360 objects_checked_count += hp->snoop_stat.objects_checked_count;
20361 zero_ref_count += hp->snoop_stat.zero_ref_count;
20362 objects_marked_count += hp->snoop_stat.objects_marked_count;
20363 check_level_count += hp->snoop_stat.check_level_count;
20364 busy_count += hp->snoop_stat.busy_count;
20365 interlocked_count += hp->snoop_stat.interlocked_count;
20366 partial_mark_parent_count += hp->snoop_stat.partial_mark_parent_count;
20367 stolen_or_pm_count += hp->snoop_stat.stolen_or_pm_count;
20368 stolen_entry_count += hp->snoop_stat.stolen_entry_count;
20369 pm_not_ready_count += hp->snoop_stat.pm_not_ready_count;
20370 normal_count += hp->snoop_stat.normal_count;
20371 stack_bottom_clear_count += hp->snoop_stat.stack_bottom_clear_count;
20376 printf ("-------total stats-------\n");
20377 printf ("%8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s\n",
20378 "checked", "zero", "marked", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "normal", "clear");
20379 printf ("%8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
20380 objects_checked_count,
20382 objects_marked_count,
20386 partial_mark_parent_count,
20387 stolen_or_pm_count,
20388 stolen_entry_count,
20389 pm_not_ready_count,
20391 stack_bottom_clear_count);
20393 #endif //SNOOP_STATS
20395 //start all threads.
20396 dprintf(3, ("Starting all threads for end of mark phase"));
20397 gc_t_join.restart();
20398 #else //MULTIPLE_HEAPS
20400 //decide on promotion
20401 if (!settings.promotion)
20404 for (int n = 0; n <= condemned_gen_number;n++)
20406 m += (size_t)(dd_min_size (dynamic_data_of (n))*(n+1)*0.06);
20408 dynamic_data* dd = dynamic_data_of (min (condemned_gen_number +1,
20410 size_t older_gen_size = (dd_current_size (dd) +
20411 (dd_desired_allocation (dd) -
20412 dd_new_allocation (dd)));
20414 dprintf (2, ("promotion threshold: %Id, promoted bytes: %Id size n+1: %Id",
20415 m, promoted_bytes (heap_number), older_gen_size));
20417 if ((m > older_gen_size) ||
20418 (promoted_bytes (heap_number) > m))
20420 settings.promotion = TRUE;
20424 #endif //MULTIPLE_HEAPS
20427 #ifdef MULTIPLE_HEAPS
20429 #ifdef PARALLEL_MARK_LIST_SORT
20430 // start = GetCycleCount32();
20431 merge_mark_lists();
20432 // printf("merge_mark_lists took %u cycles\n", GetCycleCount32() - start);
20433 #endif //PARALLEL_MARK_LIST_SORT
20435 #endif //MULTIPLE_HEAPS
20437 #ifdef BACKGROUND_GC
20438 total_promoted_bytes = promoted_bytes (heap_number);
20439 #endif //BACKGROUND_GC
20441 promoted_bytes (heap_number) -= promoted_bytes_live;
20444 finish = GetCycleCount32();
20445 mark_time = finish - start;
20448 dprintf(2,("---- End of mark phase ----"));
20452 void gc_heap::pin_object (uint8_t* o, uint8_t** ppObject, uint8_t* low, uint8_t* high)
20454 dprintf (3, ("Pinning %Ix", (size_t)o));
20455 if ((o >= low) && (o < high))
20457 dprintf(3,("^%Ix^", (size_t)o));
20460 #ifdef FEATURE_EVENT_TRACE
20461 if(EVENT_ENABLED(PinObjectAtGCTime))
20463 fire_etw_pin_object_event(o, ppObject);
20465 #endif // FEATURE_EVENT_TRACE
20467 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
20468 num_pinned_objects++;
20469 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
20473 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
20474 size_t gc_heap::get_total_pinned_objects()
20476 #ifdef MULTIPLE_HEAPS
20477 size_t total_num_pinned_objects = 0;
20478 for (int i = 0; i < gc_heap::n_heaps; i++)
20480 gc_heap* hp = gc_heap::g_heaps[i];
20481 total_num_pinned_objects += hp->num_pinned_objects;
20483 return total_num_pinned_objects;
20484 #else //MULTIPLE_HEAPS
20485 return num_pinned_objects;
20486 #endif //MULTIPLE_HEAPS
20488 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
20490 void gc_heap::reset_mark_stack ()
20492 reset_pinned_queue();
20493 max_overflow_address = 0;
20494 min_overflow_address = MAX_PTR;
20497 #ifdef FEATURE_STRUCTALIGN
20499 // The word with left child, right child, and align info is laid out as follows:
20501 // | upper short word | lower short word |
20502 // |<------------> <----->|<------------> <----->|
20503 // | left child info hi| right child info lo|
20504 // x86: | 10 bits 6 bits| 10 bits 6 bits|
20506 // where left/right child are signed values and concat(info hi, info lo) is unsigned.
20508 // The "align info" encodes two numbers: the required alignment (a power of two)
20509 // and the misalignment (the number of machine words the destination address needs
20510 // to be adjusted by to provide alignment - so this number is always smaller than
20511 // the required alignment). Thus, the two can be represented as the "logical or"
20512 // of the two numbers. Note that the actual pad is computed from the misalignment
20513 // by adding the alignment iff the misalignment is non-zero and less than min_obj_size.
20516 // The number of bits in a brick.
20517 #if defined (_TARGET_AMD64_)
20518 #define brick_bits (12)
20520 #define brick_bits (11)
20521 #endif //_TARGET_AMD64_
20522 C_ASSERT(brick_size == (1 << brick_bits));
20524 // The number of bits needed to represent the offset to a child node.
20525 // "brick_bits + 1" allows us to represent a signed offset within a brick.
20526 #define child_bits (brick_bits + 1 - LOG2_PTRSIZE)
20528 // The number of bits in each of the pad hi, pad lo fields.
20529 #define pad_bits (sizeof(short) * 8 - child_bits)
20531 #define child_from_short(w) (((signed short)(w) / (1 << (pad_bits - LOG2_PTRSIZE))) & ~((1 << LOG2_PTRSIZE) - 1))
20532 #define pad_mask ((1 << pad_bits) - 1)
20533 #define pad_from_short(w) ((size_t)(w) & pad_mask)
20534 #else // FEATURE_STRUCTALIGN
20535 #define child_from_short(w) (w)
20536 #endif // FEATURE_STRUCTALIGN
20539 short node_left_child(uint8_t* node)
20541 return child_from_short(((plug_and_pair*)node)[-1].m_pair.left);
20545 void set_node_left_child(uint8_t* node, ptrdiff_t val)
20547 assert (val > -(ptrdiff_t)brick_size);
20548 assert (val < (ptrdiff_t)brick_size);
20549 assert (Aligned (val));
20550 #ifdef FEATURE_STRUCTALIGN
20551 size_t pad = pad_from_short(((plug_and_pair*)node)[-1].m_pair.left);
20552 ((plug_and_pair*)node)[-1].m_pair.left = ((short)val << (pad_bits - LOG2_PTRSIZE)) | (short)pad;
20553 #else // FEATURE_STRUCTALIGN
20554 ((plug_and_pair*)node)[-1].m_pair.left = (short)val;
20555 #endif // FEATURE_STRUCTALIGN
20556 assert (node_left_child (node) == val);
20560 short node_right_child(uint8_t* node)
20562 return child_from_short(((plug_and_pair*)node)[-1].m_pair.right);
20566 void set_node_right_child(uint8_t* node, ptrdiff_t val)
20568 assert (val > -(ptrdiff_t)brick_size);
20569 assert (val < (ptrdiff_t)brick_size);
20570 assert (Aligned (val));
20571 #ifdef FEATURE_STRUCTALIGN
20572 size_t pad = pad_from_short(((plug_and_pair*)node)[-1].m_pair.right);
20573 ((plug_and_pair*)node)[-1].m_pair.right = ((short)val << (pad_bits - LOG2_PTRSIZE)) | (short)pad;
20574 #else // FEATURE_STRUCTALIGN
20575 ((plug_and_pair*)node)[-1].m_pair.right = (short)val;
20576 #endif // FEATURE_STRUCTALIGN
20577 assert (node_right_child (node) == val);
20580 #ifdef FEATURE_STRUCTALIGN
20581 void node_aligninfo (uint8_t* node, int& requiredAlignment, ptrdiff_t& pad)
20583 // Extract the single-number aligninfo from the fields.
20584 short left = ((plug_and_pair*)node)[-1].m_pair.left;
20585 short right = ((plug_and_pair*)node)[-1].m_pair.right;
20586 ptrdiff_t pad_shifted = (pad_from_short(left) << pad_bits) | pad_from_short(right);
20587 ptrdiff_t aligninfo = pad_shifted * DATA_ALIGNMENT;
20589 // Replicate the topmost bit into all lower bits.
20590 ptrdiff_t x = aligninfo;
20596 // Clear all bits but the highest.
20597 requiredAlignment = (int)(x ^ (x >> 1));
20598 pad = aligninfo - requiredAlignment;
20599 pad += AdjustmentForMinPadSize(pad, requiredAlignment);
20603 ptrdiff_t node_alignpad (uint8_t* node)
20605 int requiredAlignment;
20606 ptrdiff_t alignpad;
20607 node_aligninfo (node, requiredAlignment, alignpad);
20611 void clear_node_aligninfo (uint8_t* node)
20613 ((plug_and_pair*)node)[-1].m_pair.left &= ~0 << pad_bits;
20614 ((plug_and_pair*)node)[-1].m_pair.right &= ~0 << pad_bits;
20617 void set_node_aligninfo (uint8_t* node, int requiredAlignment, ptrdiff_t pad)
20619 // Encode the alignment requirement and alignment offset as a single number
20620 // as described above.
20621 ptrdiff_t aligninfo = (size_t)requiredAlignment + (pad & (requiredAlignment-1));
20622 assert (Aligned (aligninfo));
20623 ptrdiff_t aligninfo_shifted = aligninfo / DATA_ALIGNMENT;
20624 assert (aligninfo_shifted < (1 << (pad_bits + pad_bits)));
20626 ptrdiff_t hi = aligninfo_shifted >> pad_bits;
20627 assert (pad_from_short(((plug_and_gap*)node)[-1].m_pair.left) == 0);
20628 ((plug_and_pair*)node)[-1].m_pair.left |= hi;
20630 ptrdiff_t lo = aligninfo_shifted & pad_mask;
20631 assert (pad_from_short(((plug_and_gap*)node)[-1].m_pair.right) == 0);
20632 ((plug_and_pair*)node)[-1].m_pair.right |= lo;
20635 int requiredAlignment2;
20637 node_aligninfo (node, requiredAlignment2, pad2);
20638 assert (requiredAlignment == requiredAlignment2);
20639 assert (pad == pad2);
20642 #endif // FEATURE_STRUCTALIGN
20645 void loh_set_node_relocation_distance(uint8_t* node, ptrdiff_t val)
20647 ptrdiff_t* place = &(((loh_obj_and_pad*)node)[-1].reloc);
20652 ptrdiff_t loh_node_relocation_distance(uint8_t* node)
20654 return (((loh_obj_and_pad*)node)[-1].reloc);
20658 ptrdiff_t node_relocation_distance (uint8_t* node)
20660 return (((plug_and_reloc*)(node))[-1].reloc & ~3);
20664 void set_node_relocation_distance(uint8_t* node, ptrdiff_t val)
20666 assert (val == (val & ~3));
20667 ptrdiff_t* place = &(((plug_and_reloc*)node)[-1].reloc);
20668 //clear the left bit and the relocation field
20674 #define node_left_p(node) (((plug_and_reloc*)(node))[-1].reloc & 2)
20676 #define set_node_left(node) ((plug_and_reloc*)(node))[-1].reloc |= 2;
20678 #ifndef FEATURE_STRUCTALIGN
20679 void set_node_realigned(uint8_t* node)
20681 ((plug_and_reloc*)(node))[-1].reloc |= 1;
20684 void clear_node_realigned(uint8_t* node)
20686 #ifdef RESPECT_LARGE_ALIGNMENT
20687 ((plug_and_reloc*)(node))[-1].reloc &= ~1;
20689 UNREFERENCED_PARAMETER(node);
20690 #endif //RESPECT_LARGE_ALIGNMENT
20692 #endif // FEATURE_STRUCTALIGN
20695 size_t node_gap_size (uint8_t* node)
20697 return ((plug_and_gap *)node)[-1].gap;
20700 void set_gap_size (uint8_t* node, size_t size)
20702 assert (Aligned (size));
20704 // clear the 2 uint32_t used by the node.
20705 ((plug_and_gap *)node)[-1].reloc = 0;
20706 ((plug_and_gap *)node)[-1].lr =0;
20707 ((plug_and_gap *)node)[-1].gap = size;
20709 assert ((size == 0 )||(size >= sizeof(plug_and_reloc)));
20713 uint8_t* gc_heap::insert_node (uint8_t* new_node, size_t sequence_number,
20714 uint8_t* tree, uint8_t* last_node)
20716 dprintf (3, ("IN: %Ix(%Ix), T: %Ix(%Ix), L: %Ix(%Ix) [%Ix]",
20717 (size_t)new_node, brick_of(new_node),
20718 (size_t)tree, brick_of(tree),
20719 (size_t)last_node, brick_of(last_node),
20721 if (power_of_two_p (sequence_number))
20723 set_node_left_child (new_node, (tree - new_node));
20724 dprintf (3, ("NT: %Ix, LC->%Ix", (size_t)new_node, (tree - new_node)));
20729 if (oddp (sequence_number))
20731 set_node_right_child (last_node, (new_node - last_node));
20732 dprintf (3, ("%Ix RC->%Ix", last_node, (new_node - last_node)));
20736 uint8_t* earlier_node = tree;
20737 size_t imax = logcount(sequence_number) - 2;
20738 for (size_t i = 0; i != imax; i++)
20740 earlier_node = earlier_node + node_right_child (earlier_node);
20742 int tmp_offset = node_right_child (earlier_node);
20743 assert (tmp_offset); // should never be empty
20744 set_node_left_child (new_node, ((earlier_node + tmp_offset ) - new_node));
20745 set_node_right_child (earlier_node, (new_node - earlier_node));
20747 dprintf (3, ("%Ix LC->%Ix, %Ix RC->%Ix",
20748 new_node, ((earlier_node + tmp_offset ) - new_node),
20749 earlier_node, (new_node - earlier_node)));
20755 size_t gc_heap::update_brick_table (uint8_t* tree, size_t current_brick,
20756 uint8_t* x, uint8_t* plug_end)
20758 dprintf (3, ("tree: %Ix, current b: %Ix, x: %Ix, plug_end: %Ix",
20759 tree, current_brick, x, plug_end));
20763 dprintf (3, ("b- %Ix->%Ix pointing to tree %Ix",
20764 current_brick, (size_t)(tree - brick_address (current_brick)), tree));
20765 set_brick (current_brick, (tree - brick_address (current_brick)));
20769 dprintf (3, ("b- %Ix->-1", current_brick));
20770 set_brick (current_brick, -1);
20772 size_t b = 1 + current_brick;
20773 ptrdiff_t offset = 0;
20774 size_t last_br = brick_of (plug_end-1);
20775 current_brick = brick_of (x-1);
20776 dprintf (3, ("ubt: %Ix->%Ix]->%Ix]", b, last_br, current_brick));
20777 while (b <= current_brick)
20781 set_brick (b, --offset);
20789 return brick_of (x);
20792 void gc_heap::plan_generation_start (generation* gen, generation* consing_gen, uint8_t* next_plug_to_allocate)
20795 // We should never demote big plugs to gen0.
20796 if (gen == youngest_generation)
20798 heap_segment* seg = ephemeral_heap_segment;
20799 size_t mark_stack_large_bos = mark_stack_bos;
20800 size_t large_plug_pos = 0;
20801 while (mark_stack_large_bos < mark_stack_tos)
20803 if (mark_stack_array[mark_stack_large_bos].len > demotion_plug_len_th)
20805 while (mark_stack_bos <= mark_stack_large_bos)
20807 size_t entry = deque_pinned_plug();
20808 size_t len = pinned_len (pinned_plug_of (entry));
20809 uint8_t* plug = pinned_plug (pinned_plug_of(entry));
20810 if (len > demotion_plug_len_th)
20812 dprintf (2, ("ps(%d): S %Ix (%Id)(%Ix)", gen->gen_num, plug, len, (plug+len)));
20814 pinned_len (pinned_plug_of (entry)) = plug - generation_allocation_pointer (consing_gen);
20815 assert(mark_stack_array[entry].len == 0 ||
20816 mark_stack_array[entry].len >= Align(min_obj_size));
20817 generation_allocation_pointer (consing_gen) = plug + len;
20818 generation_allocation_limit (consing_gen) = heap_segment_plan_allocated (seg);
20819 set_allocator_next_pin (consing_gen);
20823 mark_stack_large_bos++;
20828 generation_plan_allocation_start (gen) =
20829 allocate_in_condemned_generations (consing_gen, Align (min_obj_size), -1);
20830 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
20831 size_t allocation_left = (size_t)(generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
20832 if (next_plug_to_allocate)
20834 size_t dist_to_next_plug = (size_t)(next_plug_to_allocate - generation_allocation_pointer (consing_gen));
20835 if (allocation_left > dist_to_next_plug)
20837 allocation_left = dist_to_next_plug;
20840 if (allocation_left < Align (min_obj_size))
20842 generation_plan_allocation_start_size (gen) += allocation_left;
20843 generation_allocation_pointer (consing_gen) += allocation_left;
20846 dprintf (2, ("plan alloc gen%d(%Ix) start at %Ix (ptr: %Ix, limit: %Ix, next: %Ix)", gen->gen_num,
20847 generation_plan_allocation_start (gen),
20848 generation_plan_allocation_start_size (gen),
20849 generation_allocation_pointer (consing_gen), generation_allocation_limit (consing_gen),
20850 next_plug_to_allocate));
20853 void gc_heap::realloc_plan_generation_start (generation* gen, generation* consing_gen)
20855 BOOL adjacentp = FALSE;
20857 generation_plan_allocation_start (gen) =
20858 allocate_in_expanded_heap (consing_gen, Align(min_obj_size), adjacentp, 0,
20861 #endif //SHORT_PLUGS
20862 FALSE, -1 REQD_ALIGN_AND_OFFSET_ARG);
20864 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
20865 size_t allocation_left = (size_t)(generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
20866 if ((allocation_left < Align (min_obj_size)) &&
20867 (generation_allocation_limit (consing_gen)!=heap_segment_plan_allocated (generation_allocation_segment (consing_gen))))
20869 generation_plan_allocation_start_size (gen) += allocation_left;
20870 generation_allocation_pointer (consing_gen) += allocation_left;
20873 dprintf (1, ("plan re-alloc gen%d start at %Ix (ptr: %Ix, limit: %Ix)", gen->gen_num,
20874 generation_plan_allocation_start (consing_gen),
20875 generation_allocation_pointer (consing_gen),
20876 generation_allocation_limit (consing_gen)));
20879 void gc_heap::plan_generation_starts (generation*& consing_gen)
20881 //make sure that every generation has a planned allocation start
20882 int gen_number = settings.condemned_generation;
20883 while (gen_number >= 0)
20885 if (gen_number < max_generation)
20887 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
20889 generation* gen = generation_of (gen_number);
20890 if (0 == generation_plan_allocation_start (gen))
20892 plan_generation_start (gen, consing_gen, 0);
20893 assert (generation_plan_allocation_start (gen));
20897 // now we know the planned allocation size
20898 heap_segment_plan_allocated (ephemeral_heap_segment) =
20899 generation_allocation_pointer (consing_gen);
20902 void gc_heap::advance_pins_for_demotion (generation* gen)
20904 uint8_t* original_youngest_start = generation_allocation_start (youngest_generation);
20905 heap_segment* seg = ephemeral_heap_segment;
20907 if ((!(pinned_plug_que_empty_p())))
20909 size_t gen1_pinned_promoted = generation_pinned_allocation_compact_size (generation_of (max_generation));
20910 size_t gen1_pins_left = dd_pinned_survived_size (dynamic_data_of (max_generation - 1)) - gen1_pinned_promoted;
20911 size_t total_space_to_skip = last_gen1_pin_end - generation_allocation_pointer (gen);
20912 float pin_frag_ratio = (float)gen1_pins_left / (float)total_space_to_skip;
20913 float pin_surv_ratio = (float)gen1_pins_left / (float)(dd_survived_size (dynamic_data_of (max_generation - 1)));
20914 if ((pin_frag_ratio > 0.15) && (pin_surv_ratio > 0.30))
20916 while (!pinned_plug_que_empty_p() &&
20917 (pinned_plug (oldest_pin()) < original_youngest_start))
20919 size_t entry = deque_pinned_plug();
20920 size_t len = pinned_len (pinned_plug_of (entry));
20921 uint8_t* plug = pinned_plug (pinned_plug_of(entry));
20922 pinned_len (pinned_plug_of (entry)) = plug - generation_allocation_pointer (gen);
20923 assert(mark_stack_array[entry].len == 0 ||
20924 mark_stack_array[entry].len >= Align(min_obj_size));
20925 generation_allocation_pointer (gen) = plug + len;
20926 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20927 set_allocator_next_pin (gen);
20929 //Add the size of the pinned plug to the right pinned allocations
20930 //find out which gen this pinned plug came from
20931 int frgn = object_gennum (plug);
20932 if ((frgn != (int)max_generation) && settings.promotion)
20934 int togn = object_gennum_plan (plug);
20935 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
20938 generation_pinned_allocation_compact_size (generation_of (togn)) += len;
20942 dprintf (2, ("skipping gap %d, pin %Ix (%Id)",
20943 pinned_len (pinned_plug_of (entry)), plug, len));
20946 dprintf (2, ("ad_p_d: PL: %Id, SL: %Id, pfr: %d, psr: %d",
20947 gen1_pins_left, total_space_to_skip, (int)(pin_frag_ratio*100), (int)(pin_surv_ratio*100)));
20951 void gc_heap::process_ephemeral_boundaries (uint8_t* x,
20952 int& active_new_gen_number,
20953 int& active_old_gen_number,
20954 generation*& consing_gen,
20955 BOOL& allocate_in_condemned)
20958 if ((active_old_gen_number > 0) &&
20959 (x >= generation_allocation_start (generation_of (active_old_gen_number - 1))))
20961 dprintf (2, ("crossing gen%d, x is %Ix", active_old_gen_number - 1, x));
20963 if (!pinned_plug_que_empty_p())
20965 dprintf (2, ("oldest pin: %Ix(%Id)",
20966 pinned_plug (oldest_pin()),
20967 (x - pinned_plug (oldest_pin()))));
20970 if (active_old_gen_number <= (settings.promotion ? (max_generation - 1) : max_generation))
20972 active_new_gen_number--;
20975 active_old_gen_number--;
20976 assert ((!settings.promotion) || (active_new_gen_number>0));
20978 if (active_new_gen_number == (max_generation - 1))
20980 #ifdef FREE_USAGE_STATS
20981 if (settings.condemned_generation == max_generation)
20983 // We need to do this before we skip the rest of the pinned plugs.
20984 generation* gen_2 = generation_of (max_generation);
20985 generation* gen_1 = generation_of (max_generation - 1);
20987 size_t total_num_pinned_free_spaces_left = 0;
20989 // We are about to allocate gen1, check to see how efficient fitting in gen2 pinned free spaces is.
20990 for (int j = 0; j < NUM_GEN_POWER2; j++)
20992 dprintf (1, ("[h%d][#%Id]2^%d: current: %Id, S: 2: %Id, 1: %Id(%Id)",
20996 gen_2->gen_current_pinned_free_spaces[j],
20997 gen_2->gen_plugs[j], gen_1->gen_plugs[j],
20998 (gen_2->gen_plugs[j] + gen_1->gen_plugs[j])));
21000 total_num_pinned_free_spaces_left += gen_2->gen_current_pinned_free_spaces[j];
21003 float pinned_free_list_efficiency = 0;
21004 size_t total_pinned_free_space = generation_allocated_in_pinned_free (gen_2) + generation_pinned_free_obj_space (gen_2);
21005 if (total_pinned_free_space != 0)
21007 pinned_free_list_efficiency = (float)(generation_allocated_in_pinned_free (gen_2)) / (float)total_pinned_free_space;
21010 dprintf (1, ("[h%d] gen2 allocated %Id bytes with %Id bytes pinned free spaces (effi: %d%%), %Id (%Id) left",
21012 generation_allocated_in_pinned_free (gen_2),
21013 total_pinned_free_space,
21014 (int)(pinned_free_list_efficiency * 100),
21015 generation_pinned_free_obj_space (gen_2),
21016 total_num_pinned_free_spaces_left));
21018 #endif //FREE_USAGE_STATS
21020 //Go past all of the pinned plugs for this generation.
21021 while (!pinned_plug_que_empty_p() &&
21022 (!in_range_for_segment ((pinned_plug (oldest_pin())), ephemeral_heap_segment)))
21024 size_t entry = deque_pinned_plug();
21025 mark* m = pinned_plug_of (entry);
21026 uint8_t* plug = pinned_plug (m);
21027 size_t len = pinned_len (m);
21028 // detect pinned block in different segment (later) than
21029 // allocation segment, skip those until the oldest pin is in the ephemeral seg.
21030 // adjust the allocation segment along the way (at the end it will
21031 // be the ephemeral segment.
21032 heap_segment* nseg = heap_segment_in_range (generation_allocation_segment (consing_gen));
21034 PREFIX_ASSUME(nseg != NULL);
21036 while (!((plug >= generation_allocation_pointer (consing_gen))&&
21037 (plug < heap_segment_allocated (nseg))))
21039 //adjust the end of the segment to be the end of the plug
21040 assert (generation_allocation_pointer (consing_gen)>=
21041 heap_segment_mem (nseg));
21042 assert (generation_allocation_pointer (consing_gen)<=
21043 heap_segment_committed (nseg));
21045 heap_segment_plan_allocated (nseg) =
21046 generation_allocation_pointer (consing_gen);
21047 //switch allocation segment
21048 nseg = heap_segment_next_rw (nseg);
21049 generation_allocation_segment (consing_gen) = nseg;
21050 //reset the allocation pointer and limits
21051 generation_allocation_pointer (consing_gen) =
21052 heap_segment_mem (nseg);
21054 set_new_pin_info (m, generation_allocation_pointer (consing_gen));
21055 assert(pinned_len(m) == 0 || pinned_len(m) >= Align(min_obj_size));
21056 generation_allocation_pointer (consing_gen) = plug + len;
21057 generation_allocation_limit (consing_gen) =
21058 generation_allocation_pointer (consing_gen);
21060 allocate_in_condemned = TRUE;
21061 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
21064 if (active_new_gen_number != max_generation)
21066 if (active_new_gen_number == (max_generation - 1))
21068 maxgen_pinned_compact_before_advance = generation_pinned_allocation_compact_size (generation_of (max_generation));
21069 if (!demote_gen1_p)
21070 advance_pins_for_demotion (consing_gen);
21073 plan_generation_start (generation_of (active_new_gen_number), consing_gen, x);
21075 dprintf (2, ("process eph: allocated gen%d start at %Ix",
21076 active_new_gen_number,
21077 generation_plan_allocation_start (generation_of (active_new_gen_number))));
21079 if ((demotion_low == MAX_PTR) && !pinned_plug_que_empty_p())
21081 uint8_t* pplug = pinned_plug (oldest_pin());
21082 if (object_gennum (pplug) > 0)
21084 demotion_low = pplug;
21085 dprintf (3, ("process eph: dlow->%Ix", demotion_low));
21089 assert (generation_plan_allocation_start (generation_of (active_new_gen_number)));
21097 void gc_heap::seg_clear_mark_bits (heap_segment* seg)
21099 uint8_t* o = heap_segment_mem (seg);
21100 while (o < heap_segment_allocated (seg))
21106 o = o + Align (size (o));
21110 #ifdef FEATURE_BASICFREEZE
21111 void gc_heap::sweep_ro_segments (heap_segment* start_seg)
21113 //go through all of the segment in range and reset the mark bit
21114 //TODO works only on small object segments
21116 heap_segment* seg = start_seg;
21120 if (heap_segment_read_only_p (seg) &&
21121 heap_segment_in_range_p (seg))
21123 #ifdef BACKGROUND_GC
21124 if (settings.concurrent)
21126 seg_clear_mark_array_bits_soh (seg);
21130 seg_clear_mark_bits (seg);
21132 #else //BACKGROUND_GC
21135 if(gc_can_use_concurrent)
21137 clear_mark_array (max (heap_segment_mem (seg), lowest_address),
21138 min (heap_segment_allocated (seg), highest_address),
21139 FALSE); // read_only segments need the mark clear
21142 seg_clear_mark_bits (seg);
21143 #endif //MARK_ARRAY
21145 #endif //BACKGROUND_GC
21147 seg = heap_segment_next (seg);
21150 #endif // FEATURE_BASICFREEZE
21152 #ifdef FEATURE_LOH_COMPACTION
21154 BOOL gc_heap::loh_pinned_plug_que_empty_p()
21156 return (loh_pinned_queue_bos == loh_pinned_queue_tos);
21159 void gc_heap::loh_set_allocator_next_pin()
21161 if (!(loh_pinned_plug_que_empty_p()))
21163 mark* oldest_entry = loh_oldest_pin();
21164 uint8_t* plug = pinned_plug (oldest_entry);
21165 generation* gen = large_object_generation;
21166 if ((plug >= generation_allocation_pointer (gen)) &&
21167 (plug < generation_allocation_limit (gen)))
21169 generation_allocation_limit (gen) = pinned_plug (oldest_entry);
21172 assert (!((plug < generation_allocation_pointer (gen)) &&
21173 (plug >= heap_segment_mem (generation_allocation_segment (gen)))));
21177 size_t gc_heap::loh_deque_pinned_plug ()
21179 size_t m = loh_pinned_queue_bos;
21180 loh_pinned_queue_bos++;
21185 mark* gc_heap::loh_pinned_plug_of (size_t bos)
21187 return &loh_pinned_queue[bos];
21191 mark* gc_heap::loh_oldest_pin()
21193 return loh_pinned_plug_of (loh_pinned_queue_bos);
21196 // If we can't grow the queue, then don't compact.
21197 BOOL gc_heap::loh_enque_pinned_plug (uint8_t* plug, size_t len)
21199 assert(len >= Align(min_obj_size, get_alignment_constant (FALSE)));
21201 if (loh_pinned_queue_length <= loh_pinned_queue_tos)
21203 if (!grow_mark_stack (loh_pinned_queue, loh_pinned_queue_length, LOH_PIN_QUEUE_LENGTH))
21208 dprintf (3, (" P: %Ix(%Id)", plug, len));
21209 mark& m = loh_pinned_queue[loh_pinned_queue_tos];
21212 loh_pinned_queue_tos++;
21213 loh_set_allocator_next_pin();
21218 BOOL gc_heap::loh_size_fit_p (size_t size, uint8_t* alloc_pointer, uint8_t* alloc_limit)
21220 dprintf (1235, ("trying to fit %Id(%Id) between %Ix and %Ix (%Id)",
21222 (2* AlignQword (loh_padding_obj_size) + size),
21225 (alloc_limit - alloc_pointer)));
21227 return ((alloc_pointer + 2* AlignQword (loh_padding_obj_size) + size) <= alloc_limit);
21230 uint8_t* gc_heap::loh_allocate_in_condemned (uint8_t* old_loc, size_t size)
21232 UNREFERENCED_PARAMETER(old_loc);
21234 generation* gen = large_object_generation;
21235 dprintf (1235, ("E: p:%Ix, l:%Ix, s: %Id",
21236 generation_allocation_pointer (gen),
21237 generation_allocation_limit (gen),
21242 heap_segment* seg = generation_allocation_segment (gen);
21243 if (!(loh_size_fit_p (size, generation_allocation_pointer (gen), generation_allocation_limit (gen))))
21245 if ((!(loh_pinned_plug_que_empty_p()) &&
21246 (generation_allocation_limit (gen) ==
21247 pinned_plug (loh_oldest_pin()))))
21249 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
21250 size_t len = pinned_len (m);
21251 uint8_t* plug = pinned_plug (m);
21252 dprintf (1235, ("AIC: %Ix->%Ix(%Id)", generation_allocation_pointer (gen), plug, plug - generation_allocation_pointer (gen)));
21253 pinned_len (m) = plug - generation_allocation_pointer (gen);
21254 generation_allocation_pointer (gen) = plug + len;
21256 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
21257 loh_set_allocator_next_pin();
21258 dprintf (1235, ("s: p: %Ix, l: %Ix (%Id)",
21259 generation_allocation_pointer (gen),
21260 generation_allocation_limit (gen),
21261 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21266 if (generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))
21268 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
21269 dprintf (1235, ("l->pa(%Ix)", generation_allocation_limit (gen)));
21273 if (heap_segment_plan_allocated (seg) != heap_segment_committed (seg))
21275 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
21276 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
21277 dprintf (1235, ("l->c(%Ix)", generation_allocation_limit (gen)));
21281 if (loh_size_fit_p (size, generation_allocation_pointer (gen), heap_segment_reserved (seg)) &&
21282 (grow_heap_segment (seg, (generation_allocation_pointer (gen) + size + 2* AlignQword (loh_padding_obj_size)))))
21284 dprintf (1235, ("growing seg from %Ix to %Ix\n", heap_segment_committed (seg),
21285 (generation_allocation_pointer (gen) + size)));
21287 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
21288 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
21290 dprintf (1235, ("g: p: %Ix, l: %Ix (%Id)",
21291 generation_allocation_pointer (gen),
21292 generation_allocation_limit (gen),
21293 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21297 heap_segment* next_seg = heap_segment_next (seg);
21298 assert (generation_allocation_pointer (gen)>=
21299 heap_segment_mem (seg));
21300 // Verify that all pinned plugs for this segment are consumed
21301 if (!loh_pinned_plug_que_empty_p() &&
21302 ((pinned_plug (loh_oldest_pin()) <
21303 heap_segment_allocated (seg)) &&
21304 (pinned_plug (loh_oldest_pin()) >=
21305 generation_allocation_pointer (gen))))
21307 LOG((LF_GC, LL_INFO10, "remaining pinned plug %Ix while leaving segment on allocation",
21308 pinned_plug (loh_oldest_pin())));
21309 dprintf (1236, ("queue empty: %d", loh_pinned_plug_que_empty_p()));
21312 assert (generation_allocation_pointer (gen)>=
21313 heap_segment_mem (seg));
21314 assert (generation_allocation_pointer (gen)<=
21315 heap_segment_committed (seg));
21316 heap_segment_plan_allocated (seg) = generation_allocation_pointer (gen);
21320 // for LOH do we want to try starting from the first LOH every time though?
21321 generation_allocation_segment (gen) = next_seg;
21322 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
21323 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
21325 dprintf (1235, ("n: p: %Ix, l: %Ix (%Id)",
21326 generation_allocation_pointer (gen),
21327 generation_allocation_limit (gen),
21328 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21332 dprintf (1, ("We ran out of space compacting, shouldn't happen"));
21338 loh_set_allocator_next_pin();
21340 dprintf (1235, ("r: p: %Ix, l: %Ix (%Id)",
21341 generation_allocation_pointer (gen),
21342 generation_allocation_limit (gen),
21343 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21350 assert (generation_allocation_pointer (gen)>=
21351 heap_segment_mem (generation_allocation_segment (gen)));
21352 uint8_t* result = generation_allocation_pointer (gen);
21353 size_t loh_pad = AlignQword (loh_padding_obj_size);
21355 generation_allocation_pointer (gen) += size + loh_pad;
21356 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
21358 dprintf (1235, ("p: %Ix, l: %Ix (%Id)",
21359 generation_allocation_pointer (gen),
21360 generation_allocation_limit (gen),
21361 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21363 assert (result + loh_pad);
21364 return result + loh_pad;
21368 BOOL gc_heap::should_compact_loh()
21370 // If hard limit is specified GC will automatically decide if LOH needs to be compacted.
21371 return (heap_hard_limit || loh_compaction_always_p || (loh_compaction_mode != loh_compaction_default));
21375 void gc_heap::check_loh_compact_mode (BOOL all_heaps_compacted_p)
21377 if (settings.loh_compaction && (loh_compaction_mode == loh_compaction_once))
21379 if (all_heaps_compacted_p)
21381 // If the compaction mode says to compact once and we are going to compact LOH,
21382 // we need to revert it back to no compaction.
21383 loh_compaction_mode = loh_compaction_default;
21388 BOOL gc_heap::plan_loh()
21390 if (!loh_pinned_queue)
21392 loh_pinned_queue = new (nothrow) (mark [LOH_PIN_QUEUE_LENGTH]);
21393 if (!loh_pinned_queue)
21395 dprintf (1, ("Cannot allocate the LOH pinned queue (%Id bytes), no compaction",
21396 LOH_PIN_QUEUE_LENGTH * sizeof (mark)));
21400 loh_pinned_queue_length = LOH_PIN_QUEUE_LENGTH;
21403 if (heap_number == 0)
21404 loh_pinned_queue_decay = LOH_PIN_DECAY;
21406 loh_pinned_queue_tos = 0;
21407 loh_pinned_queue_bos = 0;
21409 generation* gen = large_object_generation;
21410 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
21411 PREFIX_ASSUME(start_seg != NULL);
21412 heap_segment* seg = start_seg;
21413 uint8_t* o = generation_allocation_start (gen);
21415 dprintf (1235, ("before GC LOH size: %Id, free list: %Id, free obj: %Id\n",
21416 generation_size (max_generation + 1),
21417 generation_free_list_space (gen),
21418 generation_free_obj_space (gen)));
21422 heap_segment_plan_allocated (seg) = heap_segment_mem (seg);
21423 seg = heap_segment_next (seg);
21428 //Skip the generation gap object
21429 o = o + AlignQword (size (o));
21430 // We don't need to ever realloc gen3 start so don't touch it.
21431 heap_segment_plan_allocated (seg) = o;
21432 generation_allocation_pointer (gen) = o;
21433 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
21434 generation_allocation_segment (gen) = start_seg;
21436 uint8_t* free_space_start = o;
21437 uint8_t* free_space_end = o;
21438 uint8_t* new_address = 0;
21442 if (o >= heap_segment_allocated (seg))
21444 seg = heap_segment_next (seg);
21450 o = heap_segment_mem (seg);
21455 free_space_end = o;
21456 size_t size = AlignQword (size (o));
21457 dprintf (1235, ("%Ix(%Id) M", o, size));
21461 // We don't clear the pinned bit yet so we can check in
21462 // compact phase how big a free object we should allocate
21463 // in front of the pinned object. We use the reloc address
21464 // field to store this.
21465 if (!loh_enque_pinned_plug (o, size))
21473 new_address = loh_allocate_in_condemned (o, size);
21476 loh_set_node_relocation_distance (o, (new_address - o));
21477 dprintf (1235, ("lobj %Ix-%Ix -> %Ix-%Ix (%Id)", o, (o + size), new_address, (new_address + size), (new_address - o)));
21480 free_space_start = o;
21481 if (o < heap_segment_allocated (seg))
21483 assert (!marked (o));
21488 while (o < heap_segment_allocated (seg) && !marked (o))
21490 dprintf (1235, ("%Ix(%Id) F (%d)", o, AlignQword (size (o)), ((method_table (o) == g_gc_pFreeObjectMethodTable) ? 1 : 0)));
21491 o = o + AlignQword (size (o));
21496 while (!loh_pinned_plug_que_empty_p())
21498 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
21499 size_t len = pinned_len (m);
21500 uint8_t* plug = pinned_plug (m);
21502 // detect pinned block in different segment (later) than
21503 // allocation segment
21504 heap_segment* nseg = heap_segment_rw (generation_allocation_segment (gen));
21506 while ((plug < generation_allocation_pointer (gen)) ||
21507 (plug >= heap_segment_allocated (nseg)))
21509 assert ((plug < heap_segment_mem (nseg)) ||
21510 (plug > heap_segment_reserved (nseg)));
21511 //adjust the end of the segment to be the end of the plug
21512 assert (generation_allocation_pointer (gen)>=
21513 heap_segment_mem (nseg));
21514 assert (generation_allocation_pointer (gen)<=
21515 heap_segment_committed (nseg));
21517 heap_segment_plan_allocated (nseg) =
21518 generation_allocation_pointer (gen);
21519 //switch allocation segment
21520 nseg = heap_segment_next_rw (nseg);
21521 generation_allocation_segment (gen) = nseg;
21522 //reset the allocation pointer and limits
21523 generation_allocation_pointer (gen) =
21524 heap_segment_mem (nseg);
21527 dprintf (1235, ("SP: %Ix->%Ix(%Id)", generation_allocation_pointer (gen), plug, plug - generation_allocation_pointer (gen)));
21528 pinned_len (m) = plug - generation_allocation_pointer (gen);
21529 generation_allocation_pointer (gen) = plug + len;
21532 heap_segment_plan_allocated (generation_allocation_segment (gen)) = generation_allocation_pointer (gen);
21533 generation_allocation_pointer (gen) = 0;
21534 generation_allocation_limit (gen) = 0;
21539 void gc_heap::compact_loh()
21541 assert (should_compact_loh());
21543 generation* gen = large_object_generation;
21544 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
21545 PREFIX_ASSUME(start_seg != NULL);
21546 heap_segment* seg = start_seg;
21547 heap_segment* prev_seg = 0;
21548 uint8_t* o = generation_allocation_start (gen);
21550 //Skip the generation gap object
21551 o = o + AlignQword (size (o));
21552 // We don't need to ever realloc gen3 start so don't touch it.
21553 uint8_t* free_space_start = o;
21554 uint8_t* free_space_end = o;
21555 generation_allocator (gen)->clear();
21556 generation_free_list_space (gen) = 0;
21557 generation_free_obj_space (gen) = 0;
21559 loh_pinned_queue_bos = 0;
21563 if (o >= heap_segment_allocated (seg))
21565 heap_segment* next_seg = heap_segment_next (seg);
21567 if ((heap_segment_plan_allocated (seg) == heap_segment_mem (seg)) &&
21568 (seg != start_seg) && !heap_segment_read_only_p (seg))
21570 dprintf (3, ("Preparing empty large segment %Ix", (size_t)seg));
21572 heap_segment_next (prev_seg) = next_seg;
21573 heap_segment_next (seg) = freeable_large_heap_segment;
21574 freeable_large_heap_segment = seg;
21578 if (!heap_segment_read_only_p (seg))
21580 // We grew the segment to accommodate allocations.
21581 if (heap_segment_plan_allocated (seg) > heap_segment_allocated (seg))
21583 if ((heap_segment_plan_allocated (seg) - plug_skew) > heap_segment_used (seg))
21585 heap_segment_used (seg) = heap_segment_plan_allocated (seg) - plug_skew;
21589 heap_segment_allocated (seg) = heap_segment_plan_allocated (seg);
21590 dprintf (3, ("Trimming seg to %Ix[", heap_segment_allocated (seg)));
21591 decommit_heap_segment_pages (seg, 0);
21592 dprintf (1236, ("CLOH: seg: %Ix, alloc: %Ix, used: %Ix, committed: %Ix",
21594 heap_segment_allocated (seg),
21595 heap_segment_used (seg),
21596 heap_segment_committed (seg)));
21597 //heap_segment_used (seg) = heap_segment_allocated (seg) - plug_skew;
21598 dprintf (1236, ("CLOH: used is set to %Ix", heap_segment_used (seg)));
21608 o = heap_segment_mem (seg);
21614 free_space_end = o;
21615 size_t size = AlignQword (size (o));
21618 uint8_t* reloc = o;
21623 // We are relying on the fact the pinned objects are always looked at in the same order
21624 // in plan phase and in compact phase.
21625 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
21626 uint8_t* plug = pinned_plug (m);
21627 assert (plug == o);
21629 loh_pad = pinned_len (m);
21634 loh_pad = AlignQword (loh_padding_obj_size);
21636 reloc += loh_node_relocation_distance (o);
21637 gcmemcopy (reloc, o, size, TRUE);
21640 thread_gap ((reloc - loh_pad), loh_pad, gen);
21643 free_space_start = o;
21644 if (o < heap_segment_allocated (seg))
21646 assert (!marked (o));
21651 while (o < heap_segment_allocated (seg) && !marked (o))
21653 o = o + AlignQword (size (o));
21658 assert (loh_pinned_plug_que_empty_p());
21660 dprintf (1235, ("after GC LOH size: %Id, free list: %Id, free obj: %Id\n\n",
21661 generation_size (max_generation + 1),
21662 generation_free_list_space (gen),
21663 generation_free_obj_space (gen)));
21666 void gc_heap::relocate_in_loh_compact()
21668 generation* gen = large_object_generation;
21669 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
21670 uint8_t* o = generation_allocation_start (gen);
21672 //Skip the generation gap object
21673 o = o + AlignQword (size (o));
21675 relocate_args args;
21677 args.high = gc_high;
21678 args.last_plug = 0;
21682 if (o >= heap_segment_allocated (seg))
21684 seg = heap_segment_next (seg);
21690 o = heap_segment_mem (seg);
21695 size_t size = AlignQword (size (o));
21697 check_class_object_demotion (o);
21698 if (contain_pointers (o))
21700 go_through_object_nostart (method_table (o), o, size(o), pval,
21702 reloc_survivor_helper (pval);
21707 if (o < heap_segment_allocated (seg))
21709 assert (!marked (o));
21714 while (o < heap_segment_allocated (seg) && !marked (o))
21716 o = o + AlignQword (size (o));
21721 dprintf (1235, ("after GC LOH size: %Id, free list: %Id, free obj: %Id\n\n",
21722 generation_size (max_generation + 1),
21723 generation_free_list_space (gen),
21724 generation_free_obj_space (gen)));
21727 void gc_heap::walk_relocation_for_loh (void* profiling_context, record_surv_fn fn)
21729 generation* gen = large_object_generation;
21730 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
21731 uint8_t* o = generation_allocation_start (gen);
21733 //Skip the generation gap object
21734 o = o + AlignQword (size (o));
21738 if (o >= heap_segment_allocated (seg))
21740 seg = heap_segment_next (seg);
21746 o = heap_segment_mem (seg);
21751 size_t size = AlignQword (size (o));
21753 ptrdiff_t reloc = loh_node_relocation_distance (o);
21755 STRESS_LOG_PLUG_MOVE(o, (o + size), -reloc);
21757 fn (o, (o + size), reloc, profiling_context, !!settings.compaction, false);
21760 if (o < heap_segment_allocated (seg))
21762 assert (!marked (o));
21767 while (o < heap_segment_allocated (seg) && !marked (o))
21769 o = o + AlignQword (size (o));
21775 BOOL gc_heap::loh_object_p (uint8_t* o)
21777 #ifdef MULTIPLE_HEAPS
21778 gc_heap* hp = gc_heap::g_heaps [0];
21779 int brick_entry = hp->brick_table[hp->brick_of (o)];
21780 #else //MULTIPLE_HEAPS
21781 int brick_entry = brick_table[brick_of (o)];
21782 #endif //MULTIPLE_HEAPS
21784 return (brick_entry == 0);
21786 #endif //FEATURE_LOH_COMPACTION
21788 void gc_heap::convert_to_pinned_plug (BOOL& last_npinned_plug_p,
21789 BOOL& last_pinned_plug_p,
21790 BOOL& pinned_plug_p,
21792 size_t& artificial_pinned_size)
21794 last_npinned_plug_p = FALSE;
21795 last_pinned_plug_p = TRUE;
21796 pinned_plug_p = TRUE;
21797 artificial_pinned_size = ps;
21800 // Because we have the artificial pinning, we can't guarantee that pinned and npinned
21801 // plugs are always interleaved.
21802 void gc_heap::store_plug_gap_info (uint8_t* plug_start,
21804 BOOL& last_npinned_plug_p,
21805 BOOL& last_pinned_plug_p,
21806 uint8_t*& last_pinned_plug,
21807 BOOL& pinned_plug_p,
21808 uint8_t* last_object_in_last_plug,
21809 BOOL& merge_with_last_pin_p,
21810 // this is only for verification purpose
21811 size_t last_plug_len)
21813 UNREFERENCED_PARAMETER(last_plug_len);
21815 if (!last_npinned_plug_p && !last_pinned_plug_p)
21817 //dprintf (3, ("last full plug end: %Ix, full plug start: %Ix", plug_end, plug_start));
21818 dprintf (3, ("Free: %Ix", (plug_start - plug_end)));
21819 assert ((plug_start == plug_end) || ((size_t)(plug_start - plug_end) >= Align (min_obj_size)));
21820 set_gap_size (plug_start, plug_start - plug_end);
21823 if (pinned (plug_start))
21825 BOOL save_pre_plug_info_p = FALSE;
21827 if (last_npinned_plug_p || last_pinned_plug_p)
21829 //if (last_plug_len == Align (min_obj_size))
21831 // dprintf (3, ("debugging only - last npinned plug is min, check to see if it's correct"));
21832 // GCToOSInterface::DebugBreak();
21834 save_pre_plug_info_p = TRUE;
21837 pinned_plug_p = TRUE;
21838 last_npinned_plug_p = FALSE;
21840 if (last_pinned_plug_p)
21842 dprintf (3, ("last plug %Ix was also pinned, should merge", last_pinned_plug));
21843 merge_with_last_pin_p = TRUE;
21847 last_pinned_plug_p = TRUE;
21848 last_pinned_plug = plug_start;
21850 enque_pinned_plug (last_pinned_plug, save_pre_plug_info_p, last_object_in_last_plug);
21852 if (save_pre_plug_info_p)
21854 set_gap_size (plug_start, sizeof (gap_reloc_pair));
21860 if (last_pinned_plug_p)
21862 //if (Align (last_plug_len) < min_pre_pin_obj_size)
21864 // dprintf (3, ("debugging only - last pinned plug is min, check to see if it's correct"));
21865 // GCToOSInterface::DebugBreak();
21868 save_post_plug_info (last_pinned_plug, last_object_in_last_plug, plug_start);
21869 set_gap_size (plug_start, sizeof (gap_reloc_pair));
21871 verify_pins_with_post_plug_info("after saving post plug info");
21873 last_npinned_plug_p = TRUE;
21874 last_pinned_plug_p = FALSE;
21878 void gc_heap::record_interesting_data_point (interesting_data_point idp)
21880 #ifdef GC_CONFIG_DRIVEN
21881 (interesting_data_per_gc[idp])++;
21883 UNREFERENCED_PARAMETER(idp);
21884 #endif //GC_CONFIG_DRIVEN
21888 #pragma warning(push)
21889 #pragma warning(disable:21000) // Suppress PREFast warning about overly large function
21891 void gc_heap::plan_phase (int condemned_gen_number)
21893 size_t old_gen2_allocated = 0;
21894 size_t old_gen2_size = 0;
21896 if (condemned_gen_number == (max_generation - 1))
21898 old_gen2_allocated = generation_free_list_allocated (generation_of (max_generation));
21899 old_gen2_size = generation_size (max_generation);
21902 assert (settings.concurrent == FALSE);
21904 // %type% category = quote (plan);
21908 start = GetCycleCount32();
21911 dprintf (2,("---- Plan Phase ---- Condemned generation %d, promotion: %d",
21912 condemned_gen_number, settings.promotion ? 1 : 0));
21914 generation* condemned_gen1 = generation_of (condemned_gen_number);
21917 BOOL use_mark_list = FALSE;
21918 uint8_t** mark_list_next = &mark_list[0];
21919 #ifdef GC_CONFIG_DRIVEN
21920 dprintf (3, ("total number of marked objects: %Id (%Id)",
21921 (mark_list_index - &mark_list[0]), ((mark_list_end - &mark_list[0]))));
21923 if (mark_list_index >= (mark_list_end + 1))
21924 mark_list_index = mark_list_end + 1;
21926 dprintf (3, ("mark_list length: %Id",
21927 (mark_list_index - &mark_list[0])));
21928 #endif //GC_CONFIG_DRIVEN
21930 if ((condemned_gen_number < max_generation) &&
21931 (mark_list_index <= mark_list_end)
21932 #ifdef BACKGROUND_GC
21933 && (!recursive_gc_sync::background_running_p())
21934 #endif //BACKGROUND_GC
21937 #ifndef MULTIPLE_HEAPS
21938 _sort (&mark_list[0], mark_list_index-1, 0);
21939 //printf ("using mark list at GC #%d", dd_collection_count (dynamic_data_of (0)));
21940 //verify_qsort_array (&mark_list[0], mark_list_index-1);
21941 #endif //!MULTIPLE_HEAPS
21942 use_mark_list = TRUE;
21943 get_gc_data_per_heap()->set_mechanism_bit (gc_mark_list_bit);
21947 dprintf (3, ("mark_list not used"));
21952 #ifdef FEATURE_BASICFREEZE
21953 if ((generation_start_segment (condemned_gen1) != ephemeral_heap_segment) &&
21954 ro_segments_in_range)
21956 sweep_ro_segments (generation_start_segment (condemned_gen1));
21958 #endif // FEATURE_BASICFREEZE
21960 #ifndef MULTIPLE_HEAPS
21961 if (shigh != (uint8_t*)0)
21963 heap_segment* seg = heap_segment_rw (generation_start_segment (condemned_gen1));
21965 PREFIX_ASSUME(seg != NULL);
21967 heap_segment* fseg = seg;
21970 if (slow > heap_segment_mem (seg) &&
21971 slow < heap_segment_reserved (seg))
21975 uint8_t* o = generation_allocation_start (condemned_gen1) +
21976 Align (size (generation_allocation_start (condemned_gen1)));
21979 assert ((slow - o) >= (int)Align (min_obj_size));
21980 #ifdef BACKGROUND_GC
21981 if (current_c_gc_state == c_gc_state_marking)
21983 bgc_clear_batch_mark_array_bits (o, slow);
21985 #endif //BACKGROUND_GC
21986 make_unused_array (o, slow - o);
21991 assert (condemned_gen_number == max_generation);
21992 make_unused_array (heap_segment_mem (seg),
21993 slow - heap_segment_mem (seg));
21996 if (in_range_for_segment (shigh, seg))
21998 #ifdef BACKGROUND_GC
21999 if (current_c_gc_state == c_gc_state_marking)
22001 bgc_clear_batch_mark_array_bits ((shigh + Align (size (shigh))), heap_segment_allocated (seg));
22003 #endif //BACKGROUND_GC
22004 heap_segment_allocated (seg) = shigh + Align (size (shigh));
22006 // test if the segment is in the range of [slow, shigh]
22007 if (!((heap_segment_reserved (seg) >= slow) &&
22008 (heap_segment_mem (seg) <= shigh)))
22010 // shorten it to minimum
22011 heap_segment_allocated (seg) = heap_segment_mem (seg);
22013 seg = heap_segment_next_rw (seg);
22018 heap_segment* seg = heap_segment_rw (generation_start_segment (condemned_gen1));
22020 PREFIX_ASSUME(seg != NULL);
22022 heap_segment* sseg = seg;
22025 // shorten it to minimum
22028 // no survivors make all generations look empty
22029 uint8_t* o = generation_allocation_start (condemned_gen1) +
22030 Align (size (generation_allocation_start (condemned_gen1)));
22031 #ifdef BACKGROUND_GC
22032 if (current_c_gc_state == c_gc_state_marking)
22034 bgc_clear_batch_mark_array_bits (o, heap_segment_allocated (seg));
22036 #endif //BACKGROUND_GC
22037 heap_segment_allocated (seg) = o;
22041 assert (condemned_gen_number == max_generation);
22042 #ifdef BACKGROUND_GC
22043 if (current_c_gc_state == c_gc_state_marking)
22045 bgc_clear_batch_mark_array_bits (heap_segment_mem (seg), heap_segment_allocated (seg));
22047 #endif //BACKGROUND_GC
22048 heap_segment_allocated (seg) = heap_segment_mem (seg);
22050 seg = heap_segment_next_rw (seg);
22054 #endif //MULTIPLE_HEAPS
22056 heap_segment* seg1 = heap_segment_rw (generation_start_segment (condemned_gen1));
22058 PREFIX_ASSUME(seg1 != NULL);
22060 uint8_t* end = heap_segment_allocated (seg1);
22061 uint8_t* first_condemned_address = generation_allocation_start (condemned_gen1);
22062 uint8_t* x = first_condemned_address;
22064 assert (!marked (x));
22065 uint8_t* plug_end = x;
22067 size_t sequence_number = 0;
22068 uint8_t* last_node = 0;
22069 size_t current_brick = brick_of (x);
22070 BOOL allocate_in_condemned = ((condemned_gen_number == max_generation)||
22071 (settings.promotion == FALSE));
22072 int active_old_gen_number = condemned_gen_number;
22073 int active_new_gen_number = (allocate_in_condemned ? condemned_gen_number:
22074 (1 + condemned_gen_number));
22075 generation* older_gen = 0;
22076 generation* consing_gen = condemned_gen1;
22077 alloc_list r_free_list [MAX_BUCKET_COUNT];
22079 size_t r_free_list_space = 0;
22080 size_t r_free_obj_space = 0;
22081 size_t r_older_gen_free_list_allocated = 0;
22082 size_t r_older_gen_condemned_allocated = 0;
22083 size_t r_older_gen_end_seg_allocated = 0;
22084 uint8_t* r_allocation_pointer = 0;
22085 uint8_t* r_allocation_limit = 0;
22086 uint8_t* r_allocation_start_region = 0;
22087 heap_segment* r_allocation_segment = 0;
22088 #ifdef FREE_USAGE_STATS
22089 size_t r_older_gen_free_space[NUM_GEN_POWER2];
22090 #endif //FREE_USAGE_STATS
22092 if ((condemned_gen_number < max_generation))
22094 older_gen = generation_of (min (max_generation, 1 + condemned_gen_number));
22095 generation_allocator (older_gen)->copy_to_alloc_list (r_free_list);
22097 r_free_list_space = generation_free_list_space (older_gen);
22098 r_free_obj_space = generation_free_obj_space (older_gen);
22099 #ifdef FREE_USAGE_STATS
22100 memcpy (r_older_gen_free_space, older_gen->gen_free_spaces, sizeof (r_older_gen_free_space));
22101 #endif //FREE_USAGE_STATS
22102 generation_allocate_end_seg_p (older_gen) = FALSE;
22103 r_older_gen_free_list_allocated = generation_free_list_allocated (older_gen);
22104 r_older_gen_condemned_allocated = generation_condemned_allocated (older_gen);
22105 r_older_gen_end_seg_allocated = generation_end_seg_allocated (older_gen);
22106 r_allocation_limit = generation_allocation_limit (older_gen);
22107 r_allocation_pointer = generation_allocation_pointer (older_gen);
22108 r_allocation_start_region = generation_allocation_context_start_region (older_gen);
22109 r_allocation_segment = generation_allocation_segment (older_gen);
22110 heap_segment* start_seg = heap_segment_rw (generation_start_segment (older_gen));
22112 PREFIX_ASSUME(start_seg != NULL);
22114 if (start_seg != ephemeral_heap_segment)
22116 assert (condemned_gen_number == (max_generation - 1));
22117 while (start_seg && (start_seg != ephemeral_heap_segment))
22119 assert (heap_segment_allocated (start_seg) >=
22120 heap_segment_mem (start_seg));
22121 assert (heap_segment_allocated (start_seg) <=
22122 heap_segment_reserved (start_seg));
22123 heap_segment_plan_allocated (start_seg) =
22124 heap_segment_allocated (start_seg);
22125 start_seg = heap_segment_next_rw (start_seg);
22130 //reset all of the segment allocated sizes
22132 heap_segment* seg2 = heap_segment_rw (generation_start_segment (condemned_gen1));
22134 PREFIX_ASSUME(seg2 != NULL);
22138 heap_segment_plan_allocated (seg2) =
22139 heap_segment_mem (seg2);
22140 seg2 = heap_segment_next_rw (seg2);
22143 int condemned_gn = condemned_gen_number;
22145 int bottom_gen = 0;
22146 init_free_and_plug();
22148 while (condemned_gn >= bottom_gen)
22150 generation* condemned_gen2 = generation_of (condemned_gn);
22151 generation_allocator (condemned_gen2)->clear();
22152 generation_free_list_space (condemned_gen2) = 0;
22153 generation_free_obj_space (condemned_gen2) = 0;
22154 generation_allocation_size (condemned_gen2) = 0;
22155 generation_condemned_allocated (condemned_gen2) = 0;
22156 generation_pinned_allocated (condemned_gen2) = 0;
22157 generation_free_list_allocated(condemned_gen2) = 0;
22158 generation_end_seg_allocated (condemned_gen2) = 0;
22159 generation_pinned_allocation_sweep_size (condemned_gen2) = 0;
22160 generation_pinned_allocation_compact_size (condemned_gen2) = 0;
22161 #ifdef FREE_USAGE_STATS
22162 generation_pinned_free_obj_space (condemned_gen2) = 0;
22163 generation_allocated_in_pinned_free (condemned_gen2) = 0;
22164 generation_allocated_since_last_pin (condemned_gen2) = 0;
22165 #endif //FREE_USAGE_STATS
22166 generation_plan_allocation_start (condemned_gen2) = 0;
22167 generation_allocation_segment (condemned_gen2) =
22168 heap_segment_rw (generation_start_segment (condemned_gen2));
22170 PREFIX_ASSUME(generation_allocation_segment(condemned_gen2) != NULL);
22172 if (generation_start_segment (condemned_gen2) != ephemeral_heap_segment)
22174 generation_allocation_pointer (condemned_gen2) =
22175 heap_segment_mem (generation_allocation_segment (condemned_gen2));
22179 generation_allocation_pointer (condemned_gen2) = generation_allocation_start (condemned_gen2);
22182 generation_allocation_limit (condemned_gen2) = generation_allocation_pointer (condemned_gen2);
22183 generation_allocation_context_start_region (condemned_gen2) = generation_allocation_pointer (condemned_gen2);
22188 BOOL allocate_first_generation_start = FALSE;
22190 if (allocate_in_condemned)
22192 allocate_first_generation_start = TRUE;
22195 dprintf(3,( " From %Ix to %Ix", (size_t)x, (size_t)end));
22197 demotion_low = MAX_PTR;
22198 demotion_high = heap_segment_allocated (ephemeral_heap_segment);
22200 // If we are doing a gen1 only because of cards, it means we should not demote any pinned plugs
22201 // from gen1. They should get promoted to gen2.
22202 demote_gen1_p = !(settings.promotion &&
22203 (settings.condemned_generation == (max_generation - 1)) &&
22204 gen_to_condemn_reasons.is_only_condition (gen_low_card_p));
22206 total_ephemeral_size = 0;
22208 print_free_and_plug ("BP");
22210 for (int gen_idx = 0; gen_idx <= max_generation; gen_idx++)
22212 generation* temp_gen = generation_of (gen_idx);
22214 dprintf (2, ("gen%d start %Ix, plan start %Ix",
22216 generation_allocation_start (temp_gen),
22217 generation_plan_allocation_start (temp_gen)));
22220 BOOL fire_pinned_plug_events_p = EVENT_ENABLED(PinPlugAtGCTime);
22221 size_t last_plug_len = 0;
22228 assert (heap_segment_allocated (seg1) == end);
22229 heap_segment_allocated (seg1) = plug_end;
22231 current_brick = update_brick_table (tree, current_brick, x, plug_end);
22232 dprintf (3, ("end of seg: new tree, sequence# 0"));
22233 sequence_number = 0;
22236 if (heap_segment_next_rw (seg1))
22238 seg1 = heap_segment_next_rw (seg1);
22239 end = heap_segment_allocated (seg1);
22240 plug_end = x = heap_segment_mem (seg1);
22241 current_brick = brick_of (x);
22242 dprintf(3,( " From %Ix to %Ix", (size_t)x, (size_t)end));
22251 BOOL last_npinned_plug_p = FALSE;
22252 BOOL last_pinned_plug_p = FALSE;
22254 // last_pinned_plug is the beginning of the last pinned plug. If we merge a plug into a pinned
22255 // plug we do not change the value of last_pinned_plug. This happens with artificially pinned plugs -
22256 // it can be merged with a previous pinned plug and a pinned plug after it can be merged with it.
22257 uint8_t* last_pinned_plug = 0;
22258 size_t num_pinned_plugs_in_plug = 0;
22260 uint8_t* last_object_in_plug = 0;
22262 while ((x < end) && marked (x))
22264 uint8_t* plug_start = x;
22265 uint8_t* saved_plug_end = plug_end;
22266 BOOL pinned_plug_p = FALSE;
22267 BOOL npin_before_pin_p = FALSE;
22268 BOOL saved_last_npinned_plug_p = last_npinned_plug_p;
22269 uint8_t* saved_last_object_in_plug = last_object_in_plug;
22270 BOOL merge_with_last_pin_p = FALSE;
22272 size_t added_pinning_size = 0;
22273 size_t artificial_pinned_size = 0;
22275 store_plug_gap_info (plug_start, plug_end, last_npinned_plug_p, last_pinned_plug_p,
22276 last_pinned_plug, pinned_plug_p, last_object_in_plug,
22277 merge_with_last_pin_p, last_plug_len);
22279 #ifdef FEATURE_STRUCTALIGN
22280 int requiredAlignment = ((CObjectHeader*)plug_start)->GetRequiredAlignment();
22281 size_t alignmentOffset = OBJECT_ALIGNMENT_OFFSET;
22282 #endif // FEATURE_STRUCTALIGN
22286 while ((xl < end) && marked (xl) && (pinned (xl) == pinned_plug_p))
22293 #ifdef FEATURE_STRUCTALIGN
22296 int obj_requiredAlignment = ((CObjectHeader*)xl)->GetRequiredAlignment();
22297 if (obj_requiredAlignment > requiredAlignment)
22299 requiredAlignment = obj_requiredAlignment;
22300 alignmentOffset = xl - plug_start + OBJECT_ALIGNMENT_OFFSET;
22303 #endif // FEATURE_STRUCTALIGN
22307 dprintf(4, ("+%Ix+", (size_t)xl));
22308 assert ((size (xl) > 0));
22309 assert ((size (xl) <= loh_size_threshold));
22311 last_object_in_plug = xl;
22313 xl = xl + Align (size (xl));
22317 BOOL next_object_marked_p = ((xl < end) && marked (xl));
22321 // If it is pinned we need to extend to the next marked object as we can't use part of
22322 // a pinned object to make the artificial gap (unless the last 3 ptr sized words are all
22323 // references but for now I am just using the next non pinned object for that).
22324 if (next_object_marked_p)
22327 last_object_in_plug = xl;
22328 size_t extra_size = Align (size (xl));
22329 xl = xl + extra_size;
22330 added_pinning_size = extra_size;
22335 if (next_object_marked_p)
22336 npin_before_pin_p = TRUE;
22339 assert (xl <= end);
22342 dprintf (3, ( "%Ix[", (size_t)x));
22344 size_t ps = plug_end - plug_start;
22345 last_plug_len = ps;
22346 dprintf (3, ( "%Ix[(%Ix)", (size_t)x, ps));
22347 uint8_t* new_address = 0;
22349 if (!pinned_plug_p)
22351 if (allocate_in_condemned &&
22352 (settings.condemned_generation == max_generation) &&
22353 (ps > OS_PAGE_SIZE))
22355 ptrdiff_t reloc = plug_start - generation_allocation_pointer (consing_gen);
22356 //reloc should >=0 except when we relocate
22357 //across segments and the dest seg is higher then the src
22359 if ((ps > (8*OS_PAGE_SIZE)) &&
22361 ((size_t)reloc < (ps/16)))
22363 dprintf (3, ("Pinning %Ix; reloc would have been: %Ix",
22364 (size_t)plug_start, reloc));
22365 // The last plug couldn't have been a npinned plug or it would have
22366 // included this plug.
22367 assert (!saved_last_npinned_plug_p);
22369 if (last_pinned_plug)
22371 dprintf (3, ("artificially pinned plug merged with last pinned plug"));
22372 merge_with_last_pin_p = TRUE;
22376 enque_pinned_plug (plug_start, FALSE, 0);
22377 last_pinned_plug = plug_start;
22380 convert_to_pinned_plug (last_npinned_plug_p, last_pinned_plug_p, pinned_plug_p,
22381 ps, artificial_pinned_size);
22386 if (allocate_first_generation_start)
22388 allocate_first_generation_start = FALSE;
22389 plan_generation_start (condemned_gen1, consing_gen, plug_start);
22390 assert (generation_plan_allocation_start (condemned_gen1));
22393 if (seg1 == ephemeral_heap_segment)
22395 process_ephemeral_boundaries (plug_start, active_new_gen_number,
22396 active_old_gen_number,
22398 allocate_in_condemned);
22401 dprintf (3, ("adding %Id to gen%d surv", ps, active_old_gen_number));
22403 dynamic_data* dd_active_old = dynamic_data_of (active_old_gen_number);
22404 dd_survived_size (dd_active_old) += ps;
22406 BOOL convert_to_pinned_p = FALSE;
22408 if (!pinned_plug_p)
22410 #if defined (RESPECT_LARGE_ALIGNMENT) || defined (FEATURE_STRUCTALIGN)
22411 dd_num_npinned_plugs (dd_active_old)++;
22412 #endif //RESPECT_LARGE_ALIGNMENT || FEATURE_STRUCTALIGN
22414 add_gen_plug (active_old_gen_number, ps);
22416 if (allocate_in_condemned)
22418 verify_pins_with_post_plug_info("before aic");
22421 allocate_in_condemned_generations (consing_gen,
22423 active_old_gen_number,
22425 &convert_to_pinned_p,
22426 (npin_before_pin_p ? plug_end : 0),
22428 #endif //SHORT_PLUGS
22429 plug_start REQD_ALIGN_AND_OFFSET_ARG);
22430 verify_pins_with_post_plug_info("after aic");
22434 new_address = allocate_in_older_generation (older_gen, ps, active_old_gen_number, plug_start REQD_ALIGN_AND_OFFSET_ARG);
22436 if (new_address != 0)
22438 if (settings.condemned_generation == (max_generation - 1))
22440 dprintf (3, (" NA: %Ix-%Ix -> %Ix, %Ix (%Ix)",
22441 plug_start, plug_end,
22442 (size_t)new_address, (size_t)new_address + (plug_end - plug_start),
22443 (size_t)(plug_end - plug_start)));
22448 if (generation_allocator(older_gen)->discard_if_no_fit_p())
22450 allocate_in_condemned = TRUE;
22453 new_address = allocate_in_condemned_generations (consing_gen, ps, active_old_gen_number,
22455 &convert_to_pinned_p,
22456 (npin_before_pin_p ? plug_end : 0),
22458 #endif //SHORT_PLUGS
22459 plug_start REQD_ALIGN_AND_OFFSET_ARG);
22463 if (convert_to_pinned_p)
22465 assert (last_npinned_plug_p != FALSE);
22466 assert (last_pinned_plug_p == FALSE);
22467 convert_to_pinned_plug (last_npinned_plug_p, last_pinned_plug_p, pinned_plug_p,
22468 ps, artificial_pinned_size);
22469 enque_pinned_plug (plug_start, FALSE, 0);
22470 last_pinned_plug = plug_start;
22476 //verify that we are at then end of the ephemeral segment
22477 assert (generation_allocation_segment (consing_gen) ==
22478 ephemeral_heap_segment);
22479 //verify that we are near the end
22480 assert ((generation_allocation_pointer (consing_gen) + Align (ps)) <
22481 heap_segment_allocated (ephemeral_heap_segment));
22482 assert ((generation_allocation_pointer (consing_gen) + Align (ps)) >
22483 (heap_segment_allocated (ephemeral_heap_segment) + Align (min_obj_size)));
22487 #ifdef SIMPLE_DPRINTF
22488 dprintf (3, ("(%Ix)[%Ix->%Ix, NA: [%Ix(%Id), %Ix[: %Ix(%d)",
22489 (size_t)(node_gap_size (plug_start)),
22490 plug_start, plug_end, (size_t)new_address, (size_t)(plug_start - new_address),
22491 (size_t)new_address + ps, ps,
22492 (is_plug_padded (plug_start) ? 1 : 0)));
22493 #endif //SIMPLE_DPRINTF
22496 if (is_plug_padded (plug_start))
22498 dprintf (3, ("%Ix was padded", plug_start));
22499 dd_padding_size (dd_active_old) += Align (min_obj_size);
22501 #endif //SHORT_PLUGS
22508 if (fire_pinned_plug_events_p)
22510 FIRE_EVENT(PinPlugAtGCTime, plug_start, plug_end,
22511 (merge_with_last_pin_p ? 0 : (uint8_t*)node_gap_size (plug_start)));
22514 if (merge_with_last_pin_p)
22516 merge_with_last_pinned_plug (last_pinned_plug, ps);
22520 assert (last_pinned_plug == plug_start);
22521 set_pinned_info (plug_start, ps, consing_gen);
22524 new_address = plug_start;
22526 dprintf (3, ( "(%Ix)PP: [%Ix, %Ix[%Ix](m:%d)",
22527 (size_t)(node_gap_size (plug_start)), (size_t)plug_start,
22528 (size_t)plug_end, ps,
22529 (merge_with_last_pin_p ? 1 : 0)));
22531 dprintf (3, ("adding %Id to gen%d pinned surv", plug_end - plug_start, active_old_gen_number));
22532 dd_pinned_survived_size (dd_active_old) += plug_end - plug_start;
22533 dd_added_pinned_size (dd_active_old) += added_pinning_size;
22534 dd_artificial_pinned_survived_size (dd_active_old) += artificial_pinned_size;
22536 if (!demote_gen1_p && (active_old_gen_number == (max_generation - 1)))
22538 last_gen1_pin_end = plug_end;
22543 // detect forward allocation in the same segment
22544 assert (!((new_address > plug_start) &&
22545 (new_address < heap_segment_reserved (seg1))));
22548 if (!merge_with_last_pin_p)
22550 if (current_brick != brick_of (plug_start))
22552 current_brick = update_brick_table (tree, current_brick, plug_start, saved_plug_end);
22553 sequence_number = 0;
22557 set_node_relocation_distance (plug_start, (new_address - plug_start));
22558 if (last_node && (node_relocation_distance (last_node) ==
22559 (node_relocation_distance (plug_start) +
22560 (ptrdiff_t)node_gap_size (plug_start))))
22562 //dprintf(3,( " Lb"));
22563 dprintf (3, ("%Ix Lb", plug_start));
22564 set_node_left (plug_start);
22566 if (0 == sequence_number)
22568 dprintf (2, ("sn: 0, tree is set to %Ix", plug_start));
22572 verify_pins_with_post_plug_info("before insert node");
22574 tree = insert_node (plug_start, ++sequence_number, tree, last_node);
22575 dprintf (3, ("tree is %Ix (b: %Ix) after insert_node", tree, brick_of (tree)));
22576 last_node = plug_start;
22579 // If we detect if the last plug is pinned plug right before us, we should save this gap info
22580 if (!pinned_plug_p)
22582 if (mark_stack_tos > 0)
22584 mark& m = mark_stack_array[mark_stack_tos - 1];
22585 if (m.has_post_plug_info())
22587 uint8_t* post_plug_info_start = m.saved_post_plug_info_start;
22588 size_t* current_plug_gap_start = (size_t*)(plug_start - sizeof (plug_and_gap));
22589 if ((uint8_t*)current_plug_gap_start == post_plug_info_start)
22591 dprintf (3, ("Ginfo: %Ix, %Ix, %Ix",
22592 *current_plug_gap_start, *(current_plug_gap_start + 1),
22593 *(current_plug_gap_start + 2)));
22594 memcpy (&(m.saved_post_plug_debug), current_plug_gap_start, sizeof (gap_reloc_pair));
22601 verify_pins_with_post_plug_info("after insert node");
22605 if (num_pinned_plugs_in_plug > 1)
22607 dprintf (3, ("more than %Id pinned plugs in this plug", num_pinned_plugs_in_plug));
22614 while ((mark_list_next < mark_list_index) &&
22615 (*mark_list_next <= x))
22619 if ((mark_list_next < mark_list_index)
22620 #ifdef MULTIPLE_HEAPS
22621 && (*mark_list_next < end) //for multiple segments
22622 #endif //MULTIPLE_HEAPS
22624 x = *mark_list_next;
22632 #ifdef BACKGROUND_GC
22633 if (current_c_gc_state == c_gc_state_marking)
22635 assert (recursive_gc_sync::background_running_p());
22636 while ((xl < end) && !marked (xl))
22638 dprintf (4, ("-%Ix-", (size_t)xl));
22639 assert ((size (xl) > 0));
22640 background_object_marked (xl, TRUE);
22641 xl = xl + Align (size (xl));
22646 #endif //BACKGROUND_GC
22648 while ((xl < end) && !marked (xl))
22650 dprintf (4, ("-%Ix-", (size_t)xl));
22651 assert ((size (xl) > 0));
22652 xl = xl + Align (size (xl));
22656 assert (xl <= end);
22662 while (!pinned_plug_que_empty_p())
22664 if (settings.promotion)
22666 uint8_t* pplug = pinned_plug (oldest_pin());
22667 if (in_range_for_segment (pplug, ephemeral_heap_segment))
22669 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
22670 //allocate all of the generation gaps
22671 while (active_new_gen_number > 0)
22673 active_new_gen_number--;
22675 if (active_new_gen_number == (max_generation - 1))
22677 maxgen_pinned_compact_before_advance = generation_pinned_allocation_compact_size (generation_of (max_generation));
22678 if (!demote_gen1_p)
22679 advance_pins_for_demotion (consing_gen);
22682 generation* gen = generation_of (active_new_gen_number);
22683 plan_generation_start (gen, consing_gen, 0);
22685 if (demotion_low == MAX_PTR)
22687 demotion_low = pplug;
22688 dprintf (3, ("end plan: dlow->%Ix", demotion_low));
22691 dprintf (2, ("(%d)gen%d plan start: %Ix",
22692 heap_number, active_new_gen_number, (size_t)generation_plan_allocation_start (gen)));
22693 assert (generation_plan_allocation_start (gen));
22698 if (pinned_plug_que_empty_p())
22701 size_t entry = deque_pinned_plug();
22702 mark* m = pinned_plug_of (entry);
22703 uint8_t* plug = pinned_plug (m);
22704 size_t len = pinned_len (m);
22706 // detect pinned block in different segment (later) than
22707 // allocation segment
22708 heap_segment* nseg = heap_segment_rw (generation_allocation_segment (consing_gen));
22710 while ((plug < generation_allocation_pointer (consing_gen)) ||
22711 (plug >= heap_segment_allocated (nseg)))
22713 assert ((plug < heap_segment_mem (nseg)) ||
22714 (plug > heap_segment_reserved (nseg)));
22715 //adjust the end of the segment to be the end of the plug
22716 assert (generation_allocation_pointer (consing_gen)>=
22717 heap_segment_mem (nseg));
22718 assert (generation_allocation_pointer (consing_gen)<=
22719 heap_segment_committed (nseg));
22721 heap_segment_plan_allocated (nseg) =
22722 generation_allocation_pointer (consing_gen);
22723 //switch allocation segment
22724 nseg = heap_segment_next_rw (nseg);
22725 generation_allocation_segment (consing_gen) = nseg;
22726 //reset the allocation pointer and limits
22727 generation_allocation_pointer (consing_gen) =
22728 heap_segment_mem (nseg);
22731 set_new_pin_info (m, generation_allocation_pointer (consing_gen));
22732 dprintf (2, ("pin %Ix b: %Ix->%Ix", plug, brick_of (plug),
22733 (size_t)(brick_table[brick_of (plug)])));
22735 generation_allocation_pointer (consing_gen) = plug + len;
22736 generation_allocation_limit (consing_gen) =
22737 generation_allocation_pointer (consing_gen);
22738 //Add the size of the pinned plug to the right pinned allocations
22739 //find out which gen this pinned plug came from
22740 int frgn = object_gennum (plug);
22741 if ((frgn != (int)max_generation) && settings.promotion)
22743 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
22748 plan_generation_starts (consing_gen);
22749 print_free_and_plug ("AP");
22752 #ifdef SIMPLE_DPRINTF
22753 for (int gen_idx = 0; gen_idx <= max_generation; gen_idx++)
22755 generation* temp_gen = generation_of (gen_idx);
22756 dynamic_data* temp_dd = dynamic_data_of (gen_idx);
22758 int added_pinning_ratio = 0;
22759 int artificial_pinned_ratio = 0;
22761 if (dd_pinned_survived_size (temp_dd) != 0)
22763 added_pinning_ratio = (int)((float)dd_added_pinned_size (temp_dd) * 100 / (float)dd_pinned_survived_size (temp_dd));
22764 artificial_pinned_ratio = (int)((float)dd_artificial_pinned_survived_size (temp_dd) * 100 / (float)dd_pinned_survived_size (temp_dd));
22767 size_t padding_size =
22769 dd_padding_size (temp_dd);
22772 #endif //SHORT_PLUGS
22773 dprintf (1, ("gen%d: %Ix, %Ix(%Id), NON PIN alloc: %Id, pin com: %Id, sweep: %Id, surv: %Id, pinsurv: %Id(%d%% added, %d%% art), np surv: %Id, pad: %Id",
22775 generation_allocation_start (temp_gen),
22776 generation_plan_allocation_start (temp_gen),
22777 (size_t)(generation_plan_allocation_start (temp_gen) - generation_allocation_start (temp_gen)),
22778 generation_allocation_size (temp_gen),
22779 generation_pinned_allocation_compact_size (temp_gen),
22780 generation_pinned_allocation_sweep_size (temp_gen),
22781 dd_survived_size (temp_dd),
22782 dd_pinned_survived_size (temp_dd),
22783 added_pinning_ratio,
22784 artificial_pinned_ratio,
22785 (dd_survived_size (temp_dd) - dd_pinned_survived_size (temp_dd)),
22788 #endif //SIMPLE_DPRINTF
22791 if (settings.condemned_generation == (max_generation - 1 ))
22793 size_t plan_gen2_size = generation_plan_size (max_generation);
22794 size_t growth = plan_gen2_size - old_gen2_size;
22796 generation* older_gen = generation_of (settings.condemned_generation + 1);
22797 size_t rejected_free_space = generation_free_obj_space (older_gen) - r_free_obj_space;
22798 size_t free_list_allocated = generation_free_list_allocated (older_gen) - r_older_gen_free_list_allocated;
22799 size_t end_seg_allocated = generation_end_seg_allocated (older_gen) - r_older_gen_end_seg_allocated;
22800 size_t condemned_allocated = generation_condemned_allocated (older_gen) - r_older_gen_condemned_allocated;
22804 dprintf (1, ("gen2 grew %Id (end seg alloc: %Id, condemned alloc: %Id",
22805 growth, end_seg_allocated, condemned_allocated));
22807 maxgen_size_inc_p = true;
22811 dprintf (2, ("gen2 shrank %Id (end seg alloc: %Id, , condemned alloc: %Id, gen1 c alloc: %Id",
22812 (old_gen2_size - plan_gen2_size), end_seg_allocated, condemned_allocated,
22813 generation_condemned_allocated (generation_of (max_generation - 1))));
22816 dprintf (1, ("older gen's free alloc: %Id->%Id, seg alloc: %Id->%Id, condemned alloc: %Id->%Id",
22817 r_older_gen_free_list_allocated, generation_free_list_allocated (older_gen),
22818 r_older_gen_end_seg_allocated, generation_end_seg_allocated (older_gen),
22819 r_older_gen_condemned_allocated, generation_condemned_allocated (older_gen)));
22821 dprintf (1, ("this GC did %Id free list alloc(%Id bytes free space rejected)",
22822 free_list_allocated, rejected_free_space));
22824 maxgen_size_increase* maxgen_size_info = &(get_gc_data_per_heap()->maxgen_size_info);
22825 maxgen_size_info->free_list_allocated = free_list_allocated;
22826 maxgen_size_info->free_list_rejected = rejected_free_space;
22827 maxgen_size_info->end_seg_allocated = end_seg_allocated;
22828 maxgen_size_info->condemned_allocated = condemned_allocated;
22829 maxgen_size_info->pinned_allocated = maxgen_pinned_compact_before_advance;
22830 maxgen_size_info->pinned_allocated_advance = generation_pinned_allocation_compact_size (generation_of (max_generation)) - maxgen_pinned_compact_before_advance;
22832 #ifdef FREE_USAGE_STATS
22833 int free_list_efficiency = 0;
22834 if ((free_list_allocated + rejected_free_space) != 0)
22835 free_list_efficiency = (int)(((float) (free_list_allocated) / (float)(free_list_allocated + rejected_free_space)) * (float)100);
22837 int running_free_list_efficiency = (int)(generation_allocator_efficiency(older_gen)*100);
22839 dprintf (1, ("gen%d free list alloc effi: %d%%, current effi: %d%%",
22840 older_gen->gen_num,
22841 free_list_efficiency, running_free_list_efficiency));
22843 dprintf (1, ("gen2 free list change"));
22844 for (int j = 0; j < NUM_GEN_POWER2; j++)
22846 dprintf (1, ("[h%d][#%Id]: 2^%d: F: %Id->%Id(%Id), P: %Id",
22849 (j + 10), r_older_gen_free_space[j], older_gen->gen_free_spaces[j],
22850 (ptrdiff_t)(r_older_gen_free_space[j] - older_gen->gen_free_spaces[j]),
22851 (generation_of(max_generation - 1))->gen_plugs[j]));
22853 #endif //FREE_USAGE_STATS
22856 size_t fragmentation =
22857 generation_fragmentation (generation_of (condemned_gen_number),
22859 heap_segment_allocated (ephemeral_heap_segment));
22861 dprintf (2,("Fragmentation: %Id", fragmentation));
22862 dprintf (2,("---- End of Plan phase ----"));
22865 finish = GetCycleCount32();
22866 plan_time = finish - start;
22869 // We may update write barrier code. We assume here EE has been suspended if we are on a GC thread.
22870 assert(IsGCInProgress());
22872 BOOL should_expand = FALSE;
22873 BOOL should_compact= FALSE;
22874 ephemeral_promotion = FALSE;
22877 if ((!settings.concurrent) &&
22878 !provisional_mode_triggered &&
22879 ((condemned_gen_number < max_generation) &&
22880 ((settings.gen0_reduction_count > 0) || (settings.entry_memory_load >= 95))))
22882 dprintf (GTC_LOG, ("gen0 reduction count is %d, condemning %d, mem load %d",
22883 settings.gen0_reduction_count,
22884 condemned_gen_number,
22885 settings.entry_memory_load));
22886 should_compact = TRUE;
22888 get_gc_data_per_heap()->set_mechanism (gc_heap_compact,
22889 ((settings.gen0_reduction_count > 0) ? compact_fragmented_gen0 : compact_high_mem_load));
22891 if ((condemned_gen_number >= (max_generation - 1)) &&
22892 dt_low_ephemeral_space_p (tuning_deciding_expansion))
22894 dprintf (GTC_LOG, ("Not enough space for all ephemeral generations with compaction"));
22895 should_expand = TRUE;
22901 should_compact = decide_on_compacting (condemned_gen_number, fragmentation, should_expand);
22906 #ifdef FEATURE_LOH_COMPACTION
22907 loh_compacted_p = FALSE;
22908 #endif //FEATURE_LOH_COMPACTION
22910 if (condemned_gen_number == max_generation)
22912 #ifdef FEATURE_LOH_COMPACTION
22913 if (settings.loh_compaction)
22917 should_compact = TRUE;
22918 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_loh_forced);
22919 loh_compacted_p = TRUE;
22924 if ((heap_number == 0) && (loh_pinned_queue))
22926 loh_pinned_queue_decay--;
22928 if (!loh_pinned_queue_decay)
22930 delete loh_pinned_queue;
22931 loh_pinned_queue = 0;
22936 if (!loh_compacted_p)
22937 #endif //FEATURE_LOH_COMPACTION
22939 GCToEEInterface::DiagWalkLOHSurvivors(__this);
22940 sweep_large_objects();
22945 settings.loh_compaction = FALSE;
22948 #ifdef MULTIPLE_HEAPS
22950 new_heap_segment = NULL;
22952 if (should_compact && should_expand)
22953 gc_policy = policy_expand;
22954 else if (should_compact)
22955 gc_policy = policy_compact;
22957 gc_policy = policy_sweep;
22959 //vote for result of should_compact
22960 dprintf (3, ("Joining for compaction decision"));
22961 gc_t_join.join(this, gc_join_decide_on_compaction);
22962 if (gc_t_join.joined())
22964 //safe place to delete large heap segments
22965 if (condemned_gen_number == max_generation)
22967 for (int i = 0; i < n_heaps; i++)
22969 g_heaps [i]->rearrange_large_heap_segments ();
22973 if (maxgen_size_inc_p && provisional_mode_triggered)
22975 pm_trigger_full_gc = true;
22976 dprintf (GTC_LOG, ("in PM: maxgen size inc, doing a sweeping gen1 and trigger NGC2"));
22980 settings.demotion = FALSE;
22981 int pol_max = policy_sweep;
22982 #ifdef GC_CONFIG_DRIVEN
22983 BOOL is_compaction_mandatory = FALSE;
22984 #endif //GC_CONFIG_DRIVEN
22987 for (i = 0; i < n_heaps; i++)
22989 if (pol_max < g_heaps[i]->gc_policy)
22990 pol_max = policy_compact;
22991 // set the demotion flag is any of the heap has demotion
22992 if (g_heaps[i]->demotion_high >= g_heaps[i]->demotion_low)
22994 (g_heaps[i]->get_gc_data_per_heap())->set_mechanism_bit (gc_demotion_bit);
22995 settings.demotion = TRUE;
22998 #ifdef GC_CONFIG_DRIVEN
22999 if (!is_compaction_mandatory)
23001 int compact_reason = (g_heaps[i]->get_gc_data_per_heap())->get_mechanism (gc_heap_compact);
23002 if (compact_reason >= 0)
23004 if (gc_heap_compact_reason_mandatory_p[compact_reason])
23005 is_compaction_mandatory = TRUE;
23008 #endif //GC_CONFIG_DRIVEN
23011 #ifdef GC_CONFIG_DRIVEN
23012 if (!is_compaction_mandatory)
23014 // If compaction is not mandatory we can feel free to change it to a sweeping GC.
23015 // Note that we may want to change this to only checking every so often instead of every single GC.
23016 if (should_do_sweeping_gc (pol_max >= policy_compact))
23018 pol_max = policy_sweep;
23022 if (pol_max == policy_sweep)
23023 pol_max = policy_compact;
23026 #endif //GC_CONFIG_DRIVEN
23028 for (i = 0; i < n_heaps; i++)
23030 if (pol_max > g_heaps[i]->gc_policy)
23031 g_heaps[i]->gc_policy = pol_max;
23032 //get the segment while we are serialized
23033 if (g_heaps[i]->gc_policy == policy_expand)
23035 g_heaps[i]->new_heap_segment =
23036 g_heaps[i]->soh_get_segment_to_expand();
23037 if (!g_heaps[i]->new_heap_segment)
23039 set_expand_in_full_gc (condemned_gen_number);
23040 //we are out of memory, cancel the expansion
23041 g_heaps[i]->gc_policy = policy_compact;
23046 BOOL is_full_compacting_gc = FALSE;
23048 if ((gc_policy >= policy_compact) && (condemned_gen_number == max_generation))
23050 full_gc_counts[gc_type_compacting]++;
23051 is_full_compacting_gc = TRUE;
23054 for (i = 0; i < n_heaps; i++)
23056 //copy the card and brick tables
23057 if (g_gc_card_table!= g_heaps[i]->card_table)
23059 g_heaps[i]->copy_brick_card_table();
23062 if (is_full_compacting_gc)
23064 g_heaps[i]->loh_alloc_since_cg = 0;
23069 //start all threads on the roots.
23070 dprintf(3, ("Starting all gc threads after compaction decision"));
23071 gc_t_join.restart();
23074 //reset the local variable accordingly
23075 should_compact = (gc_policy >= policy_compact);
23076 should_expand = (gc_policy >= policy_expand);
23078 #else //MULTIPLE_HEAPS
23080 //safe place to delete large heap segments
23081 if (condemned_gen_number == max_generation)
23083 rearrange_large_heap_segments ();
23086 if (maxgen_size_inc_p && provisional_mode_triggered)
23088 pm_trigger_full_gc = true;
23089 dprintf (GTC_LOG, ("in PM: maxgen size inc, doing a sweeping gen1 and trigger NGC2"));
23093 settings.demotion = ((demotion_high >= demotion_low) ? TRUE : FALSE);
23094 if (settings.demotion)
23095 get_gc_data_per_heap()->set_mechanism_bit (gc_demotion_bit);
23097 #ifdef GC_CONFIG_DRIVEN
23098 BOOL is_compaction_mandatory = FALSE;
23099 int compact_reason = get_gc_data_per_heap()->get_mechanism (gc_heap_compact);
23100 if (compact_reason >= 0)
23101 is_compaction_mandatory = gc_heap_compact_reason_mandatory_p[compact_reason];
23103 if (!is_compaction_mandatory)
23105 if (should_do_sweeping_gc (should_compact))
23106 should_compact = FALSE;
23108 should_compact = TRUE;
23110 #endif //GC_CONFIG_DRIVEN
23112 if (should_compact && (condemned_gen_number == max_generation))
23114 full_gc_counts[gc_type_compacting]++;
23115 loh_alloc_since_cg = 0;
23118 #endif //MULTIPLE_HEAPS
23120 if (!pm_trigger_full_gc && pm_stress_on && provisional_mode_triggered)
23122 if ((settings.condemned_generation == (max_generation - 1)) &&
23123 ((settings.gc_index % 5) == 0))
23125 pm_trigger_full_gc = true;
23129 if (settings.condemned_generation == (max_generation - 1))
23131 if (provisional_mode_triggered)
23135 should_expand = FALSE;
23136 dprintf (GTC_LOG, ("h%d in PM cannot expand", heap_number));
23140 if (pm_trigger_full_gc)
23142 should_compact = FALSE;
23143 dprintf (GTC_LOG, ("h%d PM doing sweeping", heap_number));
23147 if (should_compact)
23149 dprintf (2,( "**** Doing Compacting GC ****"));
23153 #ifndef MULTIPLE_HEAPS
23154 heap_segment* new_heap_segment = soh_get_segment_to_expand();
23155 #endif //!MULTIPLE_HEAPS
23156 if (new_heap_segment)
23158 consing_gen = expand_heap(condemned_gen_number,
23163 // If we couldn't get a new segment, or we were able to
23164 // reserve one but no space to commit, we couldn't
23166 if (ephemeral_heap_segment != new_heap_segment)
23168 set_expand_in_full_gc (condemned_gen_number);
23169 should_expand = FALSE;
23172 generation_allocation_limit (condemned_gen1) =
23173 generation_allocation_pointer (condemned_gen1);
23174 if ((condemned_gen_number < max_generation))
23176 generation_allocator (older_gen)->commit_alloc_list_changes();
23178 // Fix the allocation area of the older generation
23179 fix_older_allocation_area (older_gen);
23181 assert (generation_allocation_segment (consing_gen) ==
23182 ephemeral_heap_segment);
23184 GCToEEInterface::DiagWalkSurvivors(__this);
23186 relocate_phase (condemned_gen_number, first_condemned_address);
23187 compact_phase (condemned_gen_number, first_condemned_address,
23188 (!settings.demotion && settings.promotion));
23189 fix_generation_bounds (condemned_gen_number, consing_gen);
23190 assert (generation_allocation_limit (youngest_generation) ==
23191 generation_allocation_pointer (youngest_generation));
23192 if (condemned_gen_number >= (max_generation -1))
23194 #ifdef MULTIPLE_HEAPS
23195 // this needs be serialized just because we have one
23196 // segment_standby_list/seg_table for all heaps. We should make it at least
23197 // so that when hoarding is not on we don't need this join because
23198 // decommitting memory can take a long time.
23199 //must serialize on deleting segments
23200 gc_t_join.join(this, gc_join_rearrange_segs_compaction);
23201 if (gc_t_join.joined())
23203 for (int i = 0; i < n_heaps; i++)
23205 g_heaps[i]->rearrange_heap_segments(TRUE);
23207 gc_t_join.restart();
23210 rearrange_heap_segments(TRUE);
23211 #endif //MULTIPLE_HEAPS
23215 //fix the start_segment for the ephemeral generations
23216 for (int i = 0; i < max_generation; i++)
23218 generation* gen = generation_of (i);
23219 generation_start_segment (gen) = ephemeral_heap_segment;
23220 generation_allocation_segment (gen) = ephemeral_heap_segment;
23226 #ifdef FEATURE_PREMORTEM_FINALIZATION
23227 finalize_queue->UpdatePromotedGenerations (condemned_gen_number,
23228 (!settings.demotion && settings.promotion));
23229 #endif // FEATURE_PREMORTEM_FINALIZATION
23231 #ifdef MULTIPLE_HEAPS
23232 dprintf(3, ("Joining after end of compaction"));
23233 gc_t_join.join(this, gc_join_adjust_handle_age_compact);
23234 if (gc_t_join.joined())
23235 #endif //MULTIPLE_HEAPS
23237 #ifdef MULTIPLE_HEAPS
23238 //join all threads to make sure they are synchronized
23239 dprintf(3, ("Restarting after Promotion granted"));
23240 gc_t_join.restart();
23241 #endif //MULTIPLE_HEAPS
23245 sc.thread_number = heap_number;
23246 sc.promotion = FALSE;
23247 sc.concurrent = FALSE;
23248 // new generations bounds are set can call this guy
23249 if (settings.promotion && !settings.demotion)
23251 dprintf (2, ("Promoting EE roots for gen %d",
23252 condemned_gen_number));
23253 GCScan::GcPromotionsGranted(condemned_gen_number,
23254 max_generation, &sc);
23256 else if (settings.demotion)
23258 dprintf (2, ("Demoting EE roots for gen %d",
23259 condemned_gen_number));
23260 GCScan::GcDemote (condemned_gen_number, max_generation, &sc);
23265 gen0_big_free_spaces = 0;
23267 reset_pinned_queue_bos();
23268 unsigned int gen_number = min (max_generation, 1 + condemned_gen_number);
23269 generation* gen = generation_of (gen_number);
23270 uint8_t* low = generation_allocation_start (generation_of (gen_number-1));
23271 uint8_t* high = heap_segment_allocated (ephemeral_heap_segment);
23273 while (!pinned_plug_que_empty_p())
23275 mark* m = pinned_plug_of (deque_pinned_plug());
23276 size_t len = pinned_len (m);
23277 uint8_t* arr = (pinned_plug (m) - len);
23278 dprintf(3,("free [%Ix %Ix[ pin",
23279 (size_t)arr, (size_t)arr + len));
23282 assert (len >= Align (min_obj_size));
23283 make_unused_array (arr, len);
23284 // fix fully contained bricks + first one
23285 // if the array goes beyond the first brick
23286 size_t start_brick = brick_of (arr);
23287 size_t end_brick = brick_of (arr + len);
23288 if (end_brick != start_brick)
23291 ("Fixing bricks [%Ix, %Ix[ to point to unused array %Ix",
23292 start_brick, end_brick, (size_t)arr));
23293 set_brick (start_brick,
23294 arr - brick_address (start_brick));
23295 size_t brick = start_brick+1;
23296 while (brick < end_brick)
23298 set_brick (brick, start_brick - brick);
23303 //when we take an old segment to make the new
23304 //ephemeral segment. we can have a bunch of
23305 //pinned plugs out of order going to the new ephemeral seg
23306 //and then the next plugs go back to max_generation
23307 if ((heap_segment_mem (ephemeral_heap_segment) <= arr) &&
23308 (heap_segment_reserved (ephemeral_heap_segment) > arr))
23311 while ((low <= arr) && (high > arr))
23314 assert ((gen_number >= 1) || (demotion_low != MAX_PTR) ||
23315 settings.demotion || !settings.promotion);
23316 dprintf (3, ("new free list generation %d", gen_number));
23318 gen = generation_of (gen_number);
23319 if (gen_number >= 1)
23320 low = generation_allocation_start (generation_of (gen_number-1));
23327 dprintf (3, ("new free list generation %d", max_generation));
23328 gen_number = max_generation;
23329 gen = generation_of (gen_number);
23332 dprintf(3,("threading it into generation %d", gen_number));
23333 thread_gap (arr, len, gen);
23334 add_gen_free (gen_number, len);
23340 for (int x = 0; x <= max_generation; x++)
23342 assert (generation_allocation_start (generation_of (x)));
23346 if (!settings.demotion && settings.promotion)
23348 //clear card for generation 1. generation 0 is empty
23349 clear_card_for_addresses (
23350 generation_allocation_start (generation_of (1)),
23351 generation_allocation_start (generation_of (0)));
23353 if (settings.promotion && !settings.demotion)
23355 uint8_t* start = generation_allocation_start (youngest_generation);
23356 MAYBE_UNUSED_VAR(start);
23357 assert (heap_segment_allocated (ephemeral_heap_segment) ==
23358 (start + Align (size (start))));
23363 //force promotion for sweep
23364 settings.promotion = TRUE;
23365 settings.compaction = FALSE;
23368 sc.thread_number = heap_number;
23369 sc.promotion = FALSE;
23370 sc.concurrent = FALSE;
23372 dprintf (2, ("**** Doing Mark and Sweep GC****"));
23374 if ((condemned_gen_number < max_generation))
23376 generation_allocator (older_gen)->copy_from_alloc_list (r_free_list);
23377 generation_free_list_space (older_gen) = r_free_list_space;
23378 generation_free_obj_space (older_gen) = r_free_obj_space;
23379 generation_free_list_allocated (older_gen) = r_older_gen_free_list_allocated;
23380 generation_end_seg_allocated (older_gen) = r_older_gen_end_seg_allocated;
23381 generation_condemned_allocated (older_gen) = r_older_gen_condemned_allocated;
23382 generation_allocation_limit (older_gen) = r_allocation_limit;
23383 generation_allocation_pointer (older_gen) = r_allocation_pointer;
23384 generation_allocation_context_start_region (older_gen) = r_allocation_start_region;
23385 generation_allocation_segment (older_gen) = r_allocation_segment;
23388 if ((condemned_gen_number < max_generation))
23390 // Fix the allocation area of the older generation
23391 fix_older_allocation_area (older_gen);
23394 GCToEEInterface::DiagWalkSurvivors(__this);
23396 gen0_big_free_spaces = 0;
23397 make_free_lists (condemned_gen_number);
23398 recover_saved_pinned_info();
23400 #ifdef FEATURE_PREMORTEM_FINALIZATION
23401 finalize_queue->UpdatePromotedGenerations (condemned_gen_number, TRUE);
23402 #endif // FEATURE_PREMORTEM_FINALIZATION
23403 // MTHTS: leave single thread for HT processing on plan_phase
23404 #ifdef MULTIPLE_HEAPS
23405 dprintf(3, ("Joining after end of sweep"));
23406 gc_t_join.join(this, gc_join_adjust_handle_age_sweep);
23407 if (gc_t_join.joined())
23408 #endif //MULTIPLE_HEAPS
23410 GCScan::GcPromotionsGranted(condemned_gen_number,
23411 max_generation, &sc);
23412 if (condemned_gen_number >= (max_generation -1))
23414 #ifdef MULTIPLE_HEAPS
23415 for (int i = 0; i < n_heaps; i++)
23417 g_heaps[i]->rearrange_heap_segments(FALSE);
23420 rearrange_heap_segments(FALSE);
23421 #endif //MULTIPLE_HEAPS
23424 #ifdef MULTIPLE_HEAPS
23425 //join all threads to make sure they are synchronized
23426 dprintf(3, ("Restarting after Promotion granted"));
23427 gc_t_join.restart();
23428 #endif //MULTIPLE_HEAPS
23432 for (int x = 0; x <= max_generation; x++)
23434 assert (generation_allocation_start (generation_of (x)));
23438 //clear card for generation 1. generation 0 is empty
23439 clear_card_for_addresses (
23440 generation_allocation_start (generation_of (1)),
23441 generation_allocation_start (generation_of (0)));
23442 assert ((heap_segment_allocated (ephemeral_heap_segment) ==
23443 (generation_allocation_start (youngest_generation) +
23444 Align (min_obj_size))));
23447 //verify_partial();
23450 #pragma warning(pop)
23454 /*****************************
23455 Called after compact phase to fix all generation gaps
23456 ********************************/
23457 void gc_heap::fix_generation_bounds (int condemned_gen_number,
23458 generation* consing_gen)
23460 UNREFERENCED_PARAMETER(consing_gen);
23462 assert (generation_allocation_segment (consing_gen) ==
23463 ephemeral_heap_segment);
23465 //assign the planned allocation start to the generation
23466 int gen_number = condemned_gen_number;
23467 int bottom_gen = 0;
23469 while (gen_number >= bottom_gen)
23471 generation* gen = generation_of (gen_number);
23472 dprintf(3,("Fixing generation pointers for %Ix", gen_number));
23473 if ((gen_number < max_generation) && ephemeral_promotion)
23475 make_unused_array (saved_ephemeral_plan_start[gen_number],
23476 saved_ephemeral_plan_start_size[gen_number]);
23478 reset_allocation_pointers (gen, generation_plan_allocation_start (gen));
23479 make_unused_array (generation_allocation_start (gen), generation_plan_allocation_start_size (gen));
23480 dprintf(3,(" start %Ix", (size_t)generation_allocation_start (gen)));
23483 #ifdef MULTIPLE_HEAPS
23484 if (ephemeral_promotion)
23486 //we are creating a generation fault. set the cards.
23487 // and we are only doing this for multiple heaps because in the single heap scenario the
23488 // new ephemeral generations will be empty and there'll be no need to set cards for the
23489 // old ephemeral generations that got promoted into max_generation.
23490 ptrdiff_t delta = 0;
23491 #ifdef SEG_MAPPING_TABLE
23492 heap_segment* old_ephemeral_seg = seg_mapping_table_segment_of (saved_ephemeral_plan_start[max_generation-1]);
23493 #else //SEG_MAPPING_TABLE
23494 heap_segment* old_ephemeral_seg = segment_of (saved_ephemeral_plan_start[max_generation-1], delta);
23495 #endif //SEG_MAPPING_TABLE
23497 assert (in_range_for_segment (saved_ephemeral_plan_start[max_generation-1], old_ephemeral_seg));
23498 size_t end_card = card_of (align_on_card (heap_segment_plan_allocated (old_ephemeral_seg)));
23499 size_t card = card_of (saved_ephemeral_plan_start[max_generation-1]);
23500 while (card != end_card)
23506 #endif //MULTIPLE_HEAPS
23508 alloc_allocated = heap_segment_plan_allocated(ephemeral_heap_segment);
23509 //reset the allocated size
23510 uint8_t* start = generation_allocation_start (youngest_generation);
23511 MAYBE_UNUSED_VAR(start);
23512 if (settings.promotion && !settings.demotion)
23514 assert ((start + Align (size (start))) ==
23515 heap_segment_plan_allocated(ephemeral_heap_segment));
23518 heap_segment_allocated(ephemeral_heap_segment)=
23519 heap_segment_plan_allocated(ephemeral_heap_segment);
23523 uint8_t* gc_heap::generation_limit (int gen_number)
23525 if (settings.promotion)
23527 if (gen_number <= 1)
23528 return heap_segment_reserved (ephemeral_heap_segment);
23530 return generation_allocation_start (generation_of ((gen_number - 2)));
23534 if (gen_number <= 0)
23535 return heap_segment_reserved (ephemeral_heap_segment);
23537 return generation_allocation_start (generation_of ((gen_number - 1)));
23541 BOOL gc_heap::ensure_gap_allocation (int condemned_gen_number)
23543 uint8_t* start = heap_segment_allocated (ephemeral_heap_segment);
23544 size_t size = Align (min_obj_size)*(condemned_gen_number+1);
23545 assert ((start + size) <=
23546 heap_segment_reserved (ephemeral_heap_segment));
23547 if ((start + size) >
23548 heap_segment_committed (ephemeral_heap_segment))
23550 if (!grow_heap_segment (ephemeral_heap_segment, start + size))
23558 uint8_t* gc_heap::allocate_at_end (size_t size)
23560 uint8_t* start = heap_segment_allocated (ephemeral_heap_segment);
23561 size = Align (size);
23562 uint8_t* result = start;
23563 // only called to allocate a min obj so can't overflow here.
23564 assert ((start + size) <=
23565 heap_segment_reserved (ephemeral_heap_segment));
23566 //ensure_gap_allocation took care of it
23567 assert ((start + size) <=
23568 heap_segment_committed (ephemeral_heap_segment));
23569 heap_segment_allocated (ephemeral_heap_segment) += size;
23574 void gc_heap::make_free_lists (int condemned_gen_number)
23579 start = GetCycleCount32();
23582 //Promotion has to happen in sweep case.
23583 assert (settings.promotion);
23585 generation* condemned_gen = generation_of (condemned_gen_number);
23586 uint8_t* start_address = generation_allocation_start (condemned_gen);
23588 size_t current_brick = brick_of (start_address);
23589 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
23591 PREFIX_ASSUME(current_heap_segment != NULL);
23593 uint8_t* end_address = heap_segment_allocated (current_heap_segment);
23594 size_t end_brick = brick_of (end_address-1);
23595 make_free_args args;
23596 args.free_list_gen_number = min (max_generation, 1 + condemned_gen_number);
23597 args.current_gen_limit = (((condemned_gen_number == max_generation)) ?
23599 (generation_limit (args.free_list_gen_number)));
23600 args.free_list_gen = generation_of (args.free_list_gen_number);
23601 args.highest_plug = 0;
23603 if ((start_address < end_address) ||
23604 (condemned_gen_number == max_generation))
23608 if ((current_brick > end_brick))
23610 if (args.current_gen_limit == MAX_PTR)
23612 //We had an empty segment
23613 //need to allocate the generation start
23615 generation* gen = generation_of (max_generation);
23617 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
23619 PREFIX_ASSUME(start_seg != NULL);
23621 uint8_t* gap = heap_segment_mem (start_seg);
23623 generation_allocation_start (gen) = gap;
23624 heap_segment_allocated (start_seg) = gap + Align (min_obj_size);
23625 make_unused_array (gap, Align (min_obj_size));
23626 reset_allocation_pointers (gen, gap);
23627 dprintf (3, ("Start segment empty, fixing generation start of %d to: %Ix",
23628 max_generation, (size_t)gap));
23629 args.current_gen_limit = generation_limit (args.free_list_gen_number);
23631 if (heap_segment_next_rw (current_heap_segment))
23633 current_heap_segment = heap_segment_next_rw (current_heap_segment);
23634 current_brick = brick_of (heap_segment_mem (current_heap_segment));
23635 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
23645 int brick_entry = brick_table [ current_brick ];
23646 if ((brick_entry >= 0))
23648 make_free_list_in_brick (brick_address (current_brick) + brick_entry-1, &args);
23649 dprintf(3,("Fixing brick entry %Ix to %Ix",
23650 current_brick, (size_t)args.highest_plug));
23651 set_brick (current_brick,
23652 (args.highest_plug - brick_address (current_brick)));
23656 if ((brick_entry > -32768))
23660 ptrdiff_t offset = brick_of (args.highest_plug) - current_brick;
23661 if ((brick_entry != -32767) && (! ((offset == brick_entry))))
23663 assert ((brick_entry == -1));
23666 //init to -1 for faster find_first_object
23667 set_brick (current_brick, -1);
23675 int bottom_gen = 0;
23676 args.free_list_gen_number--;
23677 while (args.free_list_gen_number >= bottom_gen)
23680 generation* gen2 = generation_of (args.free_list_gen_number);
23681 gap = allocate_at_end (Align(min_obj_size));
23682 generation_allocation_start (gen2) = gap;
23683 reset_allocation_pointers (gen2, gap);
23684 dprintf(3,("Fixing generation start of %d to: %Ix",
23685 args.free_list_gen_number, (size_t)gap));
23686 PREFIX_ASSUME(gap != NULL);
23687 make_unused_array (gap, Align (min_obj_size));
23689 args.free_list_gen_number--;
23692 //reset the allocated size
23693 uint8_t* start2 = generation_allocation_start (youngest_generation);
23694 alloc_allocated = start2 + Align (size (start2));
23698 finish = GetCycleCount32();
23699 sweep_time = finish - start;
23703 void gc_heap::make_free_list_in_brick (uint8_t* tree, make_free_args* args)
23705 assert ((tree != NULL));
23707 int right_node = node_right_child (tree);
23708 int left_node = node_left_child (tree);
23709 args->highest_plug = 0;
23712 if (! (0 == left_node))
23714 make_free_list_in_brick (tree + left_node, args);
23718 uint8_t* plug = tree;
23719 size_t gap_size = node_gap_size (tree);
23720 uint8_t* gap = (plug - gap_size);
23721 dprintf (3,("Making free list %Ix len %d in %d",
23722 //dprintf (3,("F: %Ix len %Ix in %d",
23723 (size_t)gap, gap_size, args->free_list_gen_number));
23724 args->highest_plug = tree;
23726 if (is_plug_padded (plug))
23728 dprintf (3, ("%Ix padded", plug));
23729 clear_plug_padded (plug);
23731 #endif //SHORT_PLUGS
23734 if ((args->current_gen_limit == MAX_PTR) ||
23735 ((plug >= args->current_gen_limit) &&
23736 ephemeral_pointer_p (plug)))
23738 dprintf(3,(" Crossing Generation boundary at %Ix",
23739 (size_t)args->current_gen_limit));
23740 if (!(args->current_gen_limit == MAX_PTR))
23742 args->free_list_gen_number--;
23743 args->free_list_gen = generation_of (args->free_list_gen_number);
23745 dprintf(3,( " Fixing generation start of %d to: %Ix",
23746 args->free_list_gen_number, (size_t)gap));
23748 reset_allocation_pointers (args->free_list_gen, gap);
23749 args->current_gen_limit = generation_limit (args->free_list_gen_number);
23751 if ((gap_size >= (2*Align (min_obj_size))))
23753 dprintf(3,(" Splitting the gap in two %Id left",
23755 make_unused_array (gap, Align(min_obj_size));
23756 gap_size = (gap_size - Align(min_obj_size));
23757 gap = (gap + Align(min_obj_size));
23761 make_unused_array (gap, gap_size);
23768 thread_gap (gap, gap_size, args->free_list_gen);
23769 add_gen_free (args->free_list_gen->gen_num, gap_size);
23771 if (! (0 == right_node))
23773 make_free_list_in_brick (tree + right_node, args);
23779 void gc_heap::thread_gap (uint8_t* gap_start, size_t size, generation* gen)
23781 assert (generation_allocation_start (gen));
23784 if ((gen->gen_num == 0) && (size > CLR_SIZE))
23786 gen0_big_free_spaces += size;
23789 assert ((heap_segment_rw (generation_start_segment (gen))!=
23790 ephemeral_heap_segment) ||
23791 (gap_start > generation_allocation_start (gen)));
23792 // The beginning of a segment gap is not aligned
23793 assert (size >= Align (min_obj_size));
23794 make_unused_array (gap_start, size,
23795 (!settings.concurrent && (gen != youngest_generation)),
23796 (gen->gen_num == max_generation));
23797 dprintf (3, ("fr: [%Ix, %Ix[", (size_t)gap_start, (size_t)gap_start+size));
23799 if ((size >= min_free_list))
23801 generation_free_list_space (gen) += size;
23802 generation_allocator (gen)->thread_item (gap_start, size);
23806 generation_free_obj_space (gen) += size;
23811 void gc_heap::loh_thread_gap_front (uint8_t* gap_start, size_t size, generation* gen)
23813 assert (generation_allocation_start (gen));
23814 if (size >= min_free_list)
23816 generation_free_list_space (gen) += size;
23817 generation_allocator (gen)->thread_item_front (gap_start, size);
23821 void gc_heap::make_unused_array (uint8_t* x, size_t size, BOOL clearp, BOOL resetp)
23823 dprintf (3, ("Making unused array [%Ix, %Ix[",
23824 (size_t)x, (size_t)(x+size)));
23825 assert (size >= Align (min_obj_size));
23827 //#if defined (VERIFY_HEAP) && defined (BACKGROUND_GC)
23828 // check_batch_mark_array_bits (x, x+size);
23829 //#endif //VERIFY_HEAP && BACKGROUND_GC
23832 reset_memory (x, size);
23834 ((CObjectHeader*)x)->SetFree(size);
23839 #error "This won't work on big endian platforms"
23842 size_t size_as_object = (uint32_t)(size - free_object_base_size) + free_object_base_size;
23844 if (size_as_object < size)
23847 // If the size is more than 4GB, we need to create multiple objects because of
23848 // the Array::m_NumComponents is uint32_t and the high 32 bits of unused array
23849 // size is ignored in regular object size computation.
23851 uint8_t * tmp = x + size_as_object;
23852 size_t remaining_size = size - size_as_object;
23854 while (remaining_size > UINT32_MAX)
23856 // Make sure that there will be at least Align(min_obj_size) left
23857 size_t current_size = UINT32_MAX - get_alignment_constant (FALSE)
23858 - Align (min_obj_size, get_alignment_constant (FALSE));
23860 ((CObjectHeader*)tmp)->SetFree(current_size);
23862 remaining_size -= current_size;
23863 tmp += current_size;
23866 ((CObjectHeader*)tmp)->SetFree(remaining_size);
23871 clear_card_for_addresses (x, x + Align(size));
23874 // Clear memory set by make_unused_array.
23875 void gc_heap::clear_unused_array (uint8_t* x, size_t size)
23877 // Also clear the sync block
23878 *(((PTR_PTR)x)-1) = 0;
23880 ((CObjectHeader*)x)->UnsetFree();
23885 #error "This won't work on big endian platforms"
23888 // The memory could have been cleared in the meantime. We have to mirror the algorithm
23889 // from make_unused_array since we cannot depend on the object sizes in memory.
23890 size_t size_as_object = (uint32_t)(size - free_object_base_size) + free_object_base_size;
23892 if (size_as_object < size)
23894 uint8_t * tmp = x + size_as_object;
23895 size_t remaining_size = size - size_as_object;
23897 while (remaining_size > UINT32_MAX)
23899 size_t current_size = UINT32_MAX - get_alignment_constant (FALSE)
23900 - Align (min_obj_size, get_alignment_constant (FALSE));
23902 ((CObjectHeader*)tmp)->UnsetFree();
23904 remaining_size -= current_size;
23905 tmp += current_size;
23908 ((CObjectHeader*)tmp)->UnsetFree();
23911 UNREFERENCED_PARAMETER(size);
23916 uint8_t* tree_search (uint8_t* tree, uint8_t* old_address)
23918 uint8_t* candidate = 0;
23922 if (tree < old_address)
23924 if ((cn = node_right_child (tree)) != 0)
23926 assert (candidate < tree);
23929 Prefetch (tree - 8);
23935 else if (tree > old_address)
23937 if ((cn = node_left_child (tree)) != 0)
23940 Prefetch (tree - 8);
23948 if (tree <= old_address)
23950 else if (candidate)
23956 #ifdef FEATURE_BASICFREEZE
23957 bool gc_heap::frozen_object_p (Object* obj)
23959 #ifdef MULTIPLE_HEAPS
23960 ptrdiff_t delta = 0;
23961 heap_segment* pSegment = segment_of ((uint8_t*)obj, delta);
23962 #else //MULTIPLE_HEAPS
23963 heap_segment* pSegment = gc_heap::find_segment ((uint8_t*)obj, FALSE);
23964 _ASSERTE(pSegment);
23965 #endif //MULTIPLE_HEAPS
23967 return heap_segment_read_only_p(pSegment);
23969 #endif // FEATURE_BASICFREEZE
23971 #ifdef FEATURE_REDHAWK
23972 // TODO: this was added on RH, we have not done perf runs to see if this is the right
23973 // thing to do for other versions of the CLR.
23975 #endif // FEATURE_REDHAWK
23976 void gc_heap::relocate_address (uint8_t** pold_address THREAD_NUMBER_DCL)
23978 uint8_t* old_address = *pold_address;
23979 if (!((old_address >= gc_low) && (old_address < gc_high)))
23980 #ifdef MULTIPLE_HEAPS
23982 UNREFERENCED_PARAMETER(thread);
23983 if (old_address == 0)
23985 gc_heap* hp = heap_of (old_address);
23986 if ((hp == this) ||
23987 !((old_address >= hp->gc_low) && (old_address < hp->gc_high)))
23990 #else //MULTIPLE_HEAPS
23992 #endif //MULTIPLE_HEAPS
23993 // delta translates old_address into address_gc (old_address);
23994 size_t brick = brick_of (old_address);
23995 int brick_entry = brick_table [ brick ];
23996 uint8_t* new_address = old_address;
23997 if (! ((brick_entry == 0)))
24001 while (brick_entry < 0)
24003 brick = (brick + brick_entry);
24004 brick_entry = brick_table [ brick ];
24006 uint8_t* old_loc = old_address;
24008 uint8_t* node = tree_search ((brick_address (brick) + brick_entry-1),
24010 if ((node <= old_loc))
24011 new_address = (old_address + node_relocation_distance (node));
24014 if (node_left_p (node))
24016 dprintf(3,(" L: %Ix", (size_t)node));
24017 new_address = (old_address +
24018 (node_relocation_distance (node) +
24019 node_gap_size (node)));
24024 brick_entry = brick_table [ brick ];
24030 *pold_address = new_address;
24034 #ifdef FEATURE_LOH_COMPACTION
24035 if (loh_compacted_p
24036 #ifdef FEATURE_BASICFREEZE
24037 && !frozen_object_p((Object*)old_address)
24038 #endif // FEATURE_BASICFREEZE
24041 *pold_address = old_address + loh_node_relocation_distance (old_address);
24044 #endif //FEATURE_LOH_COMPACTION
24046 *pold_address = new_address;
24051 gc_heap::check_class_object_demotion (uint8_t* obj)
24053 #ifdef COLLECTIBLE_CLASS
24054 if (is_collectible(obj))
24056 check_class_object_demotion_internal (obj);
24059 UNREFERENCED_PARAMETER(obj);
24060 #endif //COLLECTIBLE_CLASS
24063 #ifdef COLLECTIBLE_CLASS
24065 gc_heap::check_class_object_demotion_internal (uint8_t* obj)
24067 if (settings.demotion)
24069 #ifdef MULTIPLE_HEAPS
24070 // We set the card without checking the demotion range 'cause at this point
24071 // the handle that points to the loader allocator object may or may not have
24072 // been relocated by other GC threads.
24073 set_card (card_of (obj));
24076 uint8_t* class_obj = get_class_object (obj);
24077 dprintf (3, ("%Ix: got classobj %Ix", obj, class_obj));
24078 uint8_t* temp_class_obj = class_obj;
24079 uint8_t** temp = &temp_class_obj;
24080 relocate_address (temp THREAD_NUMBER_ARG);
24082 check_demotion_helper (temp, obj);
24083 #endif //MULTIPLE_HEAPS
24087 #endif //COLLECTIBLE_CLASS
24090 gc_heap::check_demotion_helper (uint8_t** pval, uint8_t* parent_obj)
24092 // detect if we are demoting an object
24093 if ((*pval < demotion_high) &&
24094 (*pval >= demotion_low))
24096 dprintf(3, ("setting card %Ix:%Ix",
24097 card_of((uint8_t*)pval),
24100 set_card (card_of (parent_obj));
24102 #ifdef MULTIPLE_HEAPS
24103 else if (settings.demotion)
24105 dprintf (4, ("Demotion active, computing heap_of object"));
24106 gc_heap* hp = heap_of (*pval);
24107 if ((*pval < hp->demotion_high) &&
24108 (*pval >= hp->demotion_low))
24110 dprintf(3, ("setting card %Ix:%Ix",
24111 card_of((uint8_t*)pval),
24114 set_card (card_of (parent_obj));
24117 #endif //MULTIPLE_HEAPS
24121 gc_heap::reloc_survivor_helper (uint8_t** pval)
24124 relocate_address (pval THREAD_NUMBER_ARG);
24126 check_demotion_helper (pval, (uint8_t*)pval);
24130 gc_heap::relocate_obj_helper (uint8_t* x, size_t s)
24133 if (contain_pointers (x))
24135 dprintf (3, ("$%Ix$", (size_t)x));
24137 go_through_object_nostart (method_table(x), x, s, pval,
24139 uint8_t* child = *pval;
24140 reloc_survivor_helper (pval);
24143 dprintf (3, ("%Ix->%Ix->%Ix", (uint8_t*)pval, child, *pval));
24148 check_class_object_demotion (x);
24152 void gc_heap::reloc_ref_in_shortened_obj (uint8_t** address_to_set_card, uint8_t** address_to_reloc)
24156 uint8_t* old_val = (address_to_reloc ? *address_to_reloc : 0);
24157 relocate_address (address_to_reloc THREAD_NUMBER_ARG);
24158 if (address_to_reloc)
24160 dprintf (3, ("SR %Ix: %Ix->%Ix", (uint8_t*)address_to_reloc, old_val, *address_to_reloc));
24163 //check_demotion_helper (current_saved_info_to_relocate, (uint8_t*)pval);
24164 uint8_t* relocated_addr = *address_to_reloc;
24165 if ((relocated_addr < demotion_high) &&
24166 (relocated_addr >= demotion_low))
24168 dprintf (3, ("set card for location %Ix(%Ix)",
24169 (size_t)address_to_set_card, card_of((uint8_t*)address_to_set_card)));
24171 set_card (card_of ((uint8_t*)address_to_set_card));
24173 #ifdef MULTIPLE_HEAPS
24174 else if (settings.demotion)
24176 gc_heap* hp = heap_of (relocated_addr);
24177 if ((relocated_addr < hp->demotion_high) &&
24178 (relocated_addr >= hp->demotion_low))
24180 dprintf (3, ("%Ix on h%d, set card for location %Ix(%Ix)",
24181 relocated_addr, hp->heap_number, (size_t)address_to_set_card, card_of((uint8_t*)address_to_set_card)));
24183 set_card (card_of ((uint8_t*)address_to_set_card));
24186 #endif //MULTIPLE_HEAPS
24189 void gc_heap::relocate_pre_plug_info (mark* pinned_plug_entry)
24192 uint8_t* plug = pinned_plug (pinned_plug_entry);
24193 uint8_t* pre_plug_start = plug - sizeof (plug_and_gap);
24194 // Note that we need to add one ptr size here otherwise we may not be able to find the relocated
24195 // address. Consider this scenario:
24196 // gen1 start | 3-ptr sized NP | PP
24198 // If we are asking for the reloc address of 0x10 we will AV in relocate_address because
24199 // the first plug we saw in the brick is 0x18 which means 0x10 will cause us to go back a brick
24200 // which is 0, and then we'll AV in tree_search when we try to do node_right_child (tree).
24201 pre_plug_start += sizeof (uint8_t*);
24202 uint8_t** old_address = &pre_plug_start;
24204 uint8_t* old_val = (old_address ? *old_address : 0);
24205 relocate_address (old_address THREAD_NUMBER_ARG);
24208 dprintf (3, ("PreR %Ix: %Ix->%Ix, set reloc: %Ix",
24209 (uint8_t*)old_address, old_val, *old_address, (pre_plug_start - sizeof (uint8_t*))));
24212 pinned_plug_entry->set_pre_plug_info_reloc_start (pre_plug_start - sizeof (uint8_t*));
24216 void gc_heap::relocate_shortened_obj_helper (uint8_t* x, size_t s, uint8_t* end, mark* pinned_plug_entry, BOOL is_pinned)
24219 uint8_t* plug = pinned_plug (pinned_plug_entry);
24223 //// Temporary - we just wanna make sure we are doing things right when padding is needed.
24224 //if ((x + s) < plug)
24226 // dprintf (3, ("obj %Ix needed padding: end %Ix is %d bytes from pinned obj %Ix",
24227 // x, (x + s), (plug- (x + s)), plug));
24228 // GCToOSInterface::DebugBreak();
24231 relocate_pre_plug_info (pinned_plug_entry);
24234 verify_pins_with_post_plug_info("after relocate_pre_plug_info");
24236 uint8_t* saved_plug_info_start = 0;
24237 uint8_t** saved_info_to_relocate = 0;
24241 saved_plug_info_start = (uint8_t*)(pinned_plug_entry->get_post_plug_info_start());
24242 saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_post_plug_reloc_info());
24246 saved_plug_info_start = (plug - sizeof (plug_and_gap));
24247 saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_pre_plug_reloc_info());
24250 uint8_t** current_saved_info_to_relocate = 0;
24251 uint8_t* child = 0;
24253 dprintf (3, ("x: %Ix, pp: %Ix, end: %Ix", x, plug, end));
24255 if (contain_pointers (x))
24257 dprintf (3,("$%Ix$", (size_t)x));
24259 go_through_object_nostart (method_table(x), x, s, pval,
24261 dprintf (3, ("obj %Ix, member: %Ix->%Ix", x, (uint8_t*)pval, *pval));
24263 if ((uint8_t*)pval >= end)
24265 current_saved_info_to_relocate = saved_info_to_relocate + ((uint8_t*)pval - saved_plug_info_start) / sizeof (uint8_t**);
24266 child = *current_saved_info_to_relocate;
24267 reloc_ref_in_shortened_obj (pval, current_saved_info_to_relocate);
24268 dprintf (3, ("last part: R-%Ix(saved: %Ix)->%Ix ->%Ix",
24269 (uint8_t*)pval, current_saved_info_to_relocate, child, *current_saved_info_to_relocate));
24273 reloc_survivor_helper (pval);
24278 check_class_object_demotion (x);
24281 void gc_heap::relocate_survivor_helper (uint8_t* plug, uint8_t* plug_end)
24284 while (x < plug_end)
24286 size_t s = size (x);
24287 uint8_t* next_obj = x + Align (s);
24288 Prefetch (next_obj);
24289 relocate_obj_helper (x, s);
24295 // if we expanded, right now we are not handling it as We are not saving the new reloc info.
24296 void gc_heap::verify_pins_with_post_plug_info (const char* msg)
24298 #if defined (_DEBUG) && defined (VERIFY_HEAP)
24299 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
24301 if (!verify_pinned_queue_p)
24304 if (settings.heap_expansion)
24307 for (size_t i = 0; i < mark_stack_tos; i++)
24309 mark& m = mark_stack_array[i];
24311 mark* pinned_plug_entry = pinned_plug_of(i);
24313 if (pinned_plug_entry->has_post_plug_info() &&
24314 pinned_plug_entry->post_short_p() &&
24315 (pinned_plug_entry->saved_post_plug_debug.gap != 1))
24317 uint8_t* next_obj = pinned_plug_entry->get_post_plug_info_start() + sizeof (plug_and_gap);
24318 // object after pin
24319 dprintf (3, ("OFP: %Ix, G: %Ix, R: %Ix, LC: %d, RC: %d",
24320 next_obj, node_gap_size (next_obj), node_relocation_distance (next_obj),
24321 (int)node_left_child (next_obj), (int)node_right_child (next_obj)));
24323 size_t* post_plug_debug = (size_t*)(&m.saved_post_plug_debug);
24325 if (node_gap_size (next_obj) != *post_plug_debug)
24327 dprintf (3, ("obj: %Ix gap should be %Ix but it is %Ix",
24328 next_obj, *post_plug_debug, (size_t)(node_gap_size (next_obj))));
24332 // can't do node_relocation_distance here as it clears the left bit.
24333 //if (node_relocation_distance (next_obj) != *post_plug_debug)
24334 if (*((size_t*)(next_obj - 3 * sizeof (size_t))) != *post_plug_debug)
24336 dprintf (3, ("obj: %Ix reloc should be %Ix but it is %Ix",
24337 next_obj, *post_plug_debug, (size_t)(node_relocation_distance (next_obj))));
24340 if (node_left_child (next_obj) > 0)
24342 dprintf (3, ("obj: %Ix, vLC: %d\n", next_obj, (int)(node_left_child (next_obj))));
24348 dprintf (3, ("%s verified", msg));
24350 #else // _DEBUG && VERIFY_HEAP
24351 UNREFERENCED_PARAMETER(msg);
24352 #endif // _DEBUG && VERIFY_HEAP
24355 #ifdef COLLECTIBLE_CLASS
24356 // We don't want to burn another ptr size space for pinned plugs to record this so just
24357 // set the card unconditionally for collectible objects if we are demoting.
24359 gc_heap::unconditional_set_card_collectible (uint8_t* obj)
24361 if (settings.demotion)
24363 set_card (card_of (obj));
24366 #endif //COLLECTIBLE_CLASS
24368 void gc_heap::relocate_shortened_survivor_helper (uint8_t* plug, uint8_t* plug_end, mark* pinned_plug_entry)
24371 uint8_t* p_plug = pinned_plug (pinned_plug_entry);
24372 BOOL is_pinned = (plug == p_plug);
24373 BOOL check_short_obj_p = (is_pinned ? pinned_plug_entry->post_short_p() : pinned_plug_entry->pre_short_p());
24375 plug_end += sizeof (gap_reloc_pair);
24377 //dprintf (3, ("%s %Ix is shortened, and last object %s overwritten", (is_pinned ? "PP" : "NP"), plug, (check_short_obj_p ? "is" : "is not")));
24378 dprintf (3, ("%s %Ix-%Ix short, LO: %s OW", (is_pinned ? "PP" : "NP"), plug, plug_end, (check_short_obj_p ? "is" : "is not")));
24380 verify_pins_with_post_plug_info("begin reloc short surv");
24382 while (x < plug_end)
24384 if (check_short_obj_p && ((plug_end - x) < min_pre_pin_obj_size))
24386 dprintf (3, ("last obj %Ix is short", x));
24390 #ifdef COLLECTIBLE_CLASS
24391 if (pinned_plug_entry->post_short_collectible_p())
24392 unconditional_set_card_collectible (x);
24393 #endif //COLLECTIBLE_CLASS
24395 // Relocate the saved references based on bits set.
24396 uint8_t** saved_plug_info_start = (uint8_t**)(pinned_plug_entry->get_post_plug_info_start());
24397 uint8_t** saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_post_plug_reloc_info());
24398 for (size_t i = 0; i < pinned_plug_entry->get_max_short_bits(); i++)
24400 if (pinned_plug_entry->post_short_bit_p (i))
24402 reloc_ref_in_shortened_obj ((saved_plug_info_start + i), (saved_info_to_relocate + i));
24408 #ifdef COLLECTIBLE_CLASS
24409 if (pinned_plug_entry->pre_short_collectible_p())
24410 unconditional_set_card_collectible (x);
24411 #endif //COLLECTIBLE_CLASS
24413 relocate_pre_plug_info (pinned_plug_entry);
24415 // Relocate the saved references based on bits set.
24416 uint8_t** saved_plug_info_start = (uint8_t**)(p_plug - sizeof (plug_and_gap));
24417 uint8_t** saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_pre_plug_reloc_info());
24418 for (size_t i = 0; i < pinned_plug_entry->get_max_short_bits(); i++)
24420 if (pinned_plug_entry->pre_short_bit_p (i))
24422 reloc_ref_in_shortened_obj ((saved_plug_info_start + i), (saved_info_to_relocate + i));
24430 size_t s = size (x);
24431 uint8_t* next_obj = x + Align (s);
24432 Prefetch (next_obj);
24434 if (next_obj >= plug_end)
24436 dprintf (3, ("object %Ix is at the end of the plug %Ix->%Ix",
24437 next_obj, plug, plug_end));
24439 verify_pins_with_post_plug_info("before reloc short obj");
24441 relocate_shortened_obj_helper (x, s, (x + Align (s) - sizeof (plug_and_gap)), pinned_plug_entry, is_pinned);
24445 relocate_obj_helper (x, s);
24452 verify_pins_with_post_plug_info("end reloc short surv");
24455 void gc_heap::relocate_survivors_in_plug (uint8_t* plug, uint8_t* plug_end,
24456 BOOL check_last_object_p,
24457 mark* pinned_plug_entry)
24459 //dprintf(3,("Relocating pointers in Plug [%Ix,%Ix[", (size_t)plug, (size_t)plug_end));
24460 dprintf (3,("RP: [%Ix,%Ix[", (size_t)plug, (size_t)plug_end));
24462 if (check_last_object_p)
24464 relocate_shortened_survivor_helper (plug, plug_end, pinned_plug_entry);
24468 relocate_survivor_helper (plug, plug_end);
24472 void gc_heap::relocate_survivors_in_brick (uint8_t* tree, relocate_args* args)
24474 assert ((tree != NULL));
24476 dprintf (3, ("tree: %Ix, args->last_plug: %Ix, left: %Ix, right: %Ix, gap(t): %Ix",
24477 tree, args->last_plug,
24478 (tree + node_left_child (tree)),
24479 (tree + node_right_child (tree)),
24480 node_gap_size (tree)));
24482 if (node_left_child (tree))
24484 relocate_survivors_in_brick (tree + node_left_child (tree), args);
24487 uint8_t* plug = tree;
24488 BOOL has_post_plug_info_p = FALSE;
24489 BOOL has_pre_plug_info_p = FALSE;
24491 if (tree == oldest_pinned_plug)
24493 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
24494 &has_post_plug_info_p);
24495 assert (tree == pinned_plug (args->pinned_plug_entry));
24497 dprintf (3, ("tree is the oldest pin: %Ix", tree));
24499 if (args->last_plug)
24501 size_t gap_size = node_gap_size (tree);
24502 uint8_t* gap = (plug - gap_size);
24503 dprintf (3, ("tree: %Ix, gap: %Ix (%Ix)", tree, gap, gap_size));
24504 assert (gap_size >= Align (min_obj_size));
24505 uint8_t* last_plug_end = gap;
24507 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
24510 relocate_survivors_in_plug (args->last_plug, last_plug_end, check_last_object_p, args->pinned_plug_entry);
24515 assert (!has_pre_plug_info_p);
24518 args->last_plug = plug;
24519 args->is_shortened = has_post_plug_info_p;
24520 if (has_post_plug_info_p)
24522 dprintf (3, ("setting %Ix as shortened", plug));
24524 dprintf (3, ("last_plug: %Ix(shortened: %d)", plug, (args->is_shortened ? 1 : 0)));
24526 if (node_right_child (tree))
24528 relocate_survivors_in_brick (tree + node_right_child (tree), args);
24533 void gc_heap::update_oldest_pinned_plug()
24535 oldest_pinned_plug = (pinned_plug_que_empty_p() ? 0 : pinned_plug (oldest_pin()));
24538 void gc_heap::relocate_survivors (int condemned_gen_number,
24539 uint8_t* first_condemned_address)
24541 generation* condemned_gen = generation_of (condemned_gen_number);
24542 uint8_t* start_address = first_condemned_address;
24543 size_t current_brick = brick_of (start_address);
24544 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
24546 PREFIX_ASSUME(current_heap_segment != NULL);
24548 uint8_t* end_address = 0;
24550 reset_pinned_queue_bos();
24551 update_oldest_pinned_plug();
24553 end_address = heap_segment_allocated (current_heap_segment);
24555 size_t end_brick = brick_of (end_address - 1);
24556 relocate_args args;
24558 args.high = gc_high;
24559 args.is_shortened = FALSE;
24560 args.pinned_plug_entry = 0;
24561 args.last_plug = 0;
24564 if (current_brick > end_brick)
24566 if (args.last_plug)
24569 assert (!(args.is_shortened));
24570 relocate_survivors_in_plug (args.last_plug,
24571 heap_segment_allocated (current_heap_segment),
24573 args.pinned_plug_entry);
24576 args.last_plug = 0;
24579 if (heap_segment_next_rw (current_heap_segment))
24581 current_heap_segment = heap_segment_next_rw (current_heap_segment);
24582 current_brick = brick_of (heap_segment_mem (current_heap_segment));
24583 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24592 int brick_entry = brick_table [ current_brick ];
24594 if (brick_entry >= 0)
24596 relocate_survivors_in_brick (brick_address (current_brick) +
24605 void gc_heap::walk_plug (uint8_t* plug, size_t size, BOOL check_last_object_p, walk_relocate_args* args)
24607 if (check_last_object_p)
24609 size += sizeof (gap_reloc_pair);
24610 mark* entry = args->pinned_plug_entry;
24612 if (args->is_shortened)
24614 assert (entry->has_post_plug_info());
24615 entry->swap_post_plug_and_saved_for_profiler();
24619 assert (entry->has_pre_plug_info());
24620 entry->swap_pre_plug_and_saved_for_profiler();
24624 ptrdiff_t last_plug_relocation = node_relocation_distance (plug);
24625 STRESS_LOG_PLUG_MOVE(plug, (plug + size), -last_plug_relocation);
24626 ptrdiff_t reloc = settings.compaction ? last_plug_relocation : 0;
24628 (args->fn) (plug, (plug + size), reloc, args->profiling_context, !!settings.compaction, false);
24630 if (check_last_object_p)
24632 mark* entry = args->pinned_plug_entry;
24634 if (args->is_shortened)
24636 entry->swap_post_plug_and_saved_for_profiler();
24640 entry->swap_pre_plug_and_saved_for_profiler();
24645 void gc_heap::walk_relocation_in_brick (uint8_t* tree, walk_relocate_args* args)
24647 assert ((tree != NULL));
24648 if (node_left_child (tree))
24650 walk_relocation_in_brick (tree + node_left_child (tree), args);
24653 uint8_t* plug = tree;
24654 BOOL has_pre_plug_info_p = FALSE;
24655 BOOL has_post_plug_info_p = FALSE;
24657 if (tree == oldest_pinned_plug)
24659 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
24660 &has_post_plug_info_p);
24661 assert (tree == pinned_plug (args->pinned_plug_entry));
24664 if (args->last_plug != 0)
24666 size_t gap_size = node_gap_size (tree);
24667 uint8_t* gap = (plug - gap_size);
24668 uint8_t* last_plug_end = gap;
24669 size_t last_plug_size = (last_plug_end - args->last_plug);
24670 dprintf (3, ("tree: %Ix, last_plug: %Ix, gap: %Ix(%Ix), last_plug_end: %Ix, size: %Ix",
24671 tree, args->last_plug, gap, gap_size, last_plug_end, last_plug_size));
24673 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
24674 if (!check_last_object_p)
24676 assert (last_plug_size >= Align (min_obj_size));
24679 walk_plug (args->last_plug, last_plug_size, check_last_object_p, args);
24683 assert (!has_pre_plug_info_p);
24686 dprintf (3, ("set args last plug to plug: %Ix", plug));
24687 args->last_plug = plug;
24688 args->is_shortened = has_post_plug_info_p;
24690 if (node_right_child (tree))
24692 walk_relocation_in_brick (tree + node_right_child (tree), args);
24696 void gc_heap::walk_relocation (void* profiling_context, record_surv_fn fn)
24698 generation* condemned_gen = generation_of (settings.condemned_generation);
24699 uint8_t* start_address = generation_allocation_start (condemned_gen);
24700 size_t current_brick = brick_of (start_address);
24701 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
24703 PREFIX_ASSUME(current_heap_segment != NULL);
24705 reset_pinned_queue_bos();
24706 update_oldest_pinned_plug();
24707 size_t end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24708 walk_relocate_args args;
24709 args.is_shortened = FALSE;
24710 args.pinned_plug_entry = 0;
24711 args.last_plug = 0;
24712 args.profiling_context = profiling_context;
24717 if (current_brick > end_brick)
24719 if (args.last_plug)
24721 walk_plug (args.last_plug,
24722 (heap_segment_allocated (current_heap_segment) - args.last_plug),
24725 args.last_plug = 0;
24727 if (heap_segment_next_rw (current_heap_segment))
24729 current_heap_segment = heap_segment_next_rw (current_heap_segment);
24730 current_brick = brick_of (heap_segment_mem (current_heap_segment));
24731 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24740 int brick_entry = brick_table [ current_brick ];
24741 if (brick_entry >= 0)
24743 walk_relocation_in_brick (brick_address (current_brick) +
24752 void gc_heap::walk_survivors (record_surv_fn fn, void* context, walk_surv_type type)
24754 if (type == walk_for_gc)
24755 walk_survivors_relocation (context, fn);
24756 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24757 else if (type == walk_for_bgc)
24758 walk_survivors_for_bgc (context, fn);
24759 #endif //BACKGROUND_GC && FEATURE_EVENT_TRACE
24760 else if (type == walk_for_loh)
24761 walk_survivors_for_loh (context, fn);
24763 assert (!"unknown type!");
24766 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24767 void gc_heap::walk_survivors_for_bgc (void* profiling_context, record_surv_fn fn)
24769 // This should only be called for BGCs
24770 assert(settings.concurrent);
24772 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
24774 BOOL small_object_segments = TRUE;
24775 int align_const = get_alignment_constant (small_object_segments);
24781 if (small_object_segments)
24783 //switch to large segment
24784 small_object_segments = FALSE;
24786 align_const = get_alignment_constant (small_object_segments);
24787 seg = heap_segment_rw (generation_start_segment (large_object_generation));
24789 PREFIX_ASSUME(seg != NULL);
24797 uint8_t* o = heap_segment_mem (seg);
24798 uint8_t* end = heap_segment_allocated (seg);
24802 if (method_table(o) == g_gc_pFreeObjectMethodTable)
24804 o += Align (size (o), align_const);
24808 // It's survived. Make a fake plug, starting at o,
24809 // and send the event
24811 uint8_t* plug_start = o;
24813 while (method_table(o) != g_gc_pFreeObjectMethodTable)
24815 o += Align (size (o), align_const);
24822 uint8_t* plug_end = o;
24826 0, // Reloc distance == 0 as this is non-compacting
24828 false, // Non-compacting
24832 seg = heap_segment_next (seg);
24835 #endif // defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24837 void gc_heap::relocate_phase (int condemned_gen_number,
24838 uint8_t* first_condemned_address)
24841 sc.thread_number = heap_number;
24842 sc.promotion = FALSE;
24843 sc.concurrent = FALSE;
24849 start = GetCycleCount32();
24852 // %type% category = quote (relocate);
24853 dprintf (2,("---- Relocate phase -----"));
24855 #ifdef MULTIPLE_HEAPS
24856 //join all threads to make sure they are synchronized
24857 dprintf(3, ("Joining after end of plan"));
24858 gc_t_join.join(this, gc_join_begin_relocate_phase);
24859 if (gc_t_join.joined())
24860 #endif //MULTIPLE_HEAPS
24863 #ifdef MULTIPLE_HEAPS
24865 //join all threads to make sure they are synchronized
24866 dprintf(3, ("Restarting for relocation"));
24867 gc_t_join.restart();
24868 #endif //MULTIPLE_HEAPS
24871 dprintf(3,("Relocating roots"));
24872 GCScan::GcScanRoots(GCHeap::Relocate,
24873 condemned_gen_number, max_generation, &sc);
24875 verify_pins_with_post_plug_info("after reloc stack");
24877 #ifdef BACKGROUND_GC
24878 if (recursive_gc_sync::background_running_p())
24880 scan_background_roots (GCHeap::Relocate, heap_number, &sc);
24882 #endif //BACKGROUND_GC
24884 if (condemned_gen_number != max_generation)
24886 dprintf(3,("Relocating cross generation pointers"));
24887 mark_through_cards_for_segments (&gc_heap::relocate_address, TRUE);
24888 verify_pins_with_post_plug_info("after reloc cards");
24890 if (condemned_gen_number != max_generation)
24892 dprintf(3,("Relocating cross generation pointers for large objects"));
24893 mark_through_cards_for_large_objects (&gc_heap::relocate_address, TRUE);
24897 #ifdef FEATURE_LOH_COMPACTION
24898 if (loh_compacted_p)
24900 assert (settings.condemned_generation == max_generation);
24901 relocate_in_loh_compact();
24904 #endif //FEATURE_LOH_COMPACTION
24906 relocate_in_large_objects ();
24910 dprintf(3,("Relocating survivors"));
24911 relocate_survivors (condemned_gen_number,
24912 first_condemned_address);
24915 #ifdef FEATURE_PREMORTEM_FINALIZATION
24916 dprintf(3,("Relocating finalization data"));
24917 finalize_queue->RelocateFinalizationData (condemned_gen_number,
24919 #endif // FEATURE_PREMORTEM_FINALIZATION
24924 dprintf(3,("Relocating handle table"));
24925 GCScan::GcScanHandles(GCHeap::Relocate,
24926 condemned_gen_number, max_generation, &sc);
24929 #ifdef MULTIPLE_HEAPS
24930 //join all threads to make sure they are synchronized
24931 dprintf(3, ("Joining after end of relocation"));
24932 gc_t_join.join(this, gc_join_relocate_phase_done);
24934 #endif //MULTIPLE_HEAPS
24937 finish = GetCycleCount32();
24938 reloc_time = finish - start;
24941 dprintf(2,( "---- End of Relocate phase ----"));
24944 // This compares to see if tree is the current pinned plug and returns info
24945 // for this pinned plug. Also advances the pinned queue if that's the case.
24947 // We don't change the values of the plug info if tree is not the same as
24948 // the current pinned plug - the caller is responsible for setting the right
24949 // values to begin with.
24951 // POPO TODO: We are keeping this temporarily as this is also used by realloc
24952 // where it passes FALSE to deque_p, change it to use the same optimization
24953 // as relocate. Not as essential since realloc is already a slow path.
24954 mark* gc_heap::get_next_pinned_entry (uint8_t* tree,
24955 BOOL* has_pre_plug_info_p,
24956 BOOL* has_post_plug_info_p,
24959 if (!pinned_plug_que_empty_p())
24961 mark* oldest_entry = oldest_pin();
24962 uint8_t* oldest_plug = pinned_plug (oldest_entry);
24963 if (tree == oldest_plug)
24965 *has_pre_plug_info_p = oldest_entry->has_pre_plug_info();
24966 *has_post_plug_info_p = oldest_entry->has_post_plug_info();
24970 deque_pinned_plug();
24973 dprintf (3, ("found a pinned plug %Ix, pre: %d, post: %d",
24975 (*has_pre_plug_info_p ? 1 : 0),
24976 (*has_post_plug_info_p ? 1 : 0)));
24978 return oldest_entry;
24985 // This also deques the oldest entry and update the oldest plug
24986 mark* gc_heap::get_oldest_pinned_entry (BOOL* has_pre_plug_info_p,
24987 BOOL* has_post_plug_info_p)
24989 mark* oldest_entry = oldest_pin();
24990 *has_pre_plug_info_p = oldest_entry->has_pre_plug_info();
24991 *has_post_plug_info_p = oldest_entry->has_post_plug_info();
24993 deque_pinned_plug();
24994 update_oldest_pinned_plug();
24995 return oldest_entry;
24999 void gc_heap::copy_cards_range (uint8_t* dest, uint8_t* src, size_t len, BOOL copy_cards_p)
25002 copy_cards_for_addresses (dest, src, len);
25004 clear_card_for_addresses (dest, dest + len);
25007 // POPO TODO: We should actually just recover the artifically made gaps here..because when we copy
25008 // we always copy the earlier plugs first which means we won't need the gap sizes anymore. This way
25009 // we won't need to individually recover each overwritten part of plugs.
25011 void gc_heap::gcmemcopy (uint8_t* dest, uint8_t* src, size_t len, BOOL copy_cards_p)
25015 #ifdef BACKGROUND_GC
25016 if (current_c_gc_state == c_gc_state_marking)
25018 //TODO: should look to see whether we should consider changing this
25019 // to copy a consecutive region of the mark array instead.
25020 copy_mark_bits_for_addresses (dest, src, len);
25022 #endif //BACKGROUND_GC
25023 //dprintf(3,(" Memcopy [%Ix->%Ix, %Ix->%Ix[", (size_t)src, (size_t)dest, (size_t)src+len, (size_t)dest+len));
25024 dprintf(3,(" mc: [%Ix->%Ix, %Ix->%Ix[", (size_t)src, (size_t)dest, (size_t)src+len, (size_t)dest+len));
25025 memcopy (dest - plug_skew, src - plug_skew, len);
25026 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25027 if (SoftwareWriteWatch::IsEnabledForGCHeap())
25029 // The ranges [src - plug_kew .. src[ and [src + len - plug_skew .. src + len[ are ObjHeaders, which don't have GC
25030 // references, and are not relevant for write watch. The latter range actually corresponds to the ObjHeader for the
25031 // object at (src + len), so it can be ignored anyway.
25032 SoftwareWriteWatch::SetDirtyRegion(dest, len - plug_skew);
25034 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25035 copy_cards_range (dest, src, len, copy_cards_p);
25039 void gc_heap::compact_plug (uint8_t* plug, size_t size, BOOL check_last_object_p, compact_args* args)
25042 uint8_t* reloc_plug = plug + args->last_plug_relocation;
25044 if (check_last_object_p)
25046 size += sizeof (gap_reloc_pair);
25047 mark* entry = args->pinned_plug_entry;
25049 if (args->is_shortened)
25051 assert (entry->has_post_plug_info());
25052 entry->swap_post_plug_and_saved();
25056 assert (entry->has_pre_plug_info());
25057 entry->swap_pre_plug_and_saved();
25061 int old_brick_entry = brick_table [brick_of (plug)];
25063 assert (node_relocation_distance (plug) == args->last_plug_relocation);
25065 #ifdef FEATURE_STRUCTALIGN
25066 ptrdiff_t alignpad = node_alignpad(plug);
25069 make_unused_array (reloc_plug - alignpad, alignpad);
25070 if (brick_of (reloc_plug - alignpad) != brick_of (reloc_plug))
25072 // The alignment padding is straddling one or more bricks;
25073 // it has to be the last "object" of its first brick.
25074 fix_brick_to_highest (reloc_plug - alignpad, reloc_plug);
25077 #else // FEATURE_STRUCTALIGN
25078 size_t unused_arr_size = 0;
25079 BOOL already_padded_p = FALSE;
25081 if (is_plug_padded (plug))
25083 already_padded_p = TRUE;
25084 clear_plug_padded (plug);
25085 unused_arr_size = Align (min_obj_size);
25087 #endif //SHORT_PLUGS
25088 if (node_realigned (plug))
25090 unused_arr_size += switch_alignment_size (already_padded_p);
25093 if (unused_arr_size != 0)
25095 make_unused_array (reloc_plug - unused_arr_size, unused_arr_size);
25097 if (brick_of (reloc_plug - unused_arr_size) != brick_of (reloc_plug))
25099 dprintf (3, ("fix B for padding: %Id: %Ix->%Ix",
25100 unused_arr_size, (reloc_plug - unused_arr_size), reloc_plug));
25101 // The alignment padding is straddling one or more bricks;
25102 // it has to be the last "object" of its first brick.
25103 fix_brick_to_highest (reloc_plug - unused_arr_size, reloc_plug);
25106 #endif // FEATURE_STRUCTALIGN
25109 if (is_plug_padded (plug))
25111 make_unused_array (reloc_plug - Align (min_obj_size), Align (min_obj_size));
25113 if (brick_of (reloc_plug - Align (min_obj_size)) != brick_of (reloc_plug))
25115 // The alignment padding is straddling one or more bricks;
25116 // it has to be the last "object" of its first brick.
25117 fix_brick_to_highest (reloc_plug - Align (min_obj_size), reloc_plug);
25120 #endif //SHORT_PLUGS
25122 gcmemcopy (reloc_plug, plug, size, args->copy_cards_p);
25124 if (args->check_gennum_p)
25126 int src_gennum = args->src_gennum;
25127 if (src_gennum == -1)
25129 src_gennum = object_gennum (plug);
25132 int dest_gennum = object_gennum_plan (reloc_plug);
25134 if (src_gennum < dest_gennum)
25136 generation_allocation_size (generation_of (dest_gennum)) += size;
25140 size_t current_reloc_brick = args->current_compacted_brick;
25142 if (brick_of (reloc_plug) != current_reloc_brick)
25144 dprintf (3, ("last reloc B: %Ix, current reloc B: %Ix",
25145 current_reloc_brick, brick_of (reloc_plug)));
25147 if (args->before_last_plug)
25149 dprintf (3,(" fixing last brick %Ix to point to last plug %Ix(%Ix)",
25150 current_reloc_brick,
25151 args->before_last_plug,
25152 (args->before_last_plug - brick_address (current_reloc_brick))));
25155 set_brick (current_reloc_brick,
25156 args->before_last_plug - brick_address (current_reloc_brick));
25159 current_reloc_brick = brick_of (reloc_plug);
25161 size_t end_brick = brick_of (reloc_plug + size-1);
25162 if (end_brick != current_reloc_brick)
25164 // The plug is straddling one or more bricks
25165 // It has to be the last plug of its first brick
25166 dprintf (3,("plug spanning multiple bricks, fixing first brick %Ix to %Ix(%Ix)",
25167 current_reloc_brick, (size_t)reloc_plug,
25168 (reloc_plug - brick_address (current_reloc_brick))));
25171 set_brick (current_reloc_brick,
25172 reloc_plug - brick_address (current_reloc_brick));
25174 // update all intervening brick
25175 size_t brick = current_reloc_brick + 1;
25176 dprintf (3,("setting intervening bricks %Ix->%Ix to -1",
25177 brick, (end_brick - 1)));
25178 while (brick < end_brick)
25180 set_brick (brick, -1);
25183 // code last brick offset as a plug address
25184 args->before_last_plug = brick_address (end_brick) -1;
25185 current_reloc_brick = end_brick;
25186 dprintf (3, ("setting before last to %Ix, last brick to %Ix",
25187 args->before_last_plug, current_reloc_brick));
25191 dprintf (3, ("still in the same brick: %Ix", end_brick));
25192 args->before_last_plug = reloc_plug;
25194 args->current_compacted_brick = current_reloc_brick;
25196 if (check_last_object_p)
25198 mark* entry = args->pinned_plug_entry;
25200 if (args->is_shortened)
25202 entry->swap_post_plug_and_saved();
25206 entry->swap_pre_plug_and_saved();
25211 void gc_heap::compact_in_brick (uint8_t* tree, compact_args* args)
25213 assert (tree != NULL);
25214 int left_node = node_left_child (tree);
25215 int right_node = node_right_child (tree);
25216 ptrdiff_t relocation = node_relocation_distance (tree);
25222 dprintf (3, ("B: L: %d->%Ix", left_node, (tree + left_node)));
25223 compact_in_brick ((tree + left_node), args);
25226 uint8_t* plug = tree;
25227 BOOL has_pre_plug_info_p = FALSE;
25228 BOOL has_post_plug_info_p = FALSE;
25230 if (tree == oldest_pinned_plug)
25232 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
25233 &has_post_plug_info_p);
25234 assert (tree == pinned_plug (args->pinned_plug_entry));
25237 if (args->last_plug != 0)
25239 size_t gap_size = node_gap_size (tree);
25240 uint8_t* gap = (plug - gap_size);
25241 uint8_t* last_plug_end = gap;
25242 size_t last_plug_size = (last_plug_end - args->last_plug);
25243 dprintf (3, ("tree: %Ix, last_plug: %Ix, gap: %Ix(%Ix), last_plug_end: %Ix, size: %Ix",
25244 tree, args->last_plug, gap, gap_size, last_plug_end, last_plug_size));
25246 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
25247 if (!check_last_object_p)
25249 assert (last_plug_size >= Align (min_obj_size));
25252 compact_plug (args->last_plug, last_plug_size, check_last_object_p, args);
25256 assert (!has_pre_plug_info_p);
25259 dprintf (3, ("set args last plug to plug: %Ix, reloc: %Ix", plug, relocation));
25260 args->last_plug = plug;
25261 args->last_plug_relocation = relocation;
25262 args->is_shortened = has_post_plug_info_p;
25266 dprintf (3, ("B: R: %d->%Ix", right_node, (tree + right_node)));
25267 compact_in_brick ((tree + right_node), args);
25271 void gc_heap::recover_saved_pinned_info()
25273 reset_pinned_queue_bos();
25275 while (!(pinned_plug_que_empty_p()))
25277 mark* oldest_entry = oldest_pin();
25278 oldest_entry->recover_plug_info();
25279 #ifdef GC_CONFIG_DRIVEN
25280 if (oldest_entry->has_pre_plug_info() && oldest_entry->has_post_plug_info())
25281 record_interesting_data_point (idp_pre_and_post_pin);
25282 else if (oldest_entry->has_pre_plug_info())
25283 record_interesting_data_point (idp_pre_pin);
25284 else if (oldest_entry->has_post_plug_info())
25285 record_interesting_data_point (idp_post_pin);
25286 #endif //GC_CONFIG_DRIVEN
25288 deque_pinned_plug();
25292 void gc_heap::compact_phase (int condemned_gen_number,
25293 uint8_t* first_condemned_address,
25296 // %type% category = quote (compact);
25300 start = GetCycleCount32();
25302 generation* condemned_gen = generation_of (condemned_gen_number);
25303 uint8_t* start_address = first_condemned_address;
25304 size_t current_brick = brick_of (start_address);
25305 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
25307 PREFIX_ASSUME(current_heap_segment != NULL);
25309 reset_pinned_queue_bos();
25310 update_oldest_pinned_plug();
25312 BOOL reused_seg = expand_reused_seg_p();
25315 for (int i = 1; i <= max_generation; i++)
25317 generation_allocation_size (generation_of (i)) = 0;
25321 uint8_t* end_address = heap_segment_allocated (current_heap_segment);
25323 size_t end_brick = brick_of (end_address-1);
25325 args.last_plug = 0;
25326 args.before_last_plug = 0;
25327 args.current_compacted_brick = ~((size_t)1);
25328 args.is_shortened = FALSE;
25329 args.pinned_plug_entry = 0;
25330 args.copy_cards_p = (condemned_gen_number >= 1) || !clear_cards;
25331 args.check_gennum_p = reused_seg;
25332 if (args.check_gennum_p)
25334 args.src_gennum = ((current_heap_segment == ephemeral_heap_segment) ? -1 : 2);
25337 dprintf (2,("---- Compact Phase: %Ix(%Ix)----",
25338 first_condemned_address, brick_of (first_condemned_address)));
25340 #ifdef MULTIPLE_HEAPS
25342 if (gc_t_join.joined())
25344 #endif //MULTIPLE_HEAPS
25346 #ifdef MULTIPLE_HEAPS
25347 dprintf(3, ("Restarting for compaction"));
25348 gc_t_join.restart();
25350 #endif //MULTIPLE_HEAPS
25352 reset_pinned_queue_bos();
25354 #ifdef FEATURE_LOH_COMPACTION
25355 if (loh_compacted_p)
25359 #endif //FEATURE_LOH_COMPACTION
25361 if ((start_address < end_address) ||
25362 (condemned_gen_number == max_generation))
25366 if (current_brick > end_brick)
25368 if (args.last_plug != 0)
25370 dprintf (3, ("compacting last plug: %Ix", args.last_plug))
25371 compact_plug (args.last_plug,
25372 (heap_segment_allocated (current_heap_segment) - args.last_plug),
25377 if (heap_segment_next_rw (current_heap_segment))
25379 current_heap_segment = heap_segment_next_rw (current_heap_segment);
25380 current_brick = brick_of (heap_segment_mem (current_heap_segment));
25381 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
25382 args.last_plug = 0;
25383 if (args.check_gennum_p)
25385 args.src_gennum = ((current_heap_segment == ephemeral_heap_segment) ? -1 : 2);
25391 if (args.before_last_plug !=0)
25393 dprintf (3, ("Fixing last brick %Ix to point to plug %Ix",
25394 args.current_compacted_brick, (size_t)args.before_last_plug));
25395 assert (args.current_compacted_brick != ~1u);
25396 set_brick (args.current_compacted_brick,
25397 args.before_last_plug - brick_address (args.current_compacted_brick));
25403 int brick_entry = brick_table [ current_brick ];
25404 dprintf (3, ("B: %Ix(%Ix)->%Ix",
25405 current_brick, (size_t)brick_entry, (brick_address (current_brick) + brick_entry - 1)));
25407 if (brick_entry >= 0)
25409 compact_in_brick ((brick_address (current_brick) + brick_entry -1),
25418 recover_saved_pinned_info();
25421 finish = GetCycleCount32();
25422 compact_time = finish - start;
25425 concurrent_print_time_delta ("compact end");
25427 dprintf(2,("---- End of Compact phase ----"));
25430 #ifdef MULTIPLE_HEAPS
25433 #pragma warning(push)
25434 #pragma warning(disable:4702) // C4702: unreachable code: gc_thread_function may not return
25436 void gc_heap::gc_thread_stub (void* arg)
25438 gc_heap* heap = (gc_heap*)arg;
25439 if (!gc_thread_no_affinitize_p)
25441 GCThreadAffinity affinity;
25442 affinity.Group = GCThreadAffinity::None;
25443 affinity.Processor = GCThreadAffinity::None;
25445 // We are about to set affinity for GC threads. It is a good place to set up NUMA and
25446 // CPU groups because the process mask, processor number, and group number are all
25447 // readily available.
25448 if (GCToOSInterface::CanEnableGCCPUGroups())
25449 set_thread_group_affinity_for_heap(heap->heap_number, &affinity);
25451 set_thread_affinity_mask_for_heap(heap->heap_number, &affinity);
25453 if (!GCToOSInterface::SetThreadAffinity(&affinity))
25455 dprintf(1, ("Failed to set thread affinity for server GC thread"));
25459 // server GC threads run at a higher priority than normal.
25460 GCToOSInterface::BoostThreadPriority();
25461 _alloca (256*heap->heap_number);
25462 heap->gc_thread_function();
25465 #pragma warning(pop)
25468 #endif //MULTIPLE_HEAPS
25470 #ifdef BACKGROUND_GC
25473 #pragma warning(push)
25474 #pragma warning(disable:4702) // C4702: unreachable code: gc_thread_function may not return
25476 void gc_heap::bgc_thread_stub (void* arg)
25478 gc_heap* heap = (gc_heap*)arg;
25479 heap->bgc_thread = GCToEEInterface::GetThread();
25480 assert(heap->bgc_thread != nullptr);
25481 heap->bgc_thread_function();
25484 #pragma warning(pop)
25487 #endif //BACKGROUND_GC
25489 /*------------------ Background GC ----------------------------*/
25491 #ifdef BACKGROUND_GC
25493 void gc_heap::background_drain_mark_list (int thread)
25495 UNREFERENCED_PARAMETER(thread);
25497 size_t saved_c_mark_list_index = c_mark_list_index;
25499 if (saved_c_mark_list_index)
25501 concurrent_print_time_delta ("SML");
25503 while (c_mark_list_index != 0)
25505 size_t current_index = c_mark_list_index - 1;
25506 uint8_t* o = c_mark_list [current_index];
25507 background_mark_object (o THREAD_NUMBER_ARG);
25508 c_mark_list_index--;
25510 if (saved_c_mark_list_index)
25513 concurrent_print_time_delta ("EML");
25516 fire_drain_mark_list_event (saved_c_mark_list_index);
25520 // The background GC version of scan_dependent_handles (see that method for a more in-depth comment).
25521 #ifdef MULTIPLE_HEAPS
25522 // Since we only scan dependent handles while we are stopped we'll never interfere with FGCs scanning
25523 // them. So we can use the same static variables.
25524 void gc_heap::background_scan_dependent_handles (ScanContext *sc)
25526 // Whenever we call this method there may have been preceding object promotions. So set
25527 // s_fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
25528 // based on the how the scanning proceeded).
25529 s_fUnscannedPromotions = TRUE;
25531 // We don't know how many times we need to loop yet. In particular we can't base the loop condition on
25532 // the state of this thread's portion of the dependent handle table. That's because promotions on other
25533 // threads could cause handle promotions to become necessary here. Even if there are definitely no more
25534 // promotions possible in this thread's handles, we still have to stay in lock-step with those worker
25535 // threads that haven't finished yet (each GC worker thread has to join exactly the same number of times
25536 // as all the others or they'll get out of step).
25539 // The various worker threads are all currently racing in this code. We need to work out if at least
25540 // one of them think they have work to do this cycle. Each thread needs to rescan its portion of the
25541 // dependent handle table when both of the following conditions apply:
25542 // 1) At least one (arbitrary) object might have been promoted since the last scan (because if this
25543 // object happens to correspond to a primary in one of our handles we might potentially have to
25544 // promote the associated secondary).
25545 // 2) The table for this thread has at least one handle with a secondary that isn't promoted yet.
25547 // The first condition is represented by s_fUnscannedPromotions. This is always non-zero for the first
25548 // iteration of this loop (see comment above) and in subsequent cycles each thread updates this
25549 // whenever a mark stack overflow occurs or scanning their dependent handles results in a secondary
25550 // being promoted. This value is cleared back to zero in a synchronized fashion in the join that
25551 // follows below. Note that we can't read this outside of the join since on any iteration apart from
25552 // the first threads will be racing between reading this value and completing their previous
25553 // iteration's table scan.
25555 // The second condition is tracked by the dependent handle code itself on a per worker thread basis
25556 // (and updated by the GcDhReScan() method). We call GcDhUnpromotedHandlesExist() on each thread to
25557 // determine the local value and collect the results into the s_fUnpromotedHandles variable in what is
25558 // effectively an OR operation. As per s_fUnscannedPromotions we can't read the final result until
25559 // we're safely joined.
25560 if (GCScan::GcDhUnpromotedHandlesExist(sc))
25561 s_fUnpromotedHandles = TRUE;
25563 // Synchronize all the threads so we can read our state variables safely. The following shared
25564 // variable (indicating whether we should scan the tables or terminate the loop) will be set by a
25565 // single thread inside the join.
25566 bgc_t_join.join(this, gc_join_scan_dependent_handles);
25567 if (bgc_t_join.joined())
25569 // We're synchronized so it's safe to read our shared state variables. We update another shared
25570 // variable to indicate to all threads whether we'll be scanning for another cycle or terminating
25571 // the loop. We scan if there has been at least one object promotion since last time and at least
25572 // one thread has a dependent handle table with a potential handle promotion possible.
25573 s_fScanRequired = s_fUnscannedPromotions && s_fUnpromotedHandles;
25575 // Reset our shared state variables (ready to be set again on this scan or with a good initial
25576 // value for the next call if we're terminating the loop).
25577 s_fUnscannedPromotions = FALSE;
25578 s_fUnpromotedHandles = FALSE;
25580 if (!s_fScanRequired)
25582 uint8_t* all_heaps_max = 0;
25583 uint8_t* all_heaps_min = MAX_PTR;
25585 for (i = 0; i < n_heaps; i++)
25587 if (all_heaps_max < g_heaps[i]->background_max_overflow_address)
25588 all_heaps_max = g_heaps[i]->background_max_overflow_address;
25589 if (all_heaps_min > g_heaps[i]->background_min_overflow_address)
25590 all_heaps_min = g_heaps[i]->background_min_overflow_address;
25592 for (i = 0; i < n_heaps; i++)
25594 g_heaps[i]->background_max_overflow_address = all_heaps_max;
25595 g_heaps[i]->background_min_overflow_address = all_heaps_min;
25599 // Restart all the workers.
25600 dprintf(2, ("Starting all gc thread mark stack overflow processing"));
25601 bgc_t_join.restart();
25604 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
25605 // being visible. If there really was an overflow (process_mark_overflow returns true) then set the
25606 // global flag indicating that at least one object promotion may have occurred (the usual comment
25607 // about races applies). (Note it's OK to set this flag even if we're about to terminate the loop and
25608 // exit the method since we unconditionally set this variable on method entry anyway).
25609 if (background_process_mark_overflow (sc->concurrent))
25610 s_fUnscannedPromotions = TRUE;
25612 // If we decided that no scan was required we can terminate the loop now.
25613 if (!s_fScanRequired)
25616 // Otherwise we must join with the other workers to ensure that all mark stack overflows have been
25617 // processed before we start scanning dependent handle tables (if overflows remain while we scan we
25618 // could miss noting the promotion of some primary objects).
25619 bgc_t_join.join(this, gc_join_rescan_dependent_handles);
25620 if (bgc_t_join.joined())
25622 // Restart all the workers.
25623 dprintf(3, ("Starting all gc thread for dependent handle promotion"));
25624 bgc_t_join.restart();
25627 // If the portion of the dependent handle table managed by this worker has handles that could still be
25628 // promoted perform a rescan. If the rescan resulted in at least one promotion note this fact since it
25629 // could require a rescan of handles on this or other workers.
25630 if (GCScan::GcDhUnpromotedHandlesExist(sc))
25631 if (GCScan::GcDhReScan(sc))
25632 s_fUnscannedPromotions = TRUE;
25636 void gc_heap::background_scan_dependent_handles (ScanContext *sc)
25638 // Whenever we call this method there may have been preceding object promotions. So set
25639 // fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
25640 // based on the how the scanning proceeded).
25641 bool fUnscannedPromotions = true;
25643 // Scan dependent handles repeatedly until there are no further promotions that can be made or we made a
25644 // scan without performing any new promotions.
25645 while (GCScan::GcDhUnpromotedHandlesExist(sc) && fUnscannedPromotions)
25647 // On each iteration of the loop start with the assumption that no further objects have been promoted.
25648 fUnscannedPromotions = false;
25650 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
25651 // being visible. If there was an overflow (background_process_mark_overflow returned true) then
25652 // additional objects now appear to be promoted and we should set the flag.
25653 if (background_process_mark_overflow (sc->concurrent))
25654 fUnscannedPromotions = true;
25656 // Perform the scan and set the flag if any promotions resulted.
25657 if (GCScan::GcDhReScan (sc))
25658 fUnscannedPromotions = true;
25661 // Perform a last processing of any overflowed mark stack.
25662 background_process_mark_overflow (sc->concurrent);
25664 #endif //MULTIPLE_HEAPS
25666 void gc_heap::recover_bgc_settings()
25668 if ((settings.condemned_generation < max_generation) && recursive_gc_sync::background_running_p())
25670 dprintf (2, ("restoring bgc settings"));
25671 settings = saved_bgc_settings;
25672 GCHeap::GcCondemnedGeneration = gc_heap::settings.condemned_generation;
25676 void gc_heap::allow_fgc()
25678 assert (bgc_thread == GCToEEInterface::GetThread());
25679 bool bToggleGC = false;
25681 if (g_fSuspensionPending > 0)
25683 bToggleGC = GCToEEInterface::EnablePreemptiveGC();
25686 GCToEEInterface::DisablePreemptiveGC();
25691 BOOL gc_heap::should_commit_mark_array()
25693 return (recursive_gc_sync::background_running_p() || (current_bgc_state == bgc_initialized));
25696 void gc_heap::clear_commit_flag()
25698 generation* gen = generation_of (max_generation);
25699 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
25704 if (gen != large_object_generation)
25706 gen = large_object_generation;
25707 seg = heap_segment_in_range (generation_start_segment (gen));
25715 if (seg->flags & heap_segment_flags_ma_committed)
25717 seg->flags &= ~heap_segment_flags_ma_committed;
25720 if (seg->flags & heap_segment_flags_ma_pcommitted)
25722 seg->flags &= ~heap_segment_flags_ma_pcommitted;
25725 seg = heap_segment_next (seg);
25729 void gc_heap::clear_commit_flag_global()
25731 #ifdef MULTIPLE_HEAPS
25732 for (int i = 0; i < n_heaps; i++)
25734 g_heaps[i]->clear_commit_flag();
25737 clear_commit_flag();
25738 #endif //MULTIPLE_HEAPS
25741 void gc_heap::verify_mark_array_cleared (uint8_t* begin, uint8_t* end, uint32_t* mark_array_addr)
25744 size_t markw = mark_word_of (begin);
25745 size_t markw_end = mark_word_of (end);
25747 while (markw < markw_end)
25749 if (mark_array_addr[markw])
25751 dprintf (1, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
25752 markw, mark_array_addr[markw], mark_word_address (markw)));
25758 UNREFERENCED_PARAMETER(begin);
25759 UNREFERENCED_PARAMETER(end);
25760 UNREFERENCED_PARAMETER(mark_array_addr);
25764 void gc_heap::verify_mark_array_cleared (heap_segment* seg, uint32_t* mark_array_addr)
25766 verify_mark_array_cleared (heap_segment_mem (seg), heap_segment_reserved (seg), mark_array_addr);
25769 BOOL gc_heap::commit_mark_array_new_seg (gc_heap* hp,
25771 uint32_t* new_card_table,
25772 uint8_t* new_lowest_address)
25774 UNREFERENCED_PARAMETER(hp); // compiler bug? -- this *is*, indeed, referenced
25776 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
25777 uint8_t* end = heap_segment_reserved (seg);
25779 uint8_t* lowest = hp->background_saved_lowest_address;
25780 uint8_t* highest = hp->background_saved_highest_address;
25782 uint8_t* commit_start = NULL;
25783 uint8_t* commit_end = NULL;
25784 size_t commit_flag = 0;
25786 if ((highest >= start) &&
25789 if ((start >= lowest) && (end <= highest))
25791 dprintf (GC_TABLE_LOG, ("completely in bgc range: seg %Ix-%Ix, bgc: %Ix-%Ix",
25792 start, end, lowest, highest));
25793 commit_flag = heap_segment_flags_ma_committed;
25797 dprintf (GC_TABLE_LOG, ("partially in bgc range: seg %Ix-%Ix, bgc: %Ix-%Ix",
25798 start, end, lowest, highest));
25799 commit_flag = heap_segment_flags_ma_pcommitted;
25802 commit_start = max (lowest, start);
25803 commit_end = min (highest, end);
25805 if (!commit_mark_array_by_range (commit_start, commit_end, hp->mark_array))
25810 if (new_card_table == 0)
25812 new_card_table = g_gc_card_table;
25815 if (hp->card_table != new_card_table)
25817 if (new_lowest_address == 0)
25819 new_lowest_address = g_gc_lowest_address;
25822 uint32_t* ct = &new_card_table[card_word (gcard_of (new_lowest_address))];
25823 uint32_t* ma = (uint32_t*)((uint8_t*)card_table_mark_array (ct) - size_mark_array_of (0, new_lowest_address));
25825 dprintf (GC_TABLE_LOG, ("table realloc-ed: %Ix->%Ix, MA: %Ix->%Ix",
25826 hp->card_table, new_card_table,
25827 hp->mark_array, ma));
25829 if (!commit_mark_array_by_range (commit_start, commit_end, ma))
25835 seg->flags |= commit_flag;
25841 BOOL gc_heap::commit_mark_array_by_range (uint8_t* begin, uint8_t* end, uint32_t* mark_array_addr)
25843 size_t beg_word = mark_word_of (begin);
25844 size_t end_word = mark_word_of (align_on_mark_word (end));
25845 uint8_t* commit_start = align_lower_page ((uint8_t*)&mark_array_addr[beg_word]);
25846 uint8_t* commit_end = align_on_page ((uint8_t*)&mark_array_addr[end_word]);
25847 size_t size = (size_t)(commit_end - commit_start);
25849 #ifdef SIMPLE_DPRINTF
25850 dprintf (GC_TABLE_LOG, ("range: %Ix->%Ix mark word: %Ix->%Ix(%Id), mark array: %Ix->%Ix(%Id), commit %Ix->%Ix(%Id)",
25852 beg_word, end_word,
25853 (end_word - beg_word) * sizeof (uint32_t),
25854 &mark_array_addr[beg_word],
25855 &mark_array_addr[end_word],
25856 (size_t)(&mark_array_addr[end_word] - &mark_array_addr[beg_word]),
25857 commit_start, commit_end,
25859 #endif //SIMPLE_DPRINTF
25861 if (virtual_commit (commit_start, size))
25863 // We can only verify the mark array is cleared from begin to end, the first and the last
25864 // page aren't necessarily all cleared 'cause they could be used by other segments or
25866 verify_mark_array_cleared (begin, end, mark_array_addr);
25871 dprintf (GC_TABLE_LOG, ("failed to commit %Id bytes", (end_word - beg_word) * sizeof (uint32_t)));
25876 BOOL gc_heap::commit_mark_array_with_check (heap_segment* seg, uint32_t* new_mark_array_addr)
25878 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
25879 uint8_t* end = heap_segment_reserved (seg);
25881 #ifdef MULTIPLE_HEAPS
25882 uint8_t* lowest = heap_segment_heap (seg)->background_saved_lowest_address;
25883 uint8_t* highest = heap_segment_heap (seg)->background_saved_highest_address;
25885 uint8_t* lowest = background_saved_lowest_address;
25886 uint8_t* highest = background_saved_highest_address;
25887 #endif //MULTIPLE_HEAPS
25889 if ((highest >= start) &&
25892 start = max (lowest, start);
25893 end = min (highest, end);
25894 if (!commit_mark_array_by_range (start, end, new_mark_array_addr))
25903 BOOL gc_heap::commit_mark_array_by_seg (heap_segment* seg, uint32_t* mark_array_addr)
25905 dprintf (GC_TABLE_LOG, ("seg: %Ix->%Ix; MA: %Ix",
25907 heap_segment_reserved (seg),
25909 uint8_t* start = (heap_segment_read_only_p (seg) ? heap_segment_mem (seg) : (uint8_t*)seg);
25911 return commit_mark_array_by_range (start, heap_segment_reserved (seg), mark_array_addr);
25914 BOOL gc_heap::commit_mark_array_bgc_init (uint32_t* mark_array_addr)
25916 UNREFERENCED_PARAMETER(mark_array_addr);
25918 dprintf (GC_TABLE_LOG, ("BGC init commit: lowest: %Ix, highest: %Ix, mark_array: %Ix",
25919 lowest_address, highest_address, mark_array));
25921 generation* gen = generation_of (max_generation);
25922 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
25927 if (gen != large_object_generation)
25929 gen = large_object_generation;
25930 seg = heap_segment_in_range (generation_start_segment (gen));
25938 dprintf (GC_TABLE_LOG, ("seg: %Ix, flags: %Id", seg, seg->flags));
25940 if (!(seg->flags & heap_segment_flags_ma_committed))
25942 // For ro segments they could always be only partially in range so we'd
25943 // be calling this at the beginning of every BGC. We are not making this
25944 // more efficient right now - ro segments are currently only used by redhawk.
25945 if (heap_segment_read_only_p (seg))
25947 if ((heap_segment_mem (seg) >= lowest_address) &&
25948 (heap_segment_reserved (seg) <= highest_address))
25950 if (commit_mark_array_by_seg (seg, mark_array))
25952 seg->flags |= heap_segment_flags_ma_committed;
25961 uint8_t* start = max (lowest_address, heap_segment_mem (seg));
25962 uint8_t* end = min (highest_address, heap_segment_reserved (seg));
25963 if (commit_mark_array_by_range (start, end, mark_array))
25965 seg->flags |= heap_segment_flags_ma_pcommitted;
25975 // For normal segments they are by design completely in range so just
25976 // commit the whole mark array for each seg.
25977 if (commit_mark_array_by_seg (seg, mark_array))
25979 if (seg->flags & heap_segment_flags_ma_pcommitted)
25981 seg->flags &= ~heap_segment_flags_ma_pcommitted;
25983 seg->flags |= heap_segment_flags_ma_committed;
25992 seg = heap_segment_next (seg);
25998 // This function doesn't check the commit flag since it's for a new array -
25999 // the mark_array flag for these segments will remain the same.
26000 BOOL gc_heap::commit_new_mark_array (uint32_t* new_mark_array_addr)
26002 dprintf (GC_TABLE_LOG, ("commiting existing segs on MA %Ix", new_mark_array_addr));
26003 generation* gen = generation_of (max_generation);
26004 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
26009 if (gen != large_object_generation)
26011 gen = large_object_generation;
26012 seg = heap_segment_in_range (generation_start_segment (gen));
26020 if (!commit_mark_array_with_check (seg, new_mark_array_addr))
26025 seg = heap_segment_next (seg);
26028 #ifdef MULTIPLE_HEAPS
26029 if (new_heap_segment)
26031 if (!commit_mark_array_with_check (new_heap_segment, new_mark_array_addr))
26036 #endif //MULTIPLE_HEAPS
26041 BOOL gc_heap::commit_new_mark_array_global (uint32_t* new_mark_array)
26043 #ifdef MULTIPLE_HEAPS
26044 for (int i = 0; i < n_heaps; i++)
26046 if (!g_heaps[i]->commit_new_mark_array (new_mark_array))
26052 if (!commit_new_mark_array (new_mark_array))
26056 #endif //MULTIPLE_HEAPS
26061 void gc_heap::decommit_mark_array_by_seg (heap_segment* seg)
26063 // if BGC is disabled (the finalize watchdog does this at shutdown), the mark array could have
26064 // been set to NULL.
26065 if (mark_array == NULL)
26070 dprintf (GC_TABLE_LOG, ("decommitting seg %Ix(%Ix), MA: %Ix", seg, seg->flags, mark_array));
26072 size_t flags = seg->flags;
26074 if ((flags & heap_segment_flags_ma_committed) ||
26075 (flags & heap_segment_flags_ma_pcommitted))
26077 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
26078 uint8_t* end = heap_segment_reserved (seg);
26080 if (flags & heap_segment_flags_ma_pcommitted)
26082 start = max (lowest_address, start);
26083 end = min (highest_address, end);
26086 size_t beg_word = mark_word_of (start);
26087 size_t end_word = mark_word_of (align_on_mark_word (end));
26088 uint8_t* decommit_start = align_on_page ((uint8_t*)&mark_array[beg_word]);
26089 uint8_t* decommit_end = align_lower_page ((uint8_t*)&mark_array[end_word]);
26090 size_t size = (size_t)(decommit_end - decommit_start);
26092 #ifdef SIMPLE_DPRINTF
26093 dprintf (GC_TABLE_LOG, ("seg: %Ix mark word: %Ix->%Ix(%Id), mark array: %Ix->%Ix(%Id), decommit %Ix->%Ix(%Id)",
26095 beg_word, end_word,
26096 (end_word - beg_word) * sizeof (uint32_t),
26097 &mark_array[beg_word],
26098 &mark_array[end_word],
26099 (size_t)(&mark_array[end_word] - &mark_array[beg_word]),
26100 decommit_start, decommit_end,
26102 #endif //SIMPLE_DPRINTF
26104 if (decommit_start < decommit_end)
26106 if (!virtual_decommit (decommit_start, size))
26108 dprintf (GC_TABLE_LOG, ("decommit on %Ix for %Id bytes failed",
26109 decommit_start, size));
26110 assert (!"decommit failed");
26114 dprintf (GC_TABLE_LOG, ("decommited [%Ix for address [%Ix", beg_word, seg));
26118 void gc_heap::background_mark_phase ()
26120 verify_mark_array_cleared();
26123 sc.thread_number = heap_number;
26124 sc.promotion = TRUE;
26125 sc.concurrent = FALSE;
26128 BOOL cooperative_mode = TRUE;
26129 #ifndef MULTIPLE_HEAPS
26130 const int thread = heap_number;
26131 #endif //!MULTIPLE_HEAPS
26133 dprintf(2,("-(GC%d)BMark-", VolatileLoad(&settings.gc_index)));
26135 assert (settings.concurrent);
26140 start = GetCycleCount32();
26143 #ifdef FFIND_OBJECT
26144 if (gen0_must_clear_bricks > 0)
26145 gen0_must_clear_bricks--;
26146 #endif //FFIND_OBJECT
26148 background_soh_alloc_count = 0;
26149 background_loh_alloc_count = 0;
26150 bgc_overflow_count = 0;
26152 bpromoted_bytes (heap_number) = 0;
26153 static uint32_t num_sizedrefs = 0;
26155 background_min_overflow_address = MAX_PTR;
26156 background_max_overflow_address = 0;
26157 background_min_soh_overflow_address = MAX_PTR;
26158 background_max_soh_overflow_address = 0;
26159 processed_soh_overflow_p = FALSE;
26162 //set up the mark lists from g_mark_list
26163 assert (g_mark_list);
26164 mark_list = g_mark_list;
26165 //dont use the mark list for full gc
26166 //because multiple segments are more complex to handle and the list
26167 //is likely to overflow
26168 mark_list_end = &mark_list [0];
26169 mark_list_index = &mark_list [0];
26171 c_mark_list_index = 0;
26173 #ifndef MULTIPLE_HEAPS
26174 shigh = (uint8_t*) 0;
26176 #endif //MULTIPLE_HEAPS
26178 generation* gen = generation_of (max_generation);
26180 dprintf(3,("BGC: stack marking"));
26181 sc.concurrent = TRUE;
26183 GCScan::GcScanRoots(background_promote_callback,
26184 max_generation, max_generation,
26189 dprintf(3,("BGC: finalization marking"));
26190 finalize_queue->GcScanRoots(background_promote_callback, heap_number, 0);
26193 size_t total_loh_size = generation_size (max_generation + 1);
26194 bgc_begin_loh_size = total_loh_size;
26195 bgc_alloc_spin_loh = 0;
26196 bgc_loh_size_increased = 0;
26197 bgc_loh_allocated_in_free = 0;
26198 size_t total_soh_size = generation_sizes (generation_of (max_generation));
26200 dprintf (GTC_LOG, ("BM: h%d: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
26203 //concurrent_print_time_delta ("copying stack roots");
26204 concurrent_print_time_delta ("CS");
26206 FIRE_EVENT(BGC1stNonConEnd);
26208 expanded_in_fgc = FALSE;
26209 saved_overflow_ephemeral_seg = 0;
26210 current_bgc_state = bgc_reset_ww;
26212 // we don't need a join here - just whichever thread that gets here
26213 // first can change the states and call restart_vm.
26214 // this is not true - we can't let the EE run when we are scanning stack.
26215 // since we now allow reset ww to run concurrently and have a join for it,
26216 // we can do restart ee on the 1st thread that got here. Make sure we handle the
26217 // sizedref handles correctly.
26218 #ifdef MULTIPLE_HEAPS
26219 bgc_t_join.join(this, gc_join_restart_ee);
26220 if (bgc_t_join.joined())
26221 #endif //MULTIPLE_HEAPS
26223 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26224 // Resetting write watch for software write watch is pretty fast, much faster than for hardware write watch. Reset
26225 // can be done while the runtime is suspended or after the runtime is restarted, the preference was to reset while
26226 // the runtime is suspended. The reset for hardware write watch is done after the runtime is restarted below.
26228 concurrent_print_time_delta ("CRWW begin");
26230 #ifdef MULTIPLE_HEAPS
26231 for (int i = 0; i < n_heaps; i++)
26233 g_heaps[i]->reset_write_watch (FALSE);
26236 reset_write_watch (FALSE);
26237 #endif //MULTIPLE_HEAPS
26239 concurrent_print_time_delta ("CRWW");
26240 #endif //WRITE_WATCH
26241 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26243 num_sizedrefs = GCToEEInterface::GetTotalNumSizedRefHandles();
26245 // this c_write is not really necessary because restart_vm
26246 // has an instruction that will flush the cpu cache (interlocked
26247 // or whatever) but we don't want to rely on that.
26248 dprintf (BGC_LOG, ("setting cm_in_progress"));
26249 c_write (cm_in_progress, TRUE);
26251 //restart all thread, doing the marking from the array
26252 assert (dont_restart_ee_p);
26253 dont_restart_ee_p = FALSE;
26256 GCToOSInterface::YieldThread (0);
26257 #ifdef MULTIPLE_HEAPS
26258 dprintf(3, ("Starting all gc threads for gc"));
26259 bgc_t_join.restart();
26260 #endif //MULTIPLE_HEAPS
26263 #ifdef MULTIPLE_HEAPS
26264 bgc_t_join.join(this, gc_join_after_reset);
26265 if (bgc_t_join.joined())
26266 #endif //MULTIPLE_HEAPS
26268 disable_preemptive (true);
26270 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26271 // When software write watch is enabled, resetting write watch is done while the runtime is suspended above. The
26272 // post-reset call to revisit_written_pages is only necessary for concurrent reset_write_watch, to discard dirtied
26273 // pages during the concurrent reset.
26276 concurrent_print_time_delta ("CRWW begin");
26278 #ifdef MULTIPLE_HEAPS
26279 for (int i = 0; i < n_heaps; i++)
26281 g_heaps[i]->reset_write_watch (TRUE);
26284 reset_write_watch (TRUE);
26285 #endif //MULTIPLE_HEAPS
26287 concurrent_print_time_delta ("CRWW");
26288 #endif //WRITE_WATCH
26290 #ifdef MULTIPLE_HEAPS
26291 for (int i = 0; i < n_heaps; i++)
26293 g_heaps[i]->revisit_written_pages (TRUE, TRUE);
26296 revisit_written_pages (TRUE, TRUE);
26297 #endif //MULTIPLE_HEAPS
26299 concurrent_print_time_delta ("CRW");
26300 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26302 #ifdef MULTIPLE_HEAPS
26303 for (int i = 0; i < n_heaps; i++)
26305 g_heaps[i]->current_bgc_state = bgc_mark_handles;
26308 current_bgc_state = bgc_mark_handles;
26309 #endif //MULTIPLE_HEAPS
26311 current_c_gc_state = c_gc_state_marking;
26313 enable_preemptive ();
26315 #ifdef MULTIPLE_HEAPS
26316 dprintf(3, ("Joining BGC threads after resetting writewatch"));
26317 bgc_t_join.restart();
26318 #endif //MULTIPLE_HEAPS
26321 disable_preemptive (true);
26323 if (num_sizedrefs > 0)
26325 GCScan::GcScanSizedRefs(background_promote, max_generation, max_generation, &sc);
26327 enable_preemptive ();
26329 #ifdef MULTIPLE_HEAPS
26330 bgc_t_join.join(this, gc_join_scan_sizedref_done);
26331 if (bgc_t_join.joined())
26333 dprintf(3, ("Done with marking all sized refs. Starting all bgc thread for marking other strong roots"));
26334 bgc_t_join.restart();
26336 #endif //MULTIPLE_HEAPS
26338 disable_preemptive (true);
26341 dprintf (3,("BGC: handle table marking"));
26342 GCScan::GcScanHandles(background_promote,
26343 max_generation, max_generation,
26345 //concurrent_print_time_delta ("concurrent marking handle table");
26346 concurrent_print_time_delta ("CRH");
26348 current_bgc_state = bgc_mark_stack;
26349 dprintf (2,("concurrent draining mark list"));
26350 background_drain_mark_list (thread);
26351 //concurrent_print_time_delta ("concurrent marking stack roots");
26352 concurrent_print_time_delta ("CRS");
26354 dprintf (2,("concurrent revisiting dirtied pages"));
26355 revisit_written_pages (TRUE);
26356 revisit_written_pages (TRUE);
26357 //concurrent_print_time_delta ("concurrent marking dirtied pages on LOH");
26358 concurrent_print_time_delta ("CRre");
26360 enable_preemptive ();
26362 #ifdef MULTIPLE_HEAPS
26363 bgc_t_join.join(this, gc_join_concurrent_overflow);
26364 if (bgc_t_join.joined())
26366 uint8_t* all_heaps_max = 0;
26367 uint8_t* all_heaps_min = MAX_PTR;
26369 for (i = 0; i < n_heaps; i++)
26371 dprintf (3, ("heap %d overflow max is %Ix, min is %Ix",
26373 g_heaps[i]->background_max_overflow_address,
26374 g_heaps[i]->background_min_overflow_address));
26375 if (all_heaps_max < g_heaps[i]->background_max_overflow_address)
26376 all_heaps_max = g_heaps[i]->background_max_overflow_address;
26377 if (all_heaps_min > g_heaps[i]->background_min_overflow_address)
26378 all_heaps_min = g_heaps[i]->background_min_overflow_address;
26380 for (i = 0; i < n_heaps; i++)
26382 g_heaps[i]->background_max_overflow_address = all_heaps_max;
26383 g_heaps[i]->background_min_overflow_address = all_heaps_min;
26385 dprintf(3, ("Starting all bgc threads after updating the overflow info"));
26386 bgc_t_join.restart();
26388 #endif //MULTIPLE_HEAPS
26390 disable_preemptive (true);
26392 dprintf (2, ("before CRov count: %d", bgc_overflow_count));
26393 bgc_overflow_count = 0;
26394 background_process_mark_overflow (TRUE);
26395 dprintf (2, ("after CRov count: %d", bgc_overflow_count));
26396 bgc_overflow_count = 0;
26397 //concurrent_print_time_delta ("concurrent processing mark overflow");
26398 concurrent_print_time_delta ("CRov");
26400 // Stop all threads, crawl all stacks and revisit changed pages.
26401 FIRE_EVENT(BGC1stConEnd);
26403 dprintf (2, ("Stopping the EE"));
26405 enable_preemptive ();
26407 #ifdef MULTIPLE_HEAPS
26408 bgc_t_join.join(this, gc_join_suspend_ee);
26409 if (bgc_t_join.joined())
26411 bgc_threads_sync_event.Reset();
26413 dprintf(3, ("Joining BGC threads for non concurrent final marking"));
26414 bgc_t_join.restart();
26416 #endif //MULTIPLE_HEAPS
26418 if (heap_number == 0)
26420 enter_spin_lock (&gc_lock);
26424 bgc_threads_sync_event.Set();
26428 bgc_threads_sync_event.Wait(INFINITE, FALSE);
26429 dprintf (2, ("bgc_threads_sync_event is signalled"));
26432 assert (settings.concurrent);
26433 assert (settings.condemned_generation == max_generation);
26435 dprintf (2, ("clearing cm_in_progress"));
26436 c_write (cm_in_progress, FALSE);
26438 bgc_alloc_lock->check();
26440 current_bgc_state = bgc_final_marking;
26442 //concurrent_print_time_delta ("concurrent marking ended");
26443 concurrent_print_time_delta ("CR");
26445 FIRE_EVENT(BGC2ndNonConBegin);
26447 mark_absorb_new_alloc();
26449 // We need a join here 'cause find_object would complain if the gen0
26450 // bricks of another heap haven't been fixed up. So we need to make sure
26451 // that every heap's gen0 bricks are fixed up before we proceed.
26452 #ifdef MULTIPLE_HEAPS
26453 bgc_t_join.join(this, gc_join_after_absorb);
26454 if (bgc_t_join.joined())
26456 dprintf(3, ("Joining BGC threads after absorb"));
26457 bgc_t_join.restart();
26459 #endif //MULTIPLE_HEAPS
26461 // give VM a chance to do work
26462 GCToEEInterface::GcBeforeBGCSweepWork();
26464 //reset the flag, indicating that the EE no longer expect concurrent
26466 sc.concurrent = FALSE;
26468 total_loh_size = generation_size (max_generation + 1);
26469 total_soh_size = generation_sizes (generation_of (max_generation));
26471 dprintf (GTC_LOG, ("FM: h%d: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
26473 dprintf (2, ("nonconcurrent marking stack roots"));
26474 GCScan::GcScanRoots(background_promote,
26475 max_generation, max_generation,
26477 //concurrent_print_time_delta ("nonconcurrent marking stack roots");
26478 concurrent_print_time_delta ("NRS");
26480 // finalize_queue->EnterFinalizeLock();
26481 finalize_queue->GcScanRoots(background_promote, heap_number, 0);
26482 // finalize_queue->LeaveFinalizeLock();
26484 dprintf (2, ("nonconcurrent marking handle table"));
26485 GCScan::GcScanHandles(background_promote,
26486 max_generation, max_generation,
26488 //concurrent_print_time_delta ("nonconcurrent marking handle table");
26489 concurrent_print_time_delta ("NRH");
26491 dprintf (2,("---- (GC%d)final going through written pages ----", VolatileLoad(&settings.gc_index)));
26492 revisit_written_pages (FALSE);
26493 //concurrent_print_time_delta ("nonconcurrent revisit dirtied pages on LOH");
26494 concurrent_print_time_delta ("NRre LOH");
26496 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26497 #ifdef MULTIPLE_HEAPS
26498 bgc_t_join.join(this, gc_join_disable_software_write_watch);
26499 if (bgc_t_join.joined())
26500 #endif // MULTIPLE_HEAPS
26502 // The runtime is suspended, and we will be doing a final query of dirty pages, so pause tracking written pages to
26503 // avoid further perf penalty after the runtime is restarted
26504 SoftwareWriteWatch::DisableForGCHeap();
26506 #ifdef MULTIPLE_HEAPS
26507 dprintf(3, ("Restarting BGC threads after disabling software write watch"));
26508 bgc_t_join.restart();
26509 #endif // MULTIPLE_HEAPS
26511 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26513 dprintf (2, ("before NR 1st Hov count: %d", bgc_overflow_count));
26514 bgc_overflow_count = 0;
26516 // Dependent handles need to be scanned with a special algorithm (see the header comment on
26517 // scan_dependent_handles for more detail). We perform an initial scan without processing any mark
26518 // stack overflow. This is not guaranteed to complete the operation but in a common case (where there
26519 // are no dependent handles that are due to be collected) it allows us to optimize away further scans.
26520 // The call to background_scan_dependent_handles is what will cycle through more iterations if
26521 // required and will also perform processing of any mark stack overflow once the dependent handle
26522 // table has been fully promoted.
26523 dprintf (2, ("1st dependent handle scan and process mark overflow"));
26524 GCScan::GcDhInitialScan(background_promote, max_generation, max_generation, &sc);
26525 background_scan_dependent_handles (&sc);
26526 //concurrent_print_time_delta ("1st nonconcurrent dependent handle scan and process mark overflow");
26527 concurrent_print_time_delta ("NR 1st Hov");
26529 dprintf (2, ("after NR 1st Hov count: %d", bgc_overflow_count));
26530 bgc_overflow_count = 0;
26532 #ifdef MULTIPLE_HEAPS
26533 bgc_t_join.join(this, gc_join_null_dead_short_weak);
26534 if (bgc_t_join.joined())
26535 #endif //MULTIPLE_HEAPS
26537 GCToEEInterface::AfterGcScanRoots (max_generation, max_generation, &sc);
26539 #ifdef MULTIPLE_HEAPS
26540 dprintf(3, ("Joining BGC threads for short weak handle scan"));
26541 bgc_t_join.restart();
26542 #endif //MULTIPLE_HEAPS
26545 // null out the target of short weakref that were not promoted.
26546 GCScan::GcShortWeakPtrScan(background_promote, max_generation, max_generation,&sc);
26548 //concurrent_print_time_delta ("bgc GcShortWeakPtrScan");
26549 concurrent_print_time_delta ("NR GcShortWeakPtrScan");
26553 #ifdef MULTIPLE_HEAPS
26554 bgc_t_join.join(this, gc_join_scan_finalization);
26555 if (bgc_t_join.joined())
26557 dprintf(3, ("Joining BGC threads for finalization"));
26558 bgc_t_join.restart();
26560 #endif //MULTIPLE_HEAPS
26562 //Handle finalization.
26563 dprintf(3,("Marking finalization data"));
26564 //concurrent_print_time_delta ("bgc joined to mark finalization");
26565 concurrent_print_time_delta ("NRj");
26567 // finalize_queue->EnterFinalizeLock();
26568 finalize_queue->ScanForFinalization (background_promote, max_generation, FALSE, __this);
26569 // finalize_queue->LeaveFinalizeLock();
26571 concurrent_print_time_delta ("NRF");
26574 dprintf (2, ("before NR 2nd Hov count: %d", bgc_overflow_count));
26575 bgc_overflow_count = 0;
26577 // Scan dependent handles again to promote any secondaries associated with primaries that were promoted
26578 // for finalization. As before background_scan_dependent_handles will also process any mark stack
26580 dprintf (2, ("2nd dependent handle scan and process mark overflow"));
26581 background_scan_dependent_handles (&sc);
26582 //concurrent_print_time_delta ("2nd nonconcurrent dependent handle scan and process mark overflow");
26583 concurrent_print_time_delta ("NR 2nd Hov");
26585 #ifdef MULTIPLE_HEAPS
26586 bgc_t_join.join(this, gc_join_null_dead_long_weak);
26587 if (bgc_t_join.joined())
26589 dprintf(2, ("Joining BGC threads for weak pointer deletion"));
26590 bgc_t_join.restart();
26592 #endif //MULTIPLE_HEAPS
26594 // null out the target of long weakref that were not promoted.
26595 GCScan::GcWeakPtrScan (background_promote, max_generation, max_generation, &sc);
26596 concurrent_print_time_delta ("NR GcWeakPtrScan");
26598 #ifdef MULTIPLE_HEAPS
26599 bgc_t_join.join(this, gc_join_null_dead_syncblk);
26600 if (bgc_t_join.joined())
26601 #endif //MULTIPLE_HEAPS
26603 dprintf (2, ("calling GcWeakPtrScanBySingleThread"));
26604 // scan for deleted entries in the syncblk cache
26605 GCScan::GcWeakPtrScanBySingleThread (max_generation, max_generation, &sc);
26606 concurrent_print_time_delta ("NR GcWeakPtrScanBySingleThread");
26607 #ifdef MULTIPLE_HEAPS
26608 dprintf(2, ("Starting BGC threads for end of background mark phase"));
26609 bgc_t_join.restart();
26610 #endif //MULTIPLE_HEAPS
26613 gen0_bricks_cleared = FALSE;
26615 dprintf (2, ("end of bgc mark: loh: %d, soh: %d",
26616 generation_size (max_generation + 1),
26617 generation_sizes (generation_of (max_generation))));
26619 for (int gen_idx = max_generation; gen_idx <= (max_generation + 1); gen_idx++)
26621 generation* gen = generation_of (gen_idx);
26622 dynamic_data* dd = dynamic_data_of (gen_idx);
26623 dd_begin_data_size (dd) = generation_size (gen_idx) -
26624 (generation_free_list_space (gen) + generation_free_obj_space (gen)) -
26625 Align (size (generation_allocation_start (gen)));
26626 dd_survived_size (dd) = 0;
26627 dd_pinned_survived_size (dd) = 0;
26628 dd_artificial_pinned_survived_size (dd) = 0;
26629 dd_added_pinned_size (dd) = 0;
26632 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
26633 PREFIX_ASSUME(seg != NULL);
26637 seg->flags &= ~heap_segment_flags_swept;
26639 if (heap_segment_allocated (seg) == heap_segment_mem (seg))
26641 // This can't happen...
26645 if (seg == ephemeral_heap_segment)
26647 heap_segment_background_allocated (seg) = generation_allocation_start (generation_of (max_generation - 1));
26651 heap_segment_background_allocated (seg) = heap_segment_allocated (seg);
26654 dprintf (2, ("seg %Ix background allocated is %Ix",
26655 heap_segment_mem (seg),
26656 heap_segment_background_allocated (seg)));
26657 seg = heap_segment_next_rw (seg);
26660 // We need to void alloc contexts here 'cause while background_ephemeral_sweep is running
26661 // we can't let the user code consume the left over parts in these alloc contexts.
26662 repair_allocation_contexts (FALSE);
26665 finish = GetCycleCount32();
26666 mark_time = finish - start;
26669 dprintf (2, ("end of bgc mark: gen2 free list space: %d, free obj space: %d",
26670 generation_free_list_space (generation_of (max_generation)),
26671 generation_free_obj_space (generation_of (max_generation))));
26673 dprintf(2,("---- (GC%d)End of background mark phase ----", VolatileLoad(&settings.gc_index)));
26677 gc_heap::suspend_EE ()
26679 dprintf (2, ("suspend_EE"));
26680 #ifdef MULTIPLE_HEAPS
26681 gc_heap* hp = gc_heap::g_heaps[0];
26682 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26684 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26685 #endif //MULTIPLE_HEAPS
26688 #ifdef MULTIPLE_HEAPS
26690 gc_heap::bgc_suspend_EE ()
26692 for (int i = 0; i < n_heaps; i++)
26694 gc_heap::g_heaps[i]->reset_gc_done();
26697 dprintf (2, ("bgc_suspend_EE"));
26698 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26700 gc_started = FALSE;
26701 for (int i = 0; i < n_heaps; i++)
26703 gc_heap::g_heaps[i]->set_gc_done();
26708 gc_heap::bgc_suspend_EE ()
26712 dprintf (2, ("bgc_suspend_EE"));
26713 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26714 gc_started = FALSE;
26717 #endif //MULTIPLE_HEAPS
26720 gc_heap::restart_EE ()
26722 dprintf (2, ("restart_EE"));
26723 #ifdef MULTIPLE_HEAPS
26724 GCToEEInterface::RestartEE(FALSE);
26726 GCToEEInterface::RestartEE(FALSE);
26727 #endif //MULTIPLE_HEAPS
26730 inline uint8_t* gc_heap::high_page ( heap_segment* seg, BOOL concurrent_p)
26734 uint8_t* end = ((seg == ephemeral_heap_segment) ?
26735 generation_allocation_start (generation_of (max_generation-1)) :
26736 heap_segment_allocated (seg));
26737 return align_lower_page (end);
26741 return heap_segment_allocated (seg);
26745 void gc_heap::revisit_written_page (uint8_t* page,
26749 uint8_t*& last_page,
26750 uint8_t*& last_object,
26751 BOOL large_objects_p,
26752 size_t& num_marked_objects)
26754 UNREFERENCED_PARAMETER(seg);
26756 uint8_t* start_address = page;
26758 int align_const = get_alignment_constant (!large_objects_p);
26759 uint8_t* high_address = end;
26760 uint8_t* current_lowest_address = background_saved_lowest_address;
26761 uint8_t* current_highest_address = background_saved_highest_address;
26762 BOOL no_more_loop_p = FALSE;
26765 #ifndef MULTIPLE_HEAPS
26766 const int thread = heap_number;
26767 #endif //!MULTIPLE_HEAPS
26769 if (large_objects_p)
26775 if (((last_page + WRITE_WATCH_UNIT_SIZE) == page)
26776 || (start_address <= last_object))
26782 o = find_first_object (start_address, last_object);
26783 // We can visit the same object again, but on a different page.
26784 assert (o >= last_object);
26788 dprintf (3,("page %Ix start: %Ix, %Ix[ ",
26789 (size_t)page, (size_t)o,
26790 (size_t)(min (high_address, page + WRITE_WATCH_UNIT_SIZE))));
26792 while (o < (min (high_address, page + WRITE_WATCH_UNIT_SIZE)))
26796 if (concurrent_p && large_objects_p)
26798 bgc_alloc_lock->bgc_mark_set (o);
26800 if (((CObjectHeader*)o)->IsFree())
26802 s = unused_array_size (o);
26814 dprintf (3,("Considering object %Ix(%s)", (size_t)o, (background_object_marked (o, FALSE) ? "bm" : "nbm")));
26816 assert (Align (s) >= Align (min_obj_size));
26818 uint8_t* next_o = o + Align (s, align_const);
26820 if (next_o >= start_address)
26822 #ifdef MULTIPLE_HEAPS
26825 // We set last_object here for SVR BGC here because SVR BGC has more than
26826 // one GC thread. When we have more than one GC thread we would run into this
26827 // situation if we skipped unmarked objects:
26828 // bgc thread 1 calls GWW, and detect object X not marked so it would skip it
26830 // bgc thread 2 marks X and all its current children.
26831 // user thread comes along and dirties more (and later) pages in X.
26832 // bgc thread 1 calls GWW again and gets those later pages but it will not mark anything
26833 // on them because it had already skipped X. We need to detect that this object is now
26834 // marked and mark the children on the dirtied pages.
26835 // In the future if we have less BGC threads than we have heaps we should add
26836 // the check to the number of BGC threads.
26839 #endif //MULTIPLE_HEAPS
26841 if (contain_pointers (o) &&
26842 (!((o >= current_lowest_address) && (o < current_highest_address)) ||
26843 background_marked (o)))
26845 dprintf (3, ("going through %Ix", (size_t)o));
26846 go_through_object (method_table(o), o, s, poo, start_address, use_start, (o + s),
26847 if ((uint8_t*)poo >= min (high_address, page + WRITE_WATCH_UNIT_SIZE))
26849 no_more_loop_p = TRUE;
26852 uint8_t* oo = *poo;
26854 num_marked_objects++;
26855 background_mark_object (oo THREAD_NUMBER_ARG);
26860 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP // see comment below
26862 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26863 ((CObjectHeader*)o)->IsFree() &&
26864 (next_o > min (high_address, page + WRITE_WATCH_UNIT_SIZE)))
26866 // We need to not skip the object here because of this corner scenario:
26867 // A large object was being allocated during BGC mark so we first made it
26868 // into a free object, then cleared its memory. In this loop we would detect
26869 // that it's a free object which normally we would skip. But by the next time
26870 // we call GetWriteWatch we could still be on this object and the object had
26871 // been made into a valid object and some of its memory was changed. We need
26872 // to be sure to process those written pages so we can't skip the object just
26875 // Similarly, when using software write watch, don't advance last_object when
26876 // the current object is a free object that spans beyond the current page or
26877 // high_address. Software write watch acquires gc_lock before the concurrent
26878 // GetWriteWatch() call during revisit_written_pages(). A foreground GC may
26879 // happen at that point and allocate from this free region, so when
26880 // revisit_written_pages() continues, it cannot skip now-valid objects in this
26882 no_more_loop_p = TRUE;
26887 if (concurrent_p && large_objects_p)
26889 bgc_alloc_lock->bgc_mark_done ();
26891 if (no_more_loop_p)
26898 #ifdef MULTIPLE_HEAPS
26901 assert (last_object < (min (high_address, page + WRITE_WATCH_UNIT_SIZE)));
26904 #endif //MULTIPLE_HEAPS
26909 dprintf (3,("Last object: %Ix", (size_t)last_object));
26910 last_page = align_write_watch_lower_page (o);
26913 // When reset_only_p is TRUE, we should only reset pages that are in range
26914 // because we need to consider the segments or part of segments that were
26915 // allocated out of range all live.
26916 void gc_heap::revisit_written_pages (BOOL concurrent_p, BOOL reset_only_p)
26919 if (concurrent_p && !reset_only_p)
26921 current_bgc_state = bgc_revisit_soh;
26924 size_t total_dirtied_pages = 0;
26925 size_t total_marked_objects = 0;
26927 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
26929 PREFIX_ASSUME(seg != NULL);
26931 bool reset_watch_state = !!concurrent_p;
26932 bool is_runtime_suspended = !concurrent_p;
26933 BOOL small_object_segments = TRUE;
26934 int align_const = get_alignment_constant (small_object_segments);
26940 if (small_object_segments)
26942 //switch to large segment
26943 if (concurrent_p && !reset_only_p)
26945 current_bgc_state = bgc_revisit_loh;
26950 dprintf (GTC_LOG, ("h%d: SOH: dp:%Id; mo: %Id", heap_number, total_dirtied_pages, total_marked_objects));
26951 fire_revisit_event (total_dirtied_pages, total_marked_objects, !small_object_segments);
26952 concurrent_print_time_delta (concurrent_p ? "CR SOH" : "NR SOH");
26953 total_dirtied_pages = 0;
26954 total_marked_objects = 0;
26957 small_object_segments = FALSE;
26958 //concurrent_print_time_delta (concurrent_p ? "concurrent marking dirtied pages on SOH" : "nonconcurrent marking dirtied pages on SOH");
26960 dprintf (3, ("now revisiting large object segments"));
26961 align_const = get_alignment_constant (small_object_segments);
26962 seg = heap_segment_rw (generation_start_segment (large_object_generation));
26964 PREFIX_ASSUME(seg != NULL);
26972 dprintf (GTC_LOG, ("h%d: tdp: %Id", heap_number, total_dirtied_pages));
26976 dprintf (GTC_LOG, ("h%d: LOH: dp:%Id; mo: %Id", heap_number, total_dirtied_pages, total_marked_objects));
26977 fire_revisit_event (total_dirtied_pages, total_marked_objects, !small_object_segments);
26982 uint8_t* base_address = (uint8_t*)heap_segment_mem (seg);
26983 //we need to truncate to the base of the page because
26984 //some newly allocated could exist beyond heap_segment_allocated
26985 //and if we reset the last page write watch status,
26986 // they wouldn't be guaranteed to be visited -> gc hole.
26987 uintptr_t bcount = array_size;
26988 uint8_t* last_page = 0;
26989 uint8_t* last_object = heap_segment_mem (seg);
26990 uint8_t* high_address = 0;
26992 BOOL skip_seg_p = FALSE;
26996 if ((heap_segment_mem (seg) >= background_saved_lowest_address) ||
26997 (heap_segment_reserved (seg) <= background_saved_highest_address))
26999 dprintf (3, ("h%d: sseg: %Ix(-%Ix)", heap_number,
27000 heap_segment_mem (seg), heap_segment_reserved (seg)));
27007 dprintf (3, ("looking at seg %Ix", (size_t)last_object));
27011 base_address = max (base_address, background_saved_lowest_address);
27012 dprintf (3, ("h%d: reset only starting %Ix", heap_number, base_address));
27015 dprintf (3, ("h%d: starting: %Ix, seg %Ix-%Ix", heap_number, base_address,
27016 heap_segment_mem (seg), heap_segment_reserved (seg)));
27023 high_address = ((seg == ephemeral_heap_segment) ? alloc_allocated : heap_segment_allocated (seg));
27024 high_address = min (high_address, background_saved_highest_address);
27028 high_address = high_page (seg, concurrent_p);
27031 if ((base_address < high_address) &&
27032 (bcount >= array_size))
27034 ptrdiff_t region_size = high_address - base_address;
27035 dprintf (3, ("h%d: gw: [%Ix(%Id)", heap_number, (size_t)base_address, (size_t)region_size));
27037 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
27038 // When the runtime is not suspended, it's possible for the table to be resized concurrently with the scan
27039 // for dirty pages below. Prevent that by synchronizing with grow_brick_card_tables(). When the runtime is
27040 // suspended, it's ok to scan for dirty pages concurrently from multiple background GC threads for disjoint
27042 if (!is_runtime_suspended)
27044 enter_spin_lock(&gc_lock);
27046 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
27048 get_write_watch_for_gc_heap (reset_watch_state, base_address, region_size,
27049 (void**)background_written_addresses,
27050 &bcount, is_runtime_suspended);
27052 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
27053 if (!is_runtime_suspended)
27055 leave_spin_lock(&gc_lock);
27057 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
27061 total_dirtied_pages += bcount;
27063 dprintf (3, ("Found %d pages [%Ix, %Ix[",
27064 bcount, (size_t)base_address, (size_t)high_address));
27069 for (unsigned i = 0; i < bcount; i++)
27071 uint8_t* page = (uint8_t*)background_written_addresses[i];
27072 dprintf (3, ("looking at page %d at %Ix(h: %Ix)", i,
27073 (size_t)page, (size_t)high_address));
27074 if (page < high_address)
27076 //search for marked objects in the page
27077 revisit_written_page (page, high_address, concurrent_p,
27078 seg, last_page, last_object,
27079 !small_object_segments,
27080 total_marked_objects);
27084 dprintf (3, ("page %d at %Ix is >= %Ix!", i, (size_t)page, (size_t)high_address));
27085 assert (!"page shouldn't have exceeded limit");
27090 if (bcount >= array_size){
27091 base_address = background_written_addresses [array_size-1] + WRITE_WATCH_UNIT_SIZE;
27092 bcount = array_size;
27102 seg = heap_segment_next_rw (seg);
27105 #endif //WRITE_WATCH
27108 void gc_heap::background_grow_c_mark_list()
27110 assert (c_mark_list_index >= c_mark_list_length);
27111 BOOL should_drain_p = FALSE;
27113 #ifndef MULTIPLE_HEAPS
27114 const int thread = heap_number;
27115 #endif //!MULTIPLE_HEAPS
27117 dprintf (2, ("stack copy buffer overflow"));
27118 uint8_t** new_c_mark_list = 0;
27121 if (c_mark_list_length >= (SIZE_T_MAX / (2 * sizeof (uint8_t*))))
27123 should_drain_p = TRUE;
27127 new_c_mark_list = new (nothrow) uint8_t*[c_mark_list_length*2];
27128 if (new_c_mark_list == 0)
27130 should_drain_p = TRUE;
27134 if (should_drain_p)
27137 dprintf (2, ("No more memory for the stacks copy, draining.."));
27138 //drain the list by marking its elements
27139 background_drain_mark_list (thread);
27143 assert (new_c_mark_list);
27144 memcpy (new_c_mark_list, c_mark_list, c_mark_list_length*sizeof(uint8_t*));
27145 c_mark_list_length = c_mark_list_length*2;
27146 delete c_mark_list;
27147 c_mark_list = new_c_mark_list;
27151 void gc_heap::background_promote_callback (Object** ppObject, ScanContext* sc,
27154 UNREFERENCED_PARAMETER(sc);
27155 //in order to save space on the array, mark the object,
27156 //knowing that it will be visited later
27157 assert (settings.concurrent);
27159 THREAD_NUMBER_FROM_CONTEXT;
27160 #ifndef MULTIPLE_HEAPS
27161 const int thread = 0;
27162 #endif //!MULTIPLE_HEAPS
27164 uint8_t* o = (uint8_t*)*ppObject;
27171 gc_heap* hp = gc_heap::heap_of (o);
27173 if ((o < hp->background_saved_lowest_address) || (o >= hp->background_saved_highest_address))
27178 #ifdef INTERIOR_POINTERS
27179 if (flags & GC_CALL_INTERIOR)
27181 o = hp->find_object (o, hp->background_saved_lowest_address);
27185 #endif //INTERIOR_POINTERS
27187 #ifdef FEATURE_CONSERVATIVE_GC
27188 // For conservative GC, a value on stack may point to middle of a free object.
27189 // In this case, we don't need to promote the pointer.
27190 if (GCConfig::GetConservativeGC() && ((CObjectHeader*)o)->IsFree())
27194 #endif //FEATURE_CONSERVATIVE_GC
27197 ((CObjectHeader*)o)->Validate();
27200 dprintf (3, ("Concurrent Background Promote %Ix", (size_t)o));
27201 if (o && (size (o) > loh_size_threshold))
27203 dprintf (3, ("Brc %Ix", (size_t)o));
27206 if (hpt->c_mark_list_index >= hpt->c_mark_list_length)
27208 hpt->background_grow_c_mark_list();
27210 dprintf (3, ("pushing %08x into mark_list", (size_t)o));
27211 hpt->c_mark_list [hpt->c_mark_list_index++] = o;
27213 STRESS_LOG3(LF_GC|LF_GCROOTS, LL_INFO1000000, " GCHeap::Background Promote: Promote GC Root *%p = %p MT = %pT", ppObject, o, o ? ((Object*) o)->GetGCSafeMethodTable() : NULL);
27216 void gc_heap::mark_absorb_new_alloc()
27218 fix_allocation_contexts (FALSE);
27220 gen0_bricks_cleared = FALSE;
27222 clear_gen0_bricks();
27225 BOOL gc_heap::prepare_bgc_thread(gc_heap* gh)
27227 BOOL success = FALSE;
27228 BOOL thread_created = FALSE;
27229 dprintf (2, ("Preparing gc thread"));
27230 gh->bgc_threads_timeout_cs.Enter();
27231 if (!(gh->bgc_thread_running))
27233 dprintf (2, ("GC thread not runnning"));
27234 if ((gh->bgc_thread == 0) && create_bgc_thread(gh))
27237 thread_created = TRUE;
27242 dprintf (3, ("GC thread already running"));
27245 gh->bgc_threads_timeout_cs.Leave();
27248 FIRE_EVENT(GCCreateConcurrentThread_V1);
27253 BOOL gc_heap::create_bgc_thread(gc_heap* gh)
27255 assert (background_gc_done_event.IsValid());
27257 //dprintf (2, ("Creating BGC thread"));
27259 gh->bgc_thread_running = GCToEEInterface::CreateThread(gh->bgc_thread_stub, gh, true, ".NET Background GC");
27260 return gh->bgc_thread_running;
27263 BOOL gc_heap::create_bgc_threads_support (int number_of_heaps)
27266 dprintf (3, ("Creating concurrent GC thread for the first time"));
27267 if (!background_gc_done_event.CreateManualEventNoThrow(TRUE))
27271 if (!bgc_threads_sync_event.CreateManualEventNoThrow(FALSE))
27275 if (!ee_proceed_event.CreateAutoEventNoThrow(FALSE))
27279 if (!bgc_start_event.CreateManualEventNoThrow(FALSE))
27284 #ifdef MULTIPLE_HEAPS
27285 bgc_t_join.init (number_of_heaps, join_flavor_bgc);
27287 UNREFERENCED_PARAMETER(number_of_heaps);
27288 #endif //MULTIPLE_HEAPS
27296 if (background_gc_done_event.IsValid())
27298 background_gc_done_event.CloseEvent();
27300 if (bgc_threads_sync_event.IsValid())
27302 bgc_threads_sync_event.CloseEvent();
27304 if (ee_proceed_event.IsValid())
27306 ee_proceed_event.CloseEvent();
27308 if (bgc_start_event.IsValid())
27310 bgc_start_event.CloseEvent();
27317 BOOL gc_heap::create_bgc_thread_support()
27322 if (!gc_lh_block_event.CreateManualEventNoThrow(FALSE))
27327 //needs to have room for enough smallest objects fitting on a page
27328 parr = new (nothrow) uint8_t*[1 + OS_PAGE_SIZE / MIN_OBJECT_SIZE];
27334 make_c_mark_list (parr);
27342 if (gc_lh_block_event.IsValid())
27344 gc_lh_block_event.CloseEvent();
27351 int gc_heap::check_for_ephemeral_alloc()
27353 int gen = ((settings.reason == reason_oos_soh) ? (max_generation - 1) : -1);
27357 #ifdef MULTIPLE_HEAPS
27358 for (int heap_index = 0; heap_index < n_heaps; heap_index++)
27359 #endif //MULTIPLE_HEAPS
27361 for (int i = 0; i <= (max_generation - 1); i++)
27363 #ifdef MULTIPLE_HEAPS
27364 if (g_heaps[heap_index]->get_new_allocation (i) <= 0)
27366 if (get_new_allocation (i) <= 0)
27367 #endif //MULTIPLE_HEAPS
27369 gen = max (gen, i);
27380 // Wait for gc to finish sequential part
27381 void gc_heap::wait_to_proceed()
27383 assert (background_gc_done_event.IsValid());
27384 assert (bgc_start_event.IsValid());
27386 user_thread_wait(&ee_proceed_event, FALSE);
27389 // Start a new concurrent gc
27390 void gc_heap::start_c_gc()
27392 assert (background_gc_done_event.IsValid());
27393 assert (bgc_start_event.IsValid());
27395 //Need to make sure that the gc thread is in the right place.
27396 background_gc_done_event.Wait(INFINITE, FALSE);
27397 background_gc_done_event.Reset();
27398 bgc_start_event.Set();
27401 void gc_heap::do_background_gc()
27403 dprintf (2, ("starting a BGC"));
27404 #ifdef MULTIPLE_HEAPS
27405 for (int i = 0; i < n_heaps; i++)
27407 g_heaps[i]->init_background_gc();
27410 init_background_gc();
27411 #endif //MULTIPLE_HEAPS
27412 //start the background gc
27415 //wait until we get restarted by the BGC.
27419 void gc_heap::kill_gc_thread()
27421 //assert (settings.concurrent == FALSE);
27423 // We are doing a two-stage shutdown now.
27424 // In the first stage, we do minimum work, and call ExitProcess at the end.
27425 // In the secodn stage, we have the Loader lock and only one thread is
27426 // alive. Hence we do not need to kill gc thread.
27427 background_gc_done_event.CloseEvent();
27428 gc_lh_block_event.CloseEvent();
27429 bgc_start_event.CloseEvent();
27430 bgc_threads_timeout_cs.Destroy();
27432 recursive_gc_sync::shutdown();
27435 void gc_heap::bgc_thread_function()
27437 assert (background_gc_done_event.IsValid());
27438 assert (bgc_start_event.IsValid());
27440 dprintf (3, ("gc_thread thread starting..."));
27442 BOOL do_exit = FALSE;
27444 bool cooperative_mode = true;
27445 bgc_thread_id.SetToCurrentThread();
27446 dprintf (1, ("bgc_thread_id is set to %x", (uint32_t)GCToOSInterface::GetCurrentThreadIdForLogging()));
27449 // Wait for work to do...
27450 dprintf (3, ("bgc thread: waiting..."));
27452 cooperative_mode = enable_preemptive ();
27453 //current_thread->m_fPreemptiveGCDisabled = 0;
27455 uint32_t result = bgc_start_event.Wait(
27457 #ifdef MULTIPLE_HEAPS
27461 #endif //MULTIPLE_HEAPS
27463 #ifdef MULTIPLE_HEAPS
27467 #endif //MULTIPLE_HEAPS
27470 dprintf (2, ("gc thread: finished waiting"));
27472 // not calling disable_preemptive here 'cause we
27473 // can't wait for GC complete here - RestartEE will be called
27474 // when we've done the init work.
27476 if (result == WAIT_TIMEOUT)
27478 // Should join the bgc threads and terminate all of them
27480 dprintf (1, ("GC thread timeout"));
27481 bgc_threads_timeout_cs.Enter();
27482 if (!keep_bgc_threads_p)
27484 dprintf (2, ("GC thread exiting"));
27485 bgc_thread_running = FALSE;
27487 bgc_thread_id.Clear();
27490 bgc_threads_timeout_cs.Leave();
27495 dprintf (3, ("GC thread needed, not exiting"));
27499 // if we signal the thread with no concurrent work to do -> exit
27500 if (!settings.concurrent)
27502 dprintf (3, ("no concurrent GC needed, exiting"));
27508 recursive_gc_sync::begin_background();
27509 dprintf (2, ("beginning of bgc: gen2 FL: %d, FO: %d, frag: %d",
27510 generation_free_list_space (generation_of (max_generation)),
27511 generation_free_obj_space (generation_of (max_generation)),
27512 dd_fragmentation (dynamic_data_of (max_generation))));
27516 current_bgc_state = bgc_not_in_process;
27519 //trace_gc = FALSE;
27522 enable_preemptive ();
27523 #ifdef MULTIPLE_HEAPS
27524 bgc_t_join.join(this, gc_join_done);
27525 if (bgc_t_join.joined())
27526 #endif //MULTIPLE_HEAPS
27528 enter_spin_lock (&gc_lock);
27529 dprintf (SPINLOCK_LOG, ("bgc Egc"));
27531 bgc_start_event.Reset();
27533 #ifdef MULTIPLE_HEAPS
27534 for (int gen = max_generation; gen <= (max_generation + 1); gen++)
27536 size_t desired_per_heap = 0;
27537 size_t total_desired = 0;
27540 for (int i = 0; i < n_heaps; i++)
27543 dd = hp->dynamic_data_of (gen);
27544 size_t temp_total_desired = total_desired + dd_desired_allocation (dd);
27545 if (temp_total_desired < total_desired)
27548 total_desired = (size_t)MAX_PTR;
27551 total_desired = temp_total_desired;
27554 desired_per_heap = Align ((total_desired/n_heaps), get_alignment_constant (FALSE));
27556 for (int i = 0; i < n_heaps; i++)
27558 hp = gc_heap::g_heaps[i];
27559 dd = hp->dynamic_data_of (gen);
27560 dd_desired_allocation (dd) = desired_per_heap;
27561 dd_gc_new_allocation (dd) = desired_per_heap;
27562 dd_new_allocation (dd) = desired_per_heap;
27565 #endif //MULTIPLE_HEAPS
27566 #ifdef MULTIPLE_HEAPS
27568 #endif //MULTIPLE_HEAPS
27570 c_write (settings.concurrent, FALSE);
27571 recursive_gc_sync::end_background();
27572 keep_bgc_threads_p = FALSE;
27573 background_gc_done_event.Set();
27575 dprintf (SPINLOCK_LOG, ("bgc Lgc"));
27576 leave_spin_lock (&gc_lock);
27577 #ifdef MULTIPLE_HEAPS
27578 dprintf(1, ("End of BGC - starting all BGC threads"));
27579 bgc_t_join.restart();
27580 #endif //MULTIPLE_HEAPS
27582 // We can't disable preempt here because there might've been a GC already
27583 // started and decided to do a BGC and waiting for a BGC thread to restart
27584 // vm. That GC will be waiting in wait_to_proceed and we are waiting for it
27585 // to restart the VM so we deadlock.
27586 //gc_heap::disable_preemptive (true);
27589 FIRE_EVENT(GCTerminateConcurrentThread_V1);
27591 dprintf (3, ("bgc_thread thread exiting"));
27595 #endif //BACKGROUND_GC
27597 //Clear the cards [start_card, end_card[
27598 void gc_heap::clear_cards (size_t start_card, size_t end_card)
27600 if (start_card < end_card)
27602 size_t start_word = card_word (start_card);
27603 size_t end_word = card_word (end_card);
27604 if (start_word < end_word)
27606 // Figure out the bit positions of the cards within their words
27607 unsigned bits = card_bit (start_card);
27608 card_table [start_word] &= lowbits (~0, bits);
27609 for (size_t i = start_word+1; i < end_word; i++)
27610 card_table [i] = 0;
27611 bits = card_bit (end_card);
27612 // Don't write beyond end_card (and possibly uncommitted card table space).
27615 card_table [end_word] &= highbits (~0, bits);
27620 // If the start and end cards are in the same word, just clear the appropriate card
27621 // bits in that word.
27622 card_table [start_word] &= (lowbits (~0, card_bit (start_card)) |
27623 highbits (~0, card_bit (end_card)));
27625 #ifdef VERYSLOWDEBUG
27626 size_t card = start_card;
27627 while (card < end_card)
27629 assert (! (card_set_p (card)));
27632 #endif //VERYSLOWDEBUG
27633 dprintf (3,("Cleared cards [%Ix:%Ix, %Ix:%Ix[",
27634 start_card, (size_t)card_address (start_card),
27635 end_card, (size_t)card_address (end_card)));
27639 void gc_heap::clear_card_for_addresses (uint8_t* start_address, uint8_t* end_address)
27641 size_t start_card = card_of (align_on_card (start_address));
27642 size_t end_card = card_of (align_lower_card (end_address));
27643 clear_cards (start_card, end_card);
27646 // copy [srccard, ...[ to [dst_card, end_card[
27647 // This will set the same bit twice. Can be optimized.
27649 void gc_heap::copy_cards (size_t dst_card,
27654 // If the range is empty, this function is a no-op - with the subtlety that
27655 // either of the accesses card_table[srcwrd] or card_table[dstwrd] could be
27656 // outside the committed region. To avoid the access, leave early.
27657 if (!(dst_card < end_card))
27660 unsigned int srcbit = card_bit (src_card);
27661 unsigned int dstbit = card_bit (dst_card);
27662 size_t srcwrd = card_word (src_card);
27663 size_t dstwrd = card_word (dst_card);
27664 unsigned int srctmp = card_table[srcwrd];
27665 unsigned int dsttmp = card_table[dstwrd];
27667 for (size_t card = dst_card; card < end_card; card++)
27669 if (srctmp & (1 << srcbit))
27670 dsttmp |= 1 << dstbit;
27672 dsttmp &= ~(1 << dstbit);
27673 if (!(++srcbit % 32))
27675 srctmp = card_table[++srcwrd];
27681 if (srctmp & (1 << srcbit))
27682 dsttmp |= 1 << dstbit;
27685 if (!(++dstbit % 32))
27687 card_table[dstwrd] = dsttmp;
27689 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27692 card_bundle_set(cardw_card_bundle(dstwrd));
27697 dsttmp = card_table[dstwrd];
27702 card_table[dstwrd] = dsttmp;
27704 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27707 card_bundle_set(cardw_card_bundle(dstwrd));
27712 void gc_heap::copy_cards_for_addresses (uint8_t* dest, uint8_t* src, size_t len)
27714 ptrdiff_t relocation_distance = src - dest;
27715 size_t start_dest_card = card_of (align_on_card (dest));
27716 size_t end_dest_card = card_of (dest + len - 1);
27717 size_t dest_card = start_dest_card;
27718 size_t src_card = card_of (card_address (dest_card)+relocation_distance);
27719 dprintf (3,("Copying cards [%Ix:%Ix->%Ix:%Ix, ",
27720 src_card, (size_t)src, dest_card, (size_t)dest));
27721 dprintf (3,(" %Ix->%Ix:%Ix[",
27722 (size_t)src+len, end_dest_card, (size_t)dest+len));
27724 dprintf (3, ("dest: %Ix, src: %Ix, len: %Ix, reloc: %Ix, align_on_card(dest) is %Ix",
27725 dest, src, len, relocation_distance, (align_on_card (dest))));
27727 dprintf (3, ("start_dest_card: %Ix (address: %Ix), end_dest_card: %Ix(addr: %Ix), card_of (dest): %Ix",
27728 start_dest_card, card_address (start_dest_card), end_dest_card, card_address (end_dest_card), card_of (dest)));
27730 //First card has two boundaries
27731 if (start_dest_card != card_of (dest))
27733 if ((card_of (card_address (start_dest_card) + relocation_distance) <= card_of (src + len - 1))&&
27734 card_set_p (card_of (card_address (start_dest_card) + relocation_distance)))
27736 dprintf (3, ("card_address (start_dest_card) + reloc is %Ix, card: %Ix(set), src+len-1: %Ix, card: %Ix",
27737 (card_address (start_dest_card) + relocation_distance),
27738 card_of (card_address (start_dest_card) + relocation_distance),
27740 card_of (src + len - 1)));
27742 dprintf (3, ("setting card: %Ix", card_of (dest)));
27743 set_card (card_of (dest));
27747 if (card_set_p (card_of (src)))
27748 set_card (card_of (dest));
27751 copy_cards (dest_card, src_card, end_dest_card,
27752 ((dest - align_lower_card (dest)) != (src - align_lower_card (src))));
27754 //Last card has two boundaries.
27755 if ((card_of (card_address (end_dest_card) + relocation_distance) >= card_of (src)) &&
27756 card_set_p (card_of (card_address (end_dest_card) + relocation_distance)))
27758 dprintf (3, ("card_address (end_dest_card) + reloc is %Ix, card: %Ix(set), src: %Ix, card: %Ix",
27759 (card_address (end_dest_card) + relocation_distance),
27760 card_of (card_address (end_dest_card) + relocation_distance),
27764 dprintf (3, ("setting card: %Ix", end_dest_card));
27765 set_card (end_dest_card);
27768 if (card_set_p (card_of (src + len - 1)))
27769 set_card (end_dest_card);
27771 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27772 card_bundles_set(cardw_card_bundle(card_word(card_of(dest))), cardw_card_bundle(align_cardw_on_bundle(card_word(end_dest_card))));
27776 #ifdef BACKGROUND_GC
27777 // this does not need the Interlocked version of mark_array_set_marked.
27778 void gc_heap::copy_mark_bits_for_addresses (uint8_t* dest, uint8_t* src, size_t len)
27780 dprintf (3, ("Copying mark_bits for addresses [%Ix->%Ix, %Ix->%Ix[",
27781 (size_t)src, (size_t)dest,
27782 (size_t)src+len, (size_t)dest+len));
27784 uint8_t* src_o = src;
27786 uint8_t* src_end = src + len;
27787 int align_const = get_alignment_constant (TRUE);
27788 ptrdiff_t reloc = dest - src;
27790 while (src_o < src_end)
27792 uint8_t* next_o = src_o + Align (size (src_o), align_const);
27794 if (background_object_marked (src_o, TRUE))
27796 dest_o = src_o + reloc;
27798 //if (background_object_marked (dest_o, FALSE))
27800 // dprintf (3, ("*%Ix shouldn't have already been marked!", (size_t)(dest_o)));
27801 // FATAL_GC_ERROR();
27804 background_mark (dest_o,
27805 background_saved_lowest_address,
27806 background_saved_highest_address);
27807 dprintf (3, ("bc*%Ix*bc, b*%Ix*b", (size_t)src_o, (size_t)(dest_o)));
27813 #endif //BACKGROUND_GC
27815 void gc_heap::fix_brick_to_highest (uint8_t* o, uint8_t* next_o)
27817 size_t new_current_brick = brick_of (o);
27818 set_brick (new_current_brick,
27819 (o - brick_address (new_current_brick)));
27820 size_t b = 1 + new_current_brick;
27821 size_t limit = brick_of (next_o);
27822 //dprintf(3,(" fixing brick %Ix to point to object %Ix, till %Ix(%Ix)",
27823 dprintf(3,("b:%Ix->%Ix-%Ix",
27824 new_current_brick, (size_t)o, (size_t)next_o));
27827 set_brick (b,(new_current_brick - b));
27832 // start can not be >= heap_segment_allocated for the segment.
27833 uint8_t* gc_heap::find_first_object (uint8_t* start, uint8_t* first_object)
27835 size_t brick = brick_of (start);
27837 //last_object == null -> no search shortcut needed
27838 if ((brick == brick_of (first_object) || (start <= first_object)))
27844 ptrdiff_t min_brick = (ptrdiff_t)brick_of (first_object);
27845 ptrdiff_t prev_brick = (ptrdiff_t)brick - 1;
27846 int brick_entry = 0;
27849 if (prev_brick < min_brick)
27853 if ((brick_entry = get_brick_entry(prev_brick)) >= 0)
27857 assert (! ((brick_entry == 0)));
27858 prev_brick = (brick_entry + prev_brick);
27861 o = ((prev_brick < min_brick) ? first_object :
27862 brick_address (prev_brick) + brick_entry - 1);
27863 assert (o <= start);
27866 assert (Align (size (o)) >= Align (min_obj_size));
27867 uint8_t* next_o = o + Align (size (o));
27868 size_t curr_cl = (size_t)next_o / brick_size;
27869 size_t min_cl = (size_t)first_object / brick_size;
27871 //dprintf (3,( "Looking for intersection with %Ix from %Ix", (size_t)start, (size_t)o));
27873 unsigned int n_o = 1;
27876 uint8_t* next_b = min (align_lower_brick (next_o) + brick_size, start+1);
27878 while (next_o <= start)
27886 assert (Align (size (o)) >= Align (min_obj_size));
27887 next_o = o + Align (size (o));
27889 }while (next_o < next_b);
27891 if (((size_t)next_o / brick_size) != curr_cl)
27893 if (curr_cl >= min_cl)
27895 fix_brick_to_highest (o, next_o);
27897 curr_cl = (size_t) next_o / brick_size;
27899 next_b = min (align_lower_brick (next_o) + brick_size, start+1);
27902 size_t bo = brick_of (o);
27903 //dprintf (3, ("Looked at %Id objects, fixing brick [%Ix-[%Ix",
27904 dprintf (3, ("%Id o, [%Ix-[%Ix",
27908 set_brick (bo, (o - brick_address(bo)));
27923 // Find the first non-zero card word between cardw and cardw_end.
27924 // The index of the word we find is returned in cardw.
27925 BOOL gc_heap::find_card_dword (size_t& cardw, size_t cardw_end)
27927 dprintf (3, ("gc: %d, find_card_dword cardw: %Ix, cardw_end: %Ix",
27928 dd_collection_count (dynamic_data_of (0)), cardw, cardw_end));
27930 if (card_bundles_enabled())
27932 size_t cardb = cardw_card_bundle (cardw);
27933 size_t end_cardb = cardw_card_bundle (align_cardw_on_bundle (cardw_end));
27936 // Find a non-zero bundle
27937 while ((cardb < end_cardb) && (card_bundle_set_p (cardb) == 0))
27941 if (cardb == end_cardb)
27944 uint32_t* card_word = &card_table[max(card_bundle_cardw (cardb),cardw)];
27945 uint32_t* card_word_end = &card_table[min(card_bundle_cardw (cardb+1),cardw_end)];
27946 while ((card_word < card_word_end) && !(*card_word))
27951 if (card_word != card_word_end)
27953 cardw = (card_word - &card_table[0]);
27956 else if ((cardw <= card_bundle_cardw (cardb)) &&
27957 (card_word == &card_table [card_bundle_cardw (cardb+1)]))
27959 // a whole bundle was explored and is empty
27960 dprintf (3, ("gc: %d, find_card_dword clear bundle: %Ix cardw:[%Ix,%Ix[",
27961 dd_collection_count (dynamic_data_of (0)),
27962 cardb, card_bundle_cardw (cardb),
27963 card_bundle_cardw (cardb+1)));
27964 card_bundle_clear (cardb);
27972 uint32_t* card_word = &card_table[cardw];
27973 uint32_t* card_word_end = &card_table [cardw_end];
27975 while (card_word < card_word_end)
27977 if ((*card_word) != 0)
27979 cardw = (card_word - &card_table [0]);
27991 #endif //CARD_BUNDLE
27993 // Find cards that are set between two points in a card table.
27995 // card_table : The card table.
27996 // card : [in/out] As input, the card to start searching from.
27997 // As output, the first card that's set.
27998 // card_word_end : The card word at which to stop looking.
27999 // end_card : [out] The last card which is set.
28000 BOOL gc_heap::find_card(uint32_t* card_table,
28002 size_t card_word_end,
28005 uint32_t* last_card_word;
28006 uint32_t card_word_value;
28007 uint32_t bit_position;
28009 // Find the first card which is set
28010 last_card_word = &card_table [card_word (card)];
28011 bit_position = card_bit (card);
28012 card_word_value = (*last_card_word) >> bit_position;
28013 if (!card_word_value)
28017 // Using the card bundle, go through the remaining card words between here and
28018 // card_word_end until we find one that is non-zero.
28019 size_t lcw = card_word(card) + 1;
28020 if (gc_heap::find_card_dword (lcw, card_word_end) == FALSE)
28026 last_card_word = &card_table [lcw];
28027 card_word_value = *last_card_word;
28030 #else //CARD_BUNDLE
28031 // Go through the remaining card words between here and card_word_end until we find
28032 // one that is non-zero.
28038 while ((last_card_word < &card_table [card_word_end]) && !(*last_card_word));
28039 if (last_card_word < &card_table [card_word_end])
28041 card_word_value = *last_card_word;
28045 // We failed to find any non-zero card words before we got to card_word_end
28048 #endif //CARD_BUNDLE
28052 // Look for the lowest bit set
28053 if (card_word_value)
28055 while (!(card_word_value & 1))
28058 card_word_value = card_word_value / 2;
28062 // card is the card word index * card size + the bit index within the card
28063 card = (last_card_word - &card_table[0]) * card_word_width + bit_position;
28067 // Keep going until we get to an un-set card.
28069 card_word_value = card_word_value / 2;
28071 // If we reach the end of the card word and haven't hit a 0 yet, start going
28072 // card word by card word until we get to one that's not fully set (0xFFFF...)
28073 // or we reach card_word_end.
28074 if ((bit_position == card_word_width) && (last_card_word < &card_table [card_word_end]))
28078 card_word_value = *(++last_card_word);
28079 } while ((last_card_word < &card_table [card_word_end]) &&
28082 (card_word_value == (1 << card_word_width)-1)
28084 // if left shift count >= width of type,
28085 // gcc reports error.
28086 (card_word_value == ~0u)
28091 } while (card_word_value & 1);
28093 end_card = (last_card_word - &card_table [0])* card_word_width + bit_position;
28095 //dprintf (3, ("find_card: [%Ix, %Ix[ set", card, end_card));
28096 dprintf (3, ("fc: [%Ix, %Ix[", card, end_card));
28101 //because of heap expansion, computing end is complicated.
28102 uint8_t* compute_next_end (heap_segment* seg, uint8_t* low)
28104 if ((low >= heap_segment_mem (seg)) &&
28105 (low < heap_segment_allocated (seg)))
28108 return heap_segment_allocated (seg);
28112 gc_heap::compute_next_boundary (uint8_t* low, int gen_number,
28115 UNREFERENCED_PARAMETER(low);
28117 //when relocating, the fault line is the plan start of the younger
28118 //generation because the generation is promoted.
28119 if (relocating && (gen_number == (settings.condemned_generation + 1)))
28121 generation* gen = generation_of (gen_number - 1);
28122 uint8_t* gen_alloc = generation_plan_allocation_start (gen);
28123 assert (gen_alloc);
28128 assert (gen_number > settings.condemned_generation);
28129 return generation_allocation_start (generation_of (gen_number - 1 ));
28135 gc_heap::keep_card_live (uint8_t* o, size_t& n_gen,
28136 size_t& cg_pointers_found)
28139 if ((gc_low <= o) && (gc_high > o))
28143 #ifdef MULTIPLE_HEAPS
28146 gc_heap* hp = heap_of (o);
28149 if ((hp->gc_low <= o) &&
28156 #endif //MULTIPLE_HEAPS
28157 cg_pointers_found ++;
28158 dprintf (4, ("keep card live for %Ix", o));
28162 gc_heap::mark_through_cards_helper (uint8_t** poo, size_t& n_gen,
28163 size_t& cg_pointers_found,
28164 card_fn fn, uint8_t* nhigh,
28165 uint8_t* next_boundary)
28168 if ((gc_low <= *poo) && (gc_high > *poo))
28171 call_fn(fn) (poo THREAD_NUMBER_ARG);
28173 #ifdef MULTIPLE_HEAPS
28176 gc_heap* hp = heap_of_gc (*poo);
28179 if ((hp->gc_low <= *poo) &&
28180 (hp->gc_high > *poo))
28183 call_fn(fn) (poo THREAD_NUMBER_ARG);
28185 if ((fn == &gc_heap::relocate_address) ||
28186 ((hp->ephemeral_low <= *poo) &&
28187 (hp->ephemeral_high > *poo)))
28189 cg_pointers_found++;
28193 #endif //MULTIPLE_HEAPS
28194 if ((next_boundary <= *poo) && (nhigh > *poo))
28196 cg_pointers_found ++;
28197 dprintf (4, ("cg pointer %Ix found, %Id so far",
28198 (size_t)*poo, cg_pointers_found ));
28203 BOOL gc_heap::card_transition (uint8_t* po, uint8_t* end, size_t card_word_end,
28204 size_t& cg_pointers_found,
28205 size_t& n_eph, size_t& n_card_set,
28206 size_t& card, size_t& end_card,
28207 BOOL& foundp, uint8_t*& start_address,
28208 uint8_t*& limit, size_t& n_cards_cleared)
28210 dprintf (3, ("pointer %Ix past card %Ix", (size_t)po, (size_t)card));
28211 dprintf (3, ("ct: %Id cg", cg_pointers_found));
28212 BOOL passed_end_card_p = FALSE;
28215 if (cg_pointers_found == 0)
28217 //dprintf(3,(" Clearing cards [%Ix, %Ix[ ",
28218 dprintf(3,(" CC [%Ix, %Ix[ ",
28219 (size_t)card_address(card), (size_t)po));
28220 clear_cards (card, card_of(po));
28221 n_card_set -= (card_of (po) - card);
28222 n_cards_cleared += (card_of (po) - card);
28225 n_eph +=cg_pointers_found;
28226 cg_pointers_found = 0;
28227 card = card_of (po);
28228 if (card >= end_card)
28230 passed_end_card_p = TRUE;
28231 dprintf (3, ("card %Ix exceeding end_card %Ix",
28232 (size_t)card, (size_t)end_card));
28233 foundp = find_card (card_table, card, card_word_end, end_card);
28236 n_card_set+= end_card - card;
28237 start_address = card_address (card);
28238 dprintf (3, ("NewC: %Ix, start: %Ix, end: %Ix",
28239 (size_t)card, (size_t)start_address,
28240 (size_t)card_address (end_card)));
28242 limit = min (end, card_address (end_card));
28244 assert (!((limit == card_address (end_card))&&
28245 card_set_p (end_card)));
28248 return passed_end_card_p;
28251 void gc_heap::mark_through_cards_for_segments (card_fn fn, BOOL relocating)
28253 #ifdef BACKGROUND_GC
28254 dprintf (3, ("current_sweep_pos is %Ix, saved_sweep_ephemeral_seg is %Ix(%Ix)",
28255 current_sweep_pos, saved_sweep_ephemeral_seg, saved_sweep_ephemeral_start));
28257 heap_segment* soh_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
28258 PREFIX_ASSUME(soh_seg != NULL);
28262 dprintf (3, ("seg %Ix, bgc_alloc: %Ix, alloc: %Ix",
28264 heap_segment_background_allocated (soh_seg),
28265 heap_segment_allocated (soh_seg)));
28267 soh_seg = heap_segment_next_rw (soh_seg);
28269 #endif //BACKGROUND_GC
28271 uint8_t* low = gc_low;
28272 uint8_t* high = gc_high;
28273 size_t end_card = 0;
28275 generation* oldest_gen = generation_of (max_generation);
28276 int curr_gen_number = max_generation;
28277 uint8_t* gen_boundary = generation_allocation_start(generation_of(curr_gen_number - 1));
28278 uint8_t* next_boundary = compute_next_boundary(gc_low, curr_gen_number, relocating);
28280 heap_segment* seg = heap_segment_rw (generation_start_segment (oldest_gen));
28281 PREFIX_ASSUME(seg != NULL);
28283 uint8_t* beg = generation_allocation_start (oldest_gen);
28284 uint8_t* end = compute_next_end (seg, low);
28285 uint8_t* last_object = beg;
28287 size_t cg_pointers_found = 0;
28289 size_t card_word_end = (card_of (align_on_card_word (end)) / card_word_width);
28293 size_t n_card_set = 0;
28294 uint8_t* nhigh = (relocating ?
28295 heap_segment_plan_allocated (ephemeral_heap_segment) : high);
28297 BOOL foundp = FALSE;
28298 uint8_t* start_address = 0;
28299 uint8_t* limit = 0;
28300 size_t card = card_of (beg);
28301 #ifdef BACKGROUND_GC
28302 BOOL consider_bgc_mark_p = FALSE;
28303 BOOL check_current_sweep_p = FALSE;
28304 BOOL check_saved_sweep_p = FALSE;
28305 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
28306 #endif //BACKGROUND_GC
28308 dprintf(3, ("CMs: %Ix->%Ix", (size_t)beg, (size_t)end));
28309 size_t total_cards_cleared = 0;
28313 if (card_of(last_object) > card)
28315 dprintf (3, ("Found %Id cg pointers", cg_pointers_found));
28316 if (cg_pointers_found == 0)
28318 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card), (size_t)last_object));
28319 clear_cards (card, card_of(last_object));
28320 n_card_set -= (card_of (last_object) - card);
28321 total_cards_cleared += (card_of (last_object) - card);
28324 n_eph += cg_pointers_found;
28325 cg_pointers_found = 0;
28326 card = card_of (last_object);
28329 if (card >= end_card)
28331 foundp = find_card (card_table, card, card_word_end, end_card);
28334 n_card_set += end_card - card;
28335 start_address = max (beg, card_address (card));
28337 limit = min (end, card_address (end_card));
28339 if (!foundp || (last_object >= end) || (card_address (card) >= end))
28341 if (foundp && (cg_pointers_found == 0))
28343 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card),
28345 clear_cards (card, card_of (end));
28346 n_card_set -= (card_of (end) - card);
28347 total_cards_cleared += (card_of (end) - card);
28349 n_eph += cg_pointers_found;
28350 cg_pointers_found = 0;
28351 if ((seg = heap_segment_next_in_range (seg)) != 0)
28353 #ifdef BACKGROUND_GC
28354 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
28355 #endif //BACKGROUND_GC
28356 beg = heap_segment_mem (seg);
28357 end = compute_next_end (seg, low);
28358 card_word_end = card_of (align_on_card_word (end)) / card_word_width;
28359 card = card_of (beg);
28370 assert (card_set_p (card));
28372 uint8_t* o = last_object;
28374 o = find_first_object (start_address, last_object);
28375 // Never visit an object twice.
28376 assert (o >= last_object);
28378 //dprintf(3,("Considering card %Ix start object: %Ix, %Ix[ boundary: %Ix",
28379 dprintf(3, ("c: %Ix, o: %Ix, l: %Ix[ boundary: %Ix",
28380 card, (size_t)o, (size_t)limit, (size_t)gen_boundary));
28384 assert (Align (size (o)) >= Align (min_obj_size));
28385 size_t s = size (o);
28387 uint8_t* next_o = o + Align (s);
28390 if ((o >= gen_boundary) &&
28391 (seg == ephemeral_heap_segment))
28393 dprintf (3, ("switching gen boundary %Ix", (size_t)gen_boundary));
28395 assert ((curr_gen_number > 0));
28396 gen_boundary = generation_allocation_start
28397 (generation_of (curr_gen_number - 1));
28398 next_boundary = (compute_next_boundary
28399 (low, curr_gen_number, relocating));
28402 dprintf (4, ("|%Ix|", (size_t)o));
28404 if (next_o < start_address)
28409 #ifdef BACKGROUND_GC
28410 if (!fgc_should_consider_object (o, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p))
28414 #endif //BACKGROUND_GC
28416 #ifdef COLLECTIBLE_CLASS
28417 if (is_collectible(o))
28419 BOOL passed_end_card_p = FALSE;
28421 if (card_of (o) > card)
28423 passed_end_card_p = card_transition (o, end, card_word_end,
28427 foundp, start_address,
28428 limit, total_cards_cleared);
28431 if ((!passed_end_card_p || foundp) && (card_of (o) == card))
28433 // card is valid and it covers the head of the object
28434 if (fn == &gc_heap::relocate_address)
28436 keep_card_live (o, n_gen, cg_pointers_found);
28440 uint8_t* class_obj = get_class_object (o);
28441 mark_through_cards_helper (&class_obj, n_gen,
28442 cg_pointers_found, fn,
28443 nhigh, next_boundary);
28447 if (passed_end_card_p)
28449 if (foundp && (card_address (card) < next_o))
28451 goto go_through_refs;
28453 else if (foundp && (start_address < limit))
28455 next_o = find_first_object (start_address, o);
28464 #endif //COLLECTIBLE_CLASS
28466 if (contain_pointers (o))
28468 dprintf(3,("Going through %Ix start_address: %Ix", (size_t)o, (size_t)start_address));
28471 dprintf (4, ("normal object path"));
28473 (method_table(o), o, s, poo,
28474 start_address, use_start, (o + s),
28476 dprintf (4, ("<%Ix>:%Ix", (size_t)poo, (size_t)*poo));
28477 if (card_of ((uint8_t*)poo) > card)
28479 BOOL passed_end_card_p = card_transition ((uint8_t*)poo, end,
28484 foundp, start_address,
28485 limit, total_cards_cleared);
28487 if (passed_end_card_p)
28489 if (foundp && (card_address (card) < next_o))
28493 if (ppstop <= (uint8_t**)start_address)
28495 else if (poo < (uint8_t**)start_address)
28496 {poo = (uint8_t**)start_address;}
28499 else if (foundp && (start_address < limit))
28501 next_o = find_first_object (start_address, o);
28509 mark_through_cards_helper (poo, n_gen,
28510 cg_pointers_found, fn,
28511 nhigh, next_boundary);
28518 if (((size_t)next_o / brick_size) != ((size_t) o / brick_size))
28520 if (brick_table [brick_of (o)] <0)
28521 fix_brick_to_highest (o, next_o);
28529 // compute the efficiency ratio of the card table
28532 generation_skip_ratio = ((n_eph > 400)? (int)(((float)n_gen / (float)n_eph) * 100) : 100);
28533 dprintf (3, ("Msoh: cross: %Id, useful: %Id, cards set: %Id, cards cleared: %Id, ratio: %d",
28534 n_eph, n_gen , n_card_set, total_cards_cleared, generation_skip_ratio));
28538 dprintf (3, ("R: Msoh: cross: %Id, useful: %Id, cards set: %Id, cards cleared: %Id, ratio: %d",
28539 n_gen, n_eph, n_card_set, total_cards_cleared, generation_skip_ratio));
28543 #ifdef SEG_REUSE_STATS
28544 size_t gc_heap::dump_buckets (size_t* ordered_indices, int count, size_t* total_size)
28546 size_t total_items = 0;
28548 for (int i = 0; i < count; i++)
28550 total_items += ordered_indices[i];
28551 *total_size += ordered_indices[i] << (MIN_INDEX_POWER2 + i);
28552 dprintf (SEG_REUSE_LOG_0, ("[%d]%4d 2^%2d", heap_number, ordered_indices[i], (MIN_INDEX_POWER2 + i)));
28554 dprintf (SEG_REUSE_LOG_0, ("[%d]Total %d items, total size is 0x%Ix", heap_number, total_items, *total_size));
28555 return total_items;
28557 #endif // SEG_REUSE_STATS
28559 void gc_heap::count_plug (size_t last_plug_size, uint8_t*& last_plug)
28561 // detect pinned plugs
28562 if (!pinned_plug_que_empty_p() && (last_plug == pinned_plug (oldest_pin())))
28564 deque_pinned_plug();
28565 update_oldest_pinned_plug();
28566 dprintf (3, ("dequed pin,now oldest pin is %Ix", pinned_plug (oldest_pin())));
28570 size_t plug_size = last_plug_size + Align(min_obj_size);
28571 BOOL is_padded = FALSE;
28574 plug_size += Align (min_obj_size);
28576 #endif //SHORT_PLUGS
28578 #ifdef RESPECT_LARGE_ALIGNMENT
28579 plug_size += switch_alignment_size (is_padded);
28580 #endif //RESPECT_LARGE_ALIGNMENT
28582 total_ephemeral_plugs += plug_size;
28583 size_t plug_size_power2 = round_up_power2 (plug_size);
28584 ordered_plug_indices[relative_index_power2_plug (plug_size_power2)]++;
28585 dprintf (SEG_REUSE_LOG_1, ("[%d]count_plug: adding 0x%Ix - %Id (2^%d) to ordered plug array",
28589 (relative_index_power2_plug (plug_size_power2) + MIN_INDEX_POWER2)));
28593 void gc_heap::count_plugs_in_brick (uint8_t* tree, uint8_t*& last_plug)
28595 assert ((tree != NULL));
28596 if (node_left_child (tree))
28598 count_plugs_in_brick (tree + node_left_child (tree), last_plug);
28601 if (last_plug != 0)
28603 uint8_t* plug = tree;
28604 size_t gap_size = node_gap_size (plug);
28605 uint8_t* gap = (plug - gap_size);
28606 uint8_t* last_plug_end = gap;
28607 size_t last_plug_size = (last_plug_end - last_plug);
28608 dprintf (3, ("tree: %Ix, last plug: %Ix, gap size: %Ix, gap: %Ix, last plug size: %Ix",
28609 tree, last_plug, gap_size, gap, last_plug_size));
28611 if (tree == oldest_pinned_plug)
28613 dprintf (3, ("tree %Ix is pinned, last plug is %Ix, size is %Ix",
28614 tree, last_plug, last_plug_size));
28615 mark* m = oldest_pin();
28616 if (m->has_pre_plug_info())
28618 last_plug_size += sizeof (gap_reloc_pair);
28619 dprintf (3, ("pin %Ix has pre plug, adjusting plug size to %Ix", tree, last_plug_size));
28622 // Can't assert here - if it's a pinned plug it can be less.
28623 //assert (last_plug_size >= Align (min_obj_size));
28625 count_plug (last_plug_size, last_plug);
28630 if (node_right_child (tree))
28632 count_plugs_in_brick (tree + node_right_child (tree), last_plug);
28636 void gc_heap::build_ordered_plug_indices ()
28638 memset (ordered_plug_indices, 0, sizeof(ordered_plug_indices));
28639 memset (saved_ordered_plug_indices, 0, sizeof(saved_ordered_plug_indices));
28641 uint8_t* start_address = generation_limit (max_generation);
28642 uint8_t* end_address = heap_segment_allocated (ephemeral_heap_segment);
28643 size_t current_brick = brick_of (start_address);
28644 size_t end_brick = brick_of (end_address - 1);
28645 uint8_t* last_plug = 0;
28647 //Look for the right pinned plug to start from.
28648 reset_pinned_queue_bos();
28649 while (!pinned_plug_que_empty_p())
28651 mark* m = oldest_pin();
28652 if ((m->first >= start_address) && (m->first < end_address))
28654 dprintf (3, ("found a pin %Ix between %Ix and %Ix", m->first, start_address, end_address));
28659 deque_pinned_plug();
28662 update_oldest_pinned_plug();
28664 while (current_brick <= end_brick)
28666 int brick_entry = brick_table [ current_brick ];
28667 if (brick_entry >= 0)
28669 count_plugs_in_brick (brick_address (current_brick) + brick_entry -1, last_plug);
28677 count_plug (end_address - last_plug, last_plug);
28680 // we need to make sure that after fitting all the existing plugs, we
28681 // have big enough free space left to guarantee that the next allocation
28683 size_t extra_size = END_SPACE_AFTER_GC + Align (min_obj_size);
28684 total_ephemeral_plugs += extra_size;
28685 dprintf (SEG_REUSE_LOG_0, ("Making sure we can fit a large object after fitting all plugs"));
28686 ordered_plug_indices[relative_index_power2_plug (round_up_power2 (extra_size))]++;
28688 memcpy (saved_ordered_plug_indices, ordered_plug_indices, sizeof(ordered_plug_indices));
28690 #ifdef SEG_REUSE_STATS
28691 dprintf (SEG_REUSE_LOG_0, ("Plugs:"));
28692 size_t total_plug_power2 = 0;
28693 dump_buckets (ordered_plug_indices, MAX_NUM_BUCKETS, &total_plug_power2);
28694 dprintf (SEG_REUSE_LOG_0, ("plugs: 0x%Ix (rounded up to 0x%Ix (%d%%))",
28695 total_ephemeral_plugs,
28697 (total_ephemeral_plugs ?
28698 (total_plug_power2 * 100 / total_ephemeral_plugs) :
28700 dprintf (SEG_REUSE_LOG_0, ("-------------------"));
28701 #endif // SEG_REUSE_STATS
28704 void gc_heap::init_ordered_free_space_indices ()
28706 memset (ordered_free_space_indices, 0, sizeof(ordered_free_space_indices));
28707 memset (saved_ordered_free_space_indices, 0, sizeof(saved_ordered_free_space_indices));
28710 void gc_heap::trim_free_spaces_indices ()
28712 trimmed_free_space_index = -1;
28713 size_t max_count = max_free_space_items - 1;
28716 for (i = (MAX_NUM_BUCKETS - 1); i >= 0; i--)
28718 count += ordered_free_space_indices[i];
28720 if (count >= max_count)
28726 ptrdiff_t extra_free_space_items = count - max_count;
28728 if (extra_free_space_items > 0)
28730 ordered_free_space_indices[i] -= extra_free_space_items;
28731 free_space_items = max_count;
28732 trimmed_free_space_index = i;
28736 free_space_items = count;
28744 free_space_buckets = MAX_NUM_BUCKETS - i;
28746 for (--i; i >= 0; i--)
28748 ordered_free_space_indices[i] = 0;
28751 memcpy (saved_ordered_free_space_indices,
28752 ordered_free_space_indices,
28753 sizeof(ordered_free_space_indices));
28756 // We fit as many plugs as we can and update the number of plugs left and the number
28757 // of free spaces left.
28758 BOOL gc_heap::can_fit_in_spaces_p (size_t* ordered_blocks, int small_index, size_t* ordered_spaces, int big_index)
28760 assert (small_index <= big_index);
28761 assert (big_index < MAX_NUM_BUCKETS);
28763 size_t small_blocks = ordered_blocks[small_index];
28765 if (small_blocks == 0)
28770 size_t big_spaces = ordered_spaces[big_index];
28772 if (big_spaces == 0)
28777 dprintf (SEG_REUSE_LOG_1, ("[%d]Fitting %Id 2^%d plugs into %Id 2^%d free spaces",
28779 small_blocks, (small_index + MIN_INDEX_POWER2),
28780 big_spaces, (big_index + MIN_INDEX_POWER2)));
28782 size_t big_to_small = big_spaces << (big_index - small_index);
28784 ptrdiff_t extra_small_spaces = big_to_small - small_blocks;
28785 dprintf (SEG_REUSE_LOG_1, ("[%d]%d 2^%d spaces can fit %d 2^%d blocks",
28787 big_spaces, (big_index + MIN_INDEX_POWER2), big_to_small, (small_index + MIN_INDEX_POWER2)));
28788 BOOL can_fit = (extra_small_spaces >= 0);
28792 dprintf (SEG_REUSE_LOG_1, ("[%d]Can fit with %d 2^%d extras blocks",
28794 extra_small_spaces, (small_index + MIN_INDEX_POWER2)));
28799 dprintf (SEG_REUSE_LOG_1, ("[%d]Setting # of 2^%d spaces to 0", heap_number, (big_index + MIN_INDEX_POWER2)));
28800 ordered_spaces[big_index] = 0;
28801 if (extra_small_spaces > 0)
28803 dprintf (SEG_REUSE_LOG_1, ("[%d]Setting # of 2^%d blocks to 0", heap_number, (small_index + MIN_INDEX_POWER2)));
28804 ordered_blocks[small_index] = 0;
28805 for (i = small_index; i < big_index; i++)
28807 if (extra_small_spaces & 1)
28809 dprintf (SEG_REUSE_LOG_1, ("[%d]Increasing # of 2^%d spaces from %d to %d",
28811 (i + MIN_INDEX_POWER2), ordered_spaces[i], (ordered_spaces[i] + 1)));
28812 ordered_spaces[i] += 1;
28814 extra_small_spaces >>= 1;
28817 dprintf (SEG_REUSE_LOG_1, ("[%d]Finally increasing # of 2^%d spaces from %d to %d",
28819 (i + MIN_INDEX_POWER2), ordered_spaces[i], (ordered_spaces[i] + extra_small_spaces)));
28820 ordered_spaces[i] += extra_small_spaces;
28824 dprintf (SEG_REUSE_LOG_1, ("[%d]Decreasing # of 2^%d blocks from %d to %d",
28826 (small_index + MIN_INDEX_POWER2),
28827 ordered_blocks[small_index],
28828 (ordered_blocks[small_index] - big_to_small)));
28829 ordered_blocks[small_index] -= big_to_small;
28832 #ifdef SEG_REUSE_STATS
28834 dprintf (SEG_REUSE_LOG_1, ("[%d]Plugs became:", heap_number));
28835 dump_buckets (ordered_blocks, MAX_NUM_BUCKETS, &temp);
28837 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces became:", heap_number));
28838 dump_buckets (ordered_spaces, MAX_NUM_BUCKETS, &temp);
28839 #endif //SEG_REUSE_STATS
28844 // space_index gets updated to the biggest available space index.
28845 BOOL gc_heap::can_fit_blocks_p (size_t* ordered_blocks, int block_index, size_t* ordered_spaces, int* space_index)
28847 assert (*space_index >= block_index);
28849 while (!can_fit_in_spaces_p (ordered_blocks, block_index, ordered_spaces, *space_index))
28852 if (*space_index < block_index)
28861 BOOL gc_heap::can_fit_all_blocks_p (size_t* ordered_blocks, size_t* ordered_spaces, int count)
28863 #ifdef FEATURE_STRUCTALIGN
28864 // BARTOKTODO (4841): reenable when can_fit_in_spaces_p takes alignment requirements into account
28866 #endif // FEATURE_STRUCTALIGN
28867 int space_index = count - 1;
28868 for (int block_index = (count - 1); block_index >= 0; block_index--)
28870 if (!can_fit_blocks_p (ordered_blocks, block_index, ordered_spaces, &space_index))
28879 void gc_heap::build_ordered_free_spaces (heap_segment* seg)
28881 assert (bestfit_seg);
28883 //bestfit_seg->add_buckets (MAX_NUM_BUCKETS - free_space_buckets + MIN_INDEX_POWER2,
28884 // ordered_free_space_indices + (MAX_NUM_BUCKETS - free_space_buckets),
28885 // free_space_buckets,
28886 // free_space_items);
28888 bestfit_seg->add_buckets (MIN_INDEX_POWER2,
28889 ordered_free_space_indices,
28893 assert (settings.condemned_generation == max_generation);
28895 uint8_t* first_address = heap_segment_mem (seg);
28896 uint8_t* end_address = heap_segment_reserved (seg);
28897 //look through the pinned plugs for relevant ones.
28898 //Look for the right pinned plug to start from.
28899 reset_pinned_queue_bos();
28901 // See comment in can_expand_into_p why we need (max_generation + 1).
28902 size_t eph_gen_starts = (Align (min_obj_size)) * (max_generation + 1);
28903 BOOL has_fit_gen_starts = FALSE;
28905 while (!pinned_plug_que_empty_p())
28908 if ((pinned_plug (m) >= first_address) &&
28909 (pinned_plug (m) < end_address) &&
28910 (pinned_len (m) >= eph_gen_starts))
28913 assert ((pinned_plug (m) - pinned_len (m)) == bestfit_first_pin);
28918 deque_pinned_plug();
28922 if (!pinned_plug_que_empty_p())
28924 bestfit_seg->add ((void*)m, TRUE, TRUE);
28925 deque_pinned_plug();
28927 has_fit_gen_starts = TRUE;
28930 while (!pinned_plug_que_empty_p() &&
28931 ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address)))
28933 bestfit_seg->add ((void*)m, TRUE, FALSE);
28934 deque_pinned_plug();
28938 if (commit_end_of_seg)
28940 if (!has_fit_gen_starts)
28942 assert (bestfit_first_pin == heap_segment_plan_allocated (seg));
28944 bestfit_seg->add ((void*)seg, FALSE, (!has_fit_gen_starts));
28948 bestfit_seg->check();
28952 BOOL gc_heap::try_best_fit (BOOL end_of_segment_p)
28954 if (!end_of_segment_p)
28956 trim_free_spaces_indices ();
28959 BOOL can_bestfit = can_fit_all_blocks_p (ordered_plug_indices,
28960 ordered_free_space_indices,
28963 return can_bestfit;
28966 BOOL gc_heap::best_fit (size_t free_space,
28967 size_t largest_free_space,
28968 size_t additional_space,
28969 BOOL* use_additional_space)
28971 dprintf (SEG_REUSE_LOG_0, ("gen%d: trying best fit mechanism", settings.condemned_generation));
28973 assert (!additional_space || (additional_space && use_additional_space));
28974 if (use_additional_space)
28976 *use_additional_space = FALSE;
28979 if (ordered_plug_indices_init == FALSE)
28981 total_ephemeral_plugs = 0;
28982 build_ordered_plug_indices();
28983 ordered_plug_indices_init = TRUE;
28987 memcpy (ordered_plug_indices, saved_ordered_plug_indices, sizeof(ordered_plug_indices));
28990 if (total_ephemeral_plugs == (END_SPACE_AFTER_GC + Align (min_obj_size)))
28992 dprintf (SEG_REUSE_LOG_0, ("No ephemeral plugs to realloc, done"));
28993 size_t empty_eph = (END_SPACE_AFTER_GC + Align (min_obj_size) + (Align (min_obj_size)) * (max_generation + 1));
28994 BOOL can_fit_empty_eph = (largest_free_space >= empty_eph);
28995 if (!can_fit_empty_eph)
28997 can_fit_empty_eph = (additional_space >= empty_eph);
28999 if (can_fit_empty_eph)
29001 *use_additional_space = TRUE;
29005 return can_fit_empty_eph;
29008 if ((total_ephemeral_plugs + approximate_new_allocation()) >= (free_space + additional_space))
29010 dprintf (SEG_REUSE_LOG_0, ("We won't have enough free space left in this segment after fitting, done"));
29014 if ((free_space + additional_space) == 0)
29016 dprintf (SEG_REUSE_LOG_0, ("No free space in this segment, done"));
29020 #ifdef SEG_REUSE_STATS
29021 dprintf (SEG_REUSE_LOG_0, ("Free spaces:"));
29022 size_t total_free_space_power2 = 0;
29023 size_t total_free_space_items =
29024 dump_buckets (ordered_free_space_indices,
29026 &total_free_space_power2);
29027 dprintf (SEG_REUSE_LOG_0, ("currently max free spaces is %Id", max_free_space_items));
29029 dprintf (SEG_REUSE_LOG_0, ("Ephemeral plugs: 0x%Ix, free space: 0x%Ix (rounded down to 0x%Ix (%Id%%)), additional free_space: 0x%Ix",
29030 total_ephemeral_plugs,
29032 total_free_space_power2,
29033 (free_space ? (total_free_space_power2 * 100 / free_space) : 0),
29034 additional_space));
29036 size_t saved_all_free_space_indices[MAX_NUM_BUCKETS];
29037 memcpy (saved_all_free_space_indices,
29038 ordered_free_space_indices,
29039 sizeof(saved_all_free_space_indices));
29041 #endif // SEG_REUSE_STATS
29043 if (total_ephemeral_plugs > (free_space + additional_space))
29048 use_bestfit = try_best_fit(FALSE);
29050 if (!use_bestfit && additional_space)
29052 int relative_free_space_index = relative_index_power2_free_space (round_down_power2 (additional_space));
29054 if (relative_free_space_index != -1)
29056 int relative_plug_index = 0;
29057 size_t plugs_to_fit = 0;
29059 for (relative_plug_index = (MAX_NUM_BUCKETS - 1); relative_plug_index >= 0; relative_plug_index--)
29061 plugs_to_fit = ordered_plug_indices[relative_plug_index];
29062 if (plugs_to_fit != 0)
29068 if ((relative_plug_index > relative_free_space_index) ||
29069 ((relative_plug_index == relative_free_space_index) &&
29070 (plugs_to_fit > 1)))
29072 #ifdef SEG_REUSE_STATS
29073 dprintf (SEG_REUSE_LOG_0, ("additional space is 2^%d but we stopped at %d 2^%d plug(s)",
29074 (relative_free_space_index + MIN_INDEX_POWER2),
29076 (relative_plug_index + MIN_INDEX_POWER2)));
29077 #endif // SEG_REUSE_STATS
29081 dprintf (SEG_REUSE_LOG_0, ("Adding end of segment (2^%d)", (relative_free_space_index + MIN_INDEX_POWER2)));
29082 ordered_free_space_indices[relative_free_space_index]++;
29083 use_bestfit = try_best_fit(TRUE);
29086 free_space_items++;
29087 // Since we might've trimmed away some of the free spaces we had, we should see
29088 // if we really need to use end of seg space - if it's the same or smaller than
29089 // the largest space we trimmed we can just add that one back instead of
29090 // using end of seg.
29091 if (relative_free_space_index > trimmed_free_space_index)
29093 *use_additional_space = TRUE;
29097 // If the addition space is <= than the last trimmed space, we
29098 // should just use that last trimmed space instead.
29099 saved_ordered_free_space_indices[trimmed_free_space_index]++;
29109 dprintf (SEG_REUSE_LOG_0, ("couldn't fit..."));
29111 #ifdef SEG_REUSE_STATS
29112 size_t saved_max = max_free_space_items;
29113 BOOL temp_bestfit = FALSE;
29115 dprintf (SEG_REUSE_LOG_0, ("----Starting experiment process----"));
29116 dprintf (SEG_REUSE_LOG_0, ("----Couldn't fit with max free items %Id", max_free_space_items));
29118 // TODO: need to take the end of segment into consideration.
29119 while (max_free_space_items <= total_free_space_items)
29121 max_free_space_items += max_free_space_items / 2;
29122 dprintf (SEG_REUSE_LOG_0, ("----Temporarily increasing max free spaces to %Id", max_free_space_items));
29123 memcpy (ordered_free_space_indices,
29124 saved_all_free_space_indices,
29125 sizeof(ordered_free_space_indices));
29126 if (try_best_fit(FALSE))
29128 temp_bestfit = TRUE;
29135 dprintf (SEG_REUSE_LOG_0, ("----With %Id max free spaces we could fit", max_free_space_items));
29139 dprintf (SEG_REUSE_LOG_0, ("----Tried all free spaces and still couldn't fit, lost too much space"));
29142 dprintf (SEG_REUSE_LOG_0, ("----Restoring max free spaces to %Id", saved_max));
29143 max_free_space_items = saved_max;
29144 #endif // SEG_REUSE_STATS
29145 if (free_space_items)
29147 max_free_space_items = min (MAX_NUM_FREE_SPACES, free_space_items * 2);
29148 max_free_space_items = max (max_free_space_items, MIN_NUM_FREE_SPACES);
29152 max_free_space_items = MAX_NUM_FREE_SPACES;
29156 dprintf (SEG_REUSE_LOG_0, ("Adjusted number of max free spaces to %Id", max_free_space_items));
29157 dprintf (SEG_REUSE_LOG_0, ("------End of best fitting process------\n"));
29159 return use_bestfit;
29162 BOOL gc_heap::process_free_space (heap_segment* seg,
29164 size_t min_free_size,
29165 size_t min_cont_size,
29166 size_t* total_free_space,
29167 size_t* largest_free_space)
29169 *total_free_space += free_space;
29170 *largest_free_space = max (*largest_free_space, free_space);
29172 #ifdef SIMPLE_DPRINTF
29173 dprintf (SEG_REUSE_LOG_1, ("free space len: %Ix, total free space: %Ix, largest free space: %Ix",
29174 free_space, *total_free_space, *largest_free_space));
29175 #endif //SIMPLE_DPRINTF
29177 if ((*total_free_space >= min_free_size) && (*largest_free_space >= min_cont_size))
29179 #ifdef SIMPLE_DPRINTF
29180 dprintf (SEG_REUSE_LOG_0, ("(gen%d)total free: %Ix(min: %Ix), largest free: %Ix(min: %Ix). Found segment %Ix to reuse without bestfit",
29181 settings.condemned_generation,
29182 *total_free_space, min_free_size, *largest_free_space, min_cont_size,
29185 UNREFERENCED_PARAMETER(seg);
29186 #endif //SIMPLE_DPRINTF
29190 int free_space_index = relative_index_power2_free_space (round_down_power2 (free_space));
29191 if (free_space_index != -1)
29193 ordered_free_space_indices[free_space_index]++;
29198 BOOL gc_heap::expand_reused_seg_p()
29200 BOOL reused_seg = FALSE;
29201 int heap_expand_mechanism = gc_data_per_heap.get_mechanism (gc_heap_expand);
29202 if ((heap_expand_mechanism == expand_reuse_bestfit) ||
29203 (heap_expand_mechanism == expand_reuse_normal))
29211 BOOL gc_heap::can_expand_into_p (heap_segment* seg, size_t min_free_size, size_t min_cont_size,
29212 allocator* gen_allocator)
29214 min_cont_size += END_SPACE_AFTER_GC;
29215 use_bestfit = FALSE;
29216 commit_end_of_seg = FALSE;
29217 bestfit_first_pin = 0;
29218 uint8_t* first_address = heap_segment_mem (seg);
29219 uint8_t* end_address = heap_segment_reserved (seg);
29220 size_t end_extra_space = end_space_after_gc();
29222 if ((heap_segment_reserved (seg) - end_extra_space) <= heap_segment_plan_allocated (seg))
29224 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: can't use segment [%Ix %Ix, has less than %d bytes at the end",
29225 first_address, end_address, end_extra_space));
29229 end_address -= end_extra_space;
29231 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p(gen%d): min free: %Ix, min continuous: %Ix",
29232 settings.condemned_generation, min_free_size, min_cont_size));
29233 size_t eph_gen_starts = eph_gen_starts_size;
29235 if (settings.condemned_generation == max_generation)
29237 size_t free_space = 0;
29238 size_t largest_free_space = free_space;
29239 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: gen2: testing segment [%Ix %Ix", first_address, end_address));
29240 //Look through the pinned plugs for relevant ones and Look for the right pinned plug to start from.
29241 //We are going to allocate the generation starts in the 1st free space,
29242 //so start from the first free space that's big enough for gen starts and a min object size.
29243 // If we see a free space that is >= gen starts but < gen starts + min obj size we just don't use it -
29244 // we could use it by allocating the last generation start a bit bigger but
29245 // the complexity isn't worth the effort (those plugs are from gen2
29246 // already anyway).
29247 reset_pinned_queue_bos();
29249 BOOL has_fit_gen_starts = FALSE;
29251 init_ordered_free_space_indices ();
29252 while (!pinned_plug_que_empty_p())
29255 if ((pinned_plug (m) >= first_address) &&
29256 (pinned_plug (m) < end_address) &&
29257 (pinned_len (m) >= (eph_gen_starts + Align (min_obj_size))))
29263 deque_pinned_plug();
29267 if (!pinned_plug_que_empty_p())
29269 bestfit_first_pin = pinned_plug (m) - pinned_len (m);
29271 if (process_free_space (seg,
29272 pinned_len (m) - eph_gen_starts,
29273 min_free_size, min_cont_size,
29274 &free_space, &largest_free_space))
29279 deque_pinned_plug();
29281 has_fit_gen_starts = TRUE;
29284 dprintf (3, ("first pin is %Ix", pinned_plug (m)));
29286 //tally up free space
29287 while (!pinned_plug_que_empty_p() &&
29288 ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address)))
29290 dprintf (3, ("looking at pin %Ix", pinned_plug (m)));
29291 if (process_free_space (seg,
29293 min_free_size, min_cont_size,
29294 &free_space, &largest_free_space))
29299 deque_pinned_plug();
29303 //try to find space at the end of the segment.
29304 size_t end_space = (end_address - heap_segment_plan_allocated (seg));
29305 size_t additional_space = ((min_free_size > free_space) ? (min_free_size - free_space) : 0);
29306 dprintf (SEG_REUSE_LOG_0, ("end space: %Ix; additional: %Ix", end_space, additional_space));
29307 if (end_space >= additional_space)
29309 BOOL can_fit = TRUE;
29310 commit_end_of_seg = TRUE;
29312 if (largest_free_space < min_cont_size)
29314 if (end_space >= min_cont_size)
29316 additional_space = max (min_cont_size, additional_space);
29317 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse without bestfit, with committing end of seg for eph",
29322 if (settings.concurrent)
29325 commit_end_of_seg = FALSE;
29329 size_t additional_space_bestfit = additional_space;
29330 if (!has_fit_gen_starts)
29332 if (additional_space_bestfit < (eph_gen_starts + Align (min_obj_size)))
29334 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, gen starts not allocated yet and end space is too small: %Id",
29335 additional_space_bestfit));
29339 bestfit_first_pin = heap_segment_plan_allocated (seg);
29340 additional_space_bestfit -= eph_gen_starts;
29343 can_fit = best_fit (free_space,
29344 largest_free_space,
29345 additional_space_bestfit,
29346 &commit_end_of_seg);
29350 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse with bestfit, %s committing end of seg",
29351 seg, (commit_end_of_seg ? "with" : "without")));
29355 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, total free space is %Ix", (free_space + end_space)));
29362 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse without bestfit, with committing end of seg", seg));
29365 assert (additional_space <= end_space);
29366 if (commit_end_of_seg)
29368 if (!grow_heap_segment (seg, heap_segment_plan_allocated (seg) + additional_space))
29370 dprintf (2, ("Couldn't commit end of segment?!"));
29371 use_bestfit = FALSE;
29378 // We increase the index here because growing heap segment could create a discrepency with
29379 // the additional space we used (could be bigger).
29380 size_t free_space_end_of_seg =
29381 heap_segment_committed (seg) - heap_segment_plan_allocated (seg);
29382 int relative_free_space_index = relative_index_power2_free_space (round_down_power2 (free_space_end_of_seg));
29383 saved_ordered_free_space_indices[relative_free_space_index]++;
29389 memcpy (ordered_free_space_indices,
29390 saved_ordered_free_space_indices,
29391 sizeof(ordered_free_space_indices));
29392 max_free_space_items = max (MIN_NUM_FREE_SPACES, free_space_items * 3 / 2);
29393 max_free_space_items = min (MAX_NUM_FREE_SPACES, max_free_space_items);
29394 dprintf (SEG_REUSE_LOG_0, ("could fit! %Id free spaces, %Id max", free_space_items, max_free_space_items));
29400 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, total free space is %Ix", (free_space + end_space)));
29405 assert (settings.condemned_generation == (max_generation-1));
29406 size_t free_space = (end_address - heap_segment_plan_allocated (seg));
29407 size_t largest_free_space = free_space;
29408 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: gen1: testing segment [%Ix %Ix", first_address, end_address));
29409 //find the first free list in range of the current segment
29410 size_t sz_list = gen_allocator->first_bucket_size();
29411 unsigned int a_l_idx = 0;
29412 uint8_t* free_list = 0;
29413 for (; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
29415 if ((eph_gen_starts <= sz_list) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
29417 free_list = gen_allocator->alloc_list_head_of (a_l_idx);
29420 if ((free_list >= first_address) &&
29421 (free_list < end_address) &&
29422 (unused_array_size (free_list) >= eph_gen_starts))
29428 free_list = free_list_slot (free_list);
29436 init_ordered_free_space_indices ();
29437 if (process_free_space (seg,
29438 unused_array_size (free_list) - eph_gen_starts + Align (min_obj_size),
29439 min_free_size, min_cont_size,
29440 &free_space, &largest_free_space))
29445 free_list = free_list_slot (free_list);
29449 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, no free list"));
29453 //tally up free space
29459 if ((free_list >= first_address) && (free_list < end_address) &&
29460 process_free_space (seg,
29461 unused_array_size (free_list),
29462 min_free_size, min_cont_size,
29463 &free_space, &largest_free_space))
29468 free_list = free_list_slot (free_list);
29471 if (a_l_idx < gen_allocator->number_of_buckets())
29473 free_list = gen_allocator->alloc_list_head_of (a_l_idx);
29479 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, total free space is %Ix", free_space));
29483 BOOL can_fit = best_fit (free_space, 0, NULL);
29486 dprintf (SEG_REUSE_LOG_0, ("(gen1)Found segment %Ix to reuse with bestfit", seg));
29490 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, total free space is %Ix", free_space));
29498 void gc_heap::realloc_plug (size_t last_plug_size, uint8_t*& last_plug,
29499 generation* gen, uint8_t* start_address,
29500 unsigned int& active_new_gen_number,
29501 uint8_t*& last_pinned_gap, BOOL& leftp,
29504 , mark* pinned_plug_entry
29505 #endif //SHORT_PLUGS
29508 // detect generation boundaries
29509 // make sure that active_new_gen_number is not the youngest generation.
29510 // because the generation_limit wouldn't return the right thing in this case.
29513 if ((active_new_gen_number > 1) &&
29514 (last_plug >= generation_limit (active_new_gen_number)))
29516 assert (last_plug >= start_address);
29517 active_new_gen_number--;
29518 realloc_plan_generation_start (generation_of (active_new_gen_number), gen);
29519 assert (generation_plan_allocation_start (generation_of (active_new_gen_number)));
29524 // detect pinned plugs
29525 if (!pinned_plug_que_empty_p() && (last_plug == pinned_plug (oldest_pin())))
29527 size_t entry = deque_pinned_plug();
29528 mark* m = pinned_plug_of (entry);
29530 size_t saved_pinned_len = pinned_len(m);
29531 pinned_len(m) = last_plug - last_pinned_gap;
29532 //dprintf (3,("Adjusting pinned gap: [%Ix, %Ix[", (size_t)last_pinned_gap, (size_t)last_plug));
29534 if (m->has_post_plug_info())
29536 last_plug_size += sizeof (gap_reloc_pair);
29537 dprintf (3, ("ra pinned %Ix was shortened, adjusting plug size to %Ix", last_plug, last_plug_size))
29540 last_pinned_gap = last_plug + last_plug_size;
29541 dprintf (3, ("ra found pin %Ix, len: %Ix->%Ix, last_p: %Ix, last_p_size: %Ix",
29542 pinned_plug (m), saved_pinned_len, pinned_len (m), last_plug, last_plug_size));
29545 //we are creating a generation fault. set the cards.
29547 size_t end_card = card_of (align_on_card (last_plug + last_plug_size));
29548 size_t card = card_of (last_plug);
29549 while (card != end_card)
29556 else if (last_plug >= start_address)
29558 #ifdef FEATURE_STRUCTALIGN
29559 int requiredAlignment;
29561 node_aligninfo (last_plug, requiredAlignment, pad);
29563 // from how we previously aligned the plug's destination address,
29564 // compute the actual alignment offset.
29565 uint8_t* reloc_plug = last_plug + node_relocation_distance (last_plug);
29566 ptrdiff_t alignmentOffset = ComputeStructAlignPad(reloc_plug, requiredAlignment, 0);
29567 if (!alignmentOffset)
29569 // allocate_in_expanded_heap doesn't expect alignmentOffset to be zero.
29570 alignmentOffset = requiredAlignment;
29573 //clear the alignment info because we are reallocating
29574 clear_node_aligninfo (last_plug);
29575 #else // FEATURE_STRUCTALIGN
29576 //clear the realignment flag because we are reallocating
29577 clear_node_realigned (last_plug);
29578 #endif // FEATURE_STRUCTALIGN
29579 BOOL adjacentp = FALSE;
29580 BOOL set_padding_on_saved_p = FALSE;
29584 last_plug_size += sizeof (gap_reloc_pair);
29587 assert (pinned_plug_entry != NULL);
29588 if (last_plug_size <= sizeof (plug_and_gap))
29590 set_padding_on_saved_p = TRUE;
29592 #endif //SHORT_PLUGS
29594 dprintf (3, ("ra plug %Ix was shortened, adjusting plug size to %Ix", last_plug, last_plug_size))
29598 clear_padding_in_expand (last_plug, set_padding_on_saved_p, pinned_plug_entry);
29599 #endif //SHORT_PLUGS
29601 uint8_t* new_address = allocate_in_expanded_heap(gen, last_plug_size, adjacentp, last_plug,
29603 set_padding_on_saved_p,
29605 #endif //SHORT_PLUGS
29606 TRUE, active_new_gen_number REQD_ALIGN_AND_OFFSET_ARG);
29608 dprintf (3, ("ra NA: [%Ix, %Ix[: %Ix", new_address, (new_address + last_plug_size), last_plug_size));
29609 assert (new_address);
29610 set_node_relocation_distance (last_plug, new_address - last_plug);
29611 #ifdef FEATURE_STRUCTALIGN
29612 if (leftp && node_alignpad (last_plug) == 0)
29613 #else // FEATURE_STRUCTALIGN
29614 if (leftp && !node_realigned (last_plug))
29615 #endif // FEATURE_STRUCTALIGN
29617 // TODO - temporarily disable L optimization because of a bug in it.
29618 //set_node_left (last_plug);
29620 dprintf (3,(" Re-allocating %Ix->%Ix len %Id", (size_t)last_plug, (size_t)new_address, last_plug_size));
29625 void gc_heap::realloc_in_brick (uint8_t* tree, uint8_t*& last_plug,
29626 uint8_t* start_address,
29628 unsigned int& active_new_gen_number,
29629 uint8_t*& last_pinned_gap, BOOL& leftp)
29631 assert (tree != NULL);
29632 int left_node = node_left_child (tree);
29633 int right_node = node_right_child (tree);
29635 dprintf (3, ("ra: tree: %Ix, last_pin_gap: %Ix, last_p: %Ix, L: %d, R: %d",
29636 tree, last_pinned_gap, last_plug, left_node, right_node));
29640 dprintf (3, ("LN: realloc %Ix(%Ix)", (tree + left_node), last_plug));
29641 realloc_in_brick ((tree + left_node), last_plug, start_address,
29642 gen, active_new_gen_number, last_pinned_gap,
29646 if (last_plug != 0)
29648 uint8_t* plug = tree;
29650 BOOL has_pre_plug_info_p = FALSE;
29651 BOOL has_post_plug_info_p = FALSE;
29652 mark* pinned_plug_entry = get_next_pinned_entry (tree,
29653 &has_pre_plug_info_p,
29654 &has_post_plug_info_p,
29657 // We only care about the pre plug info 'cause that's what decides if the last plug is shortened.
29658 // The pinned plugs are handled in realloc_plug.
29659 size_t gap_size = node_gap_size (plug);
29660 uint8_t* gap = (plug - gap_size);
29661 uint8_t* last_plug_end = gap;
29662 size_t last_plug_size = (last_plug_end - last_plug);
29663 // Cannot assert this - a plug could be less than that due to the shortened ones.
29664 //assert (last_plug_size >= Align (min_obj_size));
29665 dprintf (3, ("ra: plug %Ix, gap size: %Ix, last_pin_gap: %Ix, last_p: %Ix, last_p_end: %Ix, shortened: %d",
29666 plug, gap_size, last_pinned_gap, last_plug, last_plug_end, (has_pre_plug_info_p ? 1 : 0)));
29667 realloc_plug (last_plug_size, last_plug, gen, start_address,
29668 active_new_gen_number, last_pinned_gap,
29669 leftp, has_pre_plug_info_p
29671 , pinned_plug_entry
29672 #endif //SHORT_PLUGS
29680 dprintf (3, ("RN: realloc %Ix(%Ix)", (tree + right_node), last_plug));
29681 realloc_in_brick ((tree + right_node), last_plug, start_address,
29682 gen, active_new_gen_number, last_pinned_gap,
29688 gc_heap::realloc_plugs (generation* consing_gen, heap_segment* seg,
29689 uint8_t* start_address, uint8_t* end_address,
29690 unsigned active_new_gen_number)
29692 dprintf (3, ("--- Reallocing ---"));
29696 //make sure that every generation has a planned allocation start
29697 int gen_number = max_generation - 1;
29698 while (gen_number >= 0)
29700 generation* gen = generation_of (gen_number);
29701 if (0 == generation_plan_allocation_start (gen))
29703 generation_plan_allocation_start (gen) =
29704 bestfit_first_pin + (max_generation - gen_number - 1) * Align (min_obj_size);
29705 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
29706 assert (generation_plan_allocation_start (gen));
29712 uint8_t* first_address = start_address;
29713 //Look for the right pinned plug to start from.
29714 reset_pinned_queue_bos();
29715 uint8_t* planned_ephemeral_seg_end = heap_segment_plan_allocated (seg);
29716 while (!pinned_plug_que_empty_p())
29718 mark* m = oldest_pin();
29719 if ((pinned_plug (m) >= planned_ephemeral_seg_end) && (pinned_plug (m) < end_address))
29721 if (pinned_plug (m) < first_address)
29723 first_address = pinned_plug (m);
29728 deque_pinned_plug();
29731 size_t current_brick = brick_of (first_address);
29732 size_t end_brick = brick_of (end_address-1);
29733 uint8_t* last_plug = 0;
29735 uint8_t* last_pinned_gap = heap_segment_plan_allocated (seg);
29736 BOOL leftp = FALSE;
29738 dprintf (3, ("start addr: %Ix, first addr: %Ix, current oldest pin: %Ix",
29739 start_address, first_address, pinned_plug (oldest_pin())));
29741 while (current_brick <= end_brick)
29743 int brick_entry = brick_table [ current_brick ];
29744 if (brick_entry >= 0)
29746 realloc_in_brick ((brick_address (current_brick) + brick_entry - 1),
29747 last_plug, start_address, consing_gen,
29748 active_new_gen_number, last_pinned_gap,
29754 if (last_plug != 0)
29756 realloc_plug (end_address - last_plug, last_plug, consing_gen,
29758 active_new_gen_number, last_pinned_gap,
29762 #endif //SHORT_PLUGS
29766 //Fix the old segment allocated size
29767 assert (last_pinned_gap >= heap_segment_mem (seg));
29768 assert (last_pinned_gap <= heap_segment_committed (seg));
29769 heap_segment_plan_allocated (seg) = last_pinned_gap;
29772 void gc_heap::verify_no_pins (uint8_t* start, uint8_t* end)
29775 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
29777 BOOL contains_pinned_plugs = FALSE;
29780 while (mi != mark_stack_tos)
29782 m = pinned_plug_of (mi);
29783 if ((pinned_plug (m) >= start) && (pinned_plug (m) < end))
29785 contains_pinned_plugs = TRUE;
29792 if (contains_pinned_plugs)
29797 #endif //VERIFY_HEAP
29800 void gc_heap::set_expand_in_full_gc (int condemned_gen_number)
29802 if (!should_expand_in_full_gc)
29804 if ((condemned_gen_number != max_generation) &&
29805 (settings.pause_mode != pause_low_latency) &&
29806 (settings.pause_mode != pause_sustained_low_latency))
29808 should_expand_in_full_gc = TRUE;
29813 void gc_heap::save_ephemeral_generation_starts()
29815 for (int ephemeral_generation = 0; ephemeral_generation < max_generation; ephemeral_generation++)
29817 saved_ephemeral_plan_start[ephemeral_generation] =
29818 generation_plan_allocation_start (generation_of (ephemeral_generation));
29819 saved_ephemeral_plan_start_size[ephemeral_generation] =
29820 generation_plan_allocation_start_size (generation_of (ephemeral_generation));
29824 generation* gc_heap::expand_heap (int condemned_generation,
29825 generation* consing_gen,
29826 heap_segment* new_heap_segment)
29828 UNREFERENCED_PARAMETER(condemned_generation);
29829 assert (condemned_generation >= (max_generation -1));
29830 unsigned int active_new_gen_number = max_generation; //Set one too high to get generation gap
29831 uint8_t* start_address = generation_limit (max_generation);
29832 uint8_t* end_address = heap_segment_allocated (ephemeral_heap_segment);
29833 BOOL should_promote_ephemeral = FALSE;
29834 ptrdiff_t eph_size = total_ephemeral_size;
29835 #ifdef BACKGROUND_GC
29836 dprintf(2,("%s: ---- Heap Expansion ----", (recursive_gc_sync::background_running_p() ? "FGC" : "NGC")));
29837 #endif //BACKGROUND_GC
29838 settings.heap_expansion = TRUE;
29840 #ifdef BACKGROUND_GC
29841 if (cm_in_progress)
29843 if (!expanded_in_fgc)
29845 expanded_in_fgc = TRUE;
29848 #endif //BACKGROUND_GC
29850 //reset the elevation state for next time.
29851 dprintf (2, ("Elevation: elevation = el_none"));
29852 if (settings.should_lock_elevation && !expand_reused_seg_p())
29853 settings.should_lock_elevation = FALSE;
29855 heap_segment* new_seg = new_heap_segment;
29858 return consing_gen;
29860 //copy the card and brick tables
29861 if (g_gc_card_table!= card_table)
29862 copy_brick_card_table();
29864 BOOL new_segment_p = (heap_segment_next (new_seg) == 0);
29865 dprintf (2, ("new_segment_p %Ix", (size_t)new_segment_p));
29867 assert (generation_plan_allocation_start (generation_of (max_generation-1)));
29868 assert (generation_plan_allocation_start (generation_of (max_generation-1)) >=
29869 heap_segment_mem (ephemeral_heap_segment));
29870 assert (generation_plan_allocation_start (generation_of (max_generation-1)) <=
29871 heap_segment_committed (ephemeral_heap_segment));
29873 assert (generation_plan_allocation_start (youngest_generation));
29874 assert (generation_plan_allocation_start (youngest_generation) <
29875 heap_segment_plan_allocated (ephemeral_heap_segment));
29877 if (settings.pause_mode == pause_no_gc)
29879 // We don't reuse for no gc, so the size used on the new eph seg is eph_size.
29880 if ((size_t)(heap_segment_reserved (new_seg) - heap_segment_mem (new_seg)) < (eph_size + soh_allocation_no_gc))
29881 should_promote_ephemeral = TRUE;
29887 should_promote_ephemeral = dt_low_ephemeral_space_p (tuning_deciding_promote_ephemeral);
29891 if (should_promote_ephemeral)
29893 ephemeral_promotion = TRUE;
29894 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, expand_new_seg_ep);
29895 dprintf (2, ("promoting ephemeral"));
29896 save_ephemeral_generation_starts();
29900 // commit the new ephemeral segment all at once if it is a new one.
29901 if ((eph_size > 0) && new_segment_p)
29903 #ifdef FEATURE_STRUCTALIGN
29904 // The destination may require a larger alignment padding than the source.
29905 // Assume the worst possible alignment padding.
29906 eph_size += ComputeStructAlignPad(heap_segment_mem (new_seg), MAX_STRUCTALIGN, OBJECT_ALIGNMENT_OFFSET);
29907 #endif // FEATURE_STRUCTALIGN
29908 #ifdef RESPECT_LARGE_ALIGNMENT
29909 //Since the generation start can be larger than min_obj_size
29910 //The alignment could be switched.
29911 eph_size += switch_alignment_size(FALSE);
29912 #endif //RESPECT_LARGE_ALIGNMENT
29913 //Since the generation start can be larger than min_obj_size
29914 //Compare the alignment of the first object in gen1
29915 if (grow_heap_segment (new_seg, heap_segment_mem (new_seg) + eph_size) == 0)
29917 fgm_result.set_fgm (fgm_commit_eph_segment, eph_size, FALSE);
29918 return consing_gen;
29920 heap_segment_used (new_seg) = heap_segment_committed (new_seg);
29923 //Fix the end of the old ephemeral heap segment
29924 heap_segment_plan_allocated (ephemeral_heap_segment) =
29925 generation_plan_allocation_start (generation_of (max_generation-1));
29927 dprintf (3, ("Old ephemeral allocated set to %Ix",
29928 (size_t)heap_segment_plan_allocated (ephemeral_heap_segment)));
29933 // TODO - Is this really necessary? We should think about it.
29934 //initialize the first brick
29935 size_t first_brick = brick_of (heap_segment_mem (new_seg));
29936 set_brick (first_brick,
29937 heap_segment_mem (new_seg) - brick_address (first_brick));
29940 //From this point on, we cannot run out of memory
29942 //reset the allocation of the consing generation back to the end of the
29943 //old ephemeral segment
29944 generation_allocation_limit (consing_gen) =
29945 heap_segment_plan_allocated (ephemeral_heap_segment);
29946 generation_allocation_pointer (consing_gen) = generation_allocation_limit (consing_gen);
29947 generation_allocation_segment (consing_gen) = ephemeral_heap_segment;
29949 //clear the generation gap for all of the ephemeral generations
29951 int generation_num = max_generation-1;
29952 while (generation_num >= 0)
29954 generation* gen = generation_of (generation_num);
29955 generation_plan_allocation_start (gen) = 0;
29960 heap_segment* old_seg = ephemeral_heap_segment;
29961 ephemeral_heap_segment = new_seg;
29963 //Note: the ephemeral segment shouldn't be threaded onto the segment chain
29964 //because the relocation and compact phases shouldn't see it
29966 // set the generation members used by allocate_in_expanded_heap
29967 // and switch to ephemeral generation
29968 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
29970 if (!should_promote_ephemeral)
29972 realloc_plugs (consing_gen, old_seg, start_address, end_address,
29973 active_new_gen_number);
29978 repair_allocation_in_expanded_heap (consing_gen);
29981 // assert that the generation gap for all of the ephemeral generations were allocated.
29984 int generation_num = max_generation-1;
29985 while (generation_num >= 0)
29987 generation* gen = generation_of (generation_num);
29988 assert (generation_plan_allocation_start (gen));
29994 if (!new_segment_p)
29996 dprintf (2, ("Demoting ephemeral segment"));
29997 //demote the entire segment.
29998 settings.demotion = TRUE;
29999 get_gc_data_per_heap()->set_mechanism_bit (gc_demotion_bit);
30000 demotion_low = heap_segment_mem (ephemeral_heap_segment);
30001 demotion_high = heap_segment_reserved (ephemeral_heap_segment);
30005 demotion_low = MAX_PTR;
30007 #ifndef MULTIPLE_HEAPS
30008 settings.demotion = FALSE;
30009 get_gc_data_per_heap()->clear_mechanism_bit (gc_demotion_bit);
30010 #endif //!MULTIPLE_HEAPS
30012 ptrdiff_t eph_size1 = total_ephemeral_size;
30013 MAYBE_UNUSED_VAR(eph_size1);
30015 if (!should_promote_ephemeral && new_segment_p)
30017 assert (eph_size1 <= eph_size);
30020 if (heap_segment_mem (old_seg) == heap_segment_plan_allocated (old_seg))
30022 // This is to catch when we accidently delete a segment that has pins.
30023 verify_no_pins (heap_segment_mem (old_seg), heap_segment_reserved (old_seg));
30026 verify_no_pins (heap_segment_plan_allocated (old_seg), heap_segment_reserved(old_seg));
30028 dprintf(2,("---- End of Heap Expansion ----"));
30029 return consing_gen;
30032 void gc_heap::set_static_data()
30034 static_data* pause_mode_sdata = static_data_table[latency_level];
30035 for (int i = 0; i < NUMBERGENERATIONS; i++)
30037 dynamic_data* dd = dynamic_data_of (i);
30038 static_data* sdata = &pause_mode_sdata[i];
30041 dd->min_size = sdata->min_size;
30043 dprintf (GTC_LOG, ("PM: %d, gen%d: min: %Id, max: %Id, fr_l: %Id, fr_b: %d%%",
30044 settings.pause_mode,i,
30045 dd->min_size, dd_max_size (dd),
30046 sdata->fragmentation_limit, (int)(sdata->fragmentation_burden_limit * 100)));
30050 // Initialize the values that are not const.
30051 void gc_heap::init_static_data()
30053 size_t gen0_min_size = get_gen0_min_size();
30055 size_t gen0_max_size =
30056 #ifdef MULTIPLE_HEAPS
30057 max (6*1024*1024, min ( Align(soh_segment_size/2), 200*1024*1024));
30058 #else //MULTIPLE_HEAPS
30059 (gc_can_use_concurrent ?
30061 max (6*1024*1024, min ( Align(soh_segment_size/2), 200*1024*1024)));
30062 #endif //MULTIPLE_HEAPS
30064 if (heap_hard_limit)
30066 size_t gen0_max_size_seg = soh_segment_size / 4;
30067 dprintf (GTC_LOG, ("limit gen0 max %Id->%Id", gen0_max_size, gen0_max_size_seg));
30068 gen0_max_size = min (gen0_max_size, gen0_max_size_seg);
30071 size_t gen0_max_size_config = (size_t)GCConfig::GetGCGen0MaxBudget();
30073 if (gen0_max_size_config)
30075 gen0_max_size = min (gen0_max_size, gen0_max_size_config);
30078 gen0_max_size = Align (gen0_max_size);
30080 gen0_min_size = min (gen0_min_size, gen0_max_size);
30082 // TODO: gen0_max_size has a 200mb cap; gen1_max_size should also have a cap.
30083 size_t gen1_max_size = (size_t)
30084 #ifdef MULTIPLE_HEAPS
30085 max (6*1024*1024, Align(soh_segment_size/2));
30086 #else //MULTIPLE_HEAPS
30087 (gc_can_use_concurrent ?
30089 max (6*1024*1024, Align(soh_segment_size/2)));
30090 #endif //MULTIPLE_HEAPS
30092 dprintf (GTC_LOG, ("gen0 min: %Id, max: %Id, gen1 max: %Id",
30093 gen0_min_size, gen0_max_size, gen1_max_size));
30095 for (int i = latency_level_first; i <= latency_level_last; i++)
30097 static_data_table[i][0].min_size = gen0_min_size;
30098 static_data_table[i][0].max_size = gen0_max_size;
30099 static_data_table[i][1].max_size = gen1_max_size;
30103 bool gc_heap::init_dynamic_data()
30105 qpf = GCToOSInterface::QueryPerformanceFrequency();
30107 uint32_t now = (uint32_t)GetHighPrecisionTimeStamp();
30111 for (int i = 0; i <= max_generation+1; i++)
30113 dynamic_data* dd = dynamic_data_of (i);
30115 dd->time_clock = now;
30116 dd->current_size = 0;
30117 dd->promoted_size = 0;
30118 dd->collection_count = 0;
30119 dd->new_allocation = dd->min_size;
30120 dd->gc_new_allocation = dd->new_allocation;
30121 dd->desired_allocation = dd->new_allocation;
30122 dd->fragmentation = 0;
30125 #ifdef GC_CONFIG_DRIVEN
30126 if (heap_number == 0)
30128 #endif //GC_CONFIG_DRIVEN
30133 float gc_heap::surv_to_growth (float cst, float limit, float max_limit)
30135 if (cst < ((max_limit - limit ) / (limit * (max_limit-1.0f))))
30136 return ((limit - limit*cst) / (1.0f - (cst * limit)));
30142 //if the allocation budget wasn't exhausted, the new budget may be wrong because the survival may
30143 //not be correct (collection happened too soon). Correct with a linear estimation based on the previous
30144 //value of the budget
30145 static size_t linear_allocation_model (float allocation_fraction, size_t new_allocation,
30146 size_t previous_desired_allocation, size_t collection_count)
30148 if ((allocation_fraction < 0.95) && (allocation_fraction > 0.0))
30150 dprintf (2, ("allocation fraction: %d", (int)(allocation_fraction/100.0)));
30151 new_allocation = (size_t)(allocation_fraction*new_allocation + (1.0-allocation_fraction)*previous_desired_allocation);
30154 size_t smoothing = 3; // exponential smoothing factor
30155 if (smoothing > collection_count)
30156 smoothing = collection_count;
30157 new_allocation = new_allocation / smoothing + ((previous_desired_allocation / smoothing) * (smoothing-1));
30159 UNREFERENCED_PARAMETER(collection_count);
30161 return new_allocation;
30164 size_t gc_heap::desired_new_allocation (dynamic_data* dd,
30165 size_t out, int gen_number,
30168 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
30170 if (dd_begin_data_size (dd) == 0)
30172 size_t new_allocation = dd_min_size (dd);
30173 current_gc_data_per_heap->gen_data[gen_number].new_allocation = new_allocation;
30174 return new_allocation;
30179 size_t previous_desired_allocation = dd_desired_allocation (dd);
30180 size_t current_size = dd_current_size (dd);
30181 float max_limit = dd_max_limit (dd);
30182 float limit = dd_limit (dd);
30183 size_t min_gc_size = dd_min_size (dd);
30185 size_t max_size = dd_max_size (dd);
30186 size_t new_allocation = 0;
30187 float allocation_fraction = (float) (dd_desired_allocation (dd) - dd_gc_new_allocation (dd)) / (float) (dd_desired_allocation (dd));
30188 if (gen_number >= max_generation)
30190 size_t new_size = 0;
30192 cst = min (1.0f, float (out) / float (dd_begin_data_size (dd)));
30194 f = surv_to_growth (cst, limit, max_limit);
30195 size_t max_growth_size = (size_t)(max_size / f);
30196 if (current_size >= max_growth_size)
30198 new_size = max_size;
30202 new_size = (size_t) min (max ( (f * current_size), min_gc_size), max_size);
30205 assert ((new_size >= current_size) || (new_size == max_size));
30207 if (gen_number == max_generation)
30209 new_allocation = max((new_size - current_size), min_gc_size);
30211 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
30212 dd_desired_allocation (dd), dd_collection_count (dd));
30214 if ((dd_fragmentation (dd) > ((size_t)((f-1)*current_size))))
30216 //reducing allocation in case of fragmentation
30217 size_t new_allocation1 = max (min_gc_size,
30219 (size_t)((float)new_allocation * current_size /
30220 ((float)current_size + 2*dd_fragmentation (dd))));
30221 dprintf (2, ("Reducing max_gen allocation due to fragmentation from %Id to %Id",
30222 new_allocation, new_allocation1));
30223 new_allocation = new_allocation1;
30226 else //large object heap
30228 uint32_t memory_load = 0;
30229 uint64_t available_physical = 0;
30230 get_memory_info (&memory_load, &available_physical);
30232 if (heap_hard_limit)
30234 size_t loh_allocated = 0;
30235 size_t loh_committed = committed_size (true, &loh_allocated);
30236 dprintf (1, ("GC#%Id h%d, GMI: LOH budget, LOH commit %Id (obj %Id, frag %Id), total commit: %Id (recorded: %Id)",
30237 (size_t)settings.gc_index, heap_number,
30238 loh_committed, loh_allocated,
30239 dd_fragmentation (dynamic_data_of (max_generation + 1)),
30240 get_total_committed_size(), (current_total_committed - current_total_committed_bookkeeping)));
30243 if (heap_number == 0)
30244 settings.exit_memory_load = memory_load;
30245 if (available_physical > 1024*1024)
30246 available_physical -= 1024*1024;
30248 uint64_t available_free = available_physical + (uint64_t)generation_free_list_space (generation_of (gen_number));
30249 if (available_free > (uint64_t)MAX_PTR)
30251 available_free = (uint64_t)MAX_PTR;
30254 //try to avoid OOM during large object allocation
30255 new_allocation = max (min(max((new_size - current_size), dd_desired_allocation (dynamic_data_of (max_generation))),
30256 (size_t)available_free),
30257 max ((current_size/4), min_gc_size));
30259 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
30260 dd_desired_allocation (dd), dd_collection_count (dd));
30266 size_t survivors = out;
30267 cst = float (survivors) / float (dd_begin_data_size (dd));
30268 f = surv_to_growth (cst, limit, max_limit);
30269 new_allocation = (size_t) min (max ((f * (survivors)), min_gc_size), max_size);
30271 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
30272 dd_desired_allocation (dd), dd_collection_count (dd));
30274 if (gen_number == 0)
30279 //printf ("%f, %Id\n", cst, new_allocation);
30280 size_t free_space = generation_free_list_space (generation_of (gen_number));
30281 // DTREVIEW - is min_gc_size really a good choice?
30282 // on 64-bit this will almost always be true.
30283 dprintf (GTC_LOG, ("frag: %Id, min: %Id", free_space, min_gc_size));
30284 if (free_space > min_gc_size)
30286 settings.gen0_reduction_count = 2;
30290 if (settings.gen0_reduction_count > 0)
30291 settings.gen0_reduction_count--;
30294 if (settings.gen0_reduction_count > 0)
30296 dprintf (2, ("Reducing new allocation based on fragmentation"));
30297 new_allocation = min (new_allocation,
30298 max (min_gc_size, (max_size/3)));
30303 size_t new_allocation_ret =
30304 Align (new_allocation, get_alignment_constant (!(gen_number == (max_generation+1))));
30305 int gen_data_index = gen_number;
30306 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_data_index]);
30307 gen_data->new_allocation = new_allocation_ret;
30309 dd_surv (dd) = cst;
30311 #ifdef SIMPLE_DPRINTF
30312 dprintf (1, ("h%d g%d surv: %Id current: %Id alloc: %Id (%d%%) f: %d%% new-size: %Id new-alloc: %Id",
30313 heap_number, gen_number, out, current_size, (dd_desired_allocation (dd) - dd_gc_new_allocation (dd)),
30314 (int)(cst*100), (int)(f*100), current_size + new_allocation, new_allocation));
30316 dprintf (1,("gen: %d in: %Id out: %Id ", gen_number, generation_allocation_size (generation_of (gen_number)), out));
30317 dprintf (1,("current: %Id alloc: %Id ", current_size, (dd_desired_allocation (dd) - dd_gc_new_allocation (dd))));
30318 dprintf (1,(" surv: %d%% f: %d%% new-size: %Id new-alloc: %Id",
30319 (int)(cst*100), (int)(f*100), current_size + new_allocation, new_allocation));
30320 #endif //SIMPLE_DPRINTF
30322 return new_allocation_ret;
30326 //returns the planned size of a generation (including free list element)
30327 size_t gc_heap::generation_plan_size (int gen_number)
30329 if (0 == gen_number)
30330 return max((heap_segment_plan_allocated (ephemeral_heap_segment) -
30331 generation_plan_allocation_start (generation_of (gen_number))),
30332 (int)Align (min_obj_size));
30335 generation* gen = generation_of (gen_number);
30336 if (heap_segment_rw (generation_start_segment (gen)) == ephemeral_heap_segment)
30337 return (generation_plan_allocation_start (generation_of (gen_number - 1)) -
30338 generation_plan_allocation_start (generation_of (gen_number)));
30341 size_t gensize = 0;
30342 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
30344 PREFIX_ASSUME(seg != NULL);
30346 while (seg && (seg != ephemeral_heap_segment))
30348 gensize += heap_segment_plan_allocated (seg) -
30349 heap_segment_mem (seg);
30350 seg = heap_segment_next_rw (seg);
30354 gensize += (generation_plan_allocation_start (generation_of (gen_number - 1)) -
30355 heap_segment_mem (ephemeral_heap_segment));
30363 //returns the size of a generation (including free list element)
30364 size_t gc_heap::generation_size (int gen_number)
30366 if (0 == gen_number)
30367 return max((heap_segment_allocated (ephemeral_heap_segment) -
30368 generation_allocation_start (generation_of (gen_number))),
30369 (int)Align (min_obj_size));
30372 generation* gen = generation_of (gen_number);
30373 if (heap_segment_rw (generation_start_segment (gen)) == ephemeral_heap_segment)
30374 return (generation_allocation_start (generation_of (gen_number - 1)) -
30375 generation_allocation_start (generation_of (gen_number)));
30378 size_t gensize = 0;
30379 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
30381 PREFIX_ASSUME(seg != NULL);
30383 while (seg && (seg != ephemeral_heap_segment))
30385 gensize += heap_segment_allocated (seg) -
30386 heap_segment_mem (seg);
30387 seg = heap_segment_next_rw (seg);
30391 gensize += (generation_allocation_start (generation_of (gen_number - 1)) -
30392 heap_segment_mem (ephemeral_heap_segment));
30401 size_t gc_heap::compute_in (int gen_number)
30403 assert (gen_number != 0);
30404 dynamic_data* dd = dynamic_data_of (gen_number);
30406 size_t in = generation_allocation_size (generation_of (gen_number));
30408 if (gen_number == max_generation && ephemeral_promotion)
30411 for (int i = 0; i <= max_generation; i++)
30413 dynamic_data* dd = dynamic_data_of (i);
30414 in += dd_survived_size (dd);
30415 if (i != max_generation)
30417 generation_condemned_allocated (generation_of (gen_number)) += dd_survived_size (dd);
30422 dd_gc_new_allocation (dd) -= in;
30423 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
30425 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
30426 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_number]);
30429 generation_allocation_size (generation_of (gen_number)) = 0;
30433 void gc_heap::compute_promoted_allocation (int gen_number)
30435 compute_in (gen_number);
30440 size_t gc_heap::trim_youngest_desired (uint32_t memory_load,
30441 size_t total_new_allocation,
30442 size_t total_min_allocation)
30444 if (memory_load < MAX_ALLOWED_MEM_LOAD)
30446 // If the total of memory load and gen0 budget exceeds
30447 // our max memory load limit, trim the gen0 budget so the total
30448 // is the max memory load limit.
30449 size_t remain_memory_load = (MAX_ALLOWED_MEM_LOAD - memory_load) * mem_one_percent;
30450 return min (total_new_allocation, remain_memory_load);
30454 return max (mem_one_percent, total_min_allocation);
30458 size_t gc_heap::joined_youngest_desired (size_t new_allocation)
30460 dprintf (2, ("Entry memory load: %d; gen0 new_alloc: %Id", settings.entry_memory_load, new_allocation));
30462 size_t final_new_allocation = new_allocation;
30463 if (new_allocation > MIN_YOUNGEST_GEN_DESIRED)
30465 uint32_t num_heaps = 1;
30467 #ifdef MULTIPLE_HEAPS
30468 num_heaps = gc_heap::n_heaps;
30469 #endif //MULTIPLE_HEAPS
30471 size_t total_new_allocation = new_allocation * num_heaps;
30472 size_t total_min_allocation = MIN_YOUNGEST_GEN_DESIRED * num_heaps;
30474 if ((settings.entry_memory_load >= MAX_ALLOWED_MEM_LOAD) ||
30475 (total_new_allocation > max (youngest_gen_desired_th, total_min_allocation)))
30477 uint32_t memory_load = 0;
30478 get_memory_info (&memory_load);
30479 settings.exit_memory_load = memory_load;
30480 dprintf (2, ("Current emory load: %d", memory_load));
30482 size_t final_total =
30483 trim_youngest_desired (memory_load, total_new_allocation, total_min_allocation);
30484 size_t max_new_allocation =
30485 #ifdef MULTIPLE_HEAPS
30486 dd_max_size (g_heaps[0]->dynamic_data_of (0));
30487 #else //MULTIPLE_HEAPS
30488 dd_max_size (dynamic_data_of (0));
30489 #endif //MULTIPLE_HEAPS
30491 final_new_allocation = min (Align ((final_total / num_heaps), get_alignment_constant (TRUE)), max_new_allocation);
30495 if (final_new_allocation < new_allocation)
30497 settings.gen0_reduction_count = 2;
30500 return final_new_allocation;
30505 gc_history_per_heap* gc_heap::get_gc_data_per_heap()
30507 #ifdef BACKGROUND_GC
30508 return (settings.concurrent ? &bgc_data_per_heap : &gc_data_per_heap);
30510 return &gc_data_per_heap;
30511 #endif //BACKGROUND_GC
30514 void gc_heap::compute_new_dynamic_data (int gen_number)
30516 PREFIX_ASSUME(gen_number >= 0);
30517 PREFIX_ASSUME(gen_number <= max_generation);
30519 dynamic_data* dd = dynamic_data_of (gen_number);
30520 generation* gen = generation_of (gen_number);
30521 size_t in = (gen_number==0) ? 0 : compute_in (gen_number);
30523 size_t total_gen_size = generation_size (gen_number);
30524 //keep track of fragmentation
30525 dd_fragmentation (dd) = generation_free_list_space (gen) + generation_free_obj_space (gen);
30526 dd_current_size (dd) = total_gen_size - dd_fragmentation (dd);
30528 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
30530 size_t out = dd_survived_size (dd);
30532 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_number]);
30533 gen_data->size_after = total_gen_size;
30534 gen_data->free_list_space_after = generation_free_list_space (gen);
30535 gen_data->free_obj_space_after = generation_free_obj_space (gen);
30537 if ((settings.pause_mode == pause_low_latency) && (gen_number <= 1))
30539 // When we are in the low latency mode, we can still be
30540 // condemning more than gen1's 'cause of induced GCs.
30541 dd_desired_allocation (dd) = low_latency_alloc;
30545 if (gen_number == 0)
30547 //compensate for dead finalizable objects promotion.
30548 //they shoudn't be counted for growth.
30549 size_t final_promoted = 0;
30550 final_promoted = min (promoted_bytes (heap_number), out);
30551 // Prefast: this is clear from above but prefast needs to be told explicitly
30552 PREFIX_ASSUME(final_promoted <= out);
30554 dprintf (2, ("gen: %d final promoted: %Id", gen_number, final_promoted));
30555 dd_freach_previous_promotion (dd) = final_promoted;
30556 size_t lower_bound = desired_new_allocation (dd, out-final_promoted, gen_number, 0);
30558 if (settings.condemned_generation == 0)
30560 //there is no noise.
30561 dd_desired_allocation (dd) = lower_bound;
30565 size_t higher_bound = desired_new_allocation (dd, out, gen_number, 1);
30567 // <TODO>This assert was causing AppDomains\unload\test1n\test1nrun.bat to fail</TODO>
30568 //assert ( lower_bound <= higher_bound);
30570 //discount the noise. Change the desired allocation
30571 //only if the previous value is outside of the range.
30572 if (dd_desired_allocation (dd) < lower_bound)
30574 dd_desired_allocation (dd) = lower_bound;
30576 else if (dd_desired_allocation (dd) > higher_bound)
30578 dd_desired_allocation (dd) = higher_bound;
30580 #if defined (BIT64) && !defined (MULTIPLE_HEAPS)
30581 dd_desired_allocation (dd) = joined_youngest_desired (dd_desired_allocation (dd));
30582 #endif // BIT64 && !MULTIPLE_HEAPS
30583 trim_youngest_desired_low_memory();
30584 dprintf (2, ("final gen0 new_alloc: %Id", dd_desired_allocation (dd)));
30589 dd_desired_allocation (dd) = desired_new_allocation (dd, out, gen_number, 0);
30593 gen_data->pinned_surv = dd_pinned_survived_size (dd);
30594 gen_data->npinned_surv = dd_survived_size (dd) - dd_pinned_survived_size (dd);
30596 dd_gc_new_allocation (dd) = dd_desired_allocation (dd);
30597 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
30600 dd_promoted_size (dd) = out;
30601 if (gen_number == max_generation)
30603 dd = dynamic_data_of (max_generation+1);
30604 total_gen_size = generation_size (max_generation + 1);
30605 dd_fragmentation (dd) = generation_free_list_space (large_object_generation) +
30606 generation_free_obj_space (large_object_generation);
30607 dd_current_size (dd) = total_gen_size - dd_fragmentation (dd);
30608 dd_survived_size (dd) = dd_current_size (dd);
30610 out = dd_current_size (dd);
30611 dd_desired_allocation (dd) = desired_new_allocation (dd, out, max_generation+1, 0);
30612 dd_gc_new_allocation (dd) = Align (dd_desired_allocation (dd),
30613 get_alignment_constant (FALSE));
30614 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
30616 gen_data = &(current_gc_data_per_heap->gen_data[max_generation+1]);
30617 gen_data->size_after = total_gen_size;
30618 gen_data->free_list_space_after = generation_free_list_space (large_object_generation);
30619 gen_data->free_obj_space_after = generation_free_obj_space (large_object_generation);
30620 gen_data->npinned_surv = out;
30621 #ifdef BACKGROUND_GC
30622 end_loh_size = total_gen_size;
30623 #endif //BACKGROUND_GC
30625 dd_promoted_size (dd) = out;
30629 void gc_heap::trim_youngest_desired_low_memory()
30631 if (g_low_memory_status)
30633 size_t committed_mem = 0;
30634 heap_segment* seg = generation_start_segment (generation_of (max_generation));
30637 committed_mem += heap_segment_committed (seg) - heap_segment_mem (seg);
30638 seg = heap_segment_next (seg);
30640 seg = generation_start_segment (generation_of (max_generation + 1));
30643 committed_mem += heap_segment_committed (seg) - heap_segment_mem (seg);
30644 seg = heap_segment_next (seg);
30647 dynamic_data* dd = dynamic_data_of (0);
30648 size_t current = dd_desired_allocation (dd);
30649 size_t candidate = max (Align ((committed_mem / 10), get_alignment_constant(FALSE)), dd_min_size (dd));
30651 dd_desired_allocation (dd) = min (current, candidate);
30655 void gc_heap::decommit_ephemeral_segment_pages()
30657 if (settings.concurrent)
30662 size_t slack_space = heap_segment_committed (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment);
30664 dynamic_data* dd = dynamic_data_of (0);
30666 #ifndef MULTIPLE_HEAPS
30667 size_t extra_space = (g_low_memory_status ? 0 : (512 * 1024));
30668 size_t decommit_timeout = (g_low_memory_status ? 0 : GC_EPHEMERAL_DECOMMIT_TIMEOUT);
30669 size_t ephemeral_elapsed = dd_time_clock(dd) - gc_last_ephemeral_decommit_time;
30671 if (dd_desired_allocation (dd) > gc_gen0_desired_high)
30673 gc_gen0_desired_high = dd_desired_allocation (dd) + extra_space;
30676 if (ephemeral_elapsed >= decommit_timeout)
30678 slack_space = min (slack_space, gc_gen0_desired_high);
30680 gc_last_ephemeral_decommit_time = dd_time_clock(dd);
30681 gc_gen0_desired_high = 0;
30683 #endif //!MULTIPLE_HEAPS
30685 if (settings.condemned_generation >= (max_generation-1))
30687 size_t new_slack_space =
30689 max(min(min(soh_segment_size/32, dd_max_size(dd)), (generation_size (max_generation) / 10)), dd_desired_allocation(dd));
30691 #ifdef FEATURE_CORECLR
30692 dd_desired_allocation (dd);
30695 #endif //FEATURE_CORECLR
30698 slack_space = min (slack_space, new_slack_space);
30701 decommit_heap_segment_pages (ephemeral_heap_segment, slack_space);
30703 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
30704 current_gc_data_per_heap->extra_gen0_committed = heap_segment_committed (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment);
30707 //This is meant to be called by decide_on_compacting.
30709 size_t gc_heap::generation_fragmentation (generation* gen,
30710 generation* consing_gen,
30714 uint8_t* alloc = generation_allocation_pointer (consing_gen);
30715 // If the allocation pointer has reached the ephemeral segment
30716 // fine, otherwise the whole ephemeral segment is considered
30718 if (in_range_for_segment (alloc, ephemeral_heap_segment))
30720 if (alloc <= heap_segment_allocated(ephemeral_heap_segment))
30721 frag = end - alloc;
30724 // case when no survivors, allocated set to beginning
30727 dprintf (3, ("ephemeral frag: %Id", frag));
30730 frag = (heap_segment_allocated (ephemeral_heap_segment) -
30731 heap_segment_mem (ephemeral_heap_segment));
30732 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
30734 PREFIX_ASSUME(seg != NULL);
30736 while (seg != ephemeral_heap_segment)
30738 frag += (heap_segment_allocated (seg) -
30739 heap_segment_plan_allocated (seg));
30740 dprintf (3, ("seg: %Ix, frag: %Id", (size_t)seg,
30741 (heap_segment_allocated (seg) -
30742 heap_segment_plan_allocated (seg))));
30744 seg = heap_segment_next_rw (seg);
30747 dprintf (3, ("frag: %Id discounting pinned plugs", frag));
30748 //add the length of the dequeued plug free space
30750 while (bos < mark_stack_bos)
30752 frag += (pinned_len (pinned_plug_of (bos)));
30759 // for SOH this returns the total sizes of the generation and its
30760 // younger generation(s).
30761 // for LOH this returns just LOH size.
30762 size_t gc_heap::generation_sizes (generation* gen)
30765 if (generation_start_segment (gen ) == ephemeral_heap_segment)
30766 result = (heap_segment_allocated (ephemeral_heap_segment) -
30767 generation_allocation_start (gen));
30770 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
30772 PREFIX_ASSUME(seg != NULL);
30776 result += (heap_segment_allocated (seg) -
30777 heap_segment_mem (seg));
30778 seg = heap_segment_next_in_range (seg);
30785 size_t gc_heap::estimated_reclaim (int gen_number)
30787 dynamic_data* dd = dynamic_data_of (gen_number);
30788 size_t gen_allocated = (dd_desired_allocation (dd) - dd_new_allocation (dd));
30789 size_t gen_total_size = gen_allocated + dd_current_size (dd);
30790 size_t est_gen_surv = (size_t)((float) (gen_total_size) * dd_surv (dd));
30791 size_t est_gen_free = gen_total_size - est_gen_surv + dd_fragmentation (dd);
30793 dprintf (GTC_LOG, ("h%d gen%d total size: %Id, est dead space: %Id (s: %d, allocated: %Id), frag: %Id",
30794 heap_number, gen_number,
30797 (int)(dd_surv (dd) * 100),
30799 dd_fragmentation (dd)));
30801 return est_gen_free;
30804 BOOL gc_heap::decide_on_compacting (int condemned_gen_number,
30805 size_t fragmentation,
30806 BOOL& should_expand)
30808 BOOL should_compact = FALSE;
30809 should_expand = FALSE;
30810 generation* gen = generation_of (condemned_gen_number);
30811 dynamic_data* dd = dynamic_data_of (condemned_gen_number);
30812 size_t gen_sizes = generation_sizes(gen);
30813 float fragmentation_burden = ( ((0 == fragmentation) || (0 == gen_sizes)) ? (0.0f) :
30814 (float (fragmentation) / gen_sizes) );
30816 dprintf (GTC_LOG, ("h%d g%d fragmentation: %Id (%d%%)",
30817 heap_number, settings.condemned_generation,
30818 fragmentation, (int)(fragmentation_burden * 100.0)));
30821 // for pure GC stress runs we need compaction, for GC stress "mix"
30822 // we need to ensure a better mix of compacting and sweeping collections
30823 if (GCStress<cfg_any>::IsEnabled() && !settings.concurrent
30824 && !g_pConfig->IsGCStressMix())
30825 should_compact = TRUE;
30828 // in GC stress "mix" mode, for stress induced collections make sure we
30829 // keep sweeps and compactions relatively balanced. do not (yet) force sweeps
30830 // against the GC's determination, as it may lead to premature OOMs.
30831 if (g_pConfig->IsGCStressMix() && settings.stress_induced)
30833 int compactions = g_GCStatistics.cntCompactFGC+g_GCStatistics.cntCompactNGC;
30834 int sweeps = g_GCStatistics.cntFGC + g_GCStatistics.cntNGC - compactions;
30835 if (compactions < sweeps / 10)
30837 should_compact = TRUE;
30841 #endif //STRESS_HEAP
30843 if (GCConfig::GetForceCompact())
30844 should_compact = TRUE;
30846 if ((condemned_gen_number == max_generation) && last_gc_before_oom)
30848 should_compact = TRUE;
30849 last_gc_before_oom = FALSE;
30850 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_last_gc);
30853 if (settings.reason == reason_induced_compacting)
30855 dprintf (2, ("induced compacting GC"));
30856 should_compact = TRUE;
30857 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_induced_compacting);
30860 if (settings.reason == reason_pm_full_gc)
30862 assert (condemned_gen_number == max_generation);
30863 if (heap_number == 0)
30865 dprintf (GTC_LOG, ("PM doing compacting full GC after a gen1"));
30867 should_compact = TRUE;
30870 dprintf (2, ("Fragmentation: %d Fragmentation burden %d%%",
30871 fragmentation, (int) (100*fragmentation_burden)));
30873 if (provisional_mode_triggered && (condemned_gen_number == (max_generation - 1)))
30875 dprintf (GTC_LOG, ("gen1 in PM always compact"));
30876 should_compact = TRUE;
30879 if (!should_compact)
30881 if (dt_low_ephemeral_space_p (tuning_deciding_compaction))
30883 dprintf(GTC_LOG, ("compacting due to low ephemeral"));
30884 should_compact = TRUE;
30885 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_low_ephemeral);
30889 if (should_compact)
30891 if ((condemned_gen_number >= (max_generation - 1)))
30893 if (dt_low_ephemeral_space_p (tuning_deciding_expansion))
30895 dprintf (GTC_LOG,("Not enough space for all ephemeral generations with compaction"));
30896 should_expand = TRUE;
30902 BOOL high_memory = FALSE;
30905 if (!should_compact)
30907 // We are not putting this in dt_high_frag_p because it's not exactly
30908 // high fragmentation - it's just enough planned fragmentation for us to
30909 // want to compact. Also the "fragmentation" we are talking about here
30910 // is different from anywhere else.
30911 BOOL frag_exceeded = ((fragmentation >= dd_fragmentation_limit (dd)) &&
30912 (fragmentation_burden >= dd_fragmentation_burden_limit (dd)));
30916 #ifdef BACKGROUND_GC
30917 // do not force compaction if this was a stress-induced GC
30918 IN_STRESS_HEAP(if (!settings.stress_induced))
30920 #endif // BACKGROUND_GC
30921 assert (settings.concurrent == FALSE);
30922 should_compact = TRUE;
30923 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_high_frag);
30924 #ifdef BACKGROUND_GC
30926 #endif // BACKGROUND_GC
30930 // check for high memory situation
30931 if(!should_compact)
30933 uint32_t num_heaps = 1;
30934 #ifdef MULTIPLE_HEAPS
30935 num_heaps = gc_heap::n_heaps;
30936 #endif // MULTIPLE_HEAPS
30938 ptrdiff_t reclaim_space = generation_size(max_generation) - generation_plan_size(max_generation);
30940 if((settings.entry_memory_load >= high_memory_load_th) && (settings.entry_memory_load < v_high_memory_load_th))
30942 if(reclaim_space > (int64_t)(min_high_fragmentation_threshold (entry_available_physical_mem, num_heaps)))
30944 dprintf(GTC_LOG,("compacting due to fragmentation in high memory"));
30945 should_compact = TRUE;
30946 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_high_mem_frag);
30948 high_memory = TRUE;
30950 else if(settings.entry_memory_load >= v_high_memory_load_th)
30952 if(reclaim_space > (ptrdiff_t)(min_reclaim_fragmentation_threshold (num_heaps)))
30954 dprintf(GTC_LOG,("compacting due to fragmentation in very high memory"));
30955 should_compact = TRUE;
30956 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_vhigh_mem_frag);
30958 high_memory = TRUE;
30964 // The purpose of calling ensure_gap_allocation here is to make sure
30965 // that we actually are able to commit the memory to allocate generation
30967 if ((should_compact == FALSE) &&
30968 (ensure_gap_allocation (condemned_gen_number) == FALSE))
30970 should_compact = TRUE;
30971 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_no_gaps);
30974 if (settings.condemned_generation == max_generation)
30976 //check the progress
30979 (high_memory && !should_compact) ||
30981 (generation_plan_allocation_start (generation_of (max_generation - 1)) >=
30982 generation_allocation_start (generation_of (max_generation - 1))))
30984 dprintf (1, ("gen1 start %Ix->%Ix, gen2 size %Id->%Id, lock elevation",
30985 generation_allocation_start (generation_of (max_generation - 1)),
30986 generation_plan_allocation_start (generation_of (max_generation - 1)),
30987 generation_size (max_generation),
30988 generation_plan_size (max_generation)));
30989 //no progress -> lock
30990 settings.should_lock_elevation = TRUE;
30994 if (settings.pause_mode == pause_no_gc)
30996 should_compact = TRUE;
30997 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_plan_allocated (ephemeral_heap_segment))
30998 < soh_allocation_no_gc)
31000 should_expand = TRUE;
31004 dprintf (2, ("will %s(%s)", (should_compact ? "compact" : "sweep"), (should_expand ? "ex" : "")));
31005 return should_compact;
31008 size_t align_lower_good_size_allocation (size_t size)
31010 return (size/64)*64;
31013 size_t gc_heap::approximate_new_allocation()
31015 dynamic_data* dd0 = dynamic_data_of (0);
31016 return max (2*dd_min_size (dd0), ((dd_desired_allocation (dd0)*2)/3));
31019 BOOL gc_heap::sufficient_space_end_seg (uint8_t* start, uint8_t* seg_end, size_t end_space_required, gc_tuning_point tp)
31021 BOOL can_fit = FALSE;
31022 size_t end_seg_space = (size_t)(seg_end - start);
31023 if (end_seg_space > end_space_required)
31025 // If hard limit is specified, and if we attributed all that's left in commit to the ephemeral seg
31026 // so we treat that as segment end, do we have enough space.
31027 if (heap_hard_limit)
31029 size_t left_in_commit = heap_hard_limit - current_total_committed;
31031 #ifdef MULTIPLE_HEAPS
31032 num_heaps = n_heaps;
31033 #endif //MULTIPLE_HEAPS
31034 left_in_commit /= num_heaps;
31035 if (left_in_commit > end_space_required)
31040 dprintf (2, ("h%d end seg %Id, but only %Id left in HARD LIMIT commit, required: %Id %s on eph (%d)",
31041 heap_number, end_seg_space,
31042 left_in_commit, end_space_required,
31043 (can_fit ? "ok" : "short"), (int)tp));
31052 // After we did a GC we expect to have at least this
31053 // much space at the end of the segment to satisfy
31054 // a reasonable amount of allocation requests.
31055 size_t gc_heap::end_space_after_gc()
31057 return max ((dd_min_size (dynamic_data_of (0))/2), (END_SPACE_AFTER_GC + Align (min_obj_size)));
31060 BOOL gc_heap::ephemeral_gen_fit_p (gc_tuning_point tp)
31062 uint8_t* start = 0;
31064 if ((tp == tuning_deciding_condemned_gen) ||
31065 (tp == tuning_deciding_compaction))
31067 start = (settings.concurrent ? alloc_allocated : heap_segment_allocated (ephemeral_heap_segment));
31068 if (settings.concurrent)
31070 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment (alloc_allocated)",
31071 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated)));
31075 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment (allocated)",
31076 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment))));
31079 else if (tp == tuning_deciding_expansion)
31081 start = heap_segment_plan_allocated (ephemeral_heap_segment);
31082 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment based on plan",
31083 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - start)));
31087 assert (tp == tuning_deciding_full_gc);
31088 dprintf (GTC_LOG, ("FGC: %Id left at the end of ephemeral segment (alloc_allocated)",
31089 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated)));
31090 start = alloc_allocated;
31093 if (start == 0) // empty ephemeral generations
31095 assert (tp == tuning_deciding_expansion);
31096 // if there are no survivors in the ephemeral segment,
31097 // this should be the beginning of ephemeral segment.
31098 start = generation_allocation_pointer (generation_of (max_generation));
31099 assert (start == heap_segment_mem (ephemeral_heap_segment));
31102 if (tp == tuning_deciding_expansion)
31104 assert (settings.condemned_generation >= (max_generation-1));
31105 size_t gen0size = approximate_new_allocation();
31106 size_t eph_size = gen0size;
31107 size_t gen_min_sizes = 0;
31109 for (int j = 1; j <= max_generation-1; j++)
31111 gen_min_sizes += 2*dd_min_size (dynamic_data_of(j));
31114 eph_size += gen_min_sizes;
31116 dprintf (3, ("h%d deciding on expansion, need %Id (gen0: %Id, 2*min: %Id)",
31117 heap_number, gen0size, gen_min_sizes, eph_size));
31119 // We must find room for one large object and enough room for gen0size
31120 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - start) > eph_size)
31122 dprintf (3, ("Enough room before end of segment"));
31127 size_t room = align_lower_good_size_allocation
31128 (heap_segment_reserved (ephemeral_heap_segment) - start);
31129 size_t end_seg = room;
31131 //look at the plug free space
31132 size_t largest_alloc = END_SPACE_AFTER_GC + Align (min_obj_size);
31133 bool large_chunk_found = FALSE;
31135 uint8_t* gen0start = generation_plan_allocation_start (youngest_generation);
31136 dprintf (3, ("ephemeral_gen_fit_p: gen0 plan start: %Ix", (size_t)gen0start));
31137 if (gen0start == 0)
31139 dprintf (3, ("ephemeral_gen_fit_p: room before free list search %Id, needed: %Id",
31141 while ((bos < mark_stack_bos) &&
31142 !((room >= gen0size) && large_chunk_found))
31144 uint8_t* plug = pinned_plug (pinned_plug_of (bos));
31145 if (in_range_for_segment (plug, ephemeral_heap_segment))
31147 if (plug >= gen0start)
31149 size_t chunk = align_lower_good_size_allocation (pinned_len (pinned_plug_of (bos)));
31151 if (!large_chunk_found)
31153 large_chunk_found = (chunk >= largest_alloc);
31155 dprintf (3, ("ephemeral_gen_fit_p: room now %Id, large chunk: %Id",
31156 room, large_chunk_found));
31162 if (room >= gen0size)
31164 if (large_chunk_found)
31166 sufficient_gen0_space_p = TRUE;
31168 dprintf (3, ("Enough room"));
31173 // now we need to find largest_alloc at the end of the segment.
31174 if (end_seg >= end_space_after_gc())
31176 dprintf (3, ("Enough room (may need end of seg)"));
31182 dprintf (3, ("Not enough room"));
31188 size_t end_space = 0;
31189 dynamic_data* dd = dynamic_data_of (0);
31190 if ((tp == tuning_deciding_condemned_gen) ||
31191 (tp == tuning_deciding_full_gc))
31193 end_space = max (2*dd_min_size (dd), end_space_after_gc());
31197 assert (tp == tuning_deciding_compaction);
31198 end_space = approximate_new_allocation();
31201 BOOL can_fit = sufficient_space_end_seg (start, heap_segment_reserved (ephemeral_heap_segment), end_space, tp);
31207 CObjectHeader* gc_heap::allocate_large_object (size_t jsize, int64_t& alloc_bytes)
31209 //create a new alloc context because gen3context is shared.
31210 alloc_context acontext;
31211 acontext.alloc_ptr = 0;
31212 acontext.alloc_limit = 0;
31213 acontext.alloc_bytes = 0;
31214 #ifdef MULTIPLE_HEAPS
31215 acontext.set_alloc_heap(vm_heap);
31216 #endif //MULTIPLE_HEAPS
31219 size_t maxObjectSize = (INT64_MAX - 7 - Align(min_obj_size));
31221 size_t maxObjectSize = (INT32_MAX - 7 - Align(min_obj_size));
31224 if (jsize >= maxObjectSize)
31226 if (GCConfig::GetBreakOnOOM())
31228 GCToOSInterface::DebugBreak();
31233 size_t size = AlignQword (jsize);
31234 int align_const = get_alignment_constant (FALSE);
31235 #ifdef FEATURE_LOH_COMPACTION
31236 size_t pad = Align (loh_padding_obj_size, align_const);
31239 #endif //FEATURE_LOH_COMPACTION
31241 assert (size >= Align (min_obj_size, align_const));
31243 #pragma inline_depth(0)
31245 if (! allocate_more_space (&acontext, (size + pad), max_generation+1))
31251 #pragma inline_depth(20)
31255 uint8_t* current_lowest_address = lowest_address;
31256 uint8_t* current_highest_address = highest_address;
31257 #ifdef BACKGROUND_GC
31258 if (recursive_gc_sync::background_running_p())
31260 current_lowest_address = background_saved_lowest_address;
31261 current_highest_address = background_saved_highest_address;
31263 #endif //BACKGROUND_GC
31264 #endif // MARK_ARRAY
31266 #ifdef FEATURE_LOH_COMPACTION
31267 // The GC allocator made a free object already in this alloc context and
31268 // adjusted the alloc_ptr accordingly.
31269 #endif //FEATURE_LOH_COMPACTION
31271 uint8_t* result = acontext.alloc_ptr;
31273 assert ((size_t)(acontext.alloc_limit - acontext.alloc_ptr) == size);
31274 alloc_bytes += size;
31276 CObjectHeader* obj = (CObjectHeader*)result;
31279 if (recursive_gc_sync::background_running_p())
31281 if ((result < current_highest_address) && (result >= current_lowest_address))
31283 dprintf (3, ("Clearing mark bit at address %Ix",
31284 (size_t)(&mark_array [mark_word_of (result)])));
31286 mark_array_clear_marked (result);
31288 #ifdef BACKGROUND_GC
31289 //the object has to cover one full mark uint32_t
31290 assert (size > mark_word_size);
31291 if (current_c_gc_state != c_gc_state_free)
31293 dprintf (3, ("Concurrent allocation of a large object %Ix",
31295 //mark the new block specially so we know it is a new object
31296 if ((result < current_highest_address) && (result >= current_lowest_address))
31298 dprintf (3, ("Setting mark bit at address %Ix",
31299 (size_t)(&mark_array [mark_word_of (result)])));
31301 mark_array_set_marked (result);
31304 #endif //BACKGROUND_GC
31306 #endif //MARK_ARRAY
31309 assert ((size_t)obj == Align ((size_t)obj, align_const));
31314 void reset_memory (uint8_t* o, size_t sizeo)
31316 if (sizeo > 128 * 1024)
31318 // We cannot reset the memory for the useful part of a free object.
31319 size_t size_to_skip = min_free_list - plug_skew;
31321 size_t page_start = align_on_page ((size_t)(o + size_to_skip));
31322 size_t size = align_lower_page ((size_t)o + sizeo - size_to_skip - plug_skew) - page_start;
31323 // Note we need to compensate for an OS bug here. This bug would cause the MEM_RESET to fail
31324 // on write watched memory.
31327 #ifdef MULTIPLE_HEAPS
31328 bool unlock_p = true;
31330 // We don't do unlock because there could be many processes using workstation GC and it's
31331 // bad perf to have many threads doing unlock at the same time.
31332 bool unlock_p = false;
31333 #endif //MULTIPLE_HEAPS
31335 reset_mm_p = GCToOSInterface::VirtualReset((void*)page_start, size, unlock_p);
31340 void gc_heap::reset_large_object (uint8_t* o)
31342 // If it's a large object, allow the O/S to discard the backing store for these pages.
31343 reset_memory (o, size(o));
31346 BOOL gc_heap::large_object_marked (uint8_t* o, BOOL clearp)
31349 // It shouldn't be necessary to do these comparisons because this is only used for blocking
31350 // GCs and LOH segments cannot be out of range.
31351 if ((o >= lowest_address) && (o < highest_address))
31371 void gc_heap::walk_survivors_relocation (void* profiling_context, record_surv_fn fn)
31373 // Now walk the portion of memory that is actually being relocated.
31374 walk_relocation (profiling_context, fn);
31376 #ifdef FEATURE_LOH_COMPACTION
31377 if (loh_compacted_p)
31379 walk_relocation_for_loh (profiling_context, fn);
31381 #endif //FEATURE_LOH_COMPACTION
31384 void gc_heap::walk_survivors_for_loh (void* profiling_context, record_surv_fn fn)
31386 generation* gen = large_object_generation;
31387 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));;
31389 PREFIX_ASSUME(seg != NULL);
31391 uint8_t* o = generation_allocation_start (gen);
31392 uint8_t* plug_end = o;
31393 uint8_t* plug_start = o;
31397 if (o >= heap_segment_allocated (seg))
31399 seg = heap_segment_next (seg);
31403 o = heap_segment_mem (seg);
31405 if (large_object_marked(o, FALSE))
31412 o = o + AlignQword (size (o));
31413 if (o >= heap_segment_allocated (seg))
31417 m = large_object_marked (o, FALSE);
31422 fn (plug_start, plug_end, 0, profiling_context, false, false);
31426 while (o < heap_segment_allocated (seg) && !large_object_marked(o, FALSE))
31428 o = o + AlignQword (size (o));
31434 #ifdef BACKGROUND_GC
31436 BOOL gc_heap::background_object_marked (uint8_t* o, BOOL clearp)
31439 if ((o >= background_saved_lowest_address) && (o < background_saved_highest_address))
31441 if (mark_array_marked (o))
31445 mark_array_clear_marked (o);
31446 //dprintf (3, ("mark array bit for object %Ix is cleared", o));
31447 dprintf (3, ("CM: %Ix", o));
31457 dprintf (3, ("o %Ix(%d) %s", o, size(o), (m ? "was bm" : "was NOT bm")));
31461 void gc_heap::background_delay_delete_loh_segments()
31463 generation* gen = large_object_generation;
31464 heap_segment* seg = heap_segment_rw (generation_start_segment (large_object_generation));
31465 heap_segment* prev_seg = 0;
31469 heap_segment* next_seg = heap_segment_next (seg);
31470 if (seg->flags & heap_segment_flags_loh_delete)
31472 dprintf (3, ("deleting %Ix-%Ix-%Ix", (size_t)seg, heap_segment_allocated (seg), heap_segment_reserved (seg)));
31473 delete_heap_segment (seg, (GCConfig::GetRetainVM() != 0));
31474 heap_segment_next (prev_seg) = next_seg;
31485 uint8_t* gc_heap::background_next_end (heap_segment* seg, BOOL large_objects_p)
31488 (large_objects_p ? heap_segment_allocated (seg) : heap_segment_background_allocated (seg));
31491 void gc_heap::set_mem_verify (uint8_t* start, uint8_t* end, uint8_t b)
31496 if ((GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC) &&
31497 !(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_NO_MEM_FILL))
31499 dprintf (3, ("setting mem to %c [%Ix, [%Ix", b, start, end));
31500 memset (start, b, (end - start));
31503 #endif //VERIFY_HEAP
31506 void gc_heap::generation_delete_heap_segment (generation* gen,
31508 heap_segment* prev_seg,
31509 heap_segment* next_seg)
31511 dprintf (3, ("bgc sweep: deleting seg %Ix", seg));
31512 if (gen == large_object_generation)
31514 dprintf (3, ("Preparing empty large segment %Ix for deletion", (size_t)seg));
31516 // We cannot thread segs in here onto freeable_large_heap_segment because
31517 // grow_brick_card_tables could be committing mark array which needs to read
31518 // the seg list. So we delay it till next time we suspend EE.
31519 seg->flags |= heap_segment_flags_loh_delete;
31520 // Since we will be decommitting the seg, we need to prevent heap verification
31521 // to verify this segment.
31522 heap_segment_allocated (seg) = heap_segment_mem (seg);
31526 if (seg == ephemeral_heap_segment)
31531 heap_segment_next (next_seg) = prev_seg;
31533 dprintf (3, ("Preparing empty small segment %Ix for deletion", (size_t)seg));
31534 heap_segment_next (seg) = freeable_small_heap_segment;
31535 freeable_small_heap_segment = seg;
31538 decommit_heap_segment (seg);
31539 seg->flags |= heap_segment_flags_decommitted;
31541 set_mem_verify (heap_segment_allocated (seg) - plug_skew, heap_segment_used (seg), 0xbb);
31544 void gc_heap::process_background_segment_end (heap_segment* seg,
31546 uint8_t* last_plug_end,
31547 heap_segment* start_seg,
31551 uint8_t* allocated = heap_segment_allocated (seg);
31552 uint8_t* background_allocated = heap_segment_background_allocated (seg);
31553 BOOL loh_p = heap_segment_loh_p (seg);
31555 dprintf (3, ("Processing end of background segment [%Ix, %Ix[(%Ix[)",
31556 (size_t)heap_segment_mem (seg), background_allocated, allocated));
31558 if (!loh_p && (allocated != background_allocated))
31560 assert (gen != large_object_generation);
31562 dprintf (3, ("Make a free object before newly promoted objects [%Ix, %Ix[",
31563 (size_t)last_plug_end, background_allocated));
31564 thread_gap (last_plug_end, background_allocated - last_plug_end, generation_of (max_generation));
31567 fix_brick_to_highest (last_plug_end, background_allocated);
31569 // When we allowed fgc's during going through gaps, we could have erased the brick
31570 // that corresponds to bgc_allocated 'cause we had to update the brick there,
31571 // recover it here.
31572 fix_brick_to_highest (background_allocated, background_allocated);
31576 // by default, if allocated == background_allocated, it can't
31577 // be the ephemeral segment.
31578 if (seg == ephemeral_heap_segment)
31583 if (allocated == heap_segment_mem (seg))
31585 // this can happen with LOH segments when multiple threads
31586 // allocate new segments and not all of them were needed to
31587 // satisfy allocation requests.
31588 assert (gen == large_object_generation);
31591 if (last_plug_end == heap_segment_mem (seg))
31593 dprintf (3, ("Segment allocated is %Ix (beginning of this seg) - %s be deleted",
31594 (size_t)allocated, (*delete_p ? "should" : "should not")));
31596 if (seg != start_seg)
31603 dprintf (3, ("Trimming seg to %Ix[", (size_t)last_plug_end));
31604 heap_segment_allocated (seg) = last_plug_end;
31605 set_mem_verify (heap_segment_allocated (seg) - plug_skew, heap_segment_used (seg), 0xbb);
31607 decommit_heap_segment_pages (seg, 0);
31611 dprintf (3, ("verifying seg %Ix's mark array was completely cleared", seg));
31612 bgc_verify_mark_array_cleared (seg);
31615 void gc_heap::process_n_background_segments (heap_segment* seg,
31616 heap_segment* prev_seg,
31619 assert (gen != large_object_generation);
31623 dprintf (2, ("processing seg %Ix (not seen by bgc mark)", seg));
31624 heap_segment* next_seg = heap_segment_next (seg);
31626 if (heap_segment_read_only_p (seg))
31632 if (heap_segment_allocated (seg) == heap_segment_mem (seg))
31634 // This can happen - if we have a LOH segment where nothing survived
31635 // or a SOH segment allocated by a gen1 GC when BGC was going where
31636 // nothing survived last time we did a gen1 GC.
31637 generation_delete_heap_segment (gen, seg, prev_seg, next_seg);
31645 verify_soh_segment_list();
31651 BOOL gc_heap::fgc_should_consider_object (uint8_t* o,
31653 BOOL consider_bgc_mark_p,
31654 BOOL check_current_sweep_p,
31655 BOOL check_saved_sweep_p)
31657 // the logic for this function must be kept in sync with the analogous function
31658 // in ToolBox\SOS\Strike\gc.cpp
31660 // TRUE means we don't need to check the bgc mark bit
31661 // FALSE means we do.
31662 BOOL no_bgc_mark_p = FALSE;
31664 if (consider_bgc_mark_p)
31666 if (check_current_sweep_p && (o < current_sweep_pos))
31668 dprintf (3, ("no bgc mark - o: %Ix < cs: %Ix", o, current_sweep_pos));
31669 no_bgc_mark_p = TRUE;
31672 if (!no_bgc_mark_p)
31674 if(check_saved_sweep_p && (o >= saved_sweep_ephemeral_start))
31676 dprintf (3, ("no bgc mark - o: %Ix >= ss: %Ix", o, saved_sweep_ephemeral_start));
31677 no_bgc_mark_p = TRUE;
31680 if (!check_saved_sweep_p)
31682 uint8_t* background_allocated = heap_segment_background_allocated (seg);
31683 // if this was the saved ephemeral segment, check_saved_sweep_p
31684 // would've been true.
31685 assert (heap_segment_background_allocated (seg) != saved_sweep_ephemeral_start);
31686 // background_allocated could be 0 for the new segments acquired during bgc
31687 // sweep and we still want no_bgc_mark_p to be true.
31688 if (o >= background_allocated)
31690 dprintf (3, ("no bgc mark - o: %Ix >= ba: %Ix", o, background_allocated));
31691 no_bgc_mark_p = TRUE;
31698 no_bgc_mark_p = TRUE;
31701 dprintf (3, ("bgc mark %Ix: %s (bm: %s)", o, (no_bgc_mark_p ? "no" : "yes"), (background_object_marked (o, FALSE) ? "yes" : "no")));
31702 return (no_bgc_mark_p ? TRUE : background_object_marked (o, FALSE));
31705 // consider_bgc_mark_p tells you if you need to care about the bgc mark bit at all
31706 // if it's TRUE, check_current_sweep_p tells you if you should consider the
31707 // current sweep position or not.
31708 void gc_heap::should_check_bgc_mark (heap_segment* seg,
31709 BOOL* consider_bgc_mark_p,
31710 BOOL* check_current_sweep_p,
31711 BOOL* check_saved_sweep_p)
31713 // the logic for this function must be kept in sync with the analogous function
31714 // in ToolBox\SOS\Strike\gc.cpp
31715 *consider_bgc_mark_p = FALSE;
31716 *check_current_sweep_p = FALSE;
31717 *check_saved_sweep_p = FALSE;
31719 if (current_c_gc_state == c_gc_state_planning)
31721 // We are doing the current_sweep_pos comparison here because we have yet to
31722 // turn on the swept flag for the segment but in_range_for_segment will return
31723 // FALSE if the address is the same as reserved.
31724 if ((seg->flags & heap_segment_flags_swept) || (current_sweep_pos == heap_segment_reserved (seg)))
31726 dprintf (3, ("seg %Ix is already swept by bgc", seg));
31730 *consider_bgc_mark_p = TRUE;
31732 dprintf (3, ("seg %Ix hasn't been swept by bgc", seg));
31734 if (seg == saved_sweep_ephemeral_seg)
31736 dprintf (3, ("seg %Ix is the saved ephemeral seg", seg));
31737 *check_saved_sweep_p = TRUE;
31740 if (in_range_for_segment (current_sweep_pos, seg))
31742 dprintf (3, ("current sweep pos is %Ix and within seg %Ix",
31743 current_sweep_pos, seg));
31744 *check_current_sweep_p = TRUE;
31750 void gc_heap::background_ephemeral_sweep()
31752 dprintf (3, ("bgc ephemeral sweep"));
31754 int align_const = get_alignment_constant (TRUE);
31756 saved_sweep_ephemeral_seg = ephemeral_heap_segment;
31757 saved_sweep_ephemeral_start = generation_allocation_start (generation_of (max_generation - 1));
31759 // Since we don't want to interfere with gen0 allocation while we are threading gen0 free list,
31760 // we thread onto a list first then publish it when we are done.
31761 allocator youngest_free_list;
31762 size_t youngest_free_list_space = 0;
31763 size_t youngest_free_obj_space = 0;
31765 youngest_free_list.clear();
31767 for (int i = 0; i <= (max_generation - 1); i++)
31769 generation* gen_to_reset = generation_of (i);
31770 assert (generation_free_list_space (gen_to_reset) == 0);
31771 // Can only assert free_list_space is 0, not free_obj_space as the allocator could have added
31772 // something there.
31775 for (int i = (max_generation - 1); i >= 0; i--)
31777 generation* current_gen = generation_of (i);
31778 uint8_t* o = generation_allocation_start (current_gen);
31779 //Skip the generation gap object
31780 o = o + Align(size (o), align_const);
31781 uint8_t* end = ((i > 0) ?
31782 generation_allocation_start (generation_of (i - 1)) :
31783 heap_segment_allocated (ephemeral_heap_segment));
31785 uint8_t* plug_end = o;
31786 uint8_t* plug_start = o;
31787 BOOL marked_p = FALSE;
31791 marked_p = background_object_marked (o, TRUE);
31795 size_t plug_size = plug_start - plug_end;
31799 thread_gap (plug_end, plug_size, current_gen);
31805 make_unused_array (plug_end, plug_size);
31806 if (plug_size >= min_free_list)
31808 youngest_free_list_space += plug_size;
31809 youngest_free_list.thread_item (plug_end, plug_size);
31813 youngest_free_obj_space += plug_size;
31818 fix_brick_to_highest (plug_end, plug_start);
31819 fix_brick_to_highest (plug_start, plug_start);
31824 o = o + Align (size (o), align_const);
31830 m = background_object_marked (o, TRUE);
31833 dprintf (3, ("bgs: plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
31837 while ((o < end) && !background_object_marked (o, FALSE))
31839 o = o + Align (size (o), align_const);
31844 if (plug_end != end)
31848 thread_gap (plug_end, end - plug_end, current_gen);
31849 fix_brick_to_highest (plug_end, end);
31853 heap_segment_allocated (ephemeral_heap_segment) = plug_end;
31854 // the following line is temporary.
31855 heap_segment_saved_bg_allocated (ephemeral_heap_segment) = plug_end;
31857 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
31859 make_unused_array (plug_end, (end - plug_end));
31861 #endif //VERIFY_HEAP
31865 dd_fragmentation (dynamic_data_of (i)) =
31866 generation_free_list_space (current_gen) + generation_free_obj_space (current_gen);
31869 generation* youngest_gen = generation_of (0);
31870 generation_free_list_space (youngest_gen) = youngest_free_list_space;
31871 generation_free_obj_space (youngest_gen) = youngest_free_obj_space;
31872 dd_fragmentation (dynamic_data_of (0)) = youngest_free_list_space + youngest_free_obj_space;
31873 generation_allocator (youngest_gen)->copy_with_no_repair (&youngest_free_list);
31876 void gc_heap::background_sweep()
31878 generation* gen = generation_of (max_generation);
31879 dynamic_data* dd = dynamic_data_of (max_generation);
31880 // For SOH segments we go backwards.
31881 heap_segment* start_seg = ephemeral_heap_segment;
31882 PREFIX_ASSUME(start_seg != NULL);
31883 heap_segment* fseg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
31884 heap_segment* seg = start_seg;
31885 uint8_t* o = heap_segment_mem (seg);
31887 heap_segment* prev_seg = heap_segment_next (seg);
31888 int align_const = get_alignment_constant (TRUE);
31891 assert (o == generation_allocation_start (generation_of (max_generation)));
31892 o = o + Align(size (o), align_const);
31895 uint8_t* plug_end = o;
31896 uint8_t* plug_start = o;
31897 next_sweep_obj = o;
31898 current_sweep_pos = o;
31900 //uint8_t* end = background_next_end (seg, (gen == large_object_generation));
31901 uint8_t* end = heap_segment_background_allocated (seg);
31902 BOOL delete_p = FALSE;
31904 //concurrent_print_time_delta ("finished with mark and start with sweep");
31905 concurrent_print_time_delta ("Sw");
31906 dprintf (2, ("---- (GC%d)Background Sweep Phase ----", VolatileLoad(&settings.gc_index)));
31908 //block concurrent allocation for large objects
31909 dprintf (3, ("lh state: planning"));
31910 if (gc_lh_block_event.IsValid())
31912 gc_lh_block_event.Reset();
31915 for (int i = 0; i <= (max_generation + 1); i++)
31917 generation* gen_to_reset = generation_of (i);
31918 generation_allocator (gen_to_reset)->clear();
31919 generation_free_list_space (gen_to_reset) = 0;
31920 generation_free_obj_space (gen_to_reset) = 0;
31921 generation_free_list_allocated (gen_to_reset) = 0;
31922 generation_end_seg_allocated (gen_to_reset) = 0;
31923 generation_condemned_allocated (gen_to_reset) = 0;
31924 //reset the allocation so foreground gc can allocate into older generation
31925 generation_allocation_pointer (gen_to_reset)= 0;
31926 generation_allocation_limit (gen_to_reset) = 0;
31927 generation_allocation_segment (gen_to_reset) = heap_segment_rw (generation_start_segment (gen_to_reset));
31930 FIRE_EVENT(BGC2ndNonConEnd);
31932 loh_alloc_thread_count = 0;
31933 current_bgc_state = bgc_sweep_soh;
31934 verify_soh_segment_list();
31936 #ifdef FEATURE_BASICFREEZE
31937 if ((generation_start_segment (gen) != ephemeral_heap_segment) &&
31938 ro_segments_in_range)
31940 sweep_ro_segments (generation_start_segment (gen));
31942 #endif // FEATURE_BASICFREEZE
31944 //TODO BACKGROUND_GC: can we move this to where we switch to the LOH?
31945 if (current_c_gc_state != c_gc_state_planning)
31947 current_c_gc_state = c_gc_state_planning;
31950 concurrent_print_time_delta ("Swe");
31952 heap_segment* loh_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation + 1)));
31953 PREFIX_ASSUME(loh_seg != NULL);
31956 loh_seg->flags &= ~heap_segment_flags_swept;
31957 heap_segment_background_allocated (loh_seg) = heap_segment_allocated (loh_seg);
31958 loh_seg = heap_segment_next_rw (loh_seg);
31961 #ifdef MULTIPLE_HEAPS
31962 bgc_t_join.join(this, gc_join_restart_ee);
31963 if (bgc_t_join.joined())
31964 #endif //MULTIPLE_HEAPS
31966 #ifdef MULTIPLE_HEAPS
31967 dprintf(2, ("Starting BGC threads for resuming EE"));
31968 bgc_t_join.restart();
31969 #endif //MULTIPLE_HEAPS
31972 if (heap_number == 0)
31977 FIRE_EVENT(BGC2ndConBegin);
31979 background_ephemeral_sweep();
31981 concurrent_print_time_delta ("Swe eph");
31983 #ifdef MULTIPLE_HEAPS
31984 bgc_t_join.join(this, gc_join_after_ephemeral_sweep);
31985 if (bgc_t_join.joined())
31986 #endif //MULTIPLE_HEAPS
31988 #ifdef FEATURE_EVENT_TRACE
31989 bgc_heap_walk_for_etw_p = GCEventStatus::IsEnabled(GCEventProvider_Default,
31990 GCEventKeyword_GCHeapSurvivalAndMovement,
31991 GCEventLevel_Information);
31992 #endif //FEATURE_EVENT_TRACE
31994 leave_spin_lock (&gc_lock);
31996 #ifdef MULTIPLE_HEAPS
31997 dprintf(2, ("Starting BGC threads for BGC sweeping"));
31998 bgc_t_join.restart();
31999 #endif //MULTIPLE_HEAPS
32002 disable_preemptive (true);
32004 dprintf (2, ("bgs: sweeping gen2 objects"));
32005 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
32006 (size_t)heap_segment_mem (seg),
32007 (size_t)heap_segment_allocated (seg),
32008 (size_t)heap_segment_background_allocated (seg)));
32010 int num_objs = 256;
32011 int current_num_objs = 0;
32012 heap_segment* next_seg = 0;
32018 if (gen == large_object_generation)
32020 next_seg = heap_segment_next (seg);
32024 next_seg = heap_segment_prev (fseg, seg);
32029 if (!heap_segment_read_only_p (seg))
32031 if (gen == large_object_generation)
32033 // we can treat all LOH segments as in the bgc domain
32034 // regardless of whether we saw in bgc mark or not
32035 // because we don't allow LOH allocations during bgc
32036 // sweep anyway - the LOH segments can't change.
32037 process_background_segment_end (seg, gen, plug_end,
32038 start_seg, &delete_p);
32042 assert (heap_segment_background_allocated (seg) != 0);
32043 process_background_segment_end (seg, gen, plug_end,
32044 start_seg, &delete_p);
32046 assert (next_seg || !delete_p);
32052 generation_delete_heap_segment (gen, seg, prev_seg, next_seg);
32057 dprintf (2, ("seg %Ix has been swept", seg));
32058 seg->flags |= heap_segment_flags_swept;
32061 verify_soh_segment_list();
32065 dprintf (GTC_LOG, ("seg: %Ix, next_seg: %Ix, prev_seg: %Ix", seg, next_seg, prev_seg));
32069 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
32071 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
32073 if (gen != large_object_generation)
32075 dprintf (2, ("bgs: sweeping gen3 objects"));
32076 concurrent_print_time_delta ("Swe SOH");
32077 FIRE_EVENT(BGC1stSweepEnd, 0);
32079 enter_spin_lock (&more_space_lock_loh);
32080 add_saved_spinlock_info (true, me_acquire, mt_bgc_loh_sweep);
32082 concurrent_print_time_delta ("Swe LOH took msl");
32084 // We wait till all allocating threads are completely done.
32085 int spin_count = yp_spin_count_unit;
32086 while (loh_alloc_thread_count)
32088 spin_and_switch (spin_count, (loh_alloc_thread_count == 0));
32091 current_bgc_state = bgc_sweep_loh;
32092 gen = generation_of (max_generation+1);
32093 start_seg = heap_segment_rw (generation_start_segment (gen));
32095 PREFIX_ASSUME(start_seg != NULL);
32099 o = generation_allocation_start (gen);
32100 assert (method_table (o) == g_gc_pFreeObjectMethodTable);
32101 align_const = get_alignment_constant (FALSE);
32102 o = o + Align(size (o), align_const);
32104 end = heap_segment_allocated (seg);
32105 dprintf (2, ("sweeping gen3 objects"));
32106 generation_free_obj_space (gen) = 0;
32107 generation_allocator (gen)->clear();
32108 generation_free_list_space (gen) = 0;
32110 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
32111 (size_t)heap_segment_mem (seg),
32112 (size_t)heap_segment_allocated (seg),
32113 (size_t)heap_segment_background_allocated (seg)));
32120 o = heap_segment_mem (seg);
32123 assert (gen != large_object_generation);
32124 assert (o == generation_allocation_start (generation_of (max_generation)));
32125 align_const = get_alignment_constant (TRUE);
32126 o = o + Align(size (o), align_const);
32130 current_sweep_pos = o;
32131 next_sweep_obj = o;
32134 end = background_next_end (seg, (gen == large_object_generation));
32135 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
32136 (size_t)heap_segment_mem (seg),
32137 (size_t)heap_segment_allocated (seg),
32138 (size_t)heap_segment_background_allocated (seg)));
32142 if ((o < end) && background_object_marked (o, TRUE))
32145 if (gen == large_object_generation)
32147 dprintf (2, ("loh fr: [%Ix-%Ix[(%Id)", plug_end, plug_start, plug_start-plug_end));
32150 thread_gap (plug_end, plug_start-plug_end, gen);
32151 if (gen != large_object_generation)
32153 add_gen_free (max_generation, plug_start-plug_end);
32154 fix_brick_to_highest (plug_end, plug_start);
32155 // we need to fix the brick for the next plug here 'cause an FGC can
32156 // happen and can't read a stale brick.
32157 fix_brick_to_highest (plug_start, plug_start);
32164 next_sweep_obj = o + Align(size (o), align_const);
32165 current_num_objs++;
32166 if (current_num_objs >= num_objs)
32168 current_sweep_pos = next_sweep_obj;
32171 current_num_objs = 0;
32174 o = next_sweep_obj;
32180 m = background_object_marked (o, TRUE);
32183 if (gen != large_object_generation)
32185 add_gen_plug (max_generation, plug_end-plug_start);
32186 dd_survived_size (dd) += (plug_end - plug_start);
32188 dprintf (3, ("bgs: plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
32192 while ((o < end) && !background_object_marked (o, FALSE))
32194 next_sweep_obj = o + Align(size (o), align_const);;
32195 current_num_objs++;
32196 if (current_num_objs >= num_objs)
32198 current_sweep_pos = plug_end;
32199 dprintf (1234, ("f: swept till %Ix", current_sweep_pos));
32201 current_num_objs = 0;
32204 o = next_sweep_obj;
32209 size_t total_loh_size = generation_size (max_generation + 1);
32210 size_t total_soh_size = generation_sizes (generation_of (max_generation));
32212 dprintf (GTC_LOG, ("h%d: S: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
32214 dprintf (GTC_LOG, ("end of bgc sweep: gen2 FL: %Id, FO: %Id",
32215 generation_free_list_space (generation_of (max_generation)),
32216 generation_free_obj_space (generation_of (max_generation))));
32217 dprintf (GTC_LOG, ("h%d: end of bgc sweep: gen3 FL: %Id, FO: %Id",
32219 generation_free_list_space (generation_of (max_generation + 1)),
32220 generation_free_obj_space (generation_of (max_generation + 1))));
32222 FIRE_EVENT(BGC2ndConEnd);
32223 concurrent_print_time_delta ("background sweep");
32225 heap_segment* reset_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
32226 PREFIX_ASSUME(reset_seg != NULL);
32230 heap_segment_saved_bg_allocated (reset_seg) = heap_segment_background_allocated (reset_seg);
32231 heap_segment_background_allocated (reset_seg) = 0;
32232 reset_seg = heap_segment_next_rw (reset_seg);
32235 generation* loh_gen = generation_of (max_generation + 1);
32236 generation_allocation_segment (loh_gen) = heap_segment_rw (generation_start_segment (loh_gen));
32238 // We calculate dynamic data here because if we wait till we signal the lh event,
32239 // the allocation thread can change the fragmentation and we may read an intermediate
32240 // value (which can be greater than the generation size). Plus by that time it won't
32242 compute_new_dynamic_data (max_generation);
32244 enable_preemptive ();
32246 #ifdef MULTIPLE_HEAPS
32247 bgc_t_join.join(this, gc_join_set_state_free);
32248 if (bgc_t_join.joined())
32249 #endif //MULTIPLE_HEAPS
32251 // TODO: We are using this join just to set the state. Should
32252 // look into eliminating it - check to make sure things that use
32253 // this state can live with per heap state like should_check_bgc_mark.
32254 current_c_gc_state = c_gc_state_free;
32256 #ifdef MULTIPLE_HEAPS
32257 dprintf(2, ("Starting BGC threads after background sweep phase"));
32258 bgc_t_join.restart();
32259 #endif //MULTIPLE_HEAPS
32262 disable_preemptive (true);
32264 if (gc_lh_block_event.IsValid())
32266 gc_lh_block_event.Set();
32269 add_saved_spinlock_info (true, me_release, mt_bgc_loh_sweep);
32270 leave_spin_lock (&more_space_lock_loh);
32272 //dprintf (GTC_LOG, ("---- (GC%d)End Background Sweep Phase ----", VolatileLoad(&settings.gc_index)));
32273 dprintf (GTC_LOG, ("---- (GC%d)ESw ----", VolatileLoad(&settings.gc_index)));
32275 #endif //BACKGROUND_GC
32277 void gc_heap::sweep_large_objects ()
32279 //this min value is for the sake of the dynamic tuning.
32280 //so we know that we are not starting even if we have no
32282 generation* gen = large_object_generation;
32283 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
32285 PREFIX_ASSUME(start_seg != NULL);
32287 heap_segment* seg = start_seg;
32288 heap_segment* prev_seg = 0;
32289 uint8_t* o = generation_allocation_start (gen);
32290 int align_const = get_alignment_constant (FALSE);
32292 //Skip the generation gap object
32293 o = o + Align(size (o), align_const);
32295 uint8_t* plug_end = o;
32296 uint8_t* plug_start = o;
32298 generation_allocator (gen)->clear();
32299 generation_free_list_space (gen) = 0;
32300 generation_free_obj_space (gen) = 0;
32303 dprintf (3, ("sweeping large objects"));
32304 dprintf (3, ("seg: %Ix, [%Ix, %Ix[, starting from %Ix",
32306 (size_t)heap_segment_mem (seg),
32307 (size_t)heap_segment_allocated (seg),
32312 if (o >= heap_segment_allocated (seg))
32314 heap_segment* next_seg = heap_segment_next (seg);
32315 //delete the empty segment if not the only one
32316 if ((plug_end == heap_segment_mem (seg)) &&
32317 (seg != start_seg) && !heap_segment_read_only_p (seg))
32319 //prepare for deletion
32320 dprintf (3, ("Preparing empty large segment %Ix", (size_t)seg));
32322 heap_segment_next (prev_seg) = next_seg;
32323 heap_segment_next (seg) = freeable_large_heap_segment;
32324 freeable_large_heap_segment = seg;
32328 if (!heap_segment_read_only_p (seg))
32330 dprintf (3, ("Trimming seg to %Ix[", (size_t)plug_end));
32331 heap_segment_allocated (seg) = plug_end;
32332 decommit_heap_segment_pages (seg, 0);
32341 o = heap_segment_mem (seg);
32343 dprintf (3, ("seg: %Ix, [%Ix, %Ix[", (size_t)seg,
32344 (size_t)heap_segment_mem (seg),
32345 (size_t)heap_segment_allocated (seg)));
32348 if (large_object_marked(o, TRUE))
32351 //everything between plug_end and plug_start is free
32352 thread_gap (plug_end, plug_start-plug_end, gen);
32357 o = o + AlignQword (size (o));
32358 if (o >= heap_segment_allocated (seg))
32362 m = large_object_marked (o, TRUE);
32365 dprintf (3, ("plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
32369 while (o < heap_segment_allocated (seg) && !large_object_marked(o, FALSE))
32371 o = o + AlignQword (size (o));
32376 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
32378 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
32381 void gc_heap::relocate_in_large_objects ()
32383 relocate_args args;
32385 args.high = gc_high;
32386 args.last_plug = 0;
32388 generation* gen = large_object_generation;
32390 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32392 PREFIX_ASSUME(seg != NULL);
32394 uint8_t* o = generation_allocation_start (gen);
32398 if (o >= heap_segment_allocated (seg))
32400 seg = heap_segment_next_rw (seg);
32405 o = heap_segment_mem (seg);
32408 while (o < heap_segment_allocated (seg))
32410 check_class_object_demotion (o);
32411 if (contain_pointers (o))
32413 dprintf(3, ("Relocating through large object %Ix", (size_t)o));
32414 go_through_object_nostart (method_table (o), o, size(o), pval,
32416 reloc_survivor_helper (pval);
32419 o = o + AlignQword (size (o));
32424 void gc_heap::mark_through_cards_for_large_objects (card_fn fn,
32427 uint8_t* low = gc_low;
32428 size_t end_card = 0;
32429 generation* oldest_gen = generation_of (max_generation+1);
32430 heap_segment* seg = heap_segment_rw (generation_start_segment (oldest_gen));
32432 PREFIX_ASSUME(seg != NULL);
32434 uint8_t* beg = generation_allocation_start (oldest_gen);
32435 uint8_t* end = heap_segment_allocated (seg);
32437 size_t cg_pointers_found = 0;
32439 size_t card_word_end = (card_of (align_on_card_word (end)) /
32444 size_t n_card_set = 0;
32445 uint8_t* next_boundary = (relocating ?
32446 generation_plan_allocation_start (generation_of (max_generation -1)) :
32449 uint8_t* nhigh = (relocating ?
32450 heap_segment_plan_allocated (ephemeral_heap_segment) :
32453 BOOL foundp = FALSE;
32454 uint8_t* start_address = 0;
32455 uint8_t* limit = 0;
32456 size_t card = card_of (beg);
32458 #ifdef BACKGROUND_GC
32459 BOOL consider_bgc_mark_p = FALSE;
32460 BOOL check_current_sweep_p = FALSE;
32461 BOOL check_saved_sweep_p = FALSE;
32462 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
32463 #endif //BACKGROUND_GC
32465 size_t total_cards_cleared = 0;
32467 //dprintf(3,( "scanning large objects from %Ix to %Ix", (size_t)beg, (size_t)end));
32468 dprintf(3, ("CMl: %Ix->%Ix", (size_t)beg, (size_t)end));
32471 if ((o < end) && (card_of(o) > card))
32473 dprintf (3, ("Found %Id cg pointers", cg_pointers_found));
32474 if (cg_pointers_found == 0)
32476 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card), (size_t)o));
32477 clear_cards (card, card_of((uint8_t*)o));
32478 total_cards_cleared += (card_of((uint8_t*)o) - card);
32480 n_eph +=cg_pointers_found;
32481 cg_pointers_found = 0;
32482 card = card_of ((uint8_t*)o);
32484 if ((o < end) &&(card >= end_card))
32486 foundp = find_card (card_table, card, card_word_end, end_card);
32489 n_card_set+= end_card - card;
32490 start_address = max (beg, card_address (card));
32492 limit = min (end, card_address (end_card));
32494 if ((!foundp) || (o >= end) || (card_address (card) >= end))
32496 if ((foundp) && (cg_pointers_found == 0))
32498 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card),
32499 (size_t)card_address(card+1)));
32500 clear_cards (card, card+1);
32501 total_cards_cleared += 1;
32503 n_eph +=cg_pointers_found;
32504 cg_pointers_found = 0;
32505 if ((seg = heap_segment_next_rw (seg)) != 0)
32507 #ifdef BACKGROUND_GC
32508 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
32509 #endif //BACKGROUND_GC
32510 beg = heap_segment_mem (seg);
32511 end = compute_next_end (seg, low);
32512 card_word_end = card_of (align_on_card_word (end)) / card_word_width;
32513 card = card_of (beg);
32524 assert (card_set_p (card));
32526 dprintf(3,("card %Ix: o: %Ix, l: %Ix[ ",
32527 card, (size_t)o, (size_t)limit));
32529 assert (Align (size (o)) >= Align (min_obj_size));
32530 size_t s = size (o);
32531 uint8_t* next_o = o + AlignQword (s);
32537 assert (Align (s) >= Align (min_obj_size));
32538 next_o = o + AlignQword (s);
32541 dprintf (4, ("|%Ix|", (size_t)o));
32542 if (next_o < start_address)
32547 #ifdef BACKGROUND_GC
32548 if (!fgc_should_consider_object (o, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p))
32552 #endif //BACKGROUND_GC
32554 #ifdef COLLECTIBLE_CLASS
32555 if (is_collectible(o))
32557 BOOL passed_end_card_p = FALSE;
32559 if (card_of (o) > card)
32561 passed_end_card_p = card_transition (o, end, card_word_end,
32565 foundp, start_address,
32566 limit, total_cards_cleared);
32569 if ((!passed_end_card_p || foundp) && (card_of (o) == card))
32571 // card is valid and it covers the head of the object
32572 if (fn == &gc_heap::relocate_address)
32574 keep_card_live (o, n_gen, cg_pointers_found);
32578 uint8_t* class_obj = get_class_object (o);
32579 mark_through_cards_helper (&class_obj, n_gen,
32580 cg_pointers_found, fn,
32581 nhigh, next_boundary);
32585 if (passed_end_card_p)
32587 if (foundp && (card_address (card) < next_o))
32589 goto go_through_refs;
32599 #endif //COLLECTIBLE_CLASS
32601 if (contain_pointers (o))
32603 dprintf(3,("Going through %Ix", (size_t)o));
32605 go_through_object (method_table(o), o, s, poo,
32606 start_address, use_start, (o + s),
32608 if (card_of ((uint8_t*)poo) > card)
32610 BOOL passed_end_card_p = card_transition ((uint8_t*)poo, end,
32615 foundp, start_address,
32616 limit, total_cards_cleared);
32618 if (passed_end_card_p)
32620 if (foundp && (card_address (card) < next_o))
32624 if (ppstop <= (uint8_t**)start_address)
32626 else if (poo < (uint8_t**)start_address)
32627 {poo = (uint8_t**)start_address;}
32637 mark_through_cards_helper (poo, n_gen,
32638 cg_pointers_found, fn,
32639 nhigh, next_boundary);
32651 // compute the efficiency ratio of the card table
32654 generation_skip_ratio = min (((n_eph > 800) ?
32655 (int)(((float)n_gen / (float)n_eph) * 100) : 100),
32656 generation_skip_ratio);
32658 dprintf (3, ("Mloh: cross: %Id, useful: %Id, cards cleared: %Id, cards set: %Id, ratio: %d",
32659 n_eph, n_gen, total_cards_cleared, n_card_set, generation_skip_ratio));
32663 dprintf (3, ("R: Mloh: cross: %Id, useful: %Id, cards set: %Id, ratio: %d",
32664 n_eph, n_gen, n_card_set, generation_skip_ratio));
32668 void gc_heap::descr_segment (heap_segment* seg )
32671 uint8_t* x = heap_segment_mem (seg);
32672 while (x < heap_segment_allocated (seg))
32674 dprintf(2, ( "%Ix: %d ", (size_t)x, size (x)));
32675 x = x + Align(size (x));
32678 UNREFERENCED_PARAMETER(seg);
32682 void gc_heap::descr_generations_to_profiler (gen_walk_fn fn, void *context)
32684 #ifdef MULTIPLE_HEAPS
32685 int n_heaps = g_theGCHeap->GetNumberOfHeaps ();
32686 for (int i = 0; i < n_heaps; i++)
32688 gc_heap* hp = GCHeap::GetHeap(i)->pGenGCHeap;
32689 #else //MULTIPLE_HEAPS
32691 gc_heap* hp = NULL;
32693 // prefix complains about us dereferencing hp in wks build even though we only access static members
32694 // this way. not sure how to shut it up except for this ugly workaround:
32695 PREFIX_ASSUME(hp != NULL);
32696 #endif // _PREFAST_
32697 #endif //MULTIPLE_HEAPS
32699 int curr_gen_number0 = max_generation+1;
32700 while (curr_gen_number0 >= 0)
32702 generation* gen = hp->generation_of (curr_gen_number0);
32703 heap_segment* seg = generation_start_segment (gen);
32704 while (seg && (seg != hp->ephemeral_heap_segment))
32706 assert (curr_gen_number0 > 0);
32708 // report bounds from heap_segment_mem (seg) to
32709 // heap_segment_allocated (seg);
32710 // for generation # curr_gen_number0
32711 // for heap # heap_no
32713 fn(context, curr_gen_number0, heap_segment_mem (seg),
32714 heap_segment_allocated (seg),
32715 curr_gen_number0 == max_generation+1 ? heap_segment_reserved (seg) : heap_segment_allocated (seg));
32717 seg = heap_segment_next (seg);
32721 assert (seg == hp->ephemeral_heap_segment);
32722 assert (curr_gen_number0 <= max_generation);
32724 if (curr_gen_number0 == max_generation)
32726 if (heap_segment_mem (seg) < generation_allocation_start (hp->generation_of (max_generation-1)))
32728 // report bounds from heap_segment_mem (seg) to
32729 // generation_allocation_start (generation_of (max_generation-1))
32730 // for heap # heap_number
32732 fn(context, curr_gen_number0, heap_segment_mem (seg),
32733 generation_allocation_start (hp->generation_of (max_generation-1)),
32734 generation_allocation_start (hp->generation_of (max_generation-1)) );
32737 else if (curr_gen_number0 != 0)
32739 //report bounds from generation_allocation_start (generation_of (curr_gen_number0))
32740 // to generation_allocation_start (generation_of (curr_gen_number0-1))
32741 // for heap # heap_number
32743 fn(context, curr_gen_number0, generation_allocation_start (hp->generation_of (curr_gen_number0)),
32744 generation_allocation_start (hp->generation_of (curr_gen_number0-1)),
32745 generation_allocation_start (hp->generation_of (curr_gen_number0-1)));
32749 //report bounds from generation_allocation_start (generation_of (curr_gen_number0))
32750 // to heap_segment_allocated (ephemeral_heap_segment);
32751 // for heap # heap_number
32753 fn(context, curr_gen_number0, generation_allocation_start (hp->generation_of (curr_gen_number0)),
32754 heap_segment_allocated (hp->ephemeral_heap_segment),
32755 heap_segment_reserved (hp->ephemeral_heap_segment) );
32758 curr_gen_number0--;
32764 // Note that when logging is on it can take a long time to go through the free items.
32765 void gc_heap::print_free_list (int gen, heap_segment* seg)
32767 UNREFERENCED_PARAMETER(gen);
32768 UNREFERENCED_PARAMETER(seg);
32770 if (settings.concurrent == FALSE)
32772 uint8_t* seg_start = heap_segment_mem (seg);
32773 uint8_t* seg_end = heap_segment_allocated (seg);
32775 dprintf (3, ("Free list in seg %Ix:", seg_start));
32777 size_t total_free_item = 0;
32779 allocator* gen_allocator = generation_allocator (generation_of (gen));
32780 for (unsigned int b = 0; b < gen_allocator->number_of_buckets(); b++)
32782 uint8_t* fo = gen_allocator->alloc_list_head_of (b);
32785 if (fo >= seg_start && fo < seg_end)
32789 size_t free_item_len = size(fo);
32791 dprintf (3, ("[%Ix, %Ix[:%Id",
32793 (size_t)(fo + free_item_len),
32797 fo = free_list_slot (fo);
32801 dprintf (3, ("total %Id free items", total_free_item));
32807 void gc_heap::descr_generations (BOOL begin_gc_p)
32809 UNREFERENCED_PARAMETER(begin_gc_p);
32811 if (StressLog::StressLogOn(LF_GC, LL_INFO10))
32814 #ifdef MULTIPLE_HEAPS
32816 #endif //MULTIPLE_HEAPS
32818 STRESS_LOG1(LF_GC, LL_INFO10, "GC Heap %p\n", hp);
32819 for (int n = max_generation; n >= 0; --n)
32821 STRESS_LOG4(LF_GC, LL_INFO10, " Generation %d [%p, %p] cur = %p\n",
32823 generation_allocation_start(generation_of(n)),
32824 generation_allocation_limit(generation_of(n)),
32825 generation_allocation_pointer(generation_of(n)));
32827 heap_segment* seg = generation_start_segment(generation_of(n));
32830 STRESS_LOG4(LF_GC, LL_INFO10, " Segment mem %p alloc = %p used %p committed %p\n",
32831 heap_segment_mem(seg),
32832 heap_segment_allocated(seg),
32833 heap_segment_used(seg),
32834 heap_segment_committed(seg));
32835 seg = heap_segment_next(seg);
32839 #endif // STRESS_LOG
32842 dprintf (2, ("lowest_address: %Ix highest_address: %Ix",
32843 (size_t) lowest_address, (size_t) highest_address));
32844 #ifdef BACKGROUND_GC
32845 dprintf (2, ("bgc lowest_address: %Ix bgc highest_address: %Ix",
32846 (size_t) background_saved_lowest_address, (size_t) background_saved_highest_address));
32847 #endif //BACKGROUND_GC
32849 if (heap_number == 0)
32851 dprintf (1, ("total heap size: %Id, commit size: %Id", get_total_heap_size(), get_total_committed_size()));
32854 int curr_gen_number = max_generation+1;
32855 while (curr_gen_number >= 0)
32857 size_t total_gen_size = generation_size (curr_gen_number);
32858 #ifdef SIMPLE_DPRINTF
32859 dprintf (GTC_LOG, ("[%s][g%d]gen %d:, size: %Id, frag: %Id(L: %Id, O: %Id), f: %d%% %s %s %s",
32860 (begin_gc_p ? "BEG" : "END"),
32861 settings.condemned_generation,
32864 dd_fragmentation (dynamic_data_of (curr_gen_number)),
32865 generation_free_list_space (generation_of (curr_gen_number)),
32866 generation_free_obj_space (generation_of (curr_gen_number)),
32868 (int)(((double)dd_fragmentation (dynamic_data_of (curr_gen_number)) / (double)total_gen_size) * 100) :
32870 (begin_gc_p ? ("") : (settings.compaction ? "(compact)" : "(sweep)")),
32871 (settings.heap_expansion ? "(EX)" : " "),
32872 (settings.promotion ? "Promotion" : "NoPromotion")));
32874 dprintf (2, ( "Generation %d: gap size: %d, generation size: %Id, fragmentation: %Id",
32876 size (generation_allocation_start (generation_of (curr_gen_number))),
32878 dd_fragmentation (dynamic_data_of (curr_gen_number))));
32879 #endif //SIMPLE_DPRINTF
32881 generation* gen = generation_of (curr_gen_number);
32882 heap_segment* seg = generation_start_segment (gen);
32883 while (seg && (seg != ephemeral_heap_segment))
32885 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[-%Ix[ (%Id) (%Id)",
32887 (size_t)heap_segment_mem (seg),
32888 (size_t)heap_segment_allocated (seg),
32889 (size_t)heap_segment_committed (seg),
32890 (size_t)(heap_segment_allocated (seg) - heap_segment_mem (seg)),
32891 (size_t)(heap_segment_committed (seg) - heap_segment_allocated (seg))));
32892 print_free_list (curr_gen_number, seg);
32893 seg = heap_segment_next (seg);
32895 if (seg && (seg != generation_start_segment (gen)))
32897 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[",
32899 (size_t)heap_segment_mem (seg),
32900 (size_t)generation_allocation_start (generation_of (curr_gen_number-1))));
32901 print_free_list (curr_gen_number, seg);
32906 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[",
32908 (size_t)generation_allocation_start (generation_of (curr_gen_number)),
32909 (size_t)(((curr_gen_number == 0)) ?
32910 (heap_segment_allocated
32911 (generation_start_segment
32912 (generation_of (curr_gen_number)))) :
32913 (generation_allocation_start
32914 (generation_of (curr_gen_number - 1))))
32916 print_free_list (curr_gen_number, seg);
32928 //-----------------------------------------------------------------------------
32930 // VM Specific support
32932 //-----------------------------------------------------------------------------
32937 unsigned int PromotedObjectCount = 0;
32938 unsigned int CreatedObjectCount = 0;
32939 unsigned int AllocDuration = 0;
32940 unsigned int AllocCount = 0;
32941 unsigned int AllocBigCount = 0;
32942 unsigned int AllocSmallCount = 0;
32943 unsigned int AllocStart = 0;
32946 //Static member variables.
32947 VOLATILE(BOOL) GCHeap::GcInProgress = FALSE;
32949 //CMCSafeLock* GCHeap::fGcLock;
32950 GCEvent *GCHeap::WaitForGCEvent = NULL;
32953 unsigned int GCHeap::GcDuration;
32955 unsigned GCHeap::GcCondemnedGeneration = 0;
32956 size_t GCHeap::totalSurvivedSize = 0;
32957 #ifdef FEATURE_PREMORTEM_FINALIZATION
32958 CFinalize* GCHeap::m_Finalize = 0;
32959 BOOL GCHeap::GcCollectClasses = FALSE;
32960 VOLATILE(int32_t) GCHeap::m_GCFLock = 0;
32962 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
32964 #ifdef BACKGROUND_GC
32965 int GCHeap::gc_stress_fgcs_in_bgc = 0;
32966 #endif // BACKGROUND_GC
32967 #ifndef MULTIPLE_HEAPS
32968 OBJECTHANDLE GCHeap::m_StressObjs[NUM_HEAP_STRESS_OBJS];
32969 int GCHeap::m_CurStressObj = 0;
32970 #endif // !MULTIPLE_HEAPS
32971 #endif // STRESS_HEAP
32972 #endif // FEATURE_REDHAWK
32974 #endif //FEATURE_PREMORTEM_FINALIZATION
32976 class NoGCRegionLockHolder
32979 NoGCRegionLockHolder()
32981 enter_spin_lock_noinstru(&g_no_gc_lock);
32984 ~NoGCRegionLockHolder()
32986 leave_spin_lock_noinstru(&g_no_gc_lock);
32990 // An explanation of locking for finalization:
32992 // Multiple threads allocate objects. During the allocation, they are serialized by
32993 // the AllocLock above. But they release that lock before they register the object
32994 // for finalization. That's because there is much contention for the alloc lock, but
32995 // finalization is presumed to be a rare case.
32997 // So registering an object for finalization must be protected by the FinalizeLock.
32999 // There is another logical queue that involves finalization. When objects registered
33000 // for finalization become unreachable, they are moved from the "registered" queue to
33001 // the "unreachable" queue. Note that this only happens inside a GC, so no other
33002 // threads can be manipulating either queue at that time. Once the GC is over and
33003 // threads are resumed, the Finalizer thread will dequeue objects from the "unreachable"
33004 // queue and call their finalizers. This dequeue operation is also protected with
33005 // the finalize lock.
33007 // At first, this seems unnecessary. Only one thread is ever enqueuing or dequeuing
33008 // on the unreachable queue (either the GC thread during a GC or the finalizer thread
33009 // when a GC is not in progress). The reason we share a lock with threads enqueuing
33010 // on the "registered" queue is that the "registered" and "unreachable" queues are
33013 // They are actually two regions of a longer list, which can only grow at one end.
33014 // So to enqueue an object to the "registered" list, you actually rotate an unreachable
33015 // object at the boundary between the logical queues, out to the other end of the
33016 // unreachable queue -- where all growing takes place. Then you move the boundary
33017 // pointer so that the gap we created at the boundary is now on the "registered"
33018 // side rather than the "unreachable" side. Now the object can be placed into the
33019 // "registered" side at that point. This is much more efficient than doing moves
33020 // of arbitrarily long regions, but it causes the two queues to require a shared lock.
33022 // Notice that Enter/LeaveFinalizeLock is not a GC-aware spin lock. Instead, it relies
33023 // on the fact that the lock will only be taken for a brief period and that it will
33024 // never provoke or allow a GC while the lock is held. This is critical. If the
33025 // FinalizeLock used enter_spin_lock (and thus sometimes enters preemptive mode to
33026 // allow a GC), then the Alloc client would have to GC protect a finalizable object
33027 // to protect against that eventuality. That is too slow!
33031 BOOL IsValidObject99(uint8_t *pObject)
33034 if (!((CObjectHeader*)pObject)->IsFree())
33035 ((CObjectHeader *) pObject)->Validate();
33036 #endif //VERIFY_HEAP
33040 #ifdef BACKGROUND_GC
33041 BOOL gc_heap::bgc_mark_array_range (heap_segment* seg,
33043 uint8_t** range_beg,
33044 uint8_t** range_end)
33046 uint8_t* seg_start = heap_segment_mem (seg);
33047 uint8_t* seg_end = (whole_seg_p ? heap_segment_reserved (seg) : align_on_mark_word (heap_segment_allocated (seg)));
33049 if ((seg_start < background_saved_highest_address) &&
33050 (seg_end > background_saved_lowest_address))
33052 *range_beg = max (seg_start, background_saved_lowest_address);
33053 *range_end = min (seg_end, background_saved_highest_address);
33062 void gc_heap::bgc_verify_mark_array_cleared (heap_segment* seg)
33064 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
33065 if (recursive_gc_sync::background_running_p() && GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
33067 uint8_t* range_beg = 0;
33068 uint8_t* range_end = 0;
33070 if (bgc_mark_array_range (seg, TRUE, &range_beg, &range_end))
33072 size_t markw = mark_word_of (range_beg);
33073 size_t markw_end = mark_word_of (range_end);
33074 while (markw < markw_end)
33076 if (mark_array [markw])
33078 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
33079 markw, mark_array [markw], mark_word_address (markw)));
33084 uint8_t* p = mark_word_address (markw_end);
33085 while (p < range_end)
33087 assert (!(mark_array_marked (p)));
33092 #endif //VERIFY_HEAP && MARK_ARRAY
33095 void gc_heap::verify_mark_bits_cleared (uint8_t* obj, size_t s)
33097 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
33098 size_t start_mark_bit = mark_bit_of (obj) + 1;
33099 size_t end_mark_bit = mark_bit_of (obj + s);
33100 unsigned int startbit = mark_bit_bit (start_mark_bit);
33101 unsigned int endbit = mark_bit_bit (end_mark_bit);
33102 size_t startwrd = mark_bit_word (start_mark_bit);
33103 size_t endwrd = mark_bit_word (end_mark_bit);
33104 unsigned int result = 0;
33106 unsigned int firstwrd = ~(lowbits (~0, startbit));
33107 unsigned int lastwrd = ~(highbits (~0, endbit));
33109 if (startwrd == endwrd)
33111 unsigned int wrd = firstwrd & lastwrd;
33112 result = mark_array[startwrd] & wrd;
33120 // verify the first mark word is cleared.
33123 result = mark_array[startwrd] & firstwrd;
33131 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
33133 result = mark_array[wrdtmp];
33140 // set the last mark word.
33143 result = mark_array[endwrd] & lastwrd;
33149 #endif //VERIFY_HEAP && MARK_ARRAY
33152 void gc_heap::clear_all_mark_array()
33155 //size_t num_dwords_written = 0;
33156 //size_t begin_time = GetHighPrecisionTimeStamp();
33158 generation* gen = generation_of (max_generation);
33159 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
33165 if (gen != large_object_generation)
33167 gen = generation_of (max_generation+1);
33168 seg = heap_segment_rw (generation_start_segment (gen));
33176 uint8_t* range_beg = 0;
33177 uint8_t* range_end = 0;
33179 if (bgc_mark_array_range (seg, (seg == ephemeral_heap_segment), &range_beg, &range_end))
33181 size_t markw = mark_word_of (range_beg);
33182 size_t markw_end = mark_word_of (range_end);
33183 size_t size_total = (markw_end - markw) * sizeof (uint32_t);
33184 //num_dwords_written = markw_end - markw;
33186 size_t size_left = 0;
33188 assert (((size_t)&mark_array[markw] & (sizeof(PTR_PTR)-1)) == 0);
33190 if ((size_total & (sizeof(PTR_PTR) - 1)) != 0)
33192 size = (size_total & ~(sizeof(PTR_PTR) - 1));
33193 size_left = size_total - size;
33194 assert ((size_left & (sizeof (uint32_t) - 1)) == 0);
33201 memclr ((uint8_t*)&mark_array[markw], size);
33203 if (size_left != 0)
33205 uint32_t* markw_to_clear = &mark_array[markw + size / sizeof (uint32_t)];
33206 for (size_t i = 0; i < (size_left / sizeof (uint32_t)); i++)
33208 *markw_to_clear = 0;
33214 seg = heap_segment_next_rw (seg);
33217 //size_t end_time = GetHighPrecisionTimeStamp() - begin_time;
33219 //printf ("took %Id ms to clear %Id bytes\n", end_time, num_dwords_written*sizeof(uint32_t));
33221 #endif //MARK_ARRAY
33224 #endif //BACKGROUND_GC
33226 void gc_heap::verify_mark_array_cleared (heap_segment* seg)
33228 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
33229 assert (card_table == g_gc_card_table);
33230 size_t markw = mark_word_of (heap_segment_mem (seg));
33231 size_t markw_end = mark_word_of (heap_segment_reserved (seg));
33233 while (markw < markw_end)
33235 if (mark_array [markw])
33237 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
33238 markw, mark_array [markw], mark_word_address (markw)));
33243 #endif //VERIFY_HEAP && MARK_ARRAY
33246 void gc_heap::verify_mark_array_cleared ()
33248 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
33249 if (recursive_gc_sync::background_running_p() && GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
33251 generation* gen = generation_of (max_generation);
33252 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
33258 if (gen != large_object_generation)
33260 gen = generation_of (max_generation+1);
33261 seg = heap_segment_rw (generation_start_segment (gen));
33269 bgc_verify_mark_array_cleared (seg);
33270 seg = heap_segment_next_rw (seg);
33273 #endif //VERIFY_HEAP && MARK_ARRAY
33276 void gc_heap::verify_seg_end_mark_array_cleared()
33278 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
33279 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
33281 generation* gen = generation_of (max_generation);
33282 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
33288 if (gen != large_object_generation)
33290 gen = generation_of (max_generation+1);
33291 seg = heap_segment_rw (generation_start_segment (gen));
33299 // We already cleared all mark array bits for ephemeral generations
33300 // at the beginning of bgc sweep
33301 uint8_t* from = ((seg == ephemeral_heap_segment) ?
33302 generation_allocation_start (generation_of (max_generation - 1)) :
33303 heap_segment_allocated (seg));
33304 size_t markw = mark_word_of (align_on_mark_word (from));
33305 size_t markw_end = mark_word_of (heap_segment_reserved (seg));
33307 while (from < mark_word_address (markw))
33309 if (is_mark_bit_set (from))
33311 dprintf (3, ("mark bit for %Ix was not cleared", from));
33315 from += mark_bit_pitch;
33318 while (markw < markw_end)
33320 if (mark_array [markw])
33322 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
33323 markw, mark_array [markw], mark_word_address (markw)));
33328 seg = heap_segment_next_rw (seg);
33331 #endif //VERIFY_HEAP && MARK_ARRAY
33334 // This function is called to make sure we don't mess up the segment list
33335 // in SOH. It's called by:
33336 // 1) begin and end of ephemeral GCs
33337 // 2) during bgc sweep when we switch segments.
33338 void gc_heap::verify_soh_segment_list()
33341 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
33343 generation* gen = generation_of (max_generation);
33344 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
33345 heap_segment* last_seg = 0;
33349 seg = heap_segment_next_rw (seg);
33351 if (last_seg != ephemeral_heap_segment)
33356 #endif //VERIFY_HEAP
33359 // This function can be called at any foreground GCs or blocking GCs. For background GCs,
33360 // it can be called at the end of the final marking; and at any point during background
33362 // NOTE - to be able to call this function during background sweep, we need to temporarily
33363 // NOT clear the mark array bits as we go.
33364 void gc_heap::verify_partial ()
33366 #ifdef BACKGROUND_GC
33367 //printf ("GC#%d: Verifying loh during sweep\n", settings.gc_index);
33368 //generation* gen = large_object_generation;
33369 generation* gen = generation_of (max_generation);
33370 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
33371 int align_const = get_alignment_constant (gen != large_object_generation);
33377 // Different ways to fail.
33378 BOOL mark_missed_p = FALSE;
33379 BOOL bad_ref_p = FALSE;
33380 BOOL free_ref_p = FALSE;
33386 if (gen != large_object_generation)
33389 gen = large_object_generation;
33390 align_const = get_alignment_constant (gen != large_object_generation);
33391 seg = heap_segment_rw (generation_start_segment (gen));
33400 o = heap_segment_mem (seg);
33401 end = heap_segment_allocated (seg);
33402 //printf ("validating [%Ix-[%Ix\n", o, end);
33407 BOOL marked_p = background_object_marked (o, FALSE);
33411 go_through_object_cl (method_table (o), o, s, oo,
33415 //dprintf (3, ("VOM: verifying member %Ix in obj %Ix", (size_t)*oo, o));
33416 MethodTable *pMT = method_table (*oo);
33418 if (pMT == g_gc_pFreeObjectMethodTable)
33424 if (!pMT->SanityCheck())
33427 dprintf (3, ("Bad member of %Ix %Ix",
33428 (size_t)oo, (size_t)*oo));
33432 if (current_bgc_state == bgc_final_marking)
33434 if (marked_p && !background_object_marked (*oo, FALSE))
33436 mark_missed_p = TRUE;
33445 o = o + Align(s, align_const);
33447 seg = heap_segment_next_rw (seg);
33450 //printf ("didn't find any large object large enough...\n");
33451 //printf ("finished verifying loh\n");
33452 #endif //BACKGROUND_GC
33458 gc_heap::verify_free_lists ()
33460 for (int gen_num = 0; gen_num <= max_generation+1; gen_num++)
33462 dprintf (3, ("Verifying free list for gen:%d", gen_num));
33463 allocator* gen_alloc = generation_allocator (generation_of (gen_num));
33464 size_t sz = gen_alloc->first_bucket_size();
33465 bool verify_undo_slot = (gen_num != 0) && (gen_num != max_generation+1) && !gen_alloc->discard_if_no_fit_p();
33467 for (unsigned int a_l_number = 0; a_l_number < gen_alloc->number_of_buckets(); a_l_number++)
33469 uint8_t* free_list = gen_alloc->alloc_list_head_of (a_l_number);
33473 if (!((CObjectHeader*)free_list)->IsFree())
33475 dprintf (3, ("Verifiying Heap: curr free list item %Ix isn't a free object)",
33476 (size_t)free_list));
33479 if (((a_l_number < (gen_alloc->number_of_buckets()-1))&& (unused_array_size (free_list) >= sz))
33480 || ((a_l_number != 0) && (unused_array_size (free_list) < sz/2)))
33482 dprintf (3, ("Verifiying Heap: curr free list item %Ix isn't in the right bucket",
33483 (size_t)free_list));
33486 if (verify_undo_slot && (free_list_undo (free_list) != UNDO_EMPTY))
33488 dprintf (3, ("Verifiying Heap: curr free list item %Ix has non empty undo slot",
33489 (size_t)free_list));
33492 if ((gen_num != max_generation+1)&&(object_gennum (free_list)!= gen_num))
33494 dprintf (3, ("Verifiying Heap: curr free list item %Ix is in the wrong generation free list",
33495 (size_t)free_list));
33500 free_list = free_list_slot (free_list);
33502 //verify the sanity of the tail
33503 uint8_t* tail = gen_alloc->alloc_list_tail_of (a_l_number);
33504 if (!((tail == 0) || (tail == prev)))
33506 dprintf (3, ("Verifying Heap: tail of free list is not correct"));
33511 uint8_t* head = gen_alloc->alloc_list_head_of (a_l_number);
33512 if ((head != 0) && (free_list_slot (head) != 0))
33514 dprintf (3, ("Verifying Heap: tail of free list is not correct"));
33525 gc_heap::verify_heap (BOOL begin_gc_p)
33527 int heap_verify_level = static_cast<int>(GCConfig::GetHeapVerifyLevel());
33528 size_t last_valid_brick = 0;
33529 BOOL bCurrentBrickInvalid = FALSE;
33530 BOOL large_brick_p = TRUE;
33531 size_t curr_brick = 0;
33532 size_t prev_brick = (size_t)-1;
33533 int curr_gen_num = max_generation+1;
33534 heap_segment* seg = heap_segment_in_range (generation_start_segment (generation_of (curr_gen_num ) ));
33536 PREFIX_ASSUME(seg != NULL);
33538 uint8_t* curr_object = heap_segment_mem (seg);
33539 uint8_t* prev_object = 0;
33540 uint8_t* begin_youngest = generation_allocation_start(generation_of(0));
33541 uint8_t* end_youngest = heap_segment_allocated (ephemeral_heap_segment);
33542 uint8_t* next_boundary = generation_allocation_start (generation_of (max_generation - 1));
33543 int align_const = get_alignment_constant (FALSE);
33544 size_t total_objects_verified = 0;
33545 size_t total_objects_verified_deep = 0;
33547 #ifdef BACKGROUND_GC
33548 BOOL consider_bgc_mark_p = FALSE;
33549 BOOL check_current_sweep_p = FALSE;
33550 BOOL check_saved_sweep_p = FALSE;
33551 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
33552 #endif //BACKGROUND_GC
33554 #ifdef MULTIPLE_HEAPS
33555 t_join* current_join = &gc_t_join;
33556 #ifdef BACKGROUND_GC
33557 if (settings.concurrent && (bgc_thread_id.IsCurrentThread()))
33559 // We always call verify_heap on entry of GC on the SVR GC threads.
33560 current_join = &bgc_t_join;
33562 #endif //BACKGROUND_GC
33563 #endif //MULTIPLE_HEAPS
33565 UNREFERENCED_PARAMETER(begin_gc_p);
33566 #ifdef BACKGROUND_GC
33567 dprintf (2,("[%s]GC#%d(%s): Verifying heap - begin",
33568 (begin_gc_p ? "BEG" : "END"),
33569 VolatileLoad(&settings.gc_index),
33570 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC"))));
33572 dprintf (2,("[%s]GC#%d: Verifying heap - begin",
33573 (begin_gc_p ? "BEG" : "END"), VolatileLoad(&settings.gc_index)));
33574 #endif //BACKGROUND_GC
33576 #ifndef MULTIPLE_HEAPS
33577 if ((ephemeral_low != generation_allocation_start (generation_of (max_generation - 1))) ||
33578 (ephemeral_high != heap_segment_reserved (ephemeral_heap_segment)))
33582 #endif //MULTIPLE_HEAPS
33584 #ifdef BACKGROUND_GC
33585 //don't touch the memory because the program is allocating from it.
33586 if (!settings.concurrent)
33587 #endif //BACKGROUND_GC
33589 if (!(heap_verify_level & GCConfig::HEAPVERIFY_NO_MEM_FILL))
33591 //uninit the unused portions of segments.
33592 generation* gen1 = large_object_generation;
33593 heap_segment* seg1 = heap_segment_rw (generation_start_segment (gen1));
33594 PREFIX_ASSUME(seg1 != NULL);
33600 uint8_t* clear_start = heap_segment_allocated (seg1) - plug_skew;
33601 if (heap_segment_used (seg1) > clear_start)
33603 dprintf (3, ("setting end of seg %Ix: [%Ix-[%Ix to 0xaa",
33604 heap_segment_mem (seg1),
33606 heap_segment_used (seg1)));
33607 memset (heap_segment_allocated (seg1) - plug_skew, 0xaa,
33608 (heap_segment_used (seg1) - clear_start));
33610 seg1 = heap_segment_next_rw (seg1);
33614 if (gen1 == large_object_generation)
33616 gen1 = generation_of (max_generation);
33617 seg1 = heap_segment_rw (generation_start_segment (gen1));
33618 PREFIX_ASSUME(seg1 != NULL);
33629 #ifdef MULTIPLE_HEAPS
33630 current_join->join(this, gc_join_verify_copy_table);
33631 if (current_join->joined())
33633 // in concurrent GC, new segment could be allocated when GC is working so the card brick table might not be updated at this point
33634 for (int i = 0; i < n_heaps; i++)
33636 //copy the card and brick tables
33637 if (g_gc_card_table != g_heaps[i]->card_table)
33639 g_heaps[i]->copy_brick_card_table();
33643 current_join->restart();
33646 if (g_gc_card_table != card_table)
33647 copy_brick_card_table();
33648 #endif //MULTIPLE_HEAPS
33650 //verify that the generation structures makes sense
33652 generation* gen = generation_of (max_generation);
33654 assert (generation_allocation_start (gen) ==
33655 heap_segment_mem (heap_segment_rw (generation_start_segment (gen))));
33656 int gen_num = max_generation-1;
33657 generation* prev_gen = gen;
33658 while (gen_num >= 0)
33660 gen = generation_of (gen_num);
33661 assert (generation_allocation_segment (gen) == ephemeral_heap_segment);
33662 assert (generation_allocation_start (gen) >= heap_segment_mem (ephemeral_heap_segment));
33663 assert (generation_allocation_start (gen) < heap_segment_allocated (ephemeral_heap_segment));
33665 if (generation_start_segment (prev_gen ) ==
33666 generation_start_segment (gen))
33668 assert (generation_allocation_start (prev_gen) <
33669 generation_allocation_start (gen));
33678 // Handle segment transitions
33679 if (curr_object >= heap_segment_allocated (seg))
33681 if (curr_object > heap_segment_allocated(seg))
33683 dprintf (3, ("Verifiying Heap: curr_object: %Ix > heap_segment_allocated (seg: %Ix)",
33684 (size_t)curr_object, (size_t)seg));
33687 seg = heap_segment_next_in_range (seg);
33690 #ifdef BACKGROUND_GC
33691 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
33692 #endif //BACKGROUND_GC
33693 curr_object = heap_segment_mem(seg);
33699 if (curr_gen_num == (max_generation+1))
33702 seg = heap_segment_in_range (generation_start_segment (generation_of (curr_gen_num)));
33704 PREFIX_ASSUME(seg != NULL);
33706 #ifdef BACKGROUND_GC
33707 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
33708 #endif //BACKGROUND_GC
33709 curr_object = heap_segment_mem (seg);
33711 large_brick_p = FALSE;
33712 align_const = get_alignment_constant (TRUE);
33715 break; // Done Verifying Heap -- no more segments
33719 // Are we at the end of the youngest_generation?
33720 if (seg == ephemeral_heap_segment)
33722 if (curr_object >= end_youngest)
33724 // prev_object length is too long if we hit this int3
33725 if (curr_object > end_youngest)
33727 dprintf (3, ("Verifiying Heap: curr_object: %Ix > end_youngest: %Ix",
33728 (size_t)curr_object, (size_t)end_youngest));
33734 if ((curr_object >= next_boundary) && (curr_gen_num > 0))
33737 if (curr_gen_num > 0)
33739 next_boundary = generation_allocation_start (generation_of (curr_gen_num - 1));
33744 //if (is_mark_set (curr_object))
33746 // printf ("curr_object: %Ix is marked!",(size_t)curr_object);
33747 // FATAL_GC_ERROR();
33750 size_t s = size (curr_object);
33751 dprintf (3, ("o: %Ix, s: %d", (size_t)curr_object, s));
33754 dprintf (3, ("Verifying Heap: size of current object %Ix == 0", curr_object));
33758 // If object is not in the youngest generation, then lets
33759 // verify that the brick table is correct....
33760 if (((seg != ephemeral_heap_segment) ||
33761 (brick_of(curr_object) < brick_of(begin_youngest))))
33763 curr_brick = brick_of(curr_object);
33765 // Brick Table Verification...
33767 // On brick transition
33768 // if brick is negative
33769 // verify that brick indirects to previous valid brick
33771 // set current brick invalid flag to be flipped if we
33772 // encounter an object at the correct place
33774 if (curr_brick != prev_brick)
33776 // If the last brick we were examining had positive
33777 // entry but we never found the matching object, then
33778 // we have a problem
33779 // If prev_brick was the last one of the segment
33780 // it's ok for it to be invalid because it is never looked at
33781 if (bCurrentBrickInvalid &&
33782 (curr_brick != brick_of (heap_segment_mem (seg))) &&
33783 !heap_segment_read_only_p (seg))
33785 dprintf (3, ("curr brick %Ix invalid", curr_brick));
33791 //large objects verify the table only if they are in
33793 if ((heap_segment_reserved (seg) <= highest_address) &&
33794 (heap_segment_mem (seg) >= lowest_address) &&
33795 brick_table [curr_brick] != 0)
33797 dprintf (3, ("curr_brick %Ix for large object %Ix not set to -32768",
33798 curr_brick, (size_t)curr_object));
33803 bCurrentBrickInvalid = FALSE;
33808 // If the current brick contains a negative value make sure
33809 // that the indirection terminates at the last valid brick
33810 if (brick_table [curr_brick] <= 0)
33812 if (brick_table [curr_brick] == 0)
33814 dprintf(3, ("curr_brick %Ix for object %Ix set to 0",
33815 curr_brick, (size_t)curr_object));
33818 ptrdiff_t i = curr_brick;
33819 while ((i >= ((ptrdiff_t) brick_of (heap_segment_mem (seg)))) &&
33820 (brick_table[i] < 0))
33822 i = i + brick_table[i];
33824 if (i < ((ptrdiff_t)(brick_of (heap_segment_mem (seg))) - 1))
33826 dprintf (3, ("ptrdiff i: %Ix < brick_of (heap_segment_mem (seg)):%Ix - 1. curr_brick: %Ix",
33827 i, brick_of (heap_segment_mem (seg)),
33831 // if (i != last_valid_brick)
33832 // FATAL_GC_ERROR();
33833 bCurrentBrickInvalid = FALSE;
33835 else if (!heap_segment_read_only_p (seg))
33837 bCurrentBrickInvalid = TRUE;
33842 if (bCurrentBrickInvalid)
33844 if (curr_object == (brick_address(curr_brick) + brick_table[curr_brick] - 1))
33846 bCurrentBrickInvalid = FALSE;
33847 last_valid_brick = curr_brick;
33852 if (*((uint8_t**)curr_object) != (uint8_t *) g_gc_pFreeObjectMethodTable)
33854 #ifdef FEATURE_LOH_COMPACTION
33855 if ((curr_gen_num == (max_generation+1)) && (prev_object != 0))
33857 assert (method_table (prev_object) == g_gc_pFreeObjectMethodTable);
33859 #endif //FEATURE_LOH_COMPACTION
33861 total_objects_verified++;
33863 BOOL can_verify_deep = TRUE;
33864 #ifdef BACKGROUND_GC
33865 can_verify_deep = fgc_should_consider_object (curr_object, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p);
33866 #endif //BACKGROUND_GC
33868 BOOL deep_verify_obj = can_verify_deep;
33869 if ((heap_verify_level & GCConfig::HEAPVERIFY_DEEP_ON_COMPACT) && !settings.compaction)
33870 deep_verify_obj = FALSE;
33872 ((CObjectHeader*)curr_object)->ValidateHeap((Object*)curr_object, deep_verify_obj);
33874 if (can_verify_deep)
33876 if (curr_gen_num > 0)
33878 BOOL need_card_p = FALSE;
33879 if (contain_pointers_or_collectible (curr_object))
33881 dprintf (4, ("curr_object: %Ix", (size_t)curr_object));
33882 size_t crd = card_of (curr_object);
33883 BOOL found_card_p = card_set_p (crd);
33885 #ifdef COLLECTIBLE_CLASS
33886 if (is_collectible(curr_object))
33888 uint8_t* class_obj = get_class_object (curr_object);
33889 if ((class_obj < ephemeral_high) && (class_obj >= next_boundary))
33893 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix pointing to class object %Ix",
33894 card_of (curr_object), (size_t)curr_object, class_obj));
33900 #endif //COLLECTIBLE_CLASS
33902 if (contain_pointers(curr_object))
33904 go_through_object_nostart
33905 (method_table(curr_object), curr_object, s, oo,
33907 if ((crd != card_of ((uint8_t*)oo)) && !found_card_p)
33909 crd = card_of ((uint8_t*)oo);
33910 found_card_p = card_set_p (crd);
33911 need_card_p = FALSE;
33913 if ((*oo < ephemeral_high) && (*oo >= next_boundary))
33915 need_card_p = TRUE;
33918 if (need_card_p && !found_card_p)
33921 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix, %Ix:%Ix[",
33922 card_of (curr_object), (size_t)curr_object,
33923 card_of (curr_object+Align(s, align_const)), (size_t)curr_object+Align(s, align_const)));
33929 if (need_card_p && !found_card_p)
33931 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix, %Ix:%Ix[",
33932 card_of (curr_object), (size_t)curr_object,
33933 card_of (curr_object+Align(s, align_const)), (size_t)curr_object+Align(s, align_const)));
33938 total_objects_verified_deep++;
33942 prev_object = curr_object;
33943 prev_brick = curr_brick;
33944 curr_object = curr_object + Align(s, align_const);
33945 if (curr_object < prev_object)
33947 dprintf (3, ("overflow because of a bad object size: %Ix size %Ix", prev_object, s));
33952 #ifdef BACKGROUND_GC
33953 dprintf (2, ("(%s)(%s)(%s) total_objects_verified is %Id, total_objects_verified_deep is %Id",
33954 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p () ? "FGC" : "NGC")),
33955 (begin_gc_p ? "BEG" : "END"),
33956 ((current_c_gc_state == c_gc_state_planning) ? "in plan" : "not in plan"),
33957 total_objects_verified, total_objects_verified_deep));
33958 if (current_c_gc_state != c_gc_state_planning)
33960 assert (total_objects_verified == total_objects_verified_deep);
33962 #endif //BACKGROUND_GC
33964 verify_free_lists();
33966 #ifdef FEATURE_PREMORTEM_FINALIZATION
33967 finalize_queue->CheckFinalizerObjects();
33968 #endif // FEATURE_PREMORTEM_FINALIZATION
33971 // to be consistent with handle table APIs pass a ScanContext*
33972 // to provide the heap number. the SC isn't complete though so
33973 // limit its scope to handle table verification.
33975 sc.thread_number = heap_number;
33976 GCScan::VerifyHandleTable(max_generation, max_generation, &sc);
33979 #ifdef MULTIPLE_HEAPS
33980 current_join->join(this, gc_join_verify_objects_done);
33981 if (current_join->joined())
33982 #endif //MULTIPLE_HEAPS
33984 GCToEEInterface::VerifySyncTableEntry();
33985 #ifdef MULTIPLE_HEAPS
33986 current_join->restart();
33987 #endif //MULTIPLE_HEAPS
33990 #ifdef BACKGROUND_GC
33991 if (!settings.concurrent)
33993 if (current_c_gc_state == c_gc_state_planning)
33995 // temporarily commenting this out 'cause an FGC
33996 // could be triggered before we sweep ephemeral.
33997 //verify_seg_end_mark_array_cleared();
34001 if (settings.concurrent)
34003 verify_mark_array_cleared();
34005 dprintf (2,("GC%d(%s): Verifying heap - end",
34006 VolatileLoad(&settings.gc_index),
34007 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC"))));
34009 dprintf (2,("GC#d: Verifying heap - end", VolatileLoad(&settings.gc_index)));
34010 #endif //BACKGROUND_GC
34013 #endif //VERIFY_HEAP
34016 void GCHeap::ValidateObjectMember (Object* obj)
34019 size_t s = size (obj);
34020 uint8_t* o = (uint8_t*)obj;
34022 go_through_object_cl (method_table (obj), o, s, oo,
34024 uint8_t* child_o = *oo;
34027 dprintf (3, ("VOM: m: %Ix obj %Ix", (size_t)child_o, o));
34028 MethodTable *pMT = method_table (child_o);
34030 if (!pMT->SanityCheck()) {
34031 dprintf (3, ("Bad member of %Ix %Ix",
34032 (size_t)oo, (size_t)child_o));
34037 #endif // VERIFY_HEAP
34040 void DestructObject (CObjectHeader* hdr)
34042 UNREFERENCED_PARAMETER(hdr); // compiler bug? -- this *is*, indeed, referenced
34043 hdr->~CObjectHeader();
34046 HRESULT GCHeap::Shutdown ()
34050 GCScan::GcRuntimeStructuresValid (FALSE);
34052 // Cannot assert this, since we use SuspendEE as the mechanism to quiesce all
34053 // threads except the one performing the shutdown.
34054 // ASSERT( !GcInProgress );
34056 // Guard against any more GC occurring and against any threads blocking
34057 // for GC to complete when the GC heap is gone. This fixes a race condition
34058 // where a thread in GC is destroyed as part of process destruction and
34059 // the remaining threads block for GC complete.
34062 //EnterAllocLock();
34064 //EnterFinalizeLock();
34067 // during shutdown lot of threads are suspended
34068 // on this even, we don't want to wake them up just yet
34069 //CloseHandle (WaitForGCEvent);
34071 //find out if the global card table hasn't been used yet
34072 uint32_t* ct = &g_gc_card_table[card_word (gcard_of (g_gc_lowest_address))];
34073 if (card_table_refcount (ct) == 0)
34075 destroy_card_table (ct);
34076 g_gc_card_table = nullptr;
34078 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
34079 g_gc_card_bundle_table = nullptr;
34081 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
34082 SoftwareWriteWatch::StaticClose();
34083 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
34086 //destroy all segments on the standby list
34087 while(gc_heap::segment_standby_list != 0)
34089 heap_segment* next_seg = heap_segment_next (gc_heap::segment_standby_list);
34090 #ifdef MULTIPLE_HEAPS
34091 (gc_heap::g_heaps[0])->delete_heap_segment (gc_heap::segment_standby_list, FALSE);
34092 #else //MULTIPLE_HEAPS
34093 pGenGCHeap->delete_heap_segment (gc_heap::segment_standby_list, FALSE);
34094 #endif //MULTIPLE_HEAPS
34095 gc_heap::segment_standby_list = next_seg;
34099 #ifdef MULTIPLE_HEAPS
34101 for (int i = 0; i < gc_heap::n_heaps; i ++)
34103 delete gc_heap::g_heaps[i]->vm_heap;
34104 //destroy pure GC stuff
34105 gc_heap::destroy_gc_heap (gc_heap::g_heaps[i]);
34108 gc_heap::destroy_gc_heap (pGenGCHeap);
34110 #endif //MULTIPLE_HEAPS
34111 gc_heap::shutdown_gc();
34116 // Wait until a garbage collection is complete
34117 // returns NOERROR if wait was OK, other error code if failure.
34118 // WARNING: This will not undo the must complete state. If you are
34119 // in a must complete when you call this, you'd better know what you're
34122 #ifdef FEATURE_PREMORTEM_FINALIZATION
34124 HRESULT AllocateCFinalize(CFinalize **pCFinalize)
34126 *pCFinalize = new (nothrow) CFinalize();
34127 if (*pCFinalize == NULL || !(*pCFinalize)->Initialize())
34128 return E_OUTOFMEMORY;
34132 #endif // FEATURE_PREMORTEM_FINALIZATION
34134 // init the instance heap
34135 HRESULT GCHeap::Init(size_t hn)
34137 HRESULT hres = S_OK;
34139 #ifdef MULTIPLE_HEAPS
34140 if ((pGenGCHeap = gc_heap::make_gc_heap(this, (int)hn)) == 0)
34141 hres = E_OUTOFMEMORY;
34143 UNREFERENCED_PARAMETER(hn);
34144 if (!gc_heap::make_gc_heap())
34145 hres = E_OUTOFMEMORY;
34146 #endif //MULTIPLE_HEAPS
34152 //System wide initialization
34153 HRESULT GCHeap::Initialize()
34157 g_gc_pFreeObjectMethodTable = GCToEEInterface::GetFreeObjectMethodTable();
34158 g_num_processors = GCToOSInterface::GetTotalProcessorCount();
34159 assert(g_num_processors != 0);
34161 //Initialize the static members.
34164 CreatedObjectCount = 0;
34167 bool is_restricted;
34168 gc_heap::total_physical_mem = GCToOSInterface::GetPhysicalMemoryLimit (&is_restricted);
34171 gc_heap::heap_hard_limit = (size_t)GCConfig::GetGCHeapHardLimit();
34173 if (!(gc_heap::heap_hard_limit))
34175 uint32_t percent_of_mem = (uint32_t)GCConfig::GetGCHeapHardLimitPercent();
34176 if ((percent_of_mem > 0) && (percent_of_mem < 100))
34178 gc_heap::heap_hard_limit = (size_t)(gc_heap::total_physical_mem * (uint64_t)percent_of_mem / (uint64_t)100);
34182 // If the hard limit is specified, the user is saying even if the process is already
34183 // running in a container, use this limit for the GC heap.
34184 if (!(gc_heap::heap_hard_limit))
34188 uint64_t physical_mem_for_gc = gc_heap::total_physical_mem * (uint64_t)75 / (uint64_t)100;
34189 //printf ("returned physical mem %I64d, setting it to max (75%%: %I64d, 20mb)\n",
34190 // gc_heap::total_physical_mem, physical_mem_for_gc);
34191 gc_heap::heap_hard_limit = (size_t)max ((20 * 1024 * 1024), physical_mem_for_gc);
34195 //printf ("heap_hard_limit is %Id, total physical mem: %Id, %s restricted\n",
34196 // gc_heap::heap_hard_limit, gc_heap::total_physical_mem, (is_restricted ? "is" : "is not"));
34200 uint32_t nhp_from_config = 0;
34202 #ifdef MULTIPLE_HEAPS
34203 nhp_from_config = static_cast<uint32_t>(GCConfig::GetHeapCount());
34205 uint32_t nhp_from_process = GCToOSInterface::GetCurrentProcessCpuCount();
34207 if (nhp_from_config)
34209 // Even when the user specifies a heap count, it should not be more
34210 // than the number of procs this process can use.
34211 nhp_from_config = min (nhp_from_config, nhp_from_process);
34214 nhp = ((nhp_from_config == 0) ? nhp_from_process : nhp_from_config);
34216 nhp = min (nhp, MAX_SUPPORTED_CPUS);
34217 #ifndef FEATURE_REDHAWK
34218 gc_heap::gc_thread_no_affinitize_p = (gc_heap::heap_hard_limit ? false : (GCConfig::GetNoAffinitize() != 0));
34220 size_t gc_thread_affinity_mask = static_cast<size_t>(GCConfig::GetGCHeapAffinitizeMask());
34222 if (gc_heap::heap_hard_limit)
34224 gc_heap::gc_thread_no_affinitize_p = (gc_thread_affinity_mask == 0);
34227 if (!(gc_heap::gc_thread_no_affinitize_p))
34229 if (!(GCToOSInterface::CanEnableGCCPUGroups()))
34231 uintptr_t pmask, smask;
34232 if (GCToOSInterface::GetCurrentProcessAffinityMask(&pmask, &smask))
34236 if (gc_thread_affinity_mask)
34238 pmask &= gc_thread_affinity_mask;
34241 process_mask = pmask;
34243 unsigned int set_bits_in_pmask = 0;
34247 set_bits_in_pmask++;
34251 nhp = min (nhp, set_bits_in_pmask);
34255 gc_heap::gc_thread_no_affinitize_p = true;
34259 #endif //!FEATURE_REDHAWK
34260 #endif //MULTIPLE_HEAPS
34262 size_t seg_size = 0;
34263 size_t large_seg_size = 0;
34265 if (gc_heap::heap_hard_limit)
34267 seg_size = gc_heap::get_segment_size_hard_limit (&nhp, (nhp_from_config == 0));
34268 gc_heap::soh_segment_size = seg_size;
34269 large_seg_size = seg_size * 2;
34273 seg_size = get_valid_segment_size();
34274 gc_heap::soh_segment_size = seg_size;
34275 large_seg_size = get_valid_segment_size (TRUE);
34278 dprintf (1, ("%d heaps, soh seg size: %Id mb, loh: %Id mb\n",
34280 (seg_size / (size_t)1024 / 1024),
34281 (large_seg_size / 1024 / 1024)));
34283 gc_heap::min_loh_segment_size = large_seg_size;
34284 gc_heap::min_segment_size = min (seg_size, large_seg_size);
34285 #ifdef SEG_MAPPING_TABLE
34286 gc_heap::min_segment_size_shr = index_of_highest_set_bit (gc_heap::min_segment_size);
34287 #endif //SEG_MAPPING_TABLE
34289 #ifdef MULTIPLE_HEAPS
34290 gc_heap::n_heaps = nhp;
34291 hr = gc_heap::initialize_gc (seg_size, large_seg_size /*LHEAP_ALLOC*/, nhp);
34293 hr = gc_heap::initialize_gc (seg_size, large_seg_size /*LHEAP_ALLOC*/);
34294 #endif //MULTIPLE_HEAPS
34299 gc_heap::mem_one_percent = gc_heap::total_physical_mem / 100;
34300 #ifndef MULTIPLE_HEAPS
34301 gc_heap::mem_one_percent /= g_num_processors;
34302 #endif //!MULTIPLE_HEAPS
34304 uint32_t highmem_th_from_config = (uint32_t)GCConfig::GetGCHighMemPercent();
34305 if (highmem_th_from_config)
34307 gc_heap::high_memory_load_th = min (99, highmem_th_from_config);
34308 gc_heap::v_high_memory_load_th = min (99, (highmem_th_from_config + 7));
34312 // We should only use this if we are in the "many process" mode which really is only applicable
34313 // to very powerful machines - before that's implemented, temporarily I am only enabling this for 80GB+ memory.
34314 // For now I am using an estimate to calculate these numbers but this should really be obtained
34315 // programmatically going forward.
34316 // I am assuming 47 processes using WKS GC and 3 using SVR GC.
34317 // I am assuming 3 in part due to the "very high memory load" is 97%.
34318 int available_mem_th = 10;
34319 if (gc_heap::total_physical_mem >= ((uint64_t)80 * 1024 * 1024 * 1024))
34321 int adjusted_available_mem_th = 3 + (int)((float)47 / (float)(GCToOSInterface::GetTotalProcessorCount()));
34322 available_mem_th = min (available_mem_th, adjusted_available_mem_th);
34325 gc_heap::high_memory_load_th = 100 - available_mem_th;
34326 gc_heap::v_high_memory_load_th = 97;
34329 gc_heap::m_high_memory_load_th = min ((gc_heap::high_memory_load_th + 5), gc_heap::v_high_memory_load_th);
34331 gc_heap::pm_stress_on = (GCConfig::GetGCProvModeStress() != 0);
34334 gc_heap::youngest_gen_desired_th = gc_heap::mem_one_percent;
34337 WaitForGCEvent = new (nothrow) GCEvent;
34339 if (!WaitForGCEvent)
34341 return E_OUTOFMEMORY;
34344 if (!WaitForGCEvent->CreateManualEventNoThrow(TRUE))
34349 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
34350 #if defined (STRESS_HEAP) && !defined (MULTIPLE_HEAPS)
34351 if (GCStress<cfg_any>::IsEnabled()) {
34352 for(int i = 0; i < GCHeap::NUM_HEAP_STRESS_OBJS; i++)
34354 m_StressObjs[i] = CreateGlobalHandle(0);
34356 m_CurStressObj = 0;
34358 #endif //STRESS_HEAP && !MULTIPLE_HEAPS
34359 #endif // FEATURE_REDHAWK
34361 initGCShadow(); // If we are debugging write barriers, initialize heap shadow
34363 #ifdef MULTIPLE_HEAPS
34365 for (unsigned i = 0; i < nhp; i++)
34367 GCHeap* Hp = new (nothrow) GCHeap();
34369 return E_OUTOFMEMORY;
34371 if ((hr = Hp->Init (i))!= S_OK)
34376 // initialize numa node to heap map
34377 heap_select::init_numa_node_to_heap_map(nhp);
34380 #endif //MULTIPLE_HEAPS
34384 GCScan::GcRuntimeStructuresValid (TRUE);
34386 GCToEEInterface::DiagUpdateGenerationBounds();
34393 // GC callback functions
34394 bool GCHeap::IsPromoted(Object* object)
34397 ((CObjectHeader*)object)->Validate();
34400 uint8_t* o = (uint8_t*)object;
34402 if (gc_heap::settings.condemned_generation == max_generation)
34404 #ifdef MULTIPLE_HEAPS
34405 gc_heap* hp = gc_heap::g_heaps[0];
34407 gc_heap* hp = pGenGCHeap;
34408 #endif //MULTIPLE_HEAPS
34410 #ifdef BACKGROUND_GC
34411 if (gc_heap::settings.concurrent)
34413 bool is_marked = (!((o < hp->background_saved_highest_address) && (o >= hp->background_saved_lowest_address))||
34414 hp->background_marked (o));
34418 #endif //BACKGROUND_GC
34420 return (!((o < hp->highest_address) && (o >= hp->lowest_address))
34421 || hp->is_mark_set (o));
34426 gc_heap* hp = gc_heap::heap_of (o);
34427 return (!((o < hp->gc_high) && (o >= hp->gc_low))
34428 || hp->is_mark_set (o));
34432 size_t GCHeap::GetPromotedBytes(int heap_index)
34434 #ifdef BACKGROUND_GC
34435 if (gc_heap::settings.concurrent)
34437 return gc_heap::bpromoted_bytes (heap_index);
34440 #endif //BACKGROUND_GC
34442 return gc_heap::promoted_bytes (heap_index);
34446 void GCHeap::SetYieldProcessorScalingFactor (float scalingFactor)
34448 assert (yp_spin_count_unit != 0);
34449 int saved_yp_spin_count_unit = yp_spin_count_unit;
34450 yp_spin_count_unit = (int)((float)yp_spin_count_unit * scalingFactor / (float)9);
34452 // It's very suspicious if it becomes 0
34453 if (yp_spin_count_unit == 0)
34455 yp_spin_count_unit = saved_yp_spin_count_unit;
34459 unsigned int GCHeap::WhichGeneration (Object* object)
34461 gc_heap* hp = gc_heap::heap_of ((uint8_t*)object);
34462 unsigned int g = hp->object_gennum ((uint8_t*)object);
34463 dprintf (3, ("%Ix is in gen %d", (size_t)object, g));
34467 bool GCHeap::IsEphemeral (Object* object)
34469 uint8_t* o = (uint8_t*)object;
34470 gc_heap* hp = gc_heap::heap_of (o);
34471 return !!hp->ephemeral_pointer_p (o);
34474 // Return NULL if can't find next object. When EE is not suspended,
34475 // the result is not accurate: if the input arg is in gen0, the function could
34476 // return zeroed out memory as next object
34477 Object * GCHeap::NextObj (Object * object)
34480 uint8_t* o = (uint8_t*)object;
34482 #ifndef FEATURE_BASICFREEZE
34483 if (!((o < g_gc_highest_address) && (o >= g_gc_lowest_address)))
34487 #endif //!FEATURE_BASICFREEZE
34489 heap_segment * hs = gc_heap::find_segment (o, FALSE);
34495 BOOL large_object_p = heap_segment_loh_p (hs);
34496 if (large_object_p)
34497 return NULL; //could be racing with another core allocating.
34498 #ifdef MULTIPLE_HEAPS
34499 gc_heap* hp = heap_segment_heap (hs);
34500 #else //MULTIPLE_HEAPS
34502 #endif //MULTIPLE_HEAPS
34503 unsigned int g = hp->object_gennum ((uint8_t*)object);
34504 if ((g == 0) && hp->settings.demotion)
34505 return NULL;//could be racing with another core allocating.
34506 int align_const = get_alignment_constant (!large_object_p);
34507 uint8_t* nextobj = o + Align (size (o), align_const);
34508 if (nextobj <= o) // either overflow or 0 sized object.
34513 if ((nextobj < heap_segment_mem(hs)) ||
34514 (nextobj >= heap_segment_allocated(hs) && hs != hp->ephemeral_heap_segment) ||
34515 (nextobj >= hp->alloc_allocated))
34520 return (Object *)nextobj;
34523 #endif // VERIFY_HEAP
34528 #ifdef FEATURE_BASICFREEZE
34529 BOOL GCHeap::IsInFrozenSegment (Object * object)
34531 uint8_t* o = (uint8_t*)object;
34532 heap_segment * hs = gc_heap::find_segment (o, FALSE);
34533 //We create a frozen object for each frozen segment before the segment is inserted
34534 //to segment list; during ngen, we could also create frozen objects in segments which
34535 //don't belong to current GC heap.
34536 //So we return true if hs is NULL. It might create a hole about detecting invalidate
34537 //object. But given all other checks present, the hole should be very small
34538 return !hs || heap_segment_read_only_p (hs);
34540 #endif //FEATURE_BASICFREEZE
34542 #endif //VERIFY_HEAP
34544 // returns TRUE if the pointer is in one of the GC heaps.
34545 bool GCHeap::IsHeapPointer (void* vpObject, bool small_heap_only)
34547 // removed STATIC_CONTRACT_CAN_TAKE_LOCK here because find_segment
34548 // no longer calls GCEvent::Wait which eventually takes a lock.
34550 uint8_t* object = (uint8_t*) vpObject;
34551 #ifndef FEATURE_BASICFREEZE
34552 if (!((object < g_gc_highest_address) && (object >= g_gc_lowest_address)))
34554 #endif //!FEATURE_BASICFREEZE
34556 heap_segment * hs = gc_heap::find_segment (object, small_heap_only);
34560 #ifdef STRESS_PINNING
34561 static n_promote = 0;
34562 #endif //STRESS_PINNING
34563 // promote an object
34564 void GCHeap::Promote(Object** ppObject, ScanContext* sc, uint32_t flags)
34566 THREAD_NUMBER_FROM_CONTEXT;
34567 #ifndef MULTIPLE_HEAPS
34568 const int thread = 0;
34569 #endif //!MULTIPLE_HEAPS
34571 uint8_t* o = (uint8_t*)*ppObject;
34576 #ifdef DEBUG_DestroyedHandleValue
34577 // we can race with destroy handle during concurrent scan
34578 if (o == (uint8_t*)DEBUG_DestroyedHandleValue)
34580 #endif //DEBUG_DestroyedHandleValue
34584 gc_heap* hp = gc_heap::heap_of (o);
34586 dprintf (3, ("Promote %Ix", (size_t)o));
34588 #ifdef INTERIOR_POINTERS
34589 if (flags & GC_CALL_INTERIOR)
34591 if ((o < hp->gc_low) || (o >= hp->gc_high))
34595 if ( (o = hp->find_object (o, hp->gc_low)) == 0)
34601 #endif //INTERIOR_POINTERS
34603 #ifdef FEATURE_CONSERVATIVE_GC
34604 // For conservative GC, a value on stack may point to middle of a free object.
34605 // In this case, we don't need to promote the pointer.
34606 if (GCConfig::GetConservativeGC()
34607 && ((CObjectHeader*)o)->IsFree())
34614 ((CObjectHeader*)o)->ValidatePromote(sc, flags);
34616 UNREFERENCED_PARAMETER(sc);
34619 if (flags & GC_CALL_PINNED)
34620 hp->pin_object (o, (uint8_t**) ppObject, hp->gc_low, hp->gc_high);
34622 #ifdef STRESS_PINNING
34623 if ((++n_promote % 20) == 1)
34624 hp->pin_object (o, (uint8_t**) ppObject, hp->gc_low, hp->gc_high);
34625 #endif //STRESS_PINNING
34627 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
34628 size_t promoted_size_begin = hp->promoted_bytes (thread);
34629 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
34631 if ((o >= hp->gc_low) && (o < hp->gc_high))
34633 hpt->mark_object_simple (&o THREAD_NUMBER_ARG);
34636 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
34637 size_t promoted_size_end = hp->promoted_bytes (thread);
34638 if (g_fEnableAppDomainMonitoring)
34640 if (sc->pCurrentDomain)
34642 GCToEEInterface::RecordSurvivedBytesForHeap((promoted_size_end - promoted_size_begin), thread, sc->pCurrentDomain);
34645 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
34647 STRESS_LOG_ROOT_PROMOTE(ppObject, o, o ? header(o)->GetMethodTable() : NULL);
34650 void GCHeap::Relocate (Object** ppObject, ScanContext* sc,
34653 UNREFERENCED_PARAMETER(sc);
34655 uint8_t* object = (uint8_t*)(Object*)(*ppObject);
34657 THREAD_NUMBER_FROM_CONTEXT;
34659 //dprintf (3, ("Relocate location %Ix\n", (size_t)ppObject));
34660 dprintf (3, ("R: %Ix", (size_t)ppObject));
34665 gc_heap* hp = gc_heap::heap_of (object);
34668 if (!(flags & GC_CALL_INTERIOR))
34670 // We cannot validate this object if it's in the condemned gen because it could
34671 // be one of the objects that were overwritten by an artificial gap due to a pinned plug.
34672 if (!((object >= hp->gc_low) && (object < hp->gc_high)))
34674 ((CObjectHeader*)object)->Validate(FALSE);
34679 dprintf (3, ("Relocate %Ix\n", (size_t)object));
34683 if ((flags & GC_CALL_INTERIOR) && gc_heap::settings.loh_compaction)
34685 if (!((object >= hp->gc_low) && (object < hp->gc_high)))
34690 if (gc_heap::loh_object_p (object))
34692 pheader = hp->find_object (object, 0);
34698 ptrdiff_t ref_offset = object - pheader;
34699 hp->relocate_address(&pheader THREAD_NUMBER_ARG);
34700 *ppObject = (Object*)(pheader + ref_offset);
34707 hp->relocate_address(&pheader THREAD_NUMBER_ARG);
34708 *ppObject = (Object*)pheader;
34711 STRESS_LOG_ROOT_RELOCATE(ppObject, object, pheader, ((!(flags & GC_CALL_INTERIOR)) ? ((Object*)object)->GetGCSafeMethodTable() : 0));
34714 /*static*/ bool GCHeap::IsObjectInFixedHeap(Object *pObj)
34716 // For now we simply look at the size of the object to determine if it in the
34717 // fixed heap or not. If the bit indicating this gets set at some point
34718 // we should key off that instead.
34719 return size( pObj ) >= loh_size_threshold;
34722 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
34725 void StressHeapDummy ();
34727 static int32_t GCStressStartCount = -1;
34728 static int32_t GCStressCurCount = 0;
34729 static int32_t GCStressStartAtJit = -1;
34731 // the maximum number of foreground GCs we'll induce during one BGC
34732 // (this number does not include "naturally" occuring GCs).
34733 static int32_t GCStressMaxFGCsPerBGC = -1;
34735 // CLRRandom implementation can produce FPU exceptions if
34736 // the test/application run by CLR is enabling any FPU exceptions.
34737 // We want to avoid any unexpected exception coming from stress
34738 // infrastructure, so CLRRandom is not an option.
34739 // The code below is a replicate of CRT rand() implementation.
34740 // Using CRT rand() is not an option because we will interfere with the user application
34741 // that may also use it.
34742 int StressRNG(int iMaxValue)
34744 static BOOL bisRandInit = FALSE;
34745 static int lHoldrand = 1L;
34749 lHoldrand = (int)time(NULL);
34750 bisRandInit = TRUE;
34752 int randValue = (((lHoldrand = lHoldrand * 214013L + 2531011L) >> 16) & 0x7fff);
34753 return randValue % iMaxValue;
34755 #endif // STRESS_HEAP
34756 #endif // !FEATURE_REDHAWK
34758 // free up object so that things will move and then do a GC
34759 //return TRUE if GC actually happens, otherwise FALSE
34760 bool GCHeap::StressHeap(gc_alloc_context * context)
34762 #if defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
34763 alloc_context* acontext = static_cast<alloc_context*>(context);
34764 assert(context != nullptr);
34766 // if GC stress was dynamically disabled during this run we return FALSE
34767 if (!GCStressPolicy::IsEnabled())
34771 if (g_pConfig->FastGCStressLevel() && !GCToEEInterface::GetThread()->StressHeapIsEnabled()) {
34777 if ((g_pConfig->GetGCStressLevel() & EEConfig::GCSTRESS_UNIQUE)
34779 || g_pConfig->FastGCStressLevel() > 1
34782 if (!Thread::UniqueStack(&acontext)) {
34787 #ifdef BACKGROUND_GC
34788 // don't trigger a GC from the GC threads but still trigger GCs from user threads.
34789 if (GCToEEInterface::WasCurrentThreadCreatedByGC())
34793 #endif //BACKGROUND_GC
34795 if (GCStressStartAtJit == -1 || GCStressStartCount == -1)
34797 GCStressStartCount = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_GCStressStart);
34798 GCStressStartAtJit = CLRConfig::GetConfigValue(CLRConfig::INTERNAL_GCStressStartAtJit);
34801 if (GCStressMaxFGCsPerBGC == -1)
34803 GCStressMaxFGCsPerBGC = CLRConfig::GetConfigValue(CLRConfig::INTERNAL_GCStressMaxFGCsPerBGC);
34804 if (g_pConfig->IsGCStressMix() && GCStressMaxFGCsPerBGC == -1)
34805 GCStressMaxFGCsPerBGC = 6;
34809 if (g_JitCount < GCStressStartAtJit)
34813 // Allow programmer to skip the first N Stress GCs so that you can
34814 // get to the interesting ones faster.
34815 Interlocked::Increment(&GCStressCurCount);
34816 if (GCStressCurCount < GCStressStartCount)
34819 // throttle the number of stress-induced GCs by a factor given by GCStressStep
34820 if ((GCStressCurCount % g_pConfig->GetGCStressStep()) != 0)
34825 #ifdef BACKGROUND_GC
34826 if (IsConcurrentGCEnabled() && IsConcurrentGCInProgress())
34828 // allow a maximum number of stress induced FGCs during one BGC
34829 if (gc_stress_fgcs_in_bgc >= GCStressMaxFGCsPerBGC)
34831 ++gc_stress_fgcs_in_bgc;
34833 #endif // BACKGROUND_GC
34835 if (g_pStringClass == 0)
34837 // If the String class has not been loaded, dont do any stressing. This should
34838 // be kept to a minimum to get as complete coverage as possible.
34839 _ASSERTE(g_fEEInit);
34843 #ifndef MULTIPLE_HEAPS
34844 static int32_t OneAtATime = -1;
34846 // Only bother with this if the stress level is big enough and if nobody else is
34847 // doing it right now. Note that some callers are inside the AllocLock and are
34848 // guaranteed synchronized. But others are using AllocationContexts and have no
34849 // particular synchronization.
34851 // For this latter case, we want a very high-speed way of limiting this to one
34852 // at a time. A secondary advantage is that we release part of our StressObjs
34853 // buffer sparingly but just as effectively.
34855 if (Interlocked::Increment(&OneAtATime) == 0 &&
34856 !TrackAllocations()) // Messing with object sizes can confuse the profiler (see ICorProfilerInfo::GetObjectSize)
34860 // If the current string is used up
34861 if (HndFetchHandle(m_StressObjs[m_CurStressObj]) == 0)
34863 // Populate handles with strings
34864 int i = m_CurStressObj;
34865 while(HndFetchHandle(m_StressObjs[i]) == 0)
34867 _ASSERTE(m_StressObjs[i] != 0);
34868 unsigned strLen = ((unsigned)loh_size_threshold - 32) / sizeof(WCHAR);
34869 unsigned strSize = PtrAlign(StringObject::GetSize(strLen));
34871 // update the cached type handle before allocating
34872 SetTypeHandleOnThreadForAlloc(TypeHandle(g_pStringClass));
34873 str = (StringObject*) pGenGCHeap->allocate (strSize, acontext);
34876 str->SetMethodTable (g_pStringClass);
34877 str->SetStringLength (strLen);
34878 HndAssignHandle(m_StressObjs[i], ObjectToOBJECTREF(str));
34880 i = (i + 1) % NUM_HEAP_STRESS_OBJS;
34881 if (i == m_CurStressObj) break;
34884 // advance the current handle to the next string
34885 m_CurStressObj = (m_CurStressObj + 1) % NUM_HEAP_STRESS_OBJS;
34888 // Get the current string
34889 str = (StringObject*) OBJECTREFToObject(HndFetchHandle(m_StressObjs[m_CurStressObj]));
34892 // Chop off the end of the string and form a new object out of it.
34893 // This will 'free' an object at the begining of the heap, which will
34894 // force data movement. Note that we can only do this so many times.
34895 // before we have to move on to the next string.
34896 unsigned sizeOfNewObj = (unsigned)Align(min_obj_size * 31);
34897 if (str->GetStringLength() > sizeOfNewObj / sizeof(WCHAR))
34899 unsigned sizeToNextObj = (unsigned)Align(size(str));
34900 uint8_t* freeObj = ((uint8_t*) str) + sizeToNextObj - sizeOfNewObj;
34901 pGenGCHeap->make_unused_array (freeObj, sizeOfNewObj);
34902 str->SetStringLength(str->GetStringLength() - (sizeOfNewObj / sizeof(WCHAR)));
34906 // Let the string itself become garbage.
34907 // will be realloced next time around
34908 HndAssignHandle(m_StressObjs[m_CurStressObj], 0);
34912 Interlocked::Decrement(&OneAtATime);
34913 #endif // !MULTIPLE_HEAPS
34914 if (IsConcurrentGCEnabled())
34916 int rgen = StressRNG(10);
34918 // gen0:gen1:gen2 distribution: 40:40:20
34921 else if (rgen >= 4)
34926 GarbageCollectTry (rgen, FALSE, collection_gcstress);
34930 GarbageCollect(max_generation, FALSE, collection_gcstress);
34935 UNREFERENCED_PARAMETER(context);
34937 #endif // defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
34941 #ifdef FEATURE_PREMORTEM_FINALIZATION
34942 #define REGISTER_FOR_FINALIZATION(_object, _size) \
34943 hp->finalize_queue->RegisterForFinalization (0, (_object), (_size))
34944 #else // FEATURE_PREMORTEM_FINALIZATION
34945 #define REGISTER_FOR_FINALIZATION(_object, _size) true
34946 #endif // FEATURE_PREMORTEM_FINALIZATION
34948 #define CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(_object, _size, _register) do { \
34949 if ((_object) == NULL || ((_register) && !REGISTER_FOR_FINALIZATION(_object, _size))) \
34951 STRESS_LOG_OOM_STACK(_size); \
34957 // Small Object Allocator
34960 // Allocate small object with an alignment requirement of 8-bytes.
34962 GCHeap::AllocAlign8(gc_alloc_context* ctx, size_t size, uint32_t flags )
34964 #ifdef FEATURE_64BIT_ALIGNMENT
34970 alloc_context* acontext = static_cast<alloc_context*>(ctx);
34972 #ifdef MULTIPLE_HEAPS
34973 if (acontext->get_alloc_heap() == 0)
34975 AssignHeap (acontext);
34976 assert (acontext->get_alloc_heap());
34979 gc_heap* hp = acontext->get_alloc_heap()->pGenGCHeap;
34981 gc_heap* hp = pGenGCHeap;
34982 #endif //MULTIPLE_HEAPS
34984 return AllocAlign8Common(hp, acontext, size, flags);
34986 UNREFERENCED_PARAMETER(ctx);
34987 UNREFERENCED_PARAMETER(size);
34988 UNREFERENCED_PARAMETER(flags);
34989 assert(!"should not call GCHeap::AllocAlign8 without FEATURE_64BIT_ALIGNMENT defined!");
34991 #endif //FEATURE_64BIT_ALIGNMENT
34994 // Common code used by both variants of AllocAlign8 above.
34996 GCHeap::AllocAlign8Common(void* _hp, alloc_context* acontext, size_t size, uint32_t flags)
34998 #ifdef FEATURE_64BIT_ALIGNMENT
35004 gc_heap* hp = (gc_heap*)_hp;
35008 Object* newAlloc = NULL;
35011 #ifdef COUNT_CYCLES
35012 AllocStart = GetCycleCount32();
35014 #elif defined(ENABLE_INSTRUMENTATION)
35015 unsigned AllocStart = GetInstLogTime();
35017 #endif //COUNT_CYCLES
35020 if (size < loh_size_threshold)
35026 // Depending on where in the object the payload requiring 8-byte alignment resides we might have to
35027 // align the object header on an 8-byte boundary or midway between two such boundaries. The unaligned
35028 // case is indicated to the GC via the GC_ALLOC_ALIGN8_BIAS flag.
35029 size_t desiredAlignment = (flags & GC_ALLOC_ALIGN8_BIAS) ? 4 : 0;
35031 // Retrieve the address of the next allocation from the context (note that we're inside the alloc
35032 // lock at this point).
35033 uint8_t* result = acontext->alloc_ptr;
35035 // Will an allocation at this point yield the correct alignment and fit into the remainder of the
35037 if ((((size_t)result & 7) == desiredAlignment) && ((result + size) <= acontext->alloc_limit))
35039 // Yes, we can just go ahead and make the allocation.
35040 newAlloc = (Object*) hp->allocate (size, acontext);
35041 ASSERT(((size_t)newAlloc & 7) == desiredAlignment);
35045 // No, either the next available address is not aligned in the way we require it or there's
35046 // not enough space to allocate an object of the required size. In both cases we allocate a
35047 // padding object (marked as a free object). This object's size is such that it will reverse
35048 // the alignment of the next header (asserted below).
35050 // We allocate both together then decide based on the result whether we'll format the space as
35051 // free object + real object or real object + free object.
35052 ASSERT((Align(min_obj_size) & 7) == 4);
35053 CObjectHeader *freeobj = (CObjectHeader*) hp->allocate (Align(size) + Align(min_obj_size), acontext);
35056 if (((size_t)freeobj & 7) == desiredAlignment)
35058 // New allocation has desired alignment, return this one and place the free object at the
35059 // end of the allocated space.
35060 newAlloc = (Object*)freeobj;
35061 freeobj = (CObjectHeader*)((uint8_t*)freeobj + Align(size));
35065 // New allocation is still mis-aligned, format the initial space as a free object and the
35066 // rest of the space should be correctly aligned for the real object.
35067 newAlloc = (Object*)((uint8_t*)freeobj + Align(min_obj_size));
35068 ASSERT(((size_t)newAlloc & 7) == desiredAlignment);
35070 freeobj->SetFree(min_obj_size);
35076 // The LOH always guarantees at least 8-byte alignment, regardless of platform. Moreover it doesn't
35077 // support mis-aligned object headers so we can't support biased headers as above. Luckily for us
35078 // we've managed to arrange things so the only case where we see a bias is for boxed value types and
35079 // these can never get large enough to be allocated on the LOH.
35080 ASSERT(65536 < loh_size_threshold);
35081 ASSERT((flags & GC_ALLOC_ALIGN8_BIAS) == 0);
35083 alloc_context* acontext = generation_alloc_context (hp->generation_of (max_generation+1));
35085 newAlloc = (Object*) hp->allocate_large_object (size, acontext->alloc_bytes_loh);
35086 ASSERT(((size_t)newAlloc & 7) == 0);
35089 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
35092 #ifdef COUNT_CYCLES
35093 finish = GetCycleCount32();
35094 #elif defined(ENABLE_INSTRUMENTATION)
35095 finish = GetInstLogTime();
35096 #endif //COUNT_CYCLES
35097 AllocDuration += finish - AllocStart;
35102 UNREFERENCED_PARAMETER(_hp);
35103 UNREFERENCED_PARAMETER(acontext);
35104 UNREFERENCED_PARAMETER(size);
35105 UNREFERENCED_PARAMETER(flags);
35106 assert(!"Should not call GCHeap::AllocAlign8Common without FEATURE_64BIT_ALIGNMENT defined!");
35108 #endif // FEATURE_64BIT_ALIGNMENT
35112 GCHeap::AllocLHeap( size_t size, uint32_t flags REQD_ALIGN_DCL)
35121 Object* newAlloc = NULL;
35124 #ifdef COUNT_CYCLES
35125 AllocStart = GetCycleCount32();
35127 #elif defined(ENABLE_INSTRUMENTATION)
35128 unsigned AllocStart = GetInstLogTime();
35130 #endif //COUNT_CYCLES
35133 #ifdef MULTIPLE_HEAPS
35134 //take the first heap....
35135 gc_heap* hp = gc_heap::g_heaps[0];
35137 gc_heap* hp = pGenGCHeap;
35139 // prefix complains about us dereferencing hp in wks build even though we only access static members
35140 // this way. not sure how to shut it up except for this ugly workaround:
35141 PREFIX_ASSUME(hp != NULL);
35143 #endif //MULTIPLE_HEAPS
35145 alloc_context* acontext = generation_alloc_context (hp->generation_of (max_generation+1));
35147 newAlloc = (Object*) hp->allocate_large_object (size + ComputeMaxStructAlignPadLarge(requiredAlignment), acontext->alloc_bytes_loh);
35148 #ifdef FEATURE_STRUCTALIGN
35149 newAlloc = (Object*) hp->pad_for_alignment_large ((uint8_t*) newAlloc, requiredAlignment, size);
35150 #endif // FEATURE_STRUCTALIGN
35151 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
35154 #ifdef COUNT_CYCLES
35155 finish = GetCycleCount32();
35156 #elif defined(ENABLE_INSTRUMENTATION)
35157 finish = GetInstLogTime();
35158 #endif //COUNT_CYCLES
35159 AllocDuration += finish - AllocStart;
35166 GCHeap::Alloc(gc_alloc_context* context, size_t size, uint32_t flags REQD_ALIGN_DCL)
35175 Object* newAlloc = NULL;
35176 alloc_context* acontext = static_cast<alloc_context*>(context);
35179 #ifdef COUNT_CYCLES
35180 AllocStart = GetCycleCount32();
35182 #elif defined(ENABLE_INSTRUMENTATION)
35183 unsigned AllocStart = GetInstLogTime();
35185 #endif //COUNT_CYCLES
35188 #ifdef MULTIPLE_HEAPS
35189 if (acontext->get_alloc_heap() == 0)
35191 AssignHeap (acontext);
35192 assert (acontext->get_alloc_heap());
35194 #endif //MULTIPLE_HEAPS
35196 #ifdef MULTIPLE_HEAPS
35197 gc_heap* hp = acontext->get_alloc_heap()->pGenGCHeap;
35199 gc_heap* hp = pGenGCHeap;
35201 // prefix complains about us dereferencing hp in wks build even though we only access static members
35202 // this way. not sure how to shut it up except for this ugly workaround:
35203 PREFIX_ASSUME(hp != NULL);
35205 #endif //MULTIPLE_HEAPS
35207 if (size < loh_size_threshold)
35213 newAlloc = (Object*) hp->allocate (size + ComputeMaxStructAlignPad(requiredAlignment), acontext);
35214 #ifdef FEATURE_STRUCTALIGN
35215 newAlloc = (Object*) hp->pad_for_alignment ((uint8_t*) newAlloc, requiredAlignment, size, acontext);
35216 #endif // FEATURE_STRUCTALIGN
35217 // ASSERT (newAlloc);
35221 newAlloc = (Object*) hp->allocate_large_object (size + ComputeMaxStructAlignPadLarge(requiredAlignment), acontext->alloc_bytes_loh);
35222 #ifdef FEATURE_STRUCTALIGN
35223 newAlloc = (Object*) hp->pad_for_alignment_large ((uint8_t*) newAlloc, requiredAlignment, size);
35224 #endif // FEATURE_STRUCTALIGN
35227 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
35230 #ifdef COUNT_CYCLES
35231 finish = GetCycleCount32();
35232 #elif defined(ENABLE_INSTRUMENTATION)
35233 finish = GetInstLogTime();
35234 #endif //COUNT_CYCLES
35235 AllocDuration += finish - AllocStart;
35242 GCHeap::FixAllocContext (gc_alloc_context* context, void* arg, void *heap)
35244 alloc_context* acontext = static_cast<alloc_context*>(context);
35245 #ifdef MULTIPLE_HEAPS
35248 acontext->alloc_count = 0;
35250 uint8_t * alloc_ptr = acontext->alloc_ptr;
35255 // The acontext->alloc_heap can be out of sync with the ptrs because
35256 // of heap re-assignment in allocate
35257 gc_heap* hp = gc_heap::heap_of (alloc_ptr);
35259 gc_heap* hp = pGenGCHeap;
35260 #endif //MULTIPLE_HEAPS
35262 if (heap == NULL || heap == hp)
35264 hp->fix_allocation_context (acontext, ((arg != 0)? TRUE : FALSE),
35265 get_alignment_constant(TRUE));
35270 GCHeap::GetContainingObject (void *pInteriorPtr, bool fCollectedGenOnly)
35272 uint8_t *o = (uint8_t*)pInteriorPtr;
35274 gc_heap* hp = gc_heap::heap_of (o);
35276 uint8_t* lowest = (fCollectedGenOnly ? hp->gc_low : hp->lowest_address);
35277 uint8_t* highest = (fCollectedGenOnly ? hp->gc_high : hp->highest_address);
35279 if (o >= lowest && o < highest)
35281 o = hp->find_object (o, lowest);
35288 return (Object *)o;
35291 BOOL should_collect_optimized (dynamic_data* dd, BOOL low_memory_p)
35293 if (dd_new_allocation (dd) < 0)
35298 if (((float)(dd_new_allocation (dd)) / (float)dd_desired_allocation (dd)) < (low_memory_p ? 0.7 : 0.3))
35306 //----------------------------------------------------------------------------
35307 // #GarbageCollector
35309 // API to ensure that a complete new garbage collection takes place
35312 GCHeap::GarbageCollect (int generation, bool low_memory_p, int mode)
35317 size_t total_allocated = 0;
35318 size_t total_desired = 0;
35319 #ifdef MULTIPLE_HEAPS
35321 for (hn = 0; hn < gc_heap::n_heaps; hn++)
35323 gc_heap* hp = gc_heap::g_heaps [hn];
35324 total_desired += dd_desired_allocation (hp->dynamic_data_of (0));
35325 total_allocated += dd_desired_allocation (hp->dynamic_data_of (0))-
35326 dd_new_allocation (hp->dynamic_data_of (0));
35329 gc_heap* hp = pGenGCHeap;
35330 total_desired = dd_desired_allocation (hp->dynamic_data_of (0));
35331 total_allocated = dd_desired_allocation (hp->dynamic_data_of (0))-
35332 dd_new_allocation (hp->dynamic_data_of (0));
35333 #endif //MULTIPLE_HEAPS
35335 if ((total_desired > gc_heap::mem_one_percent) && (total_allocated < gc_heap::mem_one_percent))
35337 dprintf (2, ("Async low mem but we've only allocated %d (< 10%% of physical mem) out of %d, returning",
35338 total_allocated, total_desired));
35345 #ifdef MULTIPLE_HEAPS
35346 gc_heap* hpt = gc_heap::g_heaps[0];
35349 #endif //MULTIPLE_HEAPS
35351 generation = (generation < 0) ? max_generation : min (generation, max_generation);
35352 dynamic_data* dd = hpt->dynamic_data_of (generation);
35354 #ifdef BACKGROUND_GC
35355 if (recursive_gc_sync::background_running_p())
35357 if ((mode == collection_optimized) || (mode & collection_non_blocking))
35361 if (mode & collection_blocking)
35363 pGenGCHeap->background_gc_wait();
35364 if (mode & collection_optimized)
35370 #endif //BACKGROUND_GC
35372 if (mode & collection_optimized)
35374 if (pGenGCHeap->gc_started)
35380 BOOL should_collect = FALSE;
35381 BOOL should_check_loh = (generation == max_generation);
35382 #ifdef MULTIPLE_HEAPS
35383 for (int i = 0; i < gc_heap::n_heaps; i++)
35385 dynamic_data* dd1 = gc_heap::g_heaps [i]->dynamic_data_of (generation);
35386 dynamic_data* dd2 = (should_check_loh ?
35387 (gc_heap::g_heaps [i]->dynamic_data_of (max_generation + 1)) :
35390 if (should_collect_optimized (dd1, low_memory_p))
35392 should_collect = TRUE;
35395 if (dd2 && should_collect_optimized (dd2, low_memory_p))
35397 should_collect = TRUE;
35402 should_collect = should_collect_optimized (dd, low_memory_p);
35403 if (!should_collect && should_check_loh)
35406 should_collect_optimized (hpt->dynamic_data_of (max_generation + 1), low_memory_p);
35408 #endif //MULTIPLE_HEAPS
35409 if (!should_collect)
35416 size_t CollectionCountAtEntry = dd_collection_count (dd);
35417 size_t BlockingCollectionCountAtEntry = gc_heap::full_gc_counts[gc_type_blocking];
35418 size_t CurrentCollectionCount = 0;
35422 CurrentCollectionCount = GarbageCollectTry(generation, low_memory_p, mode);
35424 if ((mode & collection_blocking) &&
35425 (generation == max_generation) &&
35426 (gc_heap::full_gc_counts[gc_type_blocking] == BlockingCollectionCountAtEntry))
35428 #ifdef BACKGROUND_GC
35429 if (recursive_gc_sync::background_running_p())
35431 pGenGCHeap->background_gc_wait();
35433 #endif //BACKGROUND_GC
35438 if (CollectionCountAtEntry == CurrentCollectionCount)
35447 GCHeap::GarbageCollectTry (int generation, BOOL low_memory_p, int mode)
35449 int gen = (generation < 0) ?
35450 max_generation : min (generation, max_generation);
35452 gc_reason reason = reason_empty;
35456 if (mode & collection_blocking)
35458 reason = reason_lowmemory_blocking;
35462 reason = reason_lowmemory;
35467 reason = reason_induced;
35470 if (reason == reason_induced)
35472 if (mode & collection_compacting)
35474 reason = reason_induced_compacting;
35476 else if (mode & collection_non_blocking)
35478 reason = reason_induced_noforce;
35481 else if (mode & collection_gcstress)
35483 reason = reason_gcstress;
35488 return GarbageCollectGeneration (gen, reason);
35491 void gc_heap::do_pre_gc()
35493 STRESS_LOG_GC_STACK;
35496 STRESS_LOG_GC_START(VolatileLoad(&settings.gc_index),
35497 (uint32_t)settings.condemned_generation,
35498 (uint32_t)settings.reason);
35499 #endif // STRESS_LOG
35501 #ifdef MULTIPLE_HEAPS
35502 gc_heap* hp = g_heaps[0];
35505 #endif //MULTIPLE_HEAPS
35507 #ifdef BACKGROUND_GC
35508 settings.b_state = hp->current_bgc_state;
35509 #endif //BACKGROUND_GC
35512 size_t total_allocated_since_last_gc = get_total_allocated_since_last_gc();
35513 #ifdef BACKGROUND_GC
35514 dprintf (1, ("*GC* %d(gen0:%d)(%d)(alloc: %Id)(%s)(%d)",
35515 VolatileLoad(&settings.gc_index),
35516 dd_collection_count (hp->dynamic_data_of (0)),
35517 settings.condemned_generation,
35518 total_allocated_since_last_gc,
35519 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC")),
35520 settings.b_state));
35522 dprintf (1, ("*GC* %d(gen0:%d)(%d)(alloc: %Id)",
35523 VolatileLoad(&settings.gc_index),
35524 dd_collection_count(hp->dynamic_data_of(0)),
35525 settings.condemned_generation,
35526 total_allocated_since_last_gc));
35527 #endif //BACKGROUND_GC
35529 if (heap_hard_limit)
35531 size_t total_heap_committed = get_total_committed_size();
35532 size_t total_heap_committed_recorded = current_total_committed - current_total_committed_bookkeeping;
35533 dprintf (1, ("(%d)GC commit BEG #%Id: %Id (recorded: %Id)",
35534 settings.condemned_generation,
35535 (size_t)settings.gc_index, total_heap_committed, total_heap_committed_recorded));
35539 // TODO: this can happen...it's because of the way we are calling
35540 // do_pre_gc, will fix later.
35541 //if (last_gc_index > VolatileLoad(&settings.gc_index))
35543 // FATAL_GC_ERROR();
35546 last_gc_index = VolatileLoad(&settings.gc_index);
35547 GCHeap::UpdatePreGCCounters();
35549 if (settings.concurrent)
35551 #ifdef BACKGROUND_GC
35552 full_gc_counts[gc_type_background]++;
35553 #if defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
35554 GCHeap::gc_stress_fgcs_in_bgc = 0;
35555 #endif // STRESS_HEAP && !FEATURE_REDHAWK
35556 #endif // BACKGROUND_GC
35560 if (settings.condemned_generation == max_generation)
35562 full_gc_counts[gc_type_blocking]++;
35566 #ifdef BACKGROUND_GC
35567 if (settings.background_p)
35569 ephemeral_fgc_counts[settings.condemned_generation]++;
35571 #endif //BACKGROUND_GC
35575 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
35576 if (g_fEnableAppDomainMonitoring)
35578 GCToEEInterface::ResetTotalSurvivedBytes();
35580 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
35583 #ifdef GC_CONFIG_DRIVEN
35584 void gc_heap::record_interesting_info_per_heap()
35586 // datapoints are always from the last blocking GC so don't record again
35588 if (!(settings.concurrent))
35590 for (int i = 0; i < max_idp_count; i++)
35592 interesting_data_per_heap[i] += interesting_data_per_gc[i];
35596 int compact_reason = get_gc_data_per_heap()->get_mechanism (gc_heap_compact);
35597 if (compact_reason >= 0)
35598 (compact_reasons_per_heap[compact_reason])++;
35599 int expand_mechanism = get_gc_data_per_heap()->get_mechanism (gc_heap_expand);
35600 if (expand_mechanism >= 0)
35601 (expand_mechanisms_per_heap[expand_mechanism])++;
35603 for (int i = 0; i < max_gc_mechanism_bits_count; i++)
35605 if (get_gc_data_per_heap()->is_mechanism_bit_set ((gc_mechanism_bit_per_heap)i))
35606 (interesting_mechanism_bits_per_heap[i])++;
35609 // h# | GC | gen | C | EX | NF | BF | ML | DM || PreS | PostS | Merge | Conv | Pre | Post | PrPo | PreP | PostP |
35610 cprintf (("%2d | %6d | %1d | %1s | %2s | %2s | %2s | %2s | %2s || %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id |",
35612 (size_t)settings.gc_index,
35613 settings.condemned_generation,
35614 // TEMP - I am just doing this for wks GC 'cuase I wanna see the pattern of doing C/S GCs.
35615 (settings.compaction ? (((compact_reason >= 0) && gc_heap_compact_reason_mandatory_p[compact_reason]) ? "M" : "W") : ""), // compaction
35616 ((expand_mechanism >= 0)? "X" : ""), // EX
35617 ((expand_mechanism == expand_reuse_normal) ? "X" : ""), // NF
35618 ((expand_mechanism == expand_reuse_bestfit) ? "X" : ""), // BF
35619 (get_gc_data_per_heap()->is_mechanism_bit_set (gc_mark_list_bit) ? "X" : ""), // ML
35620 (get_gc_data_per_heap()->is_mechanism_bit_set (gc_demotion_bit) ? "X" : ""), // DM
35621 interesting_data_per_gc[idp_pre_short],
35622 interesting_data_per_gc[idp_post_short],
35623 interesting_data_per_gc[idp_merged_pin],
35624 interesting_data_per_gc[idp_converted_pin],
35625 interesting_data_per_gc[idp_pre_pin],
35626 interesting_data_per_gc[idp_post_pin],
35627 interesting_data_per_gc[idp_pre_and_post_pin],
35628 interesting_data_per_gc[idp_pre_short_padded],
35629 interesting_data_per_gc[idp_post_short_padded]));
35632 void gc_heap::record_global_mechanisms()
35634 for (int i = 0; i < max_global_mechanisms_count; i++)
35636 if (gc_data_global.get_mechanism_p ((gc_global_mechanism_p)i))
35638 ::record_global_mechanism (i);
35643 BOOL gc_heap::should_do_sweeping_gc (BOOL compact_p)
35645 if (!compact_ratio)
35646 return (!compact_p);
35648 size_t compact_count = compact_or_sweep_gcs[0];
35649 size_t sweep_count = compact_or_sweep_gcs[1];
35651 size_t total_count = compact_count + sweep_count;
35652 BOOL should_compact = compact_p;
35653 if (total_count > 3)
35657 int temp_ratio = (int)((compact_count + 1) * 100 / (total_count + 1));
35658 if (temp_ratio > compact_ratio)
35660 // cprintf (("compact would be: %d, total_count: %d, ratio would be %d%% > target\n",
35661 // (compact_count + 1), (total_count + 1), temp_ratio));
35662 should_compact = FALSE;
35667 int temp_ratio = (int)((sweep_count + 1) * 100 / (total_count + 1));
35668 if (temp_ratio > (100 - compact_ratio))
35670 // cprintf (("sweep would be: %d, total_count: %d, ratio would be %d%% > target\n",
35671 // (sweep_count + 1), (total_count + 1), temp_ratio));
35672 should_compact = TRUE;
35677 return !should_compact;
35679 #endif //GC_CONFIG_DRIVEN
35681 bool gc_heap::is_pm_ratio_exceeded()
35683 size_t maxgen_frag = 0;
35684 size_t maxgen_size = 0;
35685 size_t total_heap_size = get_total_heap_size();
35687 #ifdef MULTIPLE_HEAPS
35688 for (int i = 0; i < gc_heap::n_heaps; i++)
35690 gc_heap* hp = gc_heap::g_heaps[i];
35691 #else //MULTIPLE_HEAPS
35693 gc_heap* hp = pGenGCHeap;
35694 #endif //MULTIPLE_HEAPS
35696 maxgen_frag += dd_fragmentation (hp->dynamic_data_of (max_generation));
35697 maxgen_size += hp->generation_size (max_generation);
35700 double maxgen_ratio = (double)maxgen_size / (double)total_heap_size;
35701 double maxgen_frag_ratio = (double)maxgen_frag / (double)maxgen_size;
35702 dprintf (GTC_LOG, ("maxgen %Id(%d%% total heap), frag: %Id (%d%% maxgen)",
35703 maxgen_size, (int)(maxgen_ratio * 100.0),
35704 maxgen_frag, (int)(maxgen_frag_ratio * 100.0)));
35706 bool maxgen_highfrag_p = ((maxgen_ratio > 0.5) && (maxgen_frag_ratio > 0.1));
35708 // We need to adjust elevation here because if there's enough fragmentation it's not
35710 if (maxgen_highfrag_p)
35712 settings.should_lock_elevation = FALSE;
35713 dprintf (GTC_LOG, ("high frag gen2, turn off elevation"));
35716 return maxgen_highfrag_p;
35719 void gc_heap::do_post_gc()
35721 if (!settings.concurrent)
35727 #ifdef COUNT_CYCLES
35728 AllocStart = GetCycleCount32();
35730 AllocStart = clock();
35731 #endif //COUNT_CYCLES
35734 #ifdef MULTIPLE_HEAPS
35735 gc_heap* hp = g_heaps[0];
35738 #endif //MULTIPLE_HEAPS
35740 GCToEEInterface::GcDone(settings.condemned_generation);
35742 GCToEEInterface::DiagGCEnd(VolatileLoad(&settings.gc_index),
35743 (uint32_t)settings.condemned_generation,
35744 (uint32_t)settings.reason,
35745 !!settings.concurrent);
35747 //dprintf (1, (" ****end of Garbage Collection**** %d(gen0:%d)(%d)",
35748 dprintf (1, ("*EGC* %d(gen0:%d)(%d)(%s)",
35749 VolatileLoad(&settings.gc_index),
35750 dd_collection_count(hp->dynamic_data_of(0)),
35751 settings.condemned_generation,
35752 (settings.concurrent ? "BGC" : "GC")));
35754 if (settings.exit_memory_load != 0)
35755 last_gc_memory_load = settings.exit_memory_load;
35756 else if (settings.entry_memory_load != 0)
35757 last_gc_memory_load = settings.entry_memory_load;
35759 last_gc_heap_size = get_total_heap_size();
35760 last_gc_fragmentation = get_total_fragmentation();
35763 if (heap_hard_limit)
35765 size_t total_heap_committed = get_total_committed_size();
35766 size_t total_heap_committed_recorded = current_total_committed - current_total_committed_bookkeeping;
35767 dprintf (1, ("(%d)GC commit END #%Id: %Id (recorded: %Id), heap %Id, frag: %Id",
35768 settings.condemned_generation,
35769 (size_t)settings.gc_index, total_heap_committed, total_heap_committed_recorded,
35770 last_gc_heap_size, last_gc_fragmentation));
35774 // Note we only do this at the end of full blocking GCs because we do not want
35775 // to turn on this provisional mode during the middle of a BGC.
35776 if ((settings.condemned_generation == max_generation) && (!settings.concurrent))
35780 size_t full_compacting_gc_count = full_gc_counts[gc_type_compacting];
35781 if (provisional_mode_triggered)
35783 uint64_t r = gc_rand::get_rand(10);
35784 if ((full_compacting_gc_count - provisional_triggered_gc_count) >= r)
35786 provisional_mode_triggered = false;
35787 provisional_off_gc_count = full_compacting_gc_count;
35788 dprintf (GTC_LOG, ("%Id NGC2s when turned on, %Id NGCs since(%Id)",
35789 provisional_triggered_gc_count, (full_compacting_gc_count - provisional_triggered_gc_count),
35790 num_provisional_triggered));
35795 uint64_t r = gc_rand::get_rand(5);
35796 if ((full_compacting_gc_count - provisional_off_gc_count) >= r)
35798 provisional_mode_triggered = true;
35799 provisional_triggered_gc_count = full_compacting_gc_count;
35800 num_provisional_triggered++;
35801 dprintf (GTC_LOG, ("%Id NGC2s when turned off, %Id NGCs since(%Id)",
35802 provisional_off_gc_count, (full_compacting_gc_count - provisional_off_gc_count),
35803 num_provisional_triggered));
35809 if (provisional_mode_triggered)
35811 if ((settings.entry_memory_load < high_memory_load_th) ||
35812 !is_pm_ratio_exceeded())
35814 dprintf (GTC_LOG, ("turning off PM"));
35815 provisional_mode_triggered = false;
35818 else if ((settings.entry_memory_load >= high_memory_load_th) && is_pm_ratio_exceeded())
35820 dprintf (GTC_LOG, ("highmem && highfrag - turning on PM"));
35821 provisional_mode_triggered = true;
35822 num_provisional_triggered++;
35827 GCHeap::UpdatePostGCCounters();
35828 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
35829 //if (g_fEnableARM)
35831 // SystemDomain::GetADSurvivedBytes();
35833 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
35836 STRESS_LOG_GC_END(VolatileLoad(&settings.gc_index),
35837 (uint32_t)settings.condemned_generation,
35838 (uint32_t)settings.reason);
35839 #endif // STRESS_LOG
35841 #ifdef GC_CONFIG_DRIVEN
35842 if (!settings.concurrent)
35844 if (settings.compaction)
35845 (compact_or_sweep_gcs[0])++;
35847 (compact_or_sweep_gcs[1])++;
35850 #ifdef MULTIPLE_HEAPS
35851 for (int i = 0; i < n_heaps; i++)
35852 g_heaps[i]->record_interesting_info_per_heap();
35854 record_interesting_info_per_heap();
35855 #endif //MULTIPLE_HEAPS
35856 record_global_mechanisms();
35857 #endif //GC_CONFIG_DRIVEN
35860 unsigned GCHeap::GetGcCount()
35862 return (unsigned int)VolatileLoad(&pGenGCHeap->settings.gc_index);
35866 GCHeap::GarbageCollectGeneration (unsigned int gen, gc_reason reason)
35868 dprintf (2, ("triggered a GC!"));
35870 #ifdef MULTIPLE_HEAPS
35871 gc_heap* hpt = gc_heap::g_heaps[0];
35874 #endif //MULTIPLE_HEAPS
35875 bool cooperative_mode = true;
35876 dynamic_data* dd = hpt->dynamic_data_of (gen);
35877 size_t localCount = dd_collection_count (dd);
35879 enter_spin_lock (&gc_heap::gc_lock);
35880 dprintf (SPINLOCK_LOG, ("GC Egc"));
35881 ASSERT_HOLDING_SPIN_LOCK(&gc_heap::gc_lock);
35883 //don't trigger another GC if one was already in progress
35884 //while waiting for the lock
35886 size_t col_count = dd_collection_count (dd);
35888 if (localCount != col_count)
35890 #ifdef SYNCHRONIZATION_STATS
35891 gc_lock_contended++;
35892 #endif //SYNCHRONIZATION_STATS
35893 dprintf (SPINLOCK_LOG, ("no need GC Lgc"));
35894 leave_spin_lock (&gc_heap::gc_lock);
35896 // We don't need to release msl here 'cause this means a GC
35897 // has happened and would have release all msl's.
35902 #ifdef COUNT_CYCLES
35903 int gc_start = GetCycleCount32();
35904 #endif //COUNT_CYCLES
35907 #ifdef COUNT_CYCLES
35908 AllocDuration += GetCycleCount32() - AllocStart;
35910 AllocDuration += clock() - AllocStart;
35911 #endif //COUNT_CYCLES
35914 gc_heap::g_low_memory_status = (reason == reason_lowmemory) ||
35915 (reason == reason_lowmemory_blocking) ||
35916 (gc_heap::latency_level == latency_level_memory_footprint);
35918 gc_trigger_reason = reason;
35920 #ifdef MULTIPLE_HEAPS
35921 for (int i = 0; i < gc_heap::n_heaps; i++)
35923 gc_heap::g_heaps[i]->reset_gc_done();
35926 gc_heap::reset_gc_done();
35927 #endif //MULTIPLE_HEAPS
35929 gc_heap::gc_started = TRUE;
35932 init_sync_log_stats();
35934 #ifndef MULTIPLE_HEAPS
35935 cooperative_mode = gc_heap::enable_preemptive ();
35937 dprintf (2, ("Suspending EE"));
35938 BEGIN_TIMING(suspend_ee_during_log);
35939 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC);
35940 END_TIMING(suspend_ee_during_log);
35941 gc_heap::proceed_with_gc_p = gc_heap::should_proceed_with_gc();
35942 gc_heap::disable_preemptive (cooperative_mode);
35943 if (gc_heap::proceed_with_gc_p)
35944 pGenGCHeap->settings.init_mechanisms();
35946 gc_heap::update_collection_counts_for_no_gc();
35948 #endif //!MULTIPLE_HEAPS
35951 // MAP_EVENT_MONITORS(EE_MONITOR_GARBAGE_COLLECTIONS, NotifyEvent(EE_EVENT_TYPE_GC_STARTED, 0));
35954 #ifdef COUNT_CYCLES
35957 start = GetCycleCount32();
35962 #endif //COUNT_CYCLES
35963 PromotedObjectCount = 0;
35966 unsigned int condemned_generation_number = gen;
35968 // We want to get a stack from the user thread that triggered the GC
35969 // instead of on the GC thread which is the case for Server GC.
35970 // But we are doing it for Workstation GC as well to be uniform.
35971 FIRE_EVENT(GCTriggered, static_cast<uint32_t>(reason));
35973 #ifdef MULTIPLE_HEAPS
35974 GcCondemnedGeneration = condemned_generation_number;
35976 cooperative_mode = gc_heap::enable_preemptive ();
35978 BEGIN_TIMING(gc_during_log);
35979 gc_heap::ee_suspend_event.Set();
35980 gc_heap::wait_for_gc_done();
35981 END_TIMING(gc_during_log);
35983 gc_heap::disable_preemptive (cooperative_mode);
35985 condemned_generation_number = GcCondemnedGeneration;
35987 if (gc_heap::proceed_with_gc_p)
35989 BEGIN_TIMING(gc_during_log);
35990 pGenGCHeap->garbage_collect (condemned_generation_number);
35991 if (gc_heap::pm_trigger_full_gc)
35993 pGenGCHeap->garbage_collect_pm_full_gc();
35995 END_TIMING(gc_during_log);
35997 #endif //MULTIPLE_HEAPS
36000 #ifdef COUNT_CYCLES
36001 finish = GetCycleCount32();
36004 #endif //COUNT_CYCLES
36005 GcDuration += finish - start;
36007 ("<GC# %d> Condemned: %d, Duration: %d, total: %d Alloc Avg: %d, Small Objects:%d Large Objects:%d",
36008 VolatileLoad(&pGenGCHeap->settings.gc_index), condemned_generation_number,
36009 finish - start, GcDuration,
36010 AllocCount ? (AllocDuration / AllocCount) : 0,
36011 AllocSmallCount, AllocBigCount));
36016 #ifdef BACKGROUND_GC
36017 // We are deciding whether we should fire the alloc wait end event here
36018 // because in begin_foreground we could be calling end_foreground
36019 // if we need to retry.
36020 if (gc_heap::alloc_wait_event_p)
36022 hpt->fire_alloc_wait_event_end (awr_fgc_wait_for_bgc);
36023 gc_heap::alloc_wait_event_p = FALSE;
36025 #endif //BACKGROUND_GC
36027 #ifndef MULTIPLE_HEAPS
36028 #ifdef BACKGROUND_GC
36029 if (!gc_heap::dont_restart_ee_p)
36031 #endif //BACKGROUND_GC
36032 BEGIN_TIMING(restart_ee_during_log);
36033 GCToEEInterface::RestartEE(TRUE);
36034 END_TIMING(restart_ee_during_log);
36035 #ifdef BACKGROUND_GC
36037 #endif //BACKGROUND_GC
36038 #endif //!MULTIPLE_HEAPS
36040 #ifdef COUNT_CYCLES
36041 printf ("GC: %d Time: %d\n", GcCondemnedGeneration,
36042 GetCycleCount32() - gc_start);
36043 #endif //COUNT_CYCLES
36045 #ifndef MULTIPLE_HEAPS
36046 process_sync_log_stats();
36047 gc_heap::gc_started = FALSE;
36048 gc_heap::set_gc_done();
36049 dprintf (SPINLOCK_LOG, ("GC Lgc"));
36050 leave_spin_lock (&gc_heap::gc_lock);
36051 #endif //!MULTIPLE_HEAPS
36053 #ifdef FEATURE_PREMORTEM_FINALIZATION
36054 GCToEEInterface::EnableFinalization(!pGenGCHeap->settings.concurrent && pGenGCHeap->settings.found_finalizers);
36055 #endif // FEATURE_PREMORTEM_FINALIZATION
36057 return dd_collection_count (dd);
36060 size_t GCHeap::GetTotalBytesInUse ()
36062 #ifdef MULTIPLE_HEAPS
36063 //enumarate all the heaps and get their size.
36064 size_t tot_size = 0;
36065 for (int i = 0; i < gc_heap::n_heaps; i++)
36067 GCHeap* Hp = gc_heap::g_heaps [i]->vm_heap;
36068 tot_size += Hp->ApproxTotalBytesInUse (FALSE);
36072 return ApproxTotalBytesInUse ();
36073 #endif //MULTIPLE_HEAPS
36076 int GCHeap::CollectionCount (int generation, int get_bgc_fgc_count)
36078 if (get_bgc_fgc_count != 0)
36080 #ifdef BACKGROUND_GC
36081 if (generation == max_generation)
36083 return (int)(gc_heap::full_gc_counts[gc_type_background]);
36087 return (int)(gc_heap::ephemeral_fgc_counts[generation]);
36091 #endif //BACKGROUND_GC
36094 #ifdef MULTIPLE_HEAPS
36095 gc_heap* hp = gc_heap::g_heaps [0];
36096 #else //MULTIPLE_HEAPS
36097 gc_heap* hp = pGenGCHeap;
36098 #endif //MULTIPLE_HEAPS
36099 if (generation > max_generation)
36102 return (int)dd_collection_count (hp->dynamic_data_of (generation));
36105 size_t GCHeap::ApproxTotalBytesInUse(BOOL small_heap_only)
36107 size_t totsize = 0;
36109 //ASSERT(InMustComplete());
36110 enter_spin_lock (&pGenGCHeap->gc_lock);
36112 heap_segment* eph_seg = generation_allocation_segment (pGenGCHeap->generation_of (0));
36113 // Get small block heap size info
36114 totsize = (pGenGCHeap->alloc_allocated - heap_segment_mem (eph_seg));
36115 heap_segment* seg1 = generation_start_segment (pGenGCHeap->generation_of (max_generation));
36116 while (seg1 != eph_seg)
36118 totsize += heap_segment_allocated (seg1) -
36119 heap_segment_mem (seg1);
36120 seg1 = heap_segment_next (seg1);
36123 //discount the fragmentation
36124 for (int i = 0; i <= max_generation; i++)
36126 generation* gen = pGenGCHeap->generation_of (i);
36127 totsize -= (generation_free_list_space (gen) + generation_free_obj_space (gen));
36130 if (!small_heap_only)
36132 heap_segment* seg2 = generation_start_segment (pGenGCHeap->generation_of (max_generation+1));
36136 totsize += heap_segment_allocated (seg2) -
36137 heap_segment_mem (seg2);
36138 seg2 = heap_segment_next (seg2);
36141 //discount the fragmentation
36142 generation* loh_gen = pGenGCHeap->generation_of (max_generation+1);
36143 size_t frag = generation_free_list_space (loh_gen) + generation_free_obj_space (loh_gen);
36146 leave_spin_lock (&pGenGCHeap->gc_lock);
36150 #ifdef MULTIPLE_HEAPS
36151 void GCHeap::AssignHeap (alloc_context* acontext)
36153 // Assign heap based on processor
36154 acontext->set_alloc_heap(GetHeap(heap_select::select_heap(acontext, 0)));
36155 acontext->set_home_heap(acontext->get_alloc_heap());
36157 GCHeap* GCHeap::GetHeap (int n)
36159 assert (n < gc_heap::n_heaps);
36160 return gc_heap::g_heaps [n]->vm_heap;
36162 #endif //MULTIPLE_HEAPS
36164 bool GCHeap::IsThreadUsingAllocationContextHeap(gc_alloc_context* context, int thread_number)
36166 alloc_context* acontext = static_cast<alloc_context*>(context);
36167 #ifdef MULTIPLE_HEAPS
36168 return ((acontext->get_home_heap() == GetHeap(thread_number)) ||
36169 ((acontext->get_home_heap() == 0) && (thread_number == 0)));
36171 UNREFERENCED_PARAMETER(acontext);
36172 UNREFERENCED_PARAMETER(thread_number);
36174 #endif //MULTIPLE_HEAPS
36177 // Returns the number of processors required to trigger the use of thread based allocation contexts
36178 int GCHeap::GetNumberOfHeaps ()
36180 #ifdef MULTIPLE_HEAPS
36181 return gc_heap::n_heaps;
36184 #endif //MULTIPLE_HEAPS
36188 in this way we spend extra time cycling through all the heaps while create the handle
36189 it ought to be changed by keeping alloc_context.home_heap as number (equals heap_number)
36191 int GCHeap::GetHomeHeapNumber ()
36193 #ifdef MULTIPLE_HEAPS
36194 gc_alloc_context* ctx = GCToEEInterface::GetAllocContext();
36200 GCHeap *hp = static_cast<alloc_context*>(ctx)->get_home_heap();
36201 return (hp ? hp->pGenGCHeap->heap_number : 0);
36204 #endif //MULTIPLE_HEAPS
36207 unsigned int GCHeap::GetCondemnedGeneration()
36209 return gc_heap::settings.condemned_generation;
36212 void GCHeap::GetMemoryInfo(uint32_t* highMemLoadThreshold,
36213 uint64_t* totalPhysicalMem,
36214 uint32_t* lastRecordedMemLoad,
36215 size_t* lastRecordedHeapSize,
36216 size_t* lastRecordedFragmentation)
36218 *highMemLoadThreshold = gc_heap::high_memory_load_th;
36219 *totalPhysicalMem = gc_heap::total_physical_mem;
36220 *lastRecordedMemLoad = gc_heap::last_gc_memory_load;
36221 *lastRecordedHeapSize = gc_heap::last_gc_heap_size;
36222 *lastRecordedFragmentation = gc_heap::last_gc_fragmentation;
36225 int GCHeap::GetGcLatencyMode()
36227 return (int)(pGenGCHeap->settings.pause_mode);
36230 int GCHeap::SetGcLatencyMode (int newLatencyMode)
36232 if (gc_heap::settings.pause_mode == pause_no_gc)
36233 return (int)set_pause_mode_no_gc;
36235 gc_pause_mode new_mode = (gc_pause_mode)newLatencyMode;
36237 if (new_mode == pause_low_latency)
36239 #ifndef MULTIPLE_HEAPS
36240 pGenGCHeap->settings.pause_mode = new_mode;
36241 #endif //!MULTIPLE_HEAPS
36243 else if (new_mode == pause_sustained_low_latency)
36245 #ifdef BACKGROUND_GC
36246 if (gc_heap::gc_can_use_concurrent)
36248 pGenGCHeap->settings.pause_mode = new_mode;
36250 #endif //BACKGROUND_GC
36254 pGenGCHeap->settings.pause_mode = new_mode;
36257 #ifdef BACKGROUND_GC
36258 if (recursive_gc_sync::background_running_p())
36260 // If we get here, it means we are doing an FGC. If the pause
36261 // mode was altered we will need to save it in the BGC settings.
36262 if (gc_heap::saved_bgc_settings.pause_mode != new_mode)
36264 gc_heap::saved_bgc_settings.pause_mode = new_mode;
36267 #endif //BACKGROUND_GC
36269 return (int)set_pause_mode_success;
36272 int GCHeap::GetLOHCompactionMode()
36274 return pGenGCHeap->loh_compaction_mode;
36277 void GCHeap::SetLOHCompactionMode (int newLOHCompactionyMode)
36279 #ifdef FEATURE_LOH_COMPACTION
36280 pGenGCHeap->loh_compaction_mode = (gc_loh_compaction_mode)newLOHCompactionyMode;
36281 #endif //FEATURE_LOH_COMPACTION
36284 bool GCHeap::RegisterForFullGCNotification(uint32_t gen2Percentage,
36285 uint32_t lohPercentage)
36287 #ifdef MULTIPLE_HEAPS
36288 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36290 gc_heap* hp = gc_heap::g_heaps [hn];
36291 hp->fgn_last_alloc = dd_new_allocation (hp->dynamic_data_of (0));
36293 #else //MULTIPLE_HEAPS
36294 pGenGCHeap->fgn_last_alloc = dd_new_allocation (pGenGCHeap->dynamic_data_of (0));
36295 #endif //MULTIPLE_HEAPS
36297 pGenGCHeap->full_gc_approach_event.Reset();
36298 pGenGCHeap->full_gc_end_event.Reset();
36299 pGenGCHeap->full_gc_approach_event_set = false;
36301 pGenGCHeap->fgn_maxgen_percent = gen2Percentage;
36302 pGenGCHeap->fgn_loh_percent = lohPercentage;
36307 bool GCHeap::CancelFullGCNotification()
36309 pGenGCHeap->fgn_maxgen_percent = 0;
36310 pGenGCHeap->fgn_loh_percent = 0;
36312 pGenGCHeap->full_gc_approach_event.Set();
36313 pGenGCHeap->full_gc_end_event.Set();
36318 int GCHeap::WaitForFullGCApproach(int millisecondsTimeout)
36320 dprintf (2, ("WFGA: Begin wait"));
36321 int result = gc_heap::full_gc_wait (&(pGenGCHeap->full_gc_approach_event), millisecondsTimeout);
36322 dprintf (2, ("WFGA: End wait"));
36326 int GCHeap::WaitForFullGCComplete(int millisecondsTimeout)
36328 dprintf (2, ("WFGE: Begin wait"));
36329 int result = gc_heap::full_gc_wait (&(pGenGCHeap->full_gc_end_event), millisecondsTimeout);
36330 dprintf (2, ("WFGE: End wait"));
36334 int GCHeap::StartNoGCRegion(uint64_t totalSize, bool lohSizeKnown, uint64_t lohSize, bool disallowFullBlockingGC)
36336 NoGCRegionLockHolder lh;
36338 dprintf (1, ("begin no gc called"));
36339 start_no_gc_region_status status = gc_heap::prepare_for_no_gc_region (totalSize, lohSizeKnown, lohSize, disallowFullBlockingGC);
36340 if (status == start_no_gc_success)
36342 GarbageCollect (max_generation);
36343 status = gc_heap::get_start_no_gc_region_status();
36346 if (status != start_no_gc_success)
36347 gc_heap::handle_failure_for_no_gc();
36349 return (int)status;
36352 int GCHeap::EndNoGCRegion()
36354 NoGCRegionLockHolder lh;
36355 return (int)gc_heap::end_no_gc_region();
36358 void GCHeap::PublishObject (uint8_t* Obj)
36360 #ifdef BACKGROUND_GC
36361 gc_heap* hp = gc_heap::heap_of (Obj);
36362 hp->bgc_alloc_lock->loh_alloc_done (Obj);
36363 hp->bgc_untrack_loh_alloc();
36364 #endif //BACKGROUND_GC
36367 // The spec for this one isn't clear. This function
36368 // returns the size that can be allocated without
36369 // triggering a GC of any kind.
36370 size_t GCHeap::ApproxFreeBytes()
36373 //ASSERT(InMustComplete());
36374 enter_spin_lock (&pGenGCHeap->gc_lock);
36376 generation* gen = pGenGCHeap->generation_of (0);
36377 size_t res = generation_allocation_limit (gen) - generation_allocation_pointer (gen);
36379 leave_spin_lock (&pGenGCHeap->gc_lock);
36384 HRESULT GCHeap::GetGcCounters(int gen, gc_counters* counters)
36386 if ((gen < 0) || (gen > max_generation))
36388 #ifdef MULTIPLE_HEAPS
36389 counters->current_size = 0;
36390 counters->promoted_size = 0;
36391 counters->collection_count = 0;
36393 //enumarate all the heaps and get their counters.
36394 for (int i = 0; i < gc_heap::n_heaps; i++)
36396 dynamic_data* dd = gc_heap::g_heaps [i]->dynamic_data_of (gen);
36398 counters->current_size += dd_current_size (dd);
36399 counters->promoted_size += dd_promoted_size (dd);
36401 counters->collection_count += dd_collection_count (dd);
36404 dynamic_data* dd = pGenGCHeap->dynamic_data_of (gen);
36405 counters->current_size = dd_current_size (dd);
36406 counters->promoted_size = dd_promoted_size (dd);
36407 counters->collection_count = dd_collection_count (dd);
36408 #endif //MULTIPLE_HEAPS
36412 // Get the segment size to use, making sure it conforms.
36413 size_t GCHeap::GetValidSegmentSize(bool large_seg)
36415 return (large_seg ? gc_heap::min_loh_segment_size : gc_heap::soh_segment_size);
36418 // Get the max gen0 heap size, making sure it conforms.
36419 size_t gc_heap::get_gen0_min_size()
36421 size_t gen0size = static_cast<size_t>(GCConfig::GetGen0Size());
36422 bool is_config_invalid = ((gen0size == 0) || !g_theGCHeap->IsValidGen0MaxSize(gen0size));
36423 if (is_config_invalid)
36426 // performance data seems to indicate halving the size results
36427 // in optimal perf. Ask for adjusted gen0 size.
36428 gen0size = max(GCToOSInterface::GetCacheSizePerLogicalCpu(FALSE),(256*1024));
36430 // if gen0 size is too large given the available memory, reduce it.
36431 // Get true cache size, as we don't want to reduce below this.
36432 size_t trueSize = max(GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE),(256*1024));
36433 dprintf (1, ("cache: %Id-%Id",
36434 GCToOSInterface::GetCacheSizePerLogicalCpu(FALSE),
36435 GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE)));
36437 int n_heaps = gc_heap::n_heaps;
36439 size_t trueSize = GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE);
36440 gen0size = max((4*trueSize/5),(256*1024));
36441 trueSize = max(trueSize, (256*1024));
36445 dprintf (1, ("gen0size: %Id * %d = %Id, physical mem: %Id / 6 = %Id",
36446 gen0size, n_heaps, (gen0size * n_heaps),
36447 gc_heap::total_physical_mem,
36448 gc_heap::total_physical_mem / 6));
36450 // if the total min GC across heaps will exceed 1/6th of available memory,
36451 // then reduce the min GC size until it either fits or has been reduced to cache size.
36452 while ((gen0size * n_heaps) > (gc_heap::total_physical_mem / 6))
36454 gen0size = gen0size / 2;
36455 if (gen0size <= trueSize)
36457 gen0size = trueSize;
36463 size_t seg_size = gc_heap::soh_segment_size;
36466 // Generation 0 must never be more than 1/2 the segment size.
36467 if (gen0size >= (seg_size / 2))
36468 gen0size = seg_size / 2;
36470 // If the value from config is valid we use it as is without this adjustment.
36471 if (is_config_invalid)
36473 if (heap_hard_limit)
36475 size_t gen0size_seg = seg_size / 8;
36476 if (gen0size >= gen0size_seg)
36478 dprintf (1, ("gen0 limited by seg size %Id->%Id", gen0size, gen0size_seg));
36479 gen0size = gen0size_seg;
36483 gen0size = gen0size / 8 * 5;
36486 gen0size = Align (gen0size);
36491 void GCHeap::SetReservedVMLimit (size_t vmlimit)
36493 gc_heap::reserved_memory_limit = vmlimit;
36496 //versions of same method on each heap
36498 #ifdef FEATURE_PREMORTEM_FINALIZATION
36500 Object* GCHeap::GetNextFinalizableObject()
36503 #ifdef MULTIPLE_HEAPS
36505 //return the first non critical one in the first queue.
36506 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36508 gc_heap* hp = gc_heap::g_heaps [hn];
36509 Object* O = hp->finalize_queue->GetNextFinalizableObject(TRUE);
36513 //return the first non crtitical/critical one in the first queue.
36514 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36516 gc_heap* hp = gc_heap::g_heaps [hn];
36517 Object* O = hp->finalize_queue->GetNextFinalizableObject(FALSE);
36524 #else //MULTIPLE_HEAPS
36525 return pGenGCHeap->finalize_queue->GetNextFinalizableObject();
36526 #endif //MULTIPLE_HEAPS
36530 size_t GCHeap::GetNumberFinalizableObjects()
36532 #ifdef MULTIPLE_HEAPS
36534 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36536 gc_heap* hp = gc_heap::g_heaps [hn];
36537 cnt += hp->finalize_queue->GetNumberFinalizableObjects();
36542 #else //MULTIPLE_HEAPS
36543 return pGenGCHeap->finalize_queue->GetNumberFinalizableObjects();
36544 #endif //MULTIPLE_HEAPS
36547 size_t GCHeap::GetFinalizablePromotedCount()
36549 #ifdef MULTIPLE_HEAPS
36552 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36554 gc_heap* hp = gc_heap::g_heaps [hn];
36555 cnt += hp->finalize_queue->GetPromotedCount();
36559 #else //MULTIPLE_HEAPS
36560 return pGenGCHeap->finalize_queue->GetPromotedCount();
36561 #endif //MULTIPLE_HEAPS
36564 bool GCHeap::FinalizeAppDomain(void *pDomain, bool fRunFinalizers)
36566 #ifdef MULTIPLE_HEAPS
36567 bool foundp = false;
36568 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36570 gc_heap* hp = gc_heap::g_heaps [hn];
36571 if (hp->finalize_queue->FinalizeAppDomain (pDomain, fRunFinalizers))
36576 #else //MULTIPLE_HEAPS
36577 return pGenGCHeap->finalize_queue->FinalizeAppDomain (pDomain, fRunFinalizers);
36578 #endif //MULTIPLE_HEAPS
36581 bool GCHeap::ShouldRestartFinalizerWatchDog()
36583 // This condition was historically used as part of the condition to detect finalizer thread timeouts
36584 return gc_heap::gc_lock.lock != -1;
36587 void GCHeap::SetFinalizeQueueForShutdown(bool fHasLock)
36589 #ifdef MULTIPLE_HEAPS
36590 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36592 gc_heap* hp = gc_heap::g_heaps [hn];
36593 hp->finalize_queue->SetSegForShutDown(fHasLock);
36596 #else //MULTIPLE_HEAPS
36597 pGenGCHeap->finalize_queue->SetSegForShutDown(fHasLock);
36598 #endif //MULTIPLE_HEAPS
36601 //---------------------------------------------------------------------------
36602 // Finalized class tracking
36603 //---------------------------------------------------------------------------
36605 bool GCHeap::RegisterForFinalization (int gen, Object* obj)
36609 if (((((CObjectHeader*)obj)->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN))
36611 //just reset the bit
36612 ((CObjectHeader*)obj)->GetHeader()->ClrBit(BIT_SBLK_FINALIZER_RUN);
36617 gc_heap* hp = gc_heap::heap_of ((uint8_t*)obj);
36618 return hp->finalize_queue->RegisterForFinalization (gen, obj);
36622 void GCHeap::SetFinalizationRun (Object* obj)
36624 ((CObjectHeader*)obj)->GetHeader()->SetBit(BIT_SBLK_FINALIZER_RUN);
36628 //--------------------------------------------------------------------
36630 // Support for finalization
36632 //--------------------------------------------------------------------
36635 unsigned int gen_segment (int gen)
36637 assert (((signed)NUMBERGENERATIONS - gen - 1)>=0);
36638 return (NUMBERGENERATIONS - gen - 1);
36641 bool CFinalize::Initialize()
36648 m_Array = new (nothrow)(Object*[100]);
36653 STRESS_LOG_OOM_STACK(sizeof(Object*[100]));
36654 if (GCConfig::GetBreakOnOOM())
36656 GCToOSInterface::DebugBreak();
36660 m_EndArray = &m_Array[100];
36662 for (int i =0; i < FreeList; i++)
36664 SegQueueLimit (i) = m_Array;
36666 m_PromotedCount = 0;
36669 lockowner_threadid.Clear();
36675 CFinalize::~CFinalize()
36680 size_t CFinalize::GetPromotedCount ()
36682 return m_PromotedCount;
36686 void CFinalize::EnterFinalizeLock()
36688 _ASSERTE(dbgOnly_IsSpecialEEThread() ||
36689 GCToEEInterface::GetThread() == 0 ||
36690 GCToEEInterface::IsPreemptiveGCDisabled());
36693 if (Interlocked::CompareExchange(&lock, 0, -1) >= 0)
36695 unsigned int i = 0;
36698 YieldProcessor(); // indicate to the processor that we are spinning
36700 GCToOSInterface::YieldThread (0);
36702 GCToOSInterface::Sleep (5);
36708 lockowner_threadid.SetToCurrentThread();
36713 void CFinalize::LeaveFinalizeLock()
36715 _ASSERTE(dbgOnly_IsSpecialEEThread() ||
36716 GCToEEInterface::GetThread() == 0 ||
36717 GCToEEInterface::IsPreemptiveGCDisabled());
36720 lockowner_threadid.Clear();
36726 CFinalize::RegisterForFinalization (int gen, Object* obj, size_t size)
36733 EnterFinalizeLock();
36735 unsigned int dest = 0;
36737 if (g_fFinalizerRunOnShutDown)
36739 //no method table available yet,
36740 //put it in the finalizer queue and sort out when
36742 dest = FinalizerListSeg;
36746 dest = gen_segment (gen);
36748 // Adjust boundary for segments so that GC will keep objects alive.
36749 Object*** s_i = &SegQueue (FreeList);
36750 if ((*s_i) == m_EndArray)
36754 LeaveFinalizeLock();
36755 if (method_table(obj) == NULL)
36757 // If the object is uninitialized, a valid size should have been passed.
36758 assert (size >= Align (min_obj_size));
36759 dprintf (3, ("Making unused array [%Ix, %Ix[", (size_t)obj, (size_t)(obj+size)));
36760 ((CObjectHeader*)obj)->SetFree(size);
36762 STRESS_LOG_OOM_STACK(0);
36763 if (GCConfig::GetBreakOnOOM())
36765 GCToOSInterface::DebugBreak();
36770 Object*** end_si = &SegQueueLimit (dest);
36773 //is the segment empty?
36774 if (!(*s_i == *(s_i-1)))
36776 //no, swap the end elements.
36777 *(*s_i) = *(*(s_i-1));
36779 //increment the fill pointer
36781 //go to the next segment.
36783 } while (s_i > end_si);
36785 // We have reached the destination segment
36786 // store the object
36788 // increment the fill pointer
36791 LeaveFinalizeLock();
36797 CFinalize::GetNextFinalizableObject (BOOL only_non_critical)
36801 EnterFinalizeLock();
36804 if (!IsSegEmpty(FinalizerListSeg))
36806 if (g_fFinalizerRunOnShutDown)
36808 obj = *(SegQueueLimit (FinalizerListSeg)-1);
36809 if (method_table(obj)->HasCriticalFinalizer())
36811 MoveItem ((SegQueueLimit (FinalizerListSeg)-1),
36812 FinalizerListSeg, CriticalFinalizerListSeg);
36816 --SegQueueLimit (FinalizerListSeg);
36819 obj = *(--SegQueueLimit (FinalizerListSeg));
36822 else if (!only_non_critical && !IsSegEmpty(CriticalFinalizerListSeg))
36824 //the FinalizerList is empty, we can adjust both
36825 // limit instead of moving the object to the free list
36826 obj = *(--SegQueueLimit (CriticalFinalizerListSeg));
36827 --SegQueueLimit (FinalizerListSeg);
36831 dprintf (3, ("running finalizer for %Ix (mt: %Ix)", obj, method_table (obj)));
36833 LeaveFinalizeLock();
36838 CFinalize::SetSegForShutDown(BOOL fHasLock)
36843 EnterFinalizeLock();
36844 for (i = 0; i <= max_generation; i++)
36846 unsigned int seg = gen_segment (i);
36847 Object** startIndex = SegQueueLimit (seg)-1;
36848 Object** stopIndex = SegQueue (seg);
36849 for (Object** po = startIndex; po >= stopIndex; po--)
36852 if (method_table(obj)->HasCriticalFinalizer())
36854 MoveItem (po, seg, CriticalFinalizerListSeg);
36858 MoveItem (po, seg, FinalizerListSeg);
36863 LeaveFinalizeLock();
36867 CFinalize::DiscardNonCriticalObjects()
36869 //empty the finalization queue
36870 Object** startIndex = SegQueueLimit (FinalizerListSeg)-1;
36871 Object** stopIndex = SegQueue (FinalizerListSeg);
36872 for (Object** po = startIndex; po >= stopIndex; po--)
36874 MoveItem (po, FinalizerListSeg, FreeList);
36879 CFinalize::GetNumberFinalizableObjects()
36881 return SegQueueLimit (FinalizerListSeg) -
36882 (g_fFinalizerRunOnShutDown ? m_Array : SegQueue(FinalizerListSeg));
36886 CFinalize::FinalizeSegForAppDomain (void *pDomain,
36887 BOOL fRunFinalizers,
36890 BOOL finalizedFound = FALSE;
36891 Object** endIndex = SegQueue (Seg);
36892 for (Object** i = SegQueueLimit (Seg)-1; i >= endIndex ;i--)
36894 CObjectHeader* obj = (CObjectHeader*)*i;
36896 // Objects are put into the finalization queue before they are complete (ie their methodtable
36897 // may be null) so we must check that the object we found has a method table before checking
36898 // if it has the index we are looking for. If the methodtable is null, it can't be from the
36899 // unloading domain, so skip it.
36900 if (method_table(obj) == NULL)
36905 // does the EE actually want us to finalize this object?
36906 if (!GCToEEInterface::ShouldFinalizeObjectForUnload(pDomain, obj))
36911 if (!fRunFinalizers || (obj->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN)
36913 //remove the object because we don't want to
36914 //run the finalizer
36915 MoveItem (i, Seg, FreeList);
36916 //Reset the bit so it will be put back on the queue
36917 //if resurrected and re-registered.
36918 obj->GetHeader()->ClrBit (BIT_SBLK_FINALIZER_RUN);
36922 if (method_table(obj)->HasCriticalFinalizer())
36924 finalizedFound = TRUE;
36925 MoveItem (i, Seg, CriticalFinalizerListSeg);
36929 if (GCToEEInterface::AppDomainIsRudeUnload(pDomain))
36931 MoveItem (i, Seg, FreeList);
36935 finalizedFound = TRUE;
36936 MoveItem (i, Seg, FinalizerListSeg);
36942 return finalizedFound;
36946 CFinalize::FinalizeAppDomain (void *pDomain, bool fRunFinalizers)
36948 bool finalizedFound = false;
36950 unsigned int startSeg = gen_segment (max_generation);
36952 EnterFinalizeLock();
36954 for (unsigned int Seg = startSeg; Seg <= gen_segment (0); Seg++)
36956 if (FinalizeSegForAppDomain (pDomain, fRunFinalizers, Seg))
36958 finalizedFound = true;
36962 LeaveFinalizeLock();
36964 return finalizedFound;
36968 CFinalize::MoveItem (Object** fromIndex,
36969 unsigned int fromSeg,
36970 unsigned int toSeg)
36974 ASSERT (fromSeg != toSeg);
36975 if (fromSeg > toSeg)
36979 // Place the element at the boundary closest to dest
36980 Object** srcIndex = fromIndex;
36981 for (unsigned int i = fromSeg; i != toSeg; i+= step)
36983 Object**& destFill = m_FillPointers[i+(step - 1 )/2];
36984 Object** destIndex = destFill - (step + 1)/2;
36985 if (srcIndex != destIndex)
36987 Object* tmp = *srcIndex;
36988 *srcIndex = *destIndex;
36992 srcIndex = destIndex;
36997 CFinalize::GcScanRoots (promote_func* fn, int hn, ScanContext *pSC)
37003 pSC->thread_number = hn;
37005 //scan the finalization queue
37006 Object** startIndex = SegQueue (CriticalFinalizerListSeg);
37007 Object** stopIndex = SegQueueLimit (FinalizerListSeg);
37009 for (Object** po = startIndex; po < stopIndex; po++)
37012 //dprintf (3, ("scan freacheable %Ix", (size_t)o));
37013 dprintf (3, ("scan f %Ix", (size_t)o));
37014 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
37015 if (g_fEnableAppDomainMonitoring)
37017 pSC->pCurrentDomain = GCToEEInterface::GetAppDomainForObject(o);
37019 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
37025 void CFinalize::WalkFReachableObjects (fq_walk_fn fn)
37027 Object** startIndex = SegQueue (CriticalFinalizerListSeg);
37028 Object** stopCriticalIndex = SegQueueLimit (CriticalFinalizerListSeg);
37029 Object** stopIndex = SegQueueLimit (FinalizerListSeg);
37030 for (Object** po = startIndex; po < stopIndex; po++)
37033 fn(po < stopCriticalIndex, *po);
37038 CFinalize::ScanForFinalization (promote_func* pfn, int gen, BOOL mark_only_p,
37042 sc.promotion = TRUE;
37043 #ifdef MULTIPLE_HEAPS
37044 sc.thread_number = hp->heap_number;
37046 UNREFERENCED_PARAMETER(hp);
37047 #endif //MULTIPLE_HEAPS
37049 BOOL finalizedFound = FALSE;
37051 //start with gen and explore all the younger generations.
37052 unsigned int startSeg = gen_segment (gen);
37054 m_PromotedCount = 0;
37055 for (unsigned int Seg = startSeg; Seg <= gen_segment(0); Seg++)
37057 Object** endIndex = SegQueue (Seg);
37058 for (Object** i = SegQueueLimit (Seg)-1; i >= endIndex ;i--)
37060 CObjectHeader* obj = (CObjectHeader*)*i;
37061 dprintf (3, ("scanning: %Ix", (size_t)obj));
37062 if (!g_theGCHeap->IsPromoted (obj))
37064 dprintf (3, ("freacheable: %Ix", (size_t)obj));
37066 assert (method_table(obj)->HasFinalizer());
37068 if (GCToEEInterface::EagerFinalized(obj))
37070 MoveItem (i, Seg, FreeList);
37072 else if ((obj->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN)
37074 //remove the object because we don't want to
37075 //run the finalizer
37077 MoveItem (i, Seg, FreeList);
37079 //Reset the bit so it will be put back on the queue
37080 //if resurrected and re-registered.
37081 obj->GetHeader()->ClrBit (BIT_SBLK_FINALIZER_RUN);
37088 if (method_table(obj)->HasCriticalFinalizer())
37090 MoveItem (i, Seg, CriticalFinalizerListSeg);
37094 MoveItem (i, Seg, FinalizerListSeg);
37098 #ifdef BACKGROUND_GC
37101 if ((gen == max_generation) && (recursive_gc_sync::background_running_p()))
37103 // TODO - fix the following line.
37104 //assert (gc_heap::background_object_marked ((uint8_t*)obj, FALSE));
37105 dprintf (3, ("%Ix is marked", (size_t)obj));
37108 #endif //BACKGROUND_GC
37112 finalizedFound = !IsSegEmpty(FinalizerListSeg) ||
37113 !IsSegEmpty(CriticalFinalizerListSeg);
37115 if (finalizedFound)
37117 //Promote the f-reachable objects
37119 #ifdef MULTIPLE_HEAPS
37123 #endif //MULTIPLE_HEAPS
37126 hp->settings.found_finalizers = TRUE;
37128 #ifdef BACKGROUND_GC
37129 if (hp->settings.concurrent)
37131 hp->settings.found_finalizers = !(IsSegEmpty(FinalizerListSeg) && IsSegEmpty(CriticalFinalizerListSeg));
37133 #endif //BACKGROUND_GC
37134 if (hp->settings.concurrent && hp->settings.found_finalizers)
37137 GCToEEInterface::EnableFinalization(true);
37141 return finalizedFound;
37144 //Relocates all of the objects in the finalization array
37146 CFinalize::RelocateFinalizationData (int gen, gc_heap* hp)
37149 sc.promotion = FALSE;
37150 #ifdef MULTIPLE_HEAPS
37151 sc.thread_number = hp->heap_number;
37153 UNREFERENCED_PARAMETER(hp);
37154 #endif //MULTIPLE_HEAPS
37156 unsigned int Seg = gen_segment (gen);
37158 Object** startIndex = SegQueue (Seg);
37159 for (Object** po = startIndex; po < SegQueue (FreeList);po++)
37161 GCHeap::Relocate (po, &sc);
37166 CFinalize::UpdatePromotedGenerations (int gen, BOOL gen_0_empty_p)
37168 // update the generation fill pointers.
37169 // if gen_0_empty is FALSE, test each object to find out if
37170 // it was promoted or not
37173 for (int i = min (gen+1, max_generation); i > 0; i--)
37175 m_FillPointers [gen_segment(i)] = m_FillPointers [gen_segment(i-1)];
37180 //Look for demoted or promoted plugs
37182 for (int i = gen; i >= 0; i--)
37184 unsigned int Seg = gen_segment (i);
37185 Object** startIndex = SegQueue (Seg);
37187 for (Object** po = startIndex;
37188 po < SegQueueLimit (gen_segment(i)); po++)
37190 int new_gen = g_theGCHeap->WhichGeneration (*po);
37196 MoveItem (po, gen_segment (i), gen_segment (new_gen));
37201 MoveItem (po, gen_segment (i), gen_segment (new_gen));
37202 //back down in order to see all objects.
37213 CFinalize::GrowArray()
37215 size_t oldArraySize = (m_EndArray - m_Array);
37216 size_t newArraySize = (size_t)(((float)oldArraySize / 10) * 12);
37218 Object** newArray = new (nothrow) Object*[newArraySize];
37221 // It's not safe to throw here, because of the FinalizeLock. Tell our caller
37222 // to throw for us.
37223 // ASSERT (newArray);
37226 memcpy (newArray, m_Array, oldArraySize*sizeof(Object*));
37228 //adjust the fill pointers
37229 for (int i = 0; i < FreeList; i++)
37231 m_FillPointers [i] += (newArray - m_Array);
37234 m_Array = newArray;
37235 m_EndArray = &m_Array [newArraySize];
37241 void CFinalize::CheckFinalizerObjects()
37243 for (int i = 0; i <= max_generation; i++)
37245 Object **startIndex = SegQueue (gen_segment (i));
37246 Object **stopIndex = SegQueueLimit (gen_segment (i));
37248 for (Object **po = startIndex; po < stopIndex; po++)
37250 if ((int)g_theGCHeap->WhichGeneration (*po) < i)
37252 ((CObjectHeader*)*po)->Validate();
37256 #endif //VERIFY_HEAP
37258 #endif // FEATURE_PREMORTEM_FINALIZATION
37261 //------------------------------------------------------------------------------
37263 // End of VM specific support
37265 //------------------------------------------------------------------------------
37266 void gc_heap::walk_heap_per_heap (walk_fn fn, void* context, int gen_number, BOOL walk_large_object_heap_p)
37268 generation* gen = gc_heap::generation_of (gen_number);
37269 heap_segment* seg = generation_start_segment (gen);
37270 uint8_t* x = ((gen_number == max_generation) ? heap_segment_mem (seg) :
37271 generation_allocation_start (gen));
37273 uint8_t* end = heap_segment_allocated (seg);
37274 BOOL small_object_segments = TRUE;
37275 int align_const = get_alignment_constant (small_object_segments);
37282 if ((seg = heap_segment_next (seg)) != 0)
37284 x = heap_segment_mem (seg);
37285 end = heap_segment_allocated (seg);
37290 if (small_object_segments && walk_large_object_heap_p)
37293 small_object_segments = FALSE;
37294 align_const = get_alignment_constant (small_object_segments);
37295 seg = generation_start_segment (large_object_generation);
37296 x = heap_segment_mem (seg);
37297 end = heap_segment_allocated (seg);
37307 size_t s = size (x);
37308 CObjectHeader* o = (CObjectHeader*)x;
37313 _ASSERTE(((size_t)o & 0x3) == 0); // Last two bits should never be set at this point
37315 if (!fn (o->GetObjectBase(), context))
37318 x = x + Align (s, align_const);
37322 void gc_heap::walk_finalize_queue (fq_walk_fn fn)
37324 #ifdef FEATURE_PREMORTEM_FINALIZATION
37325 finalize_queue->WalkFReachableObjects (fn);
37326 #endif //FEATURE_PREMORTEM_FINALIZATION
37329 void gc_heap::walk_heap (walk_fn fn, void* context, int gen_number, BOOL walk_large_object_heap_p)
37331 #ifdef MULTIPLE_HEAPS
37332 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
37334 gc_heap* hp = gc_heap::g_heaps [hn];
37336 hp->walk_heap_per_heap (fn, context, gen_number, walk_large_object_heap_p);
37339 walk_heap_per_heap(fn, context, gen_number, walk_large_object_heap_p);
37340 #endif //MULTIPLE_HEAPS
37343 void GCHeap::DiagWalkObject (Object* obj, walk_fn fn, void* context)
37345 uint8_t* o = (uint8_t*)obj;
37348 go_through_object_cl (method_table (o), o, size(o), oo,
37352 Object *oh = (Object*)*oo;
37353 if (!fn (oh, context))
37361 void GCHeap::DiagWalkSurvivorsWithType (void* gc_context, record_surv_fn fn, void* diag_context, walk_surv_type type)
37363 gc_heap* hp = (gc_heap*)gc_context;
37364 hp->walk_survivors (fn, diag_context, type);
37367 void GCHeap::DiagWalkHeap (walk_fn fn, void* context, int gen_number, bool walk_large_object_heap_p)
37369 gc_heap::walk_heap (fn, context, gen_number, walk_large_object_heap_p);
37372 void GCHeap::DiagWalkFinalizeQueue (void* gc_context, fq_walk_fn fn)
37374 gc_heap* hp = (gc_heap*)gc_context;
37375 hp->walk_finalize_queue (fn);
37378 void GCHeap::DiagScanFinalizeQueue (fq_scan_fn fn, ScanContext* sc)
37380 #ifdef MULTIPLE_HEAPS
37381 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
37383 gc_heap* hp = gc_heap::g_heaps [hn];
37384 hp->finalize_queue->GcScanRoots(fn, hn, sc);
37387 pGenGCHeap->finalize_queue->GcScanRoots(fn, 0, sc);
37388 #endif //MULTIPLE_HEAPS
37391 void GCHeap::DiagScanHandles (handle_scan_fn fn, int gen_number, ScanContext* context)
37393 UNREFERENCED_PARAMETER(gen_number);
37394 GCScan::GcScanHandlesForProfilerAndETW (max_generation, context, fn);
37397 void GCHeap::DiagScanDependentHandles (handle_scan_fn fn, int gen_number, ScanContext* context)
37399 UNREFERENCED_PARAMETER(gen_number);
37400 GCScan::GcScanDependentHandlesForProfilerAndETW (max_generation, context, fn);
37403 // Go through and touch (read) each page straddled by a memory block.
37404 void TouchPages(void * pStart, size_t cb)
37406 const uint32_t pagesize = OS_PAGE_SIZE;
37407 _ASSERTE(0 == (pagesize & (pagesize-1))); // Must be a power of 2.
37410 VOLATILE(char)* pEnd = (VOLATILE(char)*)(cb + (char*)pStart);
37411 VOLATILE(char)* p = (VOLATILE(char)*)(((char*)pStart) - (((size_t)pStart) & (pagesize-1)));
37415 a = VolatileLoad(p);
37416 //printf("Touching page %lxh\n", (uint32_t)p);
37422 #if defined(WRITE_BARRIER_CHECK) && !defined (SERVER_GC)
37423 // This code is designed to catch the failure to update the write barrier
37424 // The way it works is to copy the whole heap right after every GC. The write
37425 // barrier code has been modified so that it updates the shadow as well as the
37426 // real GC heap. Before doing the next GC, we walk the heap, looking for pointers
37427 // that were updated in the real heap, but not the shadow. A mismatch indicates
37428 // an error. The offending code can be found by breaking after the correct GC,
37429 // and then placing a data breakpoint on the Heap location that was updated without
37430 // going through the write barrier.
37432 // Called at process shutdown
37433 void deleteGCShadow()
37435 if (g_GCShadow != 0)
37436 GCToOSInterface::VirtualRelease (g_GCShadow, g_GCShadowEnd - g_GCShadow);
37441 // Called at startup and right after a GC, get a snapshot of the GC Heap
37442 void initGCShadow()
37444 if (!(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_BARRIERCHECK))
37447 size_t len = g_gc_highest_address - g_gc_lowest_address;
37448 if (len > (size_t)(g_GCShadowEnd - g_GCShadow))
37451 g_GCShadowEnd = g_GCShadow = (uint8_t *)GCToOSInterface::VirtualReserve(len, 0, VirtualReserveFlags::None);
37452 if (g_GCShadow == NULL || !GCToOSInterface::VirtualCommit(g_GCShadow, len))
37454 _ASSERTE(!"Not enough memory to run HeapVerify level 2");
37455 // If after the assert we decide to allow the program to continue
37456 // running we need to be in a state that will not trigger any
37457 // additional AVs while we fail to allocate a shadow segment, i.e.
37458 // ensure calls to updateGCShadow() checkGCWriteBarrier() don't AV
37463 g_GCShadowEnd += len;
37466 // save the value of g_gc_lowest_address at this time. If this value changes before
37467 // the next call to checkGCWriteBarrier() it means we extended the heap (with a
37468 // large object segment most probably), and the whole shadow segment is inconsistent.
37469 g_shadow_lowest_address = g_gc_lowest_address;
37471 //****** Copy the whole GC heap ******
37473 // NOTE: This is the one situation where the combination of heap_segment_rw(gen_start_segment())
37474 // can produce a NULL result. This is because the initialization has not completed.
37476 generation* gen = gc_heap::generation_of (max_generation);
37477 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
37479 ptrdiff_t delta = g_GCShadow - g_gc_lowest_address;
37480 BOOL small_object_segments = TRUE;
37485 if (small_object_segments)
37487 small_object_segments = FALSE;
37488 seg = heap_segment_rw (generation_start_segment (gc_heap::generation_of (max_generation+1)));
37494 // Copy the segment
37495 uint8_t* start = heap_segment_mem(seg);
37496 uint8_t* end = heap_segment_allocated (seg);
37497 memcpy(start + delta, start, end - start);
37498 seg = heap_segment_next_rw (seg);
37502 #define INVALIDGCVALUE (void*)((size_t)0xcccccccd)
37504 // test to see if 'ptr' was only updated via the write barrier.
37505 inline void testGCShadow(Object** ptr)
37507 Object** shadow = (Object**) &g_GCShadow[((uint8_t*) ptr - g_gc_lowest_address)];
37508 if (*ptr != 0 && (uint8_t*) shadow < g_GCShadowEnd && *ptr != *shadow)
37511 // If you get this assertion, someone updated a GC pointer in the heap without
37512 // using the write barrier. To find out who, check the value of
37513 // dd_collection_count (dynamic_data_of (0)). Also
37514 // note the value of 'ptr'. Rerun the App that the previous GC just occurred.
37515 // Then put a data breakpoint for the value of 'ptr' Then check every write
37516 // to pointer between the two GCs. The last one is not using the write barrier.
37518 // If the memory of interest does not exist at system startup,
37519 // you need to set the data breakpoint right after the memory gets committed
37520 // Set a breakpoint at the end of grow_heap_segment, and put the value of 'ptr'
37521 // in the memory window. run until the memory gets mapped. Then you can set
37524 // Note a recent change, we've identified race conditions when updating the gc shadow.
37525 // Throughout the runtime, code will update an address in the gc heap, then erect the
37526 // write barrier, which calls updateGCShadow. With an app that pounds one heap location
37527 // from multiple threads, you can hit this assert even though all involved are using the
37528 // write barrier properly. Thusly, we detect the race and set this location to INVALIDGCVALUE.
37529 // TODO: the code in jithelp.asm doesn't call updateGCShadow, and hasn't been
37530 // TODO: fixed to detect the race. We've only seen this race from VolatileWritePtr,
37531 // TODO: so elect not to fix jithelp.asm at this time. It should be done if we start hitting
37532 // TODO: erroneous asserts in here.
37534 if(*shadow!=INVALIDGCVALUE)
37536 #ifdef FEATURE_BASICFREEZE
37537 // Write barriers for stores of references to frozen objects may be optimized away.
37538 if (!gc_heap::frozen_object_p(*ptr))
37539 #endif // FEATURE_BASICFREEZE
37541 _ASSERTE(!"Pointer updated without using write barrier");
37547 printf("saw a INVALIDGCVALUE. (just to let you know)\n");
37553 void testGCShadowHelper (uint8_t* x)
37555 size_t s = size (x);
37556 if (contain_pointers (x))
37558 go_through_object_nostart (method_table(x), x, s, oo,
37559 { testGCShadow((Object**) oo); });
37563 // Walk the whole heap, looking for pointers that were not updated with the write barrier.
37564 void checkGCWriteBarrier()
37566 // g_shadow_lowest_address != g_gc_lowest_address means the GC heap was extended by a segment
37567 // and the GC shadow segment did not track that change!
37568 if (g_GCShadowEnd <= g_GCShadow || g_shadow_lowest_address != g_gc_lowest_address)
37570 // No shadow stack, nothing to check.
37575 generation* gen = gc_heap::generation_of (max_generation);
37576 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
37578 PREFIX_ASSUME(seg != NULL);
37582 uint8_t* x = heap_segment_mem(seg);
37583 while (x < heap_segment_allocated (seg))
37585 size_t s = size (x);
37586 testGCShadowHelper (x);
37589 seg = heap_segment_next_rw (seg);
37594 // go through large object heap
37595 int alignment = get_alignment_constant(FALSE);
37596 generation* gen = gc_heap::generation_of (max_generation+1);
37597 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
37599 PREFIX_ASSUME(seg != NULL);
37603 uint8_t* x = heap_segment_mem(seg);
37604 while (x < heap_segment_allocated (seg))
37606 size_t s = size (x);
37607 testGCShadowHelper (x);
37608 x = x + Align (s, alignment);
37610 seg = heap_segment_next_rw (seg);
37614 #endif //WRITE_BARRIER_CHECK && !SERVER_GC
37616 #endif // !DACCESS_COMPILE
37618 #ifdef FEATURE_BASICFREEZE
37619 void gc_heap::walk_read_only_segment(heap_segment *seg, void *pvContext, object_callback_func pfnMethodTable, object_callback_func pfnObjRef)
37621 #ifdef DACCESS_COMPILE
37622 UNREFERENCED_PARAMETER(seg);
37623 UNREFERENCED_PARAMETER(pvContext);
37624 UNREFERENCED_PARAMETER(pfnMethodTable);
37625 UNREFERENCED_PARAMETER(pfnObjRef);
37627 uint8_t *o = heap_segment_mem(seg);
37629 // small heap alignment constant
37630 int alignment = get_alignment_constant(TRUE);
37632 while (o < heap_segment_allocated(seg))
37634 pfnMethodTable(pvContext, o);
37636 if (contain_pointers (o))
37638 go_through_object_nostart (method_table (o), o, size(o), oo,
37641 pfnObjRef(pvContext, oo);
37646 o += Align(size(o), alignment);
37648 #endif //!DACCESS_COMPILE
37650 #endif // FEATURE_BASICFREEZE
37652 #ifndef DACCESS_COMPILE
37653 HRESULT GCHeap::WaitUntilConcurrentGCCompleteAsync(int millisecondsTimeout)
37655 #ifdef BACKGROUND_GC
37656 if (recursive_gc_sync::background_running_p())
37658 uint32_t dwRet = pGenGCHeap->background_gc_wait(awr_ignored, millisecondsTimeout);
37659 if (dwRet == WAIT_OBJECT_0)
37661 else if (dwRet == WAIT_TIMEOUT)
37662 return HRESULT_FROM_WIN32(ERROR_TIMEOUT);
37664 return E_FAIL; // It is not clear if what the last error would be if the wait failed,
37665 // as there are too many layers in between. The best we can do is to return E_FAIL;
37671 #endif // !DACCESS_COMPILE
37673 void GCHeap::TemporaryEnableConcurrentGC()
37675 #ifdef BACKGROUND_GC
37676 gc_heap::temp_disable_concurrent_p = false;
37677 #endif //BACKGROUND_GC
37680 void GCHeap::TemporaryDisableConcurrentGC()
37682 #ifdef BACKGROUND_GC
37683 gc_heap::temp_disable_concurrent_p = true;
37684 #endif //BACKGROUND_GC
37687 bool GCHeap::IsConcurrentGCEnabled()
37689 #ifdef BACKGROUND_GC
37690 return (gc_heap::gc_can_use_concurrent && !(gc_heap::temp_disable_concurrent_p));
37693 #endif //BACKGROUND_GC
37696 void GCHeap::SetFinalizeRunOnShutdown(bool value)
37698 g_fFinalizerRunOnShutDown = value;
37701 void PopulateDacVars(GcDacVars *gcDacVars)
37703 #ifndef DACCESS_COMPILE
37704 assert(gcDacVars != nullptr);
37706 gcDacVars->major_version_number = 1;
37707 gcDacVars->minor_version_number = 0;
37708 gcDacVars->built_with_svr = &g_built_with_svr_gc;
37709 gcDacVars->build_variant = &g_build_variant;
37710 gcDacVars->gc_structures_invalid_cnt = const_cast<int32_t*>(&GCScan::m_GcStructuresInvalidCnt);
37711 gcDacVars->generation_size = sizeof(generation);
37712 gcDacVars->max_gen = &g_max_generation;
37713 #ifndef MULTIPLE_HEAPS
37714 gcDacVars->mark_array = &gc_heap::mark_array;
37715 gcDacVars->ephemeral_heap_segment = reinterpret_cast<dac_heap_segment**>(&gc_heap::ephemeral_heap_segment);
37716 gcDacVars->current_c_gc_state = const_cast<c_gc_state*>(&gc_heap::current_c_gc_state);
37717 gcDacVars->saved_sweep_ephemeral_seg = reinterpret_cast<dac_heap_segment**>(&gc_heap::saved_sweep_ephemeral_seg);
37718 gcDacVars->saved_sweep_ephemeral_start = &gc_heap::saved_sweep_ephemeral_start;
37719 gcDacVars->background_saved_lowest_address = &gc_heap::background_saved_lowest_address;
37720 gcDacVars->background_saved_highest_address = &gc_heap::background_saved_highest_address;
37721 gcDacVars->alloc_allocated = &gc_heap::alloc_allocated;
37722 gcDacVars->next_sweep_obj = &gc_heap::next_sweep_obj;
37723 gcDacVars->oom_info = &gc_heap::oom_info;
37724 gcDacVars->finalize_queue = reinterpret_cast<dac_finalize_queue**>(&gc_heap::finalize_queue);
37725 gcDacVars->generation_table = reinterpret_cast<dac_generation**>(&gc_heap::generation_table);
37726 #ifdef GC_CONFIG_DRIVEN
37727 gcDacVars->gc_global_mechanisms = reinterpret_cast<size_t**>(&gc_global_mechanisms);
37728 gcDacVars->interesting_data_per_heap = reinterpret_cast<size_t**>(&gc_heap::interesting_data_per_heap);
37729 gcDacVars->compact_reasons_per_heap = reinterpret_cast<size_t**>(&gc_heap::compact_reasons_per_heap);
37730 gcDacVars->expand_mechanisms_per_heap = reinterpret_cast<size_t**>(&gc_heap::expand_mechanisms_per_heap);
37731 gcDacVars->interesting_mechanism_bits_per_heap = reinterpret_cast<size_t**>(&gc_heap::interesting_mechanism_bits_per_heap);
37732 #endif // GC_CONFIG_DRIVEN
37733 #ifdef HEAP_ANALYZE
37734 gcDacVars->internal_root_array = &gc_heap::internal_root_array;
37735 gcDacVars->internal_root_array_index = &gc_heap::internal_root_array_index;
37736 gcDacVars->heap_analyze_success = &gc_heap::heap_analyze_success;
37737 #endif // HEAP_ANALYZE
37739 gcDacVars->n_heaps = &gc_heap::n_heaps;
37740 gcDacVars->g_heaps = reinterpret_cast<dac_gc_heap***>(&gc_heap::g_heaps);
37741 #endif // MULTIPLE_HEAPS
37742 #endif // DACCESS_COMPILE