1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
3 // See the LICENSE file in the project root for more information.
9 // GC automatically manages memory allocated by managed code.
10 // The design doc for GC can be found at Documentation/botr/garbage-collection.md
12 // This file includes both the code for GC and the allocator. The most common
13 // case for a GC to be triggered is from the allocator code. See
14 // code:#try_allocate_more_space where it calls GarbageCollectGeneration.
16 // Entry points for the allocator are GCHeap::Alloc* which are called by the
17 // allocation helpers in gcscan.cpp
24 // We just needed a simple random number generator for testing.
30 static uint64_t get_rand()
32 x = (314159269*x+278281) & 0x7FFFFFFF;
36 // obtain random number in the range 0 .. r-1
37 static uint64_t get_rand(uint64_t r) {
39 uint64_t x = (uint64_t)((get_rand() * r) >> 31);
44 uint64_t gc_rand::x = 0;
46 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
47 BOOL bgc_heap_walk_for_etw_p = FALSE;
48 #endif //BACKGROUND_GC && FEATURE_EVENT_TRACE
50 #if defined(FEATURE_REDHAWK)
51 #define MAYBE_UNUSED_VAR(v) v = v
53 #define MAYBE_UNUSED_VAR(v)
54 #endif // FEATURE_REDHAWK
56 #define MAX_PTR ((uint8_t*)(~(ptrdiff_t)0))
57 #define commit_min_th (16*OS_PAGE_SIZE)
60 #define partial_size_th 100
61 #define num_partial_refs 64
63 #define partial_size_th 100
64 #define num_partial_refs 32
67 #define demotion_plug_len_th (6*1024*1024)
70 #define MARK_STACK_INITIAL_LENGTH 1024
72 #define MARK_STACK_INITIAL_LENGTH 128
75 #define LOH_PIN_QUEUE_LENGTH 100
76 #define LOH_PIN_DECAY 10
79 // Right now we support maximum 1024 procs - meaning that we will create at most
80 // that many GC threads and GC heaps.
81 #define MAX_SUPPORTED_CPUS 1024
83 #define MAX_SUPPORTED_CPUS 64
86 uint32_t yp_spin_count_unit = 0;
87 size_t loh_size_threshold = LARGE_OBJECT_SIZE;
89 #ifdef GC_CONFIG_DRIVEN
90 int compact_ratio = 0;
91 #endif //GC_CONFIG_DRIVEN
93 // See comments in reset_memory.
94 BOOL reset_mm_p = TRUE;
96 bool g_fFinalizerRunOnShutDown = false;
99 bool g_built_with_svr_gc = true;
101 bool g_built_with_svr_gc = false;
102 #endif // FEATURE_SVR_GC
104 #if defined(BUILDENV_DEBUG)
105 uint8_t g_build_variant = 0;
106 #elif defined(BUILDENV_CHECKED)
107 uint8_t g_build_variant = 1;
109 uint8_t g_build_variant = 2;
110 #endif // defined(BUILDENV_DEBUG)
112 VOLATILE(int32_t) g_no_gc_lock = -1;
114 #if defined (TRACE_GC) && !defined (DACCESS_COMPILE)
115 const char * const allocation_state_str[] = {
124 "try_free_full_seg_in_bgc",
125 "try_free_after_bgc",
128 "acquire_seg_after_cg",
129 "acquire_seg_after_bgc",
130 "check_and_wait_for_bgc",
131 "trigger_full_compact_gc",
132 "trigger_ephemeral_gc",
133 "trigger_2nd_ephemeral_gc",
137 const char * const msl_take_state_str[] = {
153 #endif //TRACE_GC && !DACCESS_COMPILE
156 // Keep this in sync with the definition of gc_reason
157 #if (defined(DT_LOG) || defined(TRACE_GC)) && !defined (DACCESS_COMPILE)
158 static const char* const str_gc_reasons[] =
170 "induced_compacting",
173 "lowmemory_host_blocking"
176 static const char* const str_gc_pause_modes[] =
181 "sustained_low_latency",
184 #endif // defined(DT_LOG) || defined(TRACE_GC)
187 BOOL is_induced (gc_reason reason)
189 return ((reason == reason_induced) ||
190 (reason == reason_induced_noforce) ||
191 (reason == reason_lowmemory) ||
192 (reason == reason_lowmemory_blocking) ||
193 (reason == reason_induced_compacting) ||
194 (reason == reason_lowmemory_host) ||
195 (reason == reason_lowmemory_host_blocking));
199 BOOL is_induced_blocking (gc_reason reason)
201 return ((reason == reason_induced) ||
202 (reason == reason_lowmemory_blocking) ||
203 (reason == reason_induced_compacting) ||
204 (reason == reason_lowmemory_host_blocking));
207 #ifndef DACCESS_COMPILE
210 size_t GetHighPrecisionTimeStamp()
212 int64_t ts = GCToOSInterface::QueryPerformanceCounter();
214 return (size_t)(ts / (qpf / 1000));
219 // There is a current and a prior copy of the statistics. This allows us to display deltas per reporting
220 // interval, as well as running totals. The 'min' and 'max' values require special treatment. They are
221 // Reset (zeroed) in the current statistics when we begin a new interval and they are updated via a
222 // comparison with the global min/max.
223 GCStatistics g_GCStatistics;
224 GCStatistics g_LastGCStatistics;
226 char* GCStatistics::logFileName = NULL;
227 FILE* GCStatistics::logFile = NULL;
229 void GCStatistics::AddGCStats(const gc_mechanisms& settings, size_t timeInMSec)
232 if (settings.concurrent)
234 bgc.Accumulate((uint32_t)timeInMSec*1000);
237 else if (settings.background_p)
239 fgc.Accumulate((uint32_t)timeInMSec*1000);
241 if (settings.compaction)
243 assert(settings.condemned_generation < max_generation);
244 cntFGCGen[settings.condemned_generation]++;
247 #endif // BACKGROUND_GC
249 ngc.Accumulate((uint32_t)timeInMSec*1000);
251 if (settings.compaction)
253 cntNGCGen[settings.condemned_generation]++;
256 if (is_induced (settings.reason))
257 cntReasons[(int)reason_induced]++;
258 else if (settings.stress_induced)
259 cntReasons[(int)reason_gcstress]++;
261 cntReasons[(int)settings.reason]++;
264 if (settings.concurrent || !settings.background_p)
266 #endif // BACKGROUND_GC
270 #endif // BACKGROUND_GC
273 void GCStatistics::Initialize()
275 LIMITED_METHOD_CONTRACT;
276 // for efficiency sake we're taking a dependency on the layout of a C++ object
277 // with a vtable. protect against violations of our premise:
278 static_assert(offsetof(GCStatistics, cntDisplay) == sizeof(void*),
279 "The first field of GCStatistics follows the pointer sized vtable");
281 int podOffs = offsetof(GCStatistics, cntDisplay); // offset of the first POD field
282 memset((uint8_t*)(&g_GCStatistics)+podOffs, 0, sizeof(g_GCStatistics)-podOffs);
283 memset((uint8_t*)(&g_LastGCStatistics)+podOffs, 0, sizeof(g_LastGCStatistics)-podOffs);
286 void GCStatistics::DisplayAndUpdate()
288 LIMITED_METHOD_CONTRACT;
290 if (logFileName == NULL || logFile == NULL)
295 fprintf(logFile, "\nGCMix **** Initialize *****\n\n");
297 fprintf(logFile, "GCMix **** Summary ***** %d\n", cntDisplay);
299 // NGC summary (total, timing info)
300 ngc.DisplayAndUpdate(logFile, "NGC ", &g_LastGCStatistics.ngc, cntNGC, g_LastGCStatistics.cntNGC, msec);
302 // FGC summary (total, timing info)
303 fgc.DisplayAndUpdate(logFile, "FGC ", &g_LastGCStatistics.fgc, cntFGC, g_LastGCStatistics.cntFGC, msec);
306 bgc.DisplayAndUpdate(logFile, "BGC ", &g_LastGCStatistics.bgc, cntBGC, g_LastGCStatistics.cntBGC, msec);
308 // NGC/FGC break out by generation & compacting vs. sweeping
309 fprintf(logFile, "NGC ");
310 for (int i = max_generation; i >= 0; --i)
311 fprintf(logFile, "gen%d %d (%d). ", i, cntNGCGen[i]-g_LastGCStatistics.cntNGCGen[i], cntNGCGen[i]);
312 fprintf(logFile, "\n");
314 fprintf(logFile, "FGC ");
315 for (int i = max_generation-1; i >= 0; --i)
316 fprintf(logFile, "gen%d %d (%d). ", i, cntFGCGen[i]-g_LastGCStatistics.cntFGCGen[i], cntFGCGen[i]);
317 fprintf(logFile, "\n");
319 // Compacting vs. Sweeping break out
320 int _cntSweep = cntNGC-cntCompactNGC;
321 int _cntLastSweep = g_LastGCStatistics.cntNGC-g_LastGCStatistics.cntCompactNGC;
322 fprintf(logFile, "NGC Sweeping %d (%d) Compacting %d (%d)\n",
323 _cntSweep - _cntLastSweep, _cntSweep,
324 cntCompactNGC - g_LastGCStatistics.cntCompactNGC, cntCompactNGC);
326 _cntSweep = cntFGC-cntCompactFGC;
327 _cntLastSweep = g_LastGCStatistics.cntFGC-g_LastGCStatistics.cntCompactFGC;
328 fprintf(logFile, "FGC Sweeping %d (%d) Compacting %d (%d)\n",
329 _cntSweep - _cntLastSweep, _cntSweep,
330 cntCompactFGC - g_LastGCStatistics.cntCompactFGC, cntCompactFGC);
334 for (int reason=(int)reason_alloc_soh; reason <= (int)reason_gcstress; ++reason)
336 if (cntReasons[reason] != 0)
337 fprintf(logFile, "%s %d (%d). ", str_gc_reasons[reason],
338 cntReasons[reason]-g_LastGCStatistics.cntReasons[reason], cntReasons[reason]);
341 fprintf(logFile, "\n\n");
343 // flush the log file...
347 g_LastGCStatistics = *this;
357 size_t round_up_power2 (size_t size)
359 // Get the 0-based index of the most-significant bit in size-1.
360 // If the call failed (because size-1 is zero), size must be 1,
361 // so return 1 (because 1 rounds up to itself).
362 DWORD highest_set_bit_index;
369 &highest_set_bit_index, size - 1)) { return 1; }
371 // The size == 0 case (which would have overflowed to SIZE_MAX when decremented)
372 // is handled below by relying on the fact that highest_set_bit_index is the maximum value
373 // (31 or 63, depending on sizeof(size_t)) and left-shifting a value >= 2 by that
374 // number of bits shifts in zeros from the right, resulting in an output of zero.
375 return static_cast<size_t>(2) << highest_set_bit_index;
379 size_t round_down_power2 (size_t size)
381 // Get the 0-based index of the most-significant bit in size.
382 // If the call failed, size must be zero so return zero.
383 DWORD highest_set_bit_index;
390 &highest_set_bit_index, size)) { return 0; }
392 // Left-shift 1 by highest_set_bit_index to get back a value containing only
393 // the most-significant set bit of size, i.e. size rounded down
394 // to the next power-of-two value.
395 return static_cast<size_t>(1) << highest_set_bit_index;
398 // Get the 0-based index of the most-significant bit in the value.
399 // Returns -1 if the input value is zero (i.e. has no set bits).
401 int index_of_highest_set_bit (size_t value)
403 // Get the 0-based index of the most-significant bit in the value.
404 // If the call failed (because value is zero), return -1.
405 DWORD highest_set_bit_index;
412 &highest_set_bit_index, value)) ? -1 : static_cast<int>(highest_set_bit_index);
416 int relative_index_power2_plug (size_t power2)
418 int index = index_of_highest_set_bit (power2);
419 assert (index <= MAX_INDEX_POWER2);
421 return ((index < MIN_INDEX_POWER2) ? 0 : (index - MIN_INDEX_POWER2));
425 int relative_index_power2_free_space (size_t power2)
427 int index = index_of_highest_set_bit (power2);
428 assert (index <= MAX_INDEX_POWER2);
430 return ((index < MIN_INDEX_POWER2) ? -1 : (index - MIN_INDEX_POWER2));
434 uint32_t bgc_alloc_spin_count = 140;
435 uint32_t bgc_alloc_spin_count_loh = 16;
436 uint32_t bgc_alloc_spin = 2;
440 void c_write (uint32_t& place, uint32_t value)
442 Interlocked::Exchange (&place, value);
446 #ifndef DACCESS_COMPILE
447 // If every heap's gen2 or gen3 size is less than this threshold we will do a blocking GC.
448 const size_t bgc_min_per_heap = 4*1024*1024;
450 int gc_heap::gchist_index = 0;
451 gc_mechanisms_store gc_heap::gchist[max_history_count];
453 #ifndef MULTIPLE_HEAPS
454 size_t gc_heap::total_promoted_bytes = 0;
455 VOLATILE(bgc_state) gc_heap::current_bgc_state = bgc_not_in_process;
456 int gc_heap::gchist_index_per_heap = 0;
457 gc_heap::gc_history gc_heap::gchist_per_heap[max_history_count];
458 #endif //MULTIPLE_HEAPS
460 void gc_heap::add_to_history_per_heap()
463 gc_history* current_hist = &gchist_per_heap[gchist_index_per_heap];
464 current_hist->gc_index = settings.gc_index;
465 current_hist->current_bgc_state = current_bgc_state;
466 size_t elapsed = dd_gc_elapsed_time (dynamic_data_of (0));
467 current_hist->gc_time_ms = (uint32_t)elapsed;
468 current_hist->gc_efficiency = (elapsed ? (total_promoted_bytes / elapsed) : total_promoted_bytes);
469 current_hist->eph_low = generation_allocation_start (generation_of (max_generation-1));
470 current_hist->gen0_start = generation_allocation_start (generation_of (0));
471 current_hist->eph_high = heap_segment_allocated (ephemeral_heap_segment);
473 current_hist->bgc_lowest = background_saved_lowest_address;
474 current_hist->bgc_highest = background_saved_highest_address;
475 #endif //BACKGROUND_GC
476 current_hist->fgc_lowest = lowest_address;
477 current_hist->fgc_highest = highest_address;
478 current_hist->g_lowest = g_gc_lowest_address;
479 current_hist->g_highest = g_gc_highest_address;
481 gchist_index_per_heap++;
482 if (gchist_index_per_heap == max_history_count)
484 gchist_index_per_heap = 0;
489 void gc_heap::add_to_history()
492 gc_mechanisms_store* current_settings = &gchist[gchist_index];
493 current_settings->store (&settings);
496 if (gchist_index == max_history_count)
503 #endif //DACCESS_COMPILE
504 #endif //BACKGROUND_GC
506 #if defined(TRACE_GC) && !defined(DACCESS_COMPILE)
507 BOOL gc_log_on = TRUE;
509 size_t gc_log_file_size = 0;
511 size_t gc_buffer_index = 0;
512 size_t max_gc_buffers = 0;
514 static CLRCriticalSection gc_log_lock;
516 // we keep this much in a buffer and only flush when the buffer is full
517 #define gc_log_buffer_size (1024*1024)
518 uint8_t* gc_log_buffer = 0;
519 size_t gc_log_buffer_offset = 0;
521 void log_va_msg(const char *fmt, va_list args)
525 const int BUFFERSIZE = 512;
526 static char rgchBuffer[BUFFERSIZE];
527 char * pBuffer = &rgchBuffer[0];
530 int buffer_start = 1;
531 int pid_len = sprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, "[%5d]", (uint32_t)GCToOSInterface::GetCurrentThreadIdForLogging());
532 buffer_start += pid_len;
533 memset(&pBuffer[buffer_start], '-', BUFFERSIZE - buffer_start);
534 int msg_len = _vsnprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, _TRUNCATE, fmt, args);
537 msg_len = BUFFERSIZE - buffer_start;
540 msg_len += buffer_start;
542 if ((gc_log_buffer_offset + msg_len) > (gc_log_buffer_size - 12))
545 memset (index_str, '-', 8);
546 sprintf_s (index_str, _countof(index_str), "%d", (int)gc_buffer_index);
547 gc_log_buffer[gc_log_buffer_offset] = '\n';
548 memcpy (gc_log_buffer + (gc_log_buffer_offset + 1), index_str, 8);
551 if (gc_buffer_index > max_gc_buffers)
553 fseek (gc_log, 0, SEEK_SET);
556 fwrite(gc_log_buffer, gc_log_buffer_size, 1, gc_log);
558 memset (gc_log_buffer, '*', gc_log_buffer_size);
559 gc_log_buffer_offset = 0;
562 memcpy (gc_log_buffer + gc_log_buffer_offset, pBuffer, msg_len);
563 gc_log_buffer_offset += msg_len;
568 void GCLog (const char *fmt, ... )
570 if (gc_log_on && (gc_log != NULL))
574 log_va_msg (fmt, args);
578 #endif // TRACE_GC && !DACCESS_COMPILE
580 #if defined(GC_CONFIG_DRIVEN) && !defined(DACCESS_COMPILE)
582 BOOL gc_config_log_on = FALSE;
583 FILE* gc_config_log = NULL;
585 // we keep this much in a buffer and only flush when the buffer is full
586 #define gc_config_log_buffer_size (1*1024) // TEMP
587 uint8_t* gc_config_log_buffer = 0;
588 size_t gc_config_log_buffer_offset = 0;
590 // For config since we log so little we keep the whole history. Also it's only
591 // ever logged by one thread so no need to synchronize.
592 void log_va_msg_config(const char *fmt, va_list args)
594 const int BUFFERSIZE = 256;
595 static char rgchBuffer[BUFFERSIZE];
596 char * pBuffer = &rgchBuffer[0];
599 int buffer_start = 1;
600 int msg_len = _vsnprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, _TRUNCATE, fmt, args );
601 assert (msg_len != -1);
602 msg_len += buffer_start;
604 if ((gc_config_log_buffer_offset + msg_len) > gc_config_log_buffer_size)
606 fwrite(gc_config_log_buffer, gc_config_log_buffer_offset, 1, gc_config_log);
607 fflush(gc_config_log);
608 gc_config_log_buffer_offset = 0;
611 memcpy (gc_config_log_buffer + gc_config_log_buffer_offset, pBuffer, msg_len);
612 gc_config_log_buffer_offset += msg_len;
615 void GCLogConfig (const char *fmt, ... )
617 if (gc_config_log_on && (gc_config_log != NULL))
620 va_start( args, fmt );
621 log_va_msg_config (fmt, args);
624 #endif // GC_CONFIG_DRIVEN && !DACCESS_COMPILE
626 #ifdef SYNCHRONIZATION_STATS
628 // Number of GCs have we done since we last logged.
629 static unsigned int gc_count_during_log;
630 // In ms. This is how often we print out stats.
631 static const unsigned int log_interval = 5000;
632 // Time (in ms) when we start a new log interval.
633 static unsigned int log_start_tick;
634 static unsigned int gc_lock_contended;
635 static int64_t log_start_hires;
636 // Cycles accumulated in SuspendEE during log_interval.
637 static uint64_t suspend_ee_during_log;
638 // Cycles accumulated in RestartEE during log_interval.
639 static uint64_t restart_ee_during_log;
640 static uint64_t gc_during_log;
642 #endif //SYNCHRONIZATION_STATS
645 init_sync_log_stats()
647 #ifdef SYNCHRONIZATION_STATS
648 if (gc_count_during_log == 0)
650 gc_heap::init_sync_stats();
651 suspend_ee_during_log = 0;
652 restart_ee_during_log = 0;
654 gc_lock_contended = 0;
656 log_start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
657 log_start_hires = GCToOSInterface::QueryPerformanceCounter();
659 gc_count_during_log++;
660 #endif //SYNCHRONIZATION_STATS
664 process_sync_log_stats()
666 #ifdef SYNCHRONIZATION_STATS
668 unsigned int log_elapsed = GCToOSInterface::GetLowPrecisionTimeStamp() - log_start_tick;
670 if (log_elapsed > log_interval)
672 uint64_t total = GCToOSInterface::QueryPerformanceCounter() - log_start_hires;
673 // Print out the cycles we spent on average in each suspend and restart.
674 printf("\n_________________________________________________________________________________\n"
675 "Past %d(s): #%3d GCs; Total gc_lock contended: %8u; GC: %12u\n"
676 "SuspendEE: %8u; RestartEE: %8u GC %.3f%%\n",
680 (unsigned int)(gc_during_log / gc_count_during_log),
681 (unsigned int)(suspend_ee_during_log / gc_count_during_log),
682 (unsigned int)(restart_ee_during_log / gc_count_during_log),
683 (double)(100.0f * gc_during_log / total));
684 gc_heap::print_sync_stats(gc_count_during_log);
686 gc_count_during_log = 0;
688 #endif //SYNCHRONIZATION_STATS
691 #ifdef MULTIPLE_HEAPS
695 gc_join_init_cpu_mapping = 0,
697 gc_join_generation_determined = 2,
698 gc_join_begin_mark_phase = 3,
699 gc_join_scan_dependent_handles = 4,
700 gc_join_rescan_dependent_handles = 5,
701 gc_join_scan_sizedref_done = 6,
702 gc_join_null_dead_short_weak = 7,
703 gc_join_scan_finalization = 8,
704 gc_join_null_dead_long_weak = 9,
705 gc_join_null_dead_syncblk = 10,
706 gc_join_decide_on_compaction = 11,
707 gc_join_rearrange_segs_compaction = 12,
708 gc_join_adjust_handle_age_compact = 13,
709 gc_join_adjust_handle_age_sweep = 14,
710 gc_join_begin_relocate_phase = 15,
711 gc_join_relocate_phase_done = 16,
712 gc_join_verify_objects_done = 17,
713 gc_join_start_bgc = 18,
714 gc_join_restart_ee = 19,
715 gc_join_concurrent_overflow = 20,
716 gc_join_suspend_ee = 21,
717 gc_join_bgc_after_ephemeral = 22,
718 gc_join_allow_fgc = 23,
719 gc_join_bgc_sweep = 24,
720 gc_join_suspend_ee_verify = 25,
721 gc_join_restart_ee_verify = 26,
722 gc_join_set_state_free = 27,
723 gc_r_join_update_card_bundle = 28,
724 gc_join_after_absorb = 29,
725 gc_join_verify_copy_table = 30,
726 gc_join_after_reset = 31,
727 gc_join_after_ephemeral_sweep = 32,
728 gc_join_after_profiler_heap_walk = 33,
729 gc_join_minimal_gc = 34,
730 gc_join_after_commit_soh_no_gc = 35,
731 gc_join_expand_loh_no_gc = 36,
732 gc_join_final_no_gc = 37,
733 gc_join_disable_software_write_watch = 38,
739 join_flavor_server_gc = 0,
743 #define first_thread_arrived 2
744 #pragma warning(push)
745 #pragma warning(disable:4324) // don't complain if DECLSPEC_ALIGN actually pads
746 struct DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE) join_structure
748 // Shared non volatile keep on separate line to prevent eviction
751 // Keep polling/wait structures on separate line write once per join
752 DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE)
753 GCEvent joined_event[3]; // the last event in the array is only used for first_thread_arrived.
754 Volatile<int> lock_color;
755 VOLATILE(BOOL) wait_done;
756 VOLATILE(BOOL) joined_p;
758 // Keep volatile counted locks on separate cache line write many per join
759 DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE)
760 VOLATILE(int32_t) join_lock;
761 VOLATILE(int32_t) r_join_lock;
771 type_first_r_join = 3,
783 join_heap_restart = 100,
784 join_heap_r_restart = 200
796 join_structure join_struct;
799 gc_join_flavor flavor;
802 uint64_t start[MAX_SUPPORTED_CPUS], end[MAX_SUPPORTED_CPUS], start_seq;
803 // remember join id and last thread to arrive so restart can use these
805 // we want to print statistics every 10 seconds - this is to remember the start of the 10 sec interval
807 // counters for joins, in 1000's of clock cycles
808 uint64_t elapsed_total[gc_join_max], wake_total[gc_join_max], seq_loss_total[gc_join_max], par_loss_total[gc_join_max], in_join_total[gc_join_max];
812 BOOL init (int n_th, gc_join_flavor f)
814 dprintf (JOIN_LOG, ("Initializing join structure"));
815 join_struct.n_threads = n_th;
816 join_struct.lock_color = 0;
817 for (int i = 0; i < 3; i++)
819 if (!join_struct.joined_event[i].IsValid())
821 join_struct.joined_p = FALSE;
822 dprintf (JOIN_LOG, ("Creating join event %d", i));
823 // TODO - changing this to a non OS event
824 // because this is also used by BGC threads which are
825 // managed threads and WaitEx does not allow you to wait
826 // for an OS event on a managed thread.
827 // But we are not sure if this plays well in the hosting
829 //join_struct.joined_event[i].CreateOSManualEventNoThrow(FALSE);
830 if (!join_struct.joined_event[i].CreateManualEventNoThrow(FALSE))
834 join_struct.join_lock = join_struct.n_threads;
835 join_struct.r_join_lock = join_struct.n_threads;
836 join_struct.wait_done = FALSE;
840 start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
848 dprintf (JOIN_LOG, ("Destroying join structure"));
849 for (int i = 0; i < 3; i++)
851 if (join_struct.joined_event[i].IsValid())
852 join_struct.joined_event[i].CloseEvent();
856 inline void fire_event (int heap, join_time time, join_type type, int join_id)
858 FIRE_EVENT(GCJoin_V2, heap, time, type, join_id);
861 void join (gc_heap* gch, int join_id)
864 // parallel execution ends here
865 end[gch->heap_number] = get_ts();
868 assert (!join_struct.joined_p);
869 int color = join_struct.lock_color.LoadWithoutBarrier();
871 if (Interlocked::Decrement(&join_struct.join_lock) != 0)
873 dprintf (JOIN_LOG, ("join%d(%d): Join() Waiting...join_lock is now %d",
874 flavor, join_id, (int32_t)(join_struct.join_lock)));
876 fire_event (gch->heap_number, time_start, type_join, join_id);
878 //busy wait around the color
879 if (color == join_struct.lock_color.LoadWithoutBarrier())
882 int spin_count = 128 * yp_spin_count_unit;
883 for (int j = 0; j < spin_count; j++)
885 if (color != join_struct.lock_color.LoadWithoutBarrier())
889 YieldProcessor(); // indicate to the processor that we are spinning
892 // we've spun, and if color still hasn't changed, fall into hard wait
893 if (color == join_struct.lock_color.LoadWithoutBarrier())
895 dprintf (JOIN_LOG, ("join%d(%d): Join() hard wait on reset event %d, join_lock is now %d",
896 flavor, join_id, color, (int32_t)(join_struct.join_lock)));
898 //Thread* current_thread = GCToEEInterface::GetThread();
899 //BOOL cooperative_mode = gc_heap::enable_preemptive ();
900 uint32_t dwJoinWait = join_struct.joined_event[color].Wait(INFINITE, FALSE);
901 //gc_heap::disable_preemptive (cooperative_mode);
903 if (dwJoinWait != WAIT_OBJECT_0)
905 STRESS_LOG1 (LF_GC, LL_FATALERROR, "joined event wait failed with code: %Ix", dwJoinWait);
910 // avoid race due to the thread about to reset the event (occasionally) being preempted before ResetEvent()
911 if (color == join_struct.lock_color.LoadWithoutBarrier())
916 dprintf (JOIN_LOG, ("join%d(%d): Join() done, join_lock is %d",
917 flavor, join_id, (int32_t)(join_struct.join_lock)));
920 fire_event (gch->heap_number, time_end, type_join, join_id);
923 // parallel execution starts here
924 start[gch->heap_number] = get_ts();
925 Interlocked::ExchangeAdd(&in_join_total[join_id], (start[gch->heap_number] - end[gch->heap_number]));
930 fire_event (gch->heap_number, time_start, type_last_join, join_id);
932 join_struct.joined_p = TRUE;
933 dprintf (JOIN_LOG, ("join%d(%d): Last thread to complete the join, setting id", flavor, join_id));
934 join_struct.joined_event[!color].Reset();
936 // this one is alone so it can proceed
938 // remember the join id, the last thread arriving, the start of the sequential phase,
939 // and keep track of the cycles spent waiting in the join
940 thd = gch->heap_number;
941 start_seq = get_ts();
942 Interlocked::ExchangeAdd(&in_join_total[join_id], (start_seq - end[gch->heap_number]));
947 // Reverse join - first thread gets here does the work; other threads will only proceed
948 // after the work is done.
949 // Note that you cannot call this twice in a row on the same thread. Plus there's no
950 // need to call it twice in row - you should just merge the work.
951 BOOL r_join (gc_heap* gch, int join_id)
954 if (join_struct.n_threads == 1)
959 if (Interlocked::CompareExchange(&join_struct.r_join_lock, 0, join_struct.n_threads) == 0)
961 if (!join_struct.wait_done)
963 dprintf (JOIN_LOG, ("r_join() Waiting..."));
965 fire_event (gch->heap_number, time_start, type_join, join_id);
967 //busy wait around the color
968 if (!join_struct.wait_done)
971 int spin_count = 256 * yp_spin_count_unit;
972 for (int j = 0; j < spin_count; j++)
974 if (join_struct.wait_done)
978 YieldProcessor(); // indicate to the processor that we are spinning
981 // we've spun, and if color still hasn't changed, fall into hard wait
982 if (!join_struct.wait_done)
984 dprintf (JOIN_LOG, ("Join() hard wait on reset event %d", first_thread_arrived));
985 uint32_t dwJoinWait = join_struct.joined_event[first_thread_arrived].Wait(INFINITE, FALSE);
986 if (dwJoinWait != WAIT_OBJECT_0)
988 STRESS_LOG1 (LF_GC, LL_FATALERROR, "joined event wait failed with code: %Ix", dwJoinWait);
993 // avoid race due to the thread about to reset the event (occasionally) being preempted before ResetEvent()
994 if (!join_struct.wait_done)
999 dprintf (JOIN_LOG, ("r_join() done"));
1002 fire_event (gch->heap_number, time_end, type_join, join_id);
1009 fire_event (gch->heap_number, time_start, type_first_r_join, join_id);
1017 return GCToOSInterface::QueryPerformanceCounter();
1020 void start_ts (gc_heap* gch)
1022 // parallel execution ends here
1023 start[gch->heap_number] = get_ts();
1030 uint64_t elapsed_seq = get_ts() - start_seq;
1031 uint64_t max = 0, sum = 0, wake = 0;
1032 uint64_t min_ts = start[0];
1033 for (int i = 1; i < join_struct.n_threads; i++)
1035 if(min_ts > start[i]) min_ts = start[i];
1038 for (int i = 0; i < join_struct.n_threads; i++)
1040 uint64_t wake_delay = start[i] - min_ts;
1041 uint64_t elapsed = end[i] - start[i];
1047 uint64_t seq_loss = (join_struct.n_threads - 1)*elapsed_seq;
1048 uint64_t par_loss = join_struct.n_threads*max - sum;
1049 double efficiency = 0.0;
1051 efficiency = sum*100.0/(join_struct.n_threads*max);
1053 const double ts_scale = 1e-6;
1055 // enable this printf to get statistics on each individual join as it occurs
1056 // printf("join #%3d seq_loss = %5g par_loss = %5g efficiency = %3.0f%%\n", join_id, ts_scale*seq_loss, ts_scale*par_loss, efficiency);
1058 elapsed_total[id] += sum;
1059 wake_total[id] += wake;
1060 seq_loss_total[id] += seq_loss;
1061 par_loss_total[id] += par_loss;
1063 // every 10 seconds, print a summary of the time spent in each type of join
1064 if (GCToOSInterface::GetLowPrecisionTimeStamp() - start_tick > 10*1000)
1066 printf("**** summary *****\n");
1067 for (int i = 0; i < 16; i++)
1069 printf("join #%3d elapsed_total = %8g wake_loss = %8g seq_loss = %8g par_loss = %8g in_join_total = %8g\n",
1071 ts_scale*elapsed_total[i],
1072 ts_scale*wake_total[i],
1073 ts_scale*seq_loss_total[i],
1074 ts_scale*par_loss_total[i],
1075 ts_scale*in_join_total[i]);
1076 elapsed_total[i] = wake_total[i] = seq_loss_total[i] = par_loss_total[i] = in_join_total[i] = 0;
1078 start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
1082 fire_event (join_heap_restart, time_start, type_restart, -1);
1083 assert (join_struct.joined_p);
1084 join_struct.joined_p = FALSE;
1085 join_struct.join_lock = join_struct.n_threads;
1086 dprintf (JOIN_LOG, ("join%d(%d): Restarting from join: join_lock is %d", flavor, id, (int32_t)(join_struct.join_lock)));
1087 // printf("restart from join #%d at cycle %u from start of gc\n", join_id, GetCycleCount32() - gc_start);
1088 int color = join_struct.lock_color.LoadWithoutBarrier();
1089 join_struct.lock_color = !color;
1090 join_struct.joined_event[color].Set();
1092 // printf("Set joined_event %d\n", !join_struct.lock_color);
1094 fire_event (join_heap_restart, time_end, type_restart, -1);
1097 start[thd] = get_ts();
1103 dprintf (JOIN_LOG, ("join%d(%d): joined, join_lock is %d", flavor, id, (int32_t)(join_struct.join_lock)));
1104 return join_struct.joined_p;
1109 if (join_struct.n_threads != 1)
1111 fire_event (join_heap_r_restart, time_start, type_restart, -1);
1112 join_struct.wait_done = TRUE;
1113 join_struct.joined_event[first_thread_arrived].Set();
1114 fire_event (join_heap_r_restart, time_end, type_restart, -1);
1120 if (join_struct.n_threads != 1)
1122 join_struct.r_join_lock = join_struct.n_threads;
1123 join_struct.wait_done = FALSE;
1124 join_struct.joined_event[first_thread_arrived].Reset();
1131 #ifdef BACKGROUND_GC
1133 #endif //BACKGROUND_GC
1135 #endif //MULTIPLE_HEAPS
1137 #define spin_and_switch(count_to_spin, expr) \
1139 for (int j = 0; j < count_to_spin; j++) \
1149 GCToOSInterface::YieldThread(0); \
1153 #ifndef DACCESS_COMPILE
1154 #ifdef BACKGROUND_GC
1156 #define max_pending_allocs 64
1158 class exclusive_sync
1160 // TODO - verify that this is the right syntax for Volatile.
1161 VOLATILE(uint8_t*) rwp_object;
1162 VOLATILE(int32_t) needs_checking;
1166 uint8_t cache_separator[HS_CACHE_LINE_SIZE - sizeof (int) - sizeof (int32_t)];
1168 // TODO - perhaps each object should be on its own cache line...
1169 VOLATILE(uint8_t*) alloc_objects[max_pending_allocs];
1171 int find_free_index ()
1173 for (int i = 0; i < max_pending_allocs; i++)
1175 if (alloc_objects [i] == (uint8_t*)0)
1187 spin_count = 32 * (g_num_processors - 1);
1190 for (int i = 0; i < max_pending_allocs; i++)
1192 alloc_objects [i] = (uint8_t*)0;
1198 for (int i = 0; i < max_pending_allocs; i++)
1200 if (alloc_objects [i] != (uint8_t*)0)
1202 GCToOSInterface::DebugBreak();
1207 void bgc_mark_set (uint8_t* obj)
1209 dprintf (3, ("cm: probing %Ix", obj));
1211 if (Interlocked::CompareExchange(&needs_checking, 1, 0) == 0)
1213 // If we spend too much time spending all the allocs,
1214 // consider adding a high water mark and scan up
1215 // to that; we'll need to interlock in done when
1216 // we update the high watermark.
1217 for (int i = 0; i < max_pending_allocs; i++)
1219 if (obj == alloc_objects[i])
1222 dprintf (3, ("cm: will spin"));
1223 spin_and_switch (spin_count, (obj != alloc_objects[i]));
1230 dprintf (3, ("cm: set %Ix", obj));
1235 spin_and_switch (spin_count, (needs_checking == 0));
1240 int loh_alloc_set (uint8_t* obj)
1242 if (!gc_heap::cm_in_progress)
1248 dprintf (3, ("loh alloc: probing %Ix", obj));
1250 if (Interlocked::CompareExchange(&needs_checking, 1, 0) == 0)
1252 if (obj == rwp_object)
1255 spin_and_switch (spin_count, (obj != rwp_object));
1260 int cookie = find_free_index();
1264 alloc_objects[cookie] = obj;
1268 // GCToOSInterface::DebugBreak();
1271 dprintf (3, ("loh alloc: set %Ix at %d", obj, cookie));
1277 dprintf (3, ("loh alloc: setting %Ix will spin to acquire a free index", obj));
1278 spin_and_switch (spin_count, (find_free_index () != -1));
1285 dprintf (3, ("loh alloc: will spin on checking %Ix", obj));
1286 spin_and_switch (spin_count, (needs_checking == 0));
1291 void bgc_mark_done ()
1293 dprintf (3, ("cm: release lock on %Ix", (uint8_t *)rwp_object));
1297 void loh_alloc_done_with_index (int index)
1299 dprintf (3, ("loh alloc: release lock on %Ix based on %d", (uint8_t *)alloc_objects[index], index));
1300 assert ((index >= 0) && (index < max_pending_allocs));
1301 alloc_objects[index] = (uint8_t*)0;
1304 void loh_alloc_done (uint8_t* obj)
1306 #ifdef BACKGROUND_GC
1307 if (!gc_heap::cm_in_progress)
1312 for (int i = 0; i < max_pending_allocs; i++)
1314 if (alloc_objects [i] == obj)
1316 dprintf (3, ("loh alloc: release lock on %Ix at %d", (uint8_t *)alloc_objects[i], i));
1317 alloc_objects[i] = (uint8_t*)0;
1321 #endif //BACKGROUND_GC
1325 // Note that this class was written assuming just synchronization between
1326 // one background GC thread and multiple user threads that might request
1327 // an FGC - it does not take into account what kind of locks the multiple
1328 // user threads might be holding at the time (eg, there could only be one
1329 // user thread requesting an FGC because it needs to take gc_lock first)
1330 // so you'll see checks that may not be necessary if you take those conditions
1331 // into consideration.
1333 // With the introduction of Server Background GC we no longer use this
1334 // class to do synchronization between FGCs and BGC.
1335 class recursive_gc_sync
1337 static VOLATILE(int32_t) foreground_request_count;//initial state 0
1338 static VOLATILE(BOOL) gc_background_running; //initial state FALSE
1339 static VOLATILE(int32_t) foreground_count; // initial state 0;
1340 static VOLATILE(uint32_t) foreground_gate; // initial state FALSE;
1341 static GCEvent foreground_complete;//Auto Reset
1342 static GCEvent foreground_allowed;//Auto Reset
1344 static void begin_background();
1345 static void end_background();
1346 static void begin_foreground();
1347 static void end_foreground();
1348 BOOL allow_foreground ();
1350 static void shutdown();
1351 static BOOL background_running_p() {return gc_background_running;}
1354 VOLATILE(int32_t) recursive_gc_sync::foreground_request_count = 0;//initial state 0
1355 VOLATILE(int32_t) recursive_gc_sync::foreground_count = 0; // initial state 0;
1356 VOLATILE(BOOL) recursive_gc_sync::gc_background_running = FALSE; //initial state FALSE
1357 VOLATILE(uint32_t) recursive_gc_sync::foreground_gate = 0;
1358 GCEvent recursive_gc_sync::foreground_complete;//Auto Reset
1359 GCEvent recursive_gc_sync::foreground_allowed;//Manual Reset
1361 BOOL recursive_gc_sync::init ()
1363 foreground_request_count = 0;
1364 foreground_count = 0;
1365 gc_background_running = FALSE;
1366 foreground_gate = 0;
1368 if (!foreground_complete.CreateOSAutoEventNoThrow(FALSE))
1372 if (!foreground_allowed.CreateManualEventNoThrow(FALSE))
1384 void recursive_gc_sync::shutdown()
1386 if (foreground_complete.IsValid())
1387 foreground_complete.CloseEvent();
1388 if (foreground_allowed.IsValid())
1389 foreground_allowed.CloseEvent();
1392 void recursive_gc_sync::begin_background()
1394 dprintf (2, ("begin background"));
1395 foreground_request_count = 1;
1396 foreground_count = 1;
1397 foreground_allowed.Reset();
1398 gc_background_running = TRUE;
1400 void recursive_gc_sync::end_background()
1402 dprintf (2, ("end background"));
1403 gc_background_running = FALSE;
1404 foreground_gate = 1;
1405 foreground_allowed.Set();
1408 void recursive_gc_sync::begin_foreground()
1410 dprintf (2, ("begin_foreground"));
1412 bool cooperative_mode = false;
1413 if (gc_background_running)
1415 gc_heap::fire_alloc_wait_event_begin (awr_fgc_wait_for_bgc);
1416 gc_heap::alloc_wait_event_p = TRUE;
1420 Interlocked::Increment (&foreground_request_count);
1423 dprintf(2, ("Waiting sync gc point"));
1424 assert (foreground_allowed.IsValid());
1425 assert (foreground_complete.IsValid());
1427 cooperative_mode = gc_heap::enable_preemptive ();
1429 foreground_allowed.Wait(INFINITE, FALSE);
1431 dprintf(2, ("Waiting sync gc point is done"));
1433 gc_heap::disable_preemptive (cooperative_mode);
1435 if (foreground_gate)
1437 Interlocked::Increment (&foreground_count);
1438 dprintf (2, ("foreground_count: %d", (int32_t)foreground_count));
1439 if (foreground_gate)
1441 gc_heap::settings.concurrent = FALSE;
1452 goto try_again_no_inc;
1457 void recursive_gc_sync::end_foreground()
1459 dprintf (2, ("end_foreground"));
1460 if (gc_background_running)
1462 Interlocked::Decrement (&foreground_request_count);
1463 dprintf (2, ("foreground_count before decrement: %d", (int32_t)foreground_count));
1464 if (Interlocked::Decrement (&foreground_count) == 0)
1466 //c_write ((BOOL*)&foreground_gate, 0);
1467 // TODO - couldn't make the syntax work with Volatile<T>
1468 foreground_gate = 0;
1469 if (foreground_count == 0)
1471 foreground_allowed.Reset ();
1472 dprintf(2, ("setting foreground complete event"));
1473 foreground_complete.Set();
1480 BOOL recursive_gc_sync::allow_foreground()
1482 assert (gc_heap::settings.concurrent);
1483 dprintf (100, ("enter allow_foreground, f_req_count: %d, f_count: %d",
1484 (int32_t)foreground_request_count, (int32_t)foreground_count));
1486 BOOL did_fgc = FALSE;
1488 //if we have suspended the EE, just return because
1489 //some thread could be waiting on this to proceed.
1490 if (!GCHeap::GcInProgress)
1492 //TODO BACKGROUND_GC This is to stress the concurrency between
1493 //background and foreground
1494 // gc_heap::disallow_new_allocation (0);
1496 //GCToOSInterface::YieldThread(0);
1499 if (foreground_request_count != 0)
1501 //foreground wants to run
1502 //save the important settings
1503 //TODO BACKGROUND_GC be more selective about the important settings.
1504 gc_mechanisms saved_settings = gc_heap::settings;
1508 //c_write ((BOOL*)&foreground_gate, 1);
1509 // TODO - couldn't make the syntax work with Volatile<T>
1510 foreground_gate = 1;
1511 foreground_allowed.Set ();
1512 foreground_complete.Wait (INFINITE, FALSE);
1513 }while (/*foreground_request_count ||*/ foreground_gate);
1515 assert (!foreground_gate);
1517 //restore the important settings
1518 gc_heap::settings = saved_settings;
1519 GCHeap::GcCondemnedGeneration = gc_heap::settings.condemned_generation;
1520 //the background GC shouldn't be using gc_high and gc_low
1521 //gc_low = lowest_address;
1522 //gc_high = highest_address;
1525 //TODO BACKGROUND_GC This is to stress the concurrency between
1526 //background and foreground
1527 // gc_heap::allow_new_allocation (0);
1531 dprintf (100, ("leave allow_foreground"));
1532 assert (gc_heap::settings.concurrent);
1536 #endif //BACKGROUND_GC
1537 #endif //DACCESS_COMPILE
1540 #if defined(COUNT_CYCLES)
1542 #pragma warning(disable:4035)
1546 unsigned GetCycleCount32() // enough for about 40 seconds
1554 #pragma warning(default:4035)
1556 #endif //COUNT_CYCLES
1559 int mark_time, plan_time, sweep_time, reloc_time, compact_time;
1562 #ifndef MULTIPLE_HEAPS
1564 #endif // MULTIPLE_HEAPS
1566 void reset_memory (uint8_t* o, size_t sizeo);
1570 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1571 static bool virtual_alloc_hardware_write_watch = false;
1572 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1574 static bool hardware_write_watch_capability = false;
1576 #ifndef DACCESS_COMPILE
1578 //check if the write watch APIs are supported.
1580 void hardware_write_watch_api_supported()
1582 if (GCToOSInterface::SupportsWriteWatch())
1584 hardware_write_watch_capability = true;
1585 dprintf (2, ("WriteWatch supported"));
1589 dprintf (2,("WriteWatch not supported"));
1593 #endif //!DACCESS_COMPILE
1595 inline bool can_use_hardware_write_watch()
1597 return hardware_write_watch_capability;
1600 inline bool can_use_write_watch_for_gc_heap()
1602 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1604 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1605 return can_use_hardware_write_watch();
1606 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1609 inline bool can_use_write_watch_for_card_table()
1611 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
1614 return can_use_hardware_write_watch();
1619 #define mem_reserve (MEM_RESERVE)
1620 #endif //WRITE_WATCH
1622 //check if the low memory notification is supported
1624 #ifndef DACCESS_COMPILE
1626 void WaitLongerNoInstru (int i)
1628 // every 8th attempt:
1629 bool bToggleGC = GCToEEInterface::EnablePreemptiveGC();
1631 // if we're waiting for gc to finish, we should block immediately
1632 if (g_fSuspensionPending == 0)
1634 if (g_num_processors > 1)
1636 YieldProcessor(); // indicate to the processor that we are spinning
1638 GCToOSInterface::YieldThread (0);
1640 GCToOSInterface::Sleep (5);
1643 GCToOSInterface::Sleep (5);
1646 // If CLR is hosted, a thread may reach here while it is in preemptive GC mode,
1647 // or it has no Thread object, in order to force a task to yield, or to triger a GC.
1648 // It is important that the thread is going to wait for GC. Otherwise the thread
1649 // is in a tight loop. If the thread has high priority, the perf is going to be very BAD.
1653 // In debug builds, all enter_spin_lock operations go through this code. If a GC has
1654 // started, it is important to block until the GC thread calls set_gc_done (since it is
1655 // guaranteed to have cleared g_TrapReturningThreads by this point). This avoids livelock
1656 // conditions which can otherwise occur if threads are allowed to spin in this function
1657 // (and therefore starve the GC thread) between the point when the GC thread sets the
1658 // WaitForGC event and the point when the GC thread clears g_TrapReturningThreads.
1659 if (gc_heap::gc_started)
1661 gc_heap::wait_for_gc_done();
1664 GCToEEInterface::DisablePreemptiveGC();
1666 else if (g_fSuspensionPending > 0)
1668 g_theGCHeap->WaitUntilGCComplete();
1673 static void safe_switch_to_thread()
1675 bool cooperative_mode = gc_heap::enable_preemptive();
1677 GCToOSInterface::YieldThread(0);
1679 gc_heap::disable_preemptive(cooperative_mode);
1683 // We need the following methods to have volatile arguments, so that they can accept
1684 // raw pointers in addition to the results of the & operator on Volatile<T>.
1687 static void enter_spin_lock_noinstru (RAW_KEYWORD(volatile) int32_t* lock)
1691 if (Interlocked::CompareExchange(lock, 0, -1) >= 0)
1694 while (VolatileLoad(lock) >= 0)
1696 if ((++i & 7) && !IsGCInProgress())
1698 if (g_num_processors > 1)
1700 #ifndef MULTIPLE_HEAPS
1701 int spin_count = 32 * yp_spin_count_unit;
1702 #else //!MULTIPLE_HEAPS
1703 int spin_count = yp_spin_count_unit;
1704 #endif //!MULTIPLE_HEAPS
1705 for (int j = 0; j < spin_count; j++)
1707 if (VolatileLoad(lock) < 0 || IsGCInProgress())
1709 YieldProcessor(); // indicate to the processor that we are spinning
1711 if (VolatileLoad(lock) >= 0 && !IsGCInProgress())
1713 safe_switch_to_thread();
1718 safe_switch_to_thread();
1723 WaitLongerNoInstru(i);
1731 static BOOL try_enter_spin_lock_noinstru(RAW_KEYWORD(volatile) int32_t* lock)
1733 return (Interlocked::CompareExchange(&*lock, 0, -1) < 0);
1737 static void leave_spin_lock_noinstru (RAW_KEYWORD(volatile) int32_t* lock)
1739 VolatileStore<int32_t>((int32_t*)lock, -1);
1745 static void enter_spin_lock(GCSpinLock *pSpinLock)
1747 enter_spin_lock_noinstru(&pSpinLock->lock);
1748 assert (pSpinLock->holding_thread == (Thread*)-1);
1749 pSpinLock->holding_thread = GCToEEInterface::GetThread();
1753 static BOOL try_enter_spin_lock(GCSpinLock *pSpinLock)
1755 BOOL ret = try_enter_spin_lock_noinstru(&pSpinLock->lock);
1757 pSpinLock->holding_thread = GCToEEInterface::GetThread();
1762 static void leave_spin_lock(GCSpinLock *pSpinLock)
1764 bool gc_thread_p = GCToEEInterface::WasCurrentThreadCreatedByGC();
1765 // _ASSERTE((pSpinLock->holding_thread == GCToEEInterface::GetThread()) || gc_thread_p || pSpinLock->released_by_gc_p);
1766 pSpinLock->released_by_gc_p = gc_thread_p;
1767 pSpinLock->holding_thread = (Thread*) -1;
1768 if (pSpinLock->lock != -1)
1769 leave_spin_lock_noinstru(&pSpinLock->lock);
1772 #define ASSERT_HOLDING_SPIN_LOCK(pSpinLock) \
1773 _ASSERTE((pSpinLock)->holding_thread == GCToEEInterface::GetThread());
1775 #define ASSERT_NOT_HOLDING_SPIN_LOCK(pSpinLock) \
1776 _ASSERTE((pSpinLock)->holding_thread != GCToEEInterface::GetThread());
1780 //In the concurrent version, the Enable/DisablePreemptiveGC is optional because
1781 //the gc thread call WaitLonger.
1782 void WaitLonger (int i
1783 #ifdef SYNCHRONIZATION_STATS
1784 , GCSpinLock* spin_lock
1785 #endif //SYNCHRONIZATION_STATS
1788 #ifdef SYNCHRONIZATION_STATS
1789 (spin_lock->num_wait_longer)++;
1790 #endif //SYNCHRONIZATION_STATS
1792 // every 8th attempt:
1793 bool bToggleGC = GCToEEInterface::EnablePreemptiveGC();
1796 // if we're waiting for gc to finish, we should block immediately
1797 if (!gc_heap::gc_started)
1799 #ifdef SYNCHRONIZATION_STATS
1800 (spin_lock->num_switch_thread_w)++;
1801 #endif //SYNCHRONIZATION_STATS
1802 if (g_num_processors > 1)
1804 YieldProcessor(); // indicate to the processor that we are spinning
1806 GCToOSInterface::YieldThread (0);
1808 GCToOSInterface::Sleep (5);
1811 GCToOSInterface::Sleep (5);
1814 // If CLR is hosted, a thread may reach here while it is in preemptive GC mode,
1815 // or it has no Thread object, in order to force a task to yield, or to triger a GC.
1816 // It is important that the thread is going to wait for GC. Otherwise the thread
1817 // is in a tight loop. If the thread has high priority, the perf is going to be very BAD.
1818 if (gc_heap::gc_started)
1820 gc_heap::wait_for_gc_done();
1825 #ifdef SYNCHRONIZATION_STATS
1826 (spin_lock->num_disable_preemptive_w)++;
1827 #endif //SYNCHRONIZATION_STATS
1828 GCToEEInterface::DisablePreemptiveGC();
1833 static void enter_spin_lock (GCSpinLock* spin_lock)
1837 if (Interlocked::CompareExchange(&spin_lock->lock, 0, -1) >= 0)
1840 while (spin_lock->lock >= 0)
1842 if ((++i & 7) && !gc_heap::gc_started)
1844 if (g_num_processors > 1)
1846 #ifndef MULTIPLE_HEAPS
1847 int spin_count = 32 * yp_spin_count_unit;
1848 #else //!MULTIPLE_HEAPS
1849 int spin_count = yp_spin_count_unit;
1850 #endif //!MULTIPLE_HEAPS
1851 for (int j = 0; j < spin_count; j++)
1853 if (spin_lock->lock < 0 || gc_heap::gc_started)
1855 YieldProcessor(); // indicate to the processor that we are spinning
1857 if (spin_lock->lock >= 0 && !gc_heap::gc_started)
1859 #ifdef SYNCHRONIZATION_STATS
1860 (spin_lock->num_switch_thread)++;
1861 #endif //SYNCHRONIZATION_STATS
1862 bool cooperative_mode = gc_heap::enable_preemptive ();
1864 GCToOSInterface::YieldThread(0);
1866 gc_heap::disable_preemptive (cooperative_mode);
1870 GCToOSInterface::YieldThread(0);
1875 #ifdef SYNCHRONIZATION_STATS
1877 #endif //SYNCHRONIZATION_STATS
1885 inline BOOL try_enter_spin_lock(GCSpinLock* spin_lock)
1887 return (Interlocked::CompareExchange(&spin_lock->lock, 0, -1) < 0);
1891 static void leave_spin_lock (GCSpinLock * spin_lock)
1893 spin_lock->lock = -1;
1896 #define ASSERT_HOLDING_SPIN_LOCK(pSpinLock)
1900 bool gc_heap::enable_preemptive ()
1902 return GCToEEInterface::EnablePreemptiveGC();
1905 void gc_heap::disable_preemptive (bool restore_cooperative)
1907 if (restore_cooperative)
1909 GCToEEInterface::DisablePreemptiveGC();
1913 #endif // !DACCESS_COMPILE
1915 typedef void ** PTR_PTR;
1916 //This function clears a piece of memory
1917 // size has to be Dword aligned
1920 void memclr ( uint8_t* mem, size_t size)
1922 dprintf (3, ("MEMCLR: %Ix, %d", mem, size));
1923 assert ((size & (sizeof(PTR_PTR)-1)) == 0);
1924 assert (sizeof(PTR_PTR) == DATA_ALIGNMENT);
1927 // The compiler will recognize this pattern and replace it with memset call. We can as well just call
1928 // memset directly to make it obvious what's going on.
1929 PTR_PTR m = (PTR_PTR) mem;
1930 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
1934 memset (mem, 0, size);
1937 void memcopy (uint8_t* dmem, uint8_t* smem, size_t size)
1939 const size_t sz4ptr = sizeof(PTR_PTR)*4;
1940 const size_t sz2ptr = sizeof(PTR_PTR)*2;
1941 const size_t sz1ptr = sizeof(PTR_PTR)*1;
1943 // size must be a multiple of the pointer size
1944 assert ((size & (sizeof (PTR_PTR)-1)) == 0);
1945 assert (sizeof(PTR_PTR) == DATA_ALIGNMENT);
1947 // copy in groups of four pointer sized things at a time
1952 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1953 ((PTR_PTR)dmem)[1] = ((PTR_PTR)smem)[1];
1954 ((PTR_PTR)dmem)[2] = ((PTR_PTR)smem)[2];
1955 ((PTR_PTR)dmem)[3] = ((PTR_PTR)smem)[3];
1959 while ((size -= sz4ptr) >= sz4ptr);
1962 // still two pointer sized things or more left to copy?
1965 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1966 ((PTR_PTR)dmem)[1] = ((PTR_PTR)smem)[1];
1971 // still one pointer sized thing left to copy?
1974 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1982 ptrdiff_t round_down (ptrdiff_t add, int pitch)
1984 return ((add / pitch) * pitch);
1987 #if defined(FEATURE_STRUCTALIGN) && defined(RESPECT_LARGE_ALIGNMENT)
1988 // FEATURE_STRUCTALIGN allows the compiler to dictate the alignment,
1989 // i.e, if a larger alignment matters or is beneficial, the compiler
1990 // generated info tells us so. RESPECT_LARGE_ALIGNMENT is just the
1991 // converse - it's a heuristic for the GC to use a larger alignment.
1992 #error FEATURE_STRUCTALIGN should imply !RESPECT_LARGE_ALIGNMENT
1995 #if defined(FEATURE_STRUCTALIGN) && defined(FEATURE_LOH_COMPACTION)
1996 #error FEATURE_STRUCTALIGN and FEATURE_LOH_COMPACTION are mutually exclusive
1999 #if defined(GROWABLE_SEG_MAPPING_TABLE) && !defined(SEG_MAPPING_TABLE)
2000 #error if GROWABLE_SEG_MAPPING_TABLE is defined, SEG_MAPPING_TABLE must be defined
2004 BOOL same_large_alignment_p (uint8_t* p1, uint8_t* p2)
2006 #ifdef RESPECT_LARGE_ALIGNMENT
2007 return ((((size_t)p1 ^ (size_t)p2) & 7) == 0);
2009 UNREFERENCED_PARAMETER(p1);
2010 UNREFERENCED_PARAMETER(p2);
2012 #endif //RESPECT_LARGE_ALIGNMENT
2016 size_t switch_alignment_size (BOOL already_padded_p)
2018 if (already_padded_p)
2019 return DATA_ALIGNMENT;
2021 return (Align (min_obj_size) +((Align (min_obj_size)&DATA_ALIGNMENT)^DATA_ALIGNMENT));
2025 #ifdef FEATURE_STRUCTALIGN
2026 void set_node_aligninfo (uint8_t *node, int requiredAlignment, ptrdiff_t pad);
2027 void clear_node_aligninfo (uint8_t *node);
2028 #else // FEATURE_STRUCTALIGN
2029 #define node_realigned(node) (((plug_and_reloc*)(node))[-1].reloc & 1)
2030 void set_node_realigned (uint8_t* node);
2031 void clear_node_realigned(uint8_t* node);
2032 #endif // FEATURE_STRUCTALIGN
2035 size_t AlignQword (size_t nbytes)
2037 #ifdef FEATURE_STRUCTALIGN
2038 // This function is used to align everything on the large object
2039 // heap to an 8-byte boundary, to reduce the number of unaligned
2040 // accesses to (say) arrays of doubles. With FEATURE_STRUCTALIGN,
2041 // the compiler dictates the optimal alignment instead of having
2042 // a heuristic in the GC.
2043 return Align (nbytes);
2044 #else // FEATURE_STRUCTALIGN
2045 return (nbytes + 7) & ~7;
2046 #endif // FEATURE_STRUCTALIGN
2050 BOOL Aligned (size_t n)
2052 return (n & ALIGNCONST) == 0;
2055 #define OBJECT_ALIGNMENT_OFFSET (sizeof(MethodTable *))
2057 #ifdef FEATURE_STRUCTALIGN
2058 #define MAX_STRUCTALIGN OS_PAGE_SIZE
2059 #else // FEATURE_STRUCTALIGN
2060 #define MAX_STRUCTALIGN 0
2061 #endif // FEATURE_STRUCTALIGN
2063 #ifdef FEATURE_STRUCTALIGN
2065 ptrdiff_t AdjustmentForMinPadSize(ptrdiff_t pad, int requiredAlignment)
2067 // The resulting alignpad must be either 0 or at least min_obj_size.
2068 // Note that by computing the following difference on unsigned types,
2069 // we can do the range check 0 < alignpad < min_obj_size with a
2070 // single conditional branch.
2071 if ((size_t)(pad - DATA_ALIGNMENT) < Align (min_obj_size) - DATA_ALIGNMENT)
2073 return requiredAlignment;
2079 uint8_t* StructAlign (uint8_t* origPtr, int requiredAlignment, ptrdiff_t alignmentOffset=OBJECT_ALIGNMENT_OFFSET)
2081 // required alignment must be a power of two
2082 _ASSERTE(((size_t)origPtr & ALIGNCONST) == 0);
2083 _ASSERTE(((requiredAlignment - 1) & requiredAlignment) == 0);
2084 _ASSERTE(requiredAlignment >= sizeof(void *));
2085 _ASSERTE(requiredAlignment <= MAX_STRUCTALIGN);
2087 // When this method is invoked for individual objects (i.e., alignmentOffset
2088 // is just the size of the PostHeader), what needs to be aligned when
2089 // we're done is the pointer to the payload of the object (which means
2090 // the actual resulting object pointer is typically not aligned).
2092 uint8_t* result = (uint8_t*)Align ((size_t)origPtr + alignmentOffset, requiredAlignment-1) - alignmentOffset;
2093 ptrdiff_t alignpad = result - origPtr;
2095 return result + AdjustmentForMinPadSize (alignpad, requiredAlignment);
2099 ptrdiff_t ComputeStructAlignPad (uint8_t* plug, int requiredAlignment, size_t alignmentOffset=OBJECT_ALIGNMENT_OFFSET)
2101 return StructAlign (plug, requiredAlignment, alignmentOffset) - plug;
2104 BOOL IsStructAligned (uint8_t *ptr, int requiredAlignment)
2106 return StructAlign (ptr, requiredAlignment) == ptr;
2110 ptrdiff_t ComputeMaxStructAlignPad (int requiredAlignment)
2112 if (requiredAlignment == DATA_ALIGNMENT)
2114 // Since a non-zero alignment padding cannot be less than min_obj_size (so we can fit the
2115 // alignment padding object), the worst-case alignment padding is correspondingly larger
2116 // than the required alignment.
2117 return requiredAlignment + Align (min_obj_size) - DATA_ALIGNMENT;
2121 ptrdiff_t ComputeMaxStructAlignPadLarge (int requiredAlignment)
2123 if (requiredAlignment <= get_alignment_constant (TRUE)+1)
2125 // This is the same as ComputeMaxStructAlignPad, except that in addition to leaving space
2126 // for padding before the actual object, it also leaves space for filling a gap after the
2127 // actual object. This is needed on the large object heap, as the outer allocation functions
2128 // don't operate on an allocation context (which would have left space for the final gap).
2129 return requiredAlignment + Align (min_obj_size) * 2 - DATA_ALIGNMENT;
2132 uint8_t* gc_heap::pad_for_alignment (uint8_t* newAlloc, int requiredAlignment, size_t size, alloc_context* acontext)
2134 uint8_t* alignedPtr = StructAlign (newAlloc, requiredAlignment);
2135 if (alignedPtr != newAlloc) {
2136 make_unused_array (newAlloc, alignedPtr - newAlloc);
2138 acontext->alloc_ptr = alignedPtr + Align (size);
2142 uint8_t* gc_heap::pad_for_alignment_large (uint8_t* newAlloc, int requiredAlignment, size_t size)
2144 uint8_t* alignedPtr = StructAlign (newAlloc, requiredAlignment);
2145 if (alignedPtr != newAlloc) {
2146 make_unused_array (newAlloc, alignedPtr - newAlloc);
2148 if (alignedPtr < newAlloc + ComputeMaxStructAlignPadLarge (requiredAlignment)) {
2149 make_unused_array (alignedPtr + AlignQword (size), newAlloc + ComputeMaxStructAlignPadLarge (requiredAlignment) - alignedPtr);
2153 #else // FEATURE_STRUCTALIGN
2154 #define ComputeMaxStructAlignPad(requiredAlignment) 0
2155 #define ComputeMaxStructAlignPadLarge(requiredAlignment) 0
2156 #endif // FEATURE_STRUCTALIGN
2158 //CLR_SIZE is the max amount of bytes from gen0 that is set to 0 in one chunk
2160 #define CLR_SIZE ((size_t)(8*1024))
2162 #define CLR_SIZE ((size_t)(8*1024))
2165 #define END_SPACE_AFTER_GC (loh_size_threshold + MAX_STRUCTALIGN)
2167 #ifdef BACKGROUND_GC
2168 #define SEGMENT_INITIAL_COMMIT (2*OS_PAGE_SIZE)
2170 #define SEGMENT_INITIAL_COMMIT (OS_PAGE_SIZE)
2171 #endif //BACKGROUND_GC
2173 // This is always power of 2.
2174 const size_t min_segment_size_hard_limit = 1024*1024*16;
2180 #define INITIAL_ALLOC ((size_t)((size_t)4*1024*1024*1024))
2181 #define LHEAP_ALLOC ((size_t)(1024*1024*256))
2185 #define INITIAL_ALLOC ((size_t)(1024*1024*64))
2186 #define LHEAP_ALLOC ((size_t)(1024*1024*32))
2194 #define INITIAL_ALLOC ((size_t)(1024*1024*256))
2195 #define LHEAP_ALLOC ((size_t)(1024*1024*128))
2199 #define INITIAL_ALLOC ((size_t)(1024*1024*16))
2200 #define LHEAP_ALLOC ((size_t)(1024*1024*16))
2206 //amount in bytes of the etw allocation tick
2207 const size_t etw_allocation_tick = 100*1024;
2209 const size_t low_latency_alloc = 256*1024;
2211 const size_t fgn_check_quantum = 2*1024*1024;
2214 const int max_snoop_level = 128;
2219 //threshold of heap size to turn on card bundles.
2220 #define SH_TH_CARD_BUNDLE (40*1024*1024)
2221 #define MH_TH_CARD_BUNDLE (180*1024*1024)
2222 #endif //CARD_BUNDLE
2224 #define GC_EPHEMERAL_DECOMMIT_TIMEOUT 5000
2227 size_t align_on_page (size_t add)
2229 return ((add + OS_PAGE_SIZE - 1) & ~((size_t)OS_PAGE_SIZE - 1));
2233 uint8_t* align_on_page (uint8_t* add)
2235 return (uint8_t*)align_on_page ((size_t) add);
2239 size_t align_lower_page (size_t add)
2241 return (add & ~((size_t)OS_PAGE_SIZE - 1));
2245 uint8_t* align_lower_page (uint8_t* add)
2247 return (uint8_t*)align_lower_page ((size_t)add);
2251 size_t align_write_watch_lower_page (size_t add)
2253 return (add & ~(WRITE_WATCH_UNIT_SIZE - 1));
2257 uint8_t* align_write_watch_lower_page (uint8_t* add)
2259 return (uint8_t*)align_lower_page ((size_t)add);
2264 BOOL power_of_two_p (size_t integer)
2266 return !(integer & (integer-1));
2270 BOOL oddp (size_t integer)
2272 return (integer & 1) != 0;
2275 // we only ever use this for WORDs.
2276 size_t logcount (size_t word)
2278 //counts the number of high bits in a 16 bit word.
2279 assert (word < 0x10000);
2281 count = (word & 0x5555) + ( (word >> 1 ) & 0x5555);
2282 count = (count & 0x3333) + ( (count >> 2) & 0x3333);
2283 count = (count & 0x0F0F) + ( (count >> 4) & 0x0F0F);
2284 count = (count & 0x00FF) + ( (count >> 8) & 0x00FF);
2288 #ifndef DACCESS_COMPILE
2290 void stomp_write_barrier_resize(bool is_runtime_suspended, bool requires_upper_bounds_check)
2292 WriteBarrierParameters args = {};
2293 args.operation = WriteBarrierOp::StompResize;
2294 args.is_runtime_suspended = is_runtime_suspended;
2295 args.requires_upper_bounds_check = requires_upper_bounds_check;
2297 args.card_table = g_gc_card_table;
2298 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
2299 args.card_bundle_table = g_gc_card_bundle_table;
2302 args.lowest_address = g_gc_lowest_address;
2303 args.highest_address = g_gc_highest_address;
2305 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
2306 if (SoftwareWriteWatch::IsEnabledForGCHeap())
2308 args.write_watch_table = g_gc_sw_ww_table;
2310 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
2312 GCToEEInterface::StompWriteBarrier(&args);
2315 void stomp_write_barrier_ephemeral(uint8_t* ephemeral_low, uint8_t* ephemeral_high)
2317 WriteBarrierParameters args = {};
2318 args.operation = WriteBarrierOp::StompEphemeral;
2319 args.is_runtime_suspended = true;
2320 args.ephemeral_low = ephemeral_low;
2321 args.ephemeral_high = ephemeral_high;
2322 GCToEEInterface::StompWriteBarrier(&args);
2325 void stomp_write_barrier_initialize(uint8_t* ephemeral_low, uint8_t* ephemeral_high)
2327 WriteBarrierParameters args = {};
2328 args.operation = WriteBarrierOp::Initialize;
2329 args.is_runtime_suspended = true;
2330 args.requires_upper_bounds_check = false;
2331 args.card_table = g_gc_card_table;
2333 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
2334 args.card_bundle_table = g_gc_card_bundle_table;
2337 args.lowest_address = g_gc_lowest_address;
2338 args.highest_address = g_gc_highest_address;
2339 args.ephemeral_low = ephemeral_low;
2340 args.ephemeral_high = ephemeral_high;
2341 GCToEEInterface::StompWriteBarrier(&args);
2344 #endif // DACCESS_COMPILE
2346 //extract the low bits [0,low[ of a uint32_t
2347 #define lowbits(wrd, bits) ((wrd) & ((1 << (bits))-1))
2348 //extract the high bits [high, 32] of a uint32_t
2349 #define highbits(wrd, bits) ((wrd) & ~((1 << (bits))-1))
2351 // Things we need to manually initialize:
2352 // gen0 min_size - based on cache
2353 // gen0/1 max_size - based on segment size
2354 static static_data static_data_table[latency_level_last - latency_level_first + 1][NUMBERGENERATIONS] =
2356 // latency_level_memory_footprint
2359 {0, 0, 40000, 0.5f, 9.0f, 20.0f, 1000, 1},
2361 {160*1024, 0, 80000, 0.5f, 2.0f, 7.0f, 10000, 10},
2363 {256*1024, SSIZE_T_MAX, 200000, 0.25f, 1.2f, 1.8f, 100000, 100},
2365 {3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0}
2368 // latency_level_balanced
2372 #ifdef MULTIPLE_HEAPS
2376 #endif //MULTIPLE_HEAPS
2379 {256*1024, 0, 80000, 0.5f, 2.0f, 7.0f, 10000, 10},
2381 {256*1024, SSIZE_T_MAX, 200000, 0.25f, 1.2f, 1.8f, 100000, 100},
2383 {3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0}
2390 class CObjectHeader;
2394 class c_synchronize;
2396 #ifdef FEATURE_PREMORTEM_FINALIZATION
2397 #ifndef DACCESS_COMPILE
2399 HRESULT AllocateCFinalize(CFinalize **pCFinalize);
2400 #endif //!DACCESS_COMPILE
2401 #endif // FEATURE_PREMORTEM_FINALIZATION
2403 uint8_t* tree_search (uint8_t* tree, uint8_t* old_address);
2406 #ifdef USE_INTROSORT
2407 #define _sort introsort::sort
2408 #else //USE_INTROSORT
2409 #define _sort qsort1
2410 void qsort1(uint8_t** low, uint8_t** high, unsigned int depth);
2411 #endif //USE_INTROSORT
2413 void* virtual_alloc (size_t size);
2414 void virtual_free (void* add, size_t size);
2416 /* per heap static initialization */
2418 #ifndef MULTIPLE_HEAPS
2419 uint32_t* gc_heap::mark_array;
2420 #endif //MULTIPLE_HEAPS
2424 uint8_t** gc_heap::g_mark_list;
2426 #ifdef PARALLEL_MARK_LIST_SORT
2427 uint8_t** gc_heap::g_mark_list_copy;
2428 #endif //PARALLEL_MARK_LIST_SORT
2430 size_t gc_heap::mark_list_size;
2433 #ifdef SEG_MAPPING_TABLE
2434 seg_mapping* seg_mapping_table;
2435 #endif //SEG_MAPPING_TABLE
2437 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
2438 sorted_table* gc_heap::seg_table;
2439 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
2441 #ifdef MULTIPLE_HEAPS
2442 GCEvent gc_heap::ee_suspend_event;
2443 size_t gc_heap::min_balance_threshold = 0;
2444 #endif //MULTIPLE_HEAPS
2446 VOLATILE(BOOL) gc_heap::gc_started;
2448 #ifdef MULTIPLE_HEAPS
2450 GCEvent gc_heap::gc_start_event;
2451 bool gc_heap::gc_thread_no_affinitize_p = false;
2452 uintptr_t process_mask = 0;
2454 int gc_heap::n_heaps;
2456 gc_heap** gc_heap::g_heaps;
2458 size_t* gc_heap::g_promoted;
2461 int* gc_heap::g_mark_stack_busy;
2465 #ifdef BACKGROUND_GC
2466 size_t* gc_heap::g_bpromoted;
2467 #endif //BACKGROUND_GC
2469 #else //MULTIPLE_HEAPS
2471 size_t gc_heap::g_promoted;
2473 #ifdef BACKGROUND_GC
2474 size_t gc_heap::g_bpromoted;
2475 #endif //BACKGROUND_GC
2477 #endif //MULTIPLE_HEAPS
2479 size_t gc_heap::reserved_memory = 0;
2480 size_t gc_heap::reserved_memory_limit = 0;
2481 BOOL gc_heap::g_low_memory_status;
2483 #ifndef DACCESS_COMPILE
2484 static gc_reason gc_trigger_reason = reason_empty;
2485 #endif //DACCESS_COMPILE
2487 gc_latency_level gc_heap::latency_level = latency_level_default;
2489 gc_mechanisms gc_heap::settings;
2491 gc_history_global gc_heap::gc_data_global;
2493 size_t gc_heap::gc_last_ephemeral_decommit_time = 0;
2495 size_t gc_heap::gc_gen0_desired_high;
2497 CLRCriticalSection gc_heap::check_commit_cs;
2499 size_t gc_heap::current_total_committed = 0;
2501 size_t gc_heap::current_total_committed_bookkeeping = 0;
2504 double gc_heap::short_plugs_pad_ratio = 0;
2505 #endif //SHORT_PLUGS
2508 #define MAX_ALLOWED_MEM_LOAD 85
2510 // consider putting this in dynamic data -
2511 // we may want different values for workstation
2513 #define MIN_YOUNGEST_GEN_DESIRED (16*1024*1024)
2515 size_t gc_heap::youngest_gen_desired_th;
2518 uint32_t gc_heap::last_gc_memory_load = 0;
2520 size_t gc_heap::last_gc_heap_size = 0;
2522 size_t gc_heap::last_gc_fragmentation = 0;
2524 uint64_t gc_heap::mem_one_percent = 0;
2526 uint32_t gc_heap::high_memory_load_th = 0;
2528 uint32_t gc_heap::m_high_memory_load_th;
2530 uint32_t gc_heap::v_high_memory_load_th;
2532 uint64_t gc_heap::total_physical_mem = 0;
2534 uint64_t gc_heap::entry_available_physical_mem = 0;
2536 size_t gc_heap::heap_hard_limit = 0;
2538 #ifdef BACKGROUND_GC
2539 GCEvent gc_heap::bgc_start_event;
2541 gc_mechanisms gc_heap::saved_bgc_settings;
2543 GCEvent gc_heap::background_gc_done_event;
2545 GCEvent gc_heap::ee_proceed_event;
2547 bool gc_heap::gc_can_use_concurrent = false;
2549 bool gc_heap::temp_disable_concurrent_p = false;
2551 uint32_t gc_heap::cm_in_progress = FALSE;
2553 BOOL gc_heap::dont_restart_ee_p = FALSE;
2555 BOOL gc_heap::keep_bgc_threads_p = FALSE;
2557 GCEvent gc_heap::bgc_threads_sync_event;
2559 BOOL gc_heap::do_ephemeral_gc_p = FALSE;
2561 BOOL gc_heap::do_concurrent_p = FALSE;
2563 size_t gc_heap::ephemeral_fgc_counts[max_generation];
2565 BOOL gc_heap::alloc_wait_event_p = FALSE;
2567 VOLATILE(c_gc_state) gc_heap::current_c_gc_state = c_gc_state_free;
2569 #endif //BACKGROUND_GC
2571 #ifndef MULTIPLE_HEAPS
2572 #ifdef SPINLOCK_HISTORY
2573 int gc_heap::spinlock_info_index = 0;
2574 spinlock_info gc_heap::last_spinlock_info[max_saved_spinlock_info + 8];
2575 #endif //SPINLOCK_HISTORY
2577 size_t gc_heap::fgn_last_alloc = 0;
2579 int gc_heap::generation_skip_ratio = 100;
2581 uint64_t gc_heap::loh_alloc_since_cg = 0;
2583 BOOL gc_heap::elevation_requested = FALSE;
2585 BOOL gc_heap::last_gc_before_oom = FALSE;
2587 BOOL gc_heap::sufficient_gen0_space_p = FALSE;
2589 #ifdef BACKGROUND_GC
2590 uint8_t* gc_heap::background_saved_lowest_address = 0;
2591 uint8_t* gc_heap::background_saved_highest_address = 0;
2592 uint8_t* gc_heap::next_sweep_obj = 0;
2593 uint8_t* gc_heap::current_sweep_pos = 0;
2594 exclusive_sync* gc_heap::bgc_alloc_lock;
2595 #endif //BACKGROUND_GC
2597 oom_history gc_heap::oom_info;
2599 fgm_history gc_heap::fgm_result;
2601 size_t gc_heap::allocated_since_last_gc = 0;
2603 BOOL gc_heap::ro_segments_in_range;
2605 size_t gc_heap::gen0_big_free_spaces = 0;
2607 uint8_t* gc_heap::ephemeral_low;
2609 uint8_t* gc_heap::ephemeral_high;
2611 uint8_t* gc_heap::lowest_address;
2613 uint8_t* gc_heap::highest_address;
2615 BOOL gc_heap::ephemeral_promotion;
2617 uint8_t* gc_heap::saved_ephemeral_plan_start[NUMBERGENERATIONS-1];
2618 size_t gc_heap::saved_ephemeral_plan_start_size[NUMBERGENERATIONS-1];
2620 short* gc_heap::brick_table;
2622 uint32_t* gc_heap::card_table;
2625 uint32_t* gc_heap::card_bundle_table;
2626 #endif //CARD_BUNDLE
2628 uint8_t* gc_heap::gc_low;
2630 uint8_t* gc_heap::gc_high;
2632 uint8_t* gc_heap::demotion_low;
2634 uint8_t* gc_heap::demotion_high;
2636 BOOL gc_heap::demote_gen1_p = TRUE;
2638 uint8_t* gc_heap::last_gen1_pin_end;
2640 gen_to_condemn_tuning gc_heap::gen_to_condemn_reasons;
2642 size_t gc_heap::etw_allocation_running_amount[2];
2644 int gc_heap::gc_policy = 0;
2646 size_t gc_heap::allocation_running_time;
2648 size_t gc_heap::allocation_running_amount;
2650 heap_segment* gc_heap::ephemeral_heap_segment = 0;
2652 BOOL gc_heap::blocking_collection = FALSE;
2654 heap_segment* gc_heap::freeable_large_heap_segment = 0;
2656 size_t gc_heap::time_bgc_last = 0;
2658 size_t gc_heap::mark_stack_tos = 0;
2660 size_t gc_heap::mark_stack_bos = 0;
2662 size_t gc_heap::mark_stack_array_length = 0;
2664 mark* gc_heap::mark_stack_array = 0;
2666 #if defined (_DEBUG) && defined (VERIFY_HEAP)
2667 BOOL gc_heap::verify_pinned_queue_p = FALSE;
2668 #endif // defined (_DEBUG) && defined (VERIFY_HEAP)
2670 uint8_t* gc_heap::oldest_pinned_plug = 0;
2672 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
2673 size_t gc_heap::num_pinned_objects = 0;
2674 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
2676 #ifdef FEATURE_LOH_COMPACTION
2677 size_t gc_heap::loh_pinned_queue_tos = 0;
2679 size_t gc_heap::loh_pinned_queue_bos = 0;
2681 size_t gc_heap::loh_pinned_queue_length = 0;
2683 mark* gc_heap::loh_pinned_queue = 0;
2685 BOOL gc_heap::loh_compacted_p = FALSE;
2686 #endif //FEATURE_LOH_COMPACTION
2688 #ifdef BACKGROUND_GC
2690 EEThreadId gc_heap::bgc_thread_id;
2692 uint8_t* gc_heap::background_written_addresses [array_size+2];
2694 heap_segment* gc_heap::freeable_small_heap_segment = 0;
2696 size_t gc_heap::bgc_overflow_count = 0;
2698 size_t gc_heap::bgc_begin_loh_size = 0;
2699 size_t gc_heap::end_loh_size = 0;
2701 uint32_t gc_heap::bgc_alloc_spin_loh = 0;
2703 size_t gc_heap::bgc_loh_size_increased = 0;
2705 size_t gc_heap::bgc_loh_allocated_in_free = 0;
2707 size_t gc_heap::background_soh_alloc_count = 0;
2709 size_t gc_heap::background_loh_alloc_count = 0;
2711 uint8_t** gc_heap::background_mark_stack_tos = 0;
2713 uint8_t** gc_heap::background_mark_stack_array = 0;
2715 size_t gc_heap::background_mark_stack_array_length = 0;
2717 uint8_t* gc_heap::background_min_overflow_address =0;
2719 uint8_t* gc_heap::background_max_overflow_address =0;
2721 BOOL gc_heap::processed_soh_overflow_p = FALSE;
2723 uint8_t* gc_heap::background_min_soh_overflow_address =0;
2725 uint8_t* gc_heap::background_max_soh_overflow_address =0;
2727 heap_segment* gc_heap::saved_sweep_ephemeral_seg = 0;
2729 uint8_t* gc_heap::saved_sweep_ephemeral_start = 0;
2731 heap_segment* gc_heap::saved_overflow_ephemeral_seg = 0;
2733 Thread* gc_heap::bgc_thread = 0;
2735 BOOL gc_heap::expanded_in_fgc = FALSE;
2737 uint8_t** gc_heap::c_mark_list = 0;
2739 size_t gc_heap::c_mark_list_length = 0;
2741 size_t gc_heap::c_mark_list_index = 0;
2743 gc_history_per_heap gc_heap::bgc_data_per_heap;
2745 BOOL gc_heap::bgc_thread_running;
2747 CLRCriticalSection gc_heap::bgc_threads_timeout_cs;
2749 GCEvent gc_heap::gc_lh_block_event;
2751 #endif //BACKGROUND_GC
2754 uint8_t** gc_heap::mark_list;
2755 uint8_t** gc_heap::mark_list_index;
2756 uint8_t** gc_heap::mark_list_end;
2760 snoop_stats_data gc_heap::snoop_stat;
2761 #endif //SNOOP_STATS
2763 uint8_t* gc_heap::min_overflow_address = MAX_PTR;
2765 uint8_t* gc_heap::max_overflow_address = 0;
2767 uint8_t* gc_heap::shigh = 0;
2769 uint8_t* gc_heap::slow = MAX_PTR;
2771 size_t gc_heap::ordered_free_space_indices[MAX_NUM_BUCKETS];
2773 size_t gc_heap::saved_ordered_free_space_indices[MAX_NUM_BUCKETS];
2775 size_t gc_heap::ordered_plug_indices[MAX_NUM_BUCKETS];
2777 size_t gc_heap::saved_ordered_plug_indices[MAX_NUM_BUCKETS];
2779 BOOL gc_heap::ordered_plug_indices_init = FALSE;
2781 BOOL gc_heap::use_bestfit = FALSE;
2783 uint8_t* gc_heap::bestfit_first_pin = 0;
2785 BOOL gc_heap::commit_end_of_seg = FALSE;
2787 size_t gc_heap::max_free_space_items = 0;
2789 size_t gc_heap::free_space_buckets = 0;
2791 size_t gc_heap::free_space_items = 0;
2793 int gc_heap::trimmed_free_space_index = 0;
2795 size_t gc_heap::total_ephemeral_plugs = 0;
2797 seg_free_spaces* gc_heap::bestfit_seg = 0;
2799 size_t gc_heap::total_ephemeral_size = 0;
2803 size_t gc_heap::internal_root_array_length = initial_internal_roots;
2805 uint8_t** gc_heap::internal_root_array = 0;
2807 size_t gc_heap::internal_root_array_index = 0;
2809 BOOL gc_heap::heap_analyze_success = TRUE;
2811 uint8_t* gc_heap::current_obj = 0;
2812 size_t gc_heap::current_obj_size = 0;
2814 #endif //HEAP_ANALYZE
2816 #ifdef GC_CONFIG_DRIVEN
2817 size_t gc_heap::interesting_data_per_gc[max_idp_count];
2818 //size_t gc_heap::interesting_data_per_heap[max_idp_count];
2819 //size_t gc_heap::interesting_mechanisms_per_heap[max_im_count];
2820 #endif //GC_CONFIG_DRIVEN
2821 #endif //MULTIPLE_HEAPS
2823 no_gc_region_info gc_heap::current_no_gc_region_info;
2824 BOOL gc_heap::proceed_with_gc_p = FALSE;
2825 GCSpinLock gc_heap::gc_lock;
2827 size_t gc_heap::eph_gen_starts_size = 0;
2828 heap_segment* gc_heap::segment_standby_list;
2829 size_t gc_heap::last_gc_index = 0;
2830 #ifdef SEG_MAPPING_TABLE
2831 size_t gc_heap::min_segment_size = 0;
2832 size_t gc_heap::min_segment_size_shr = 0;
2833 #endif //SEG_MAPPING_TABLE
2834 size_t gc_heap::soh_segment_size = 0;
2835 size_t gc_heap::min_loh_segment_size = 0;
2836 size_t gc_heap::segment_info_size = 0;
2838 #ifdef GC_CONFIG_DRIVEN
2839 size_t gc_heap::time_init = 0;
2840 size_t gc_heap::time_since_init = 0;
2841 size_t gc_heap::compact_or_sweep_gcs[2];
2842 #endif //GC_CONFIG_DRIVEN
2844 #ifdef FEATURE_LOH_COMPACTION
2845 BOOL gc_heap::loh_compaction_always_p = FALSE;
2846 gc_loh_compaction_mode gc_heap::loh_compaction_mode = loh_compaction_default;
2847 int gc_heap::loh_pinned_queue_decay = LOH_PIN_DECAY;
2849 #endif //FEATURE_LOH_COMPACTION
2851 GCEvent gc_heap::full_gc_approach_event;
2853 GCEvent gc_heap::full_gc_end_event;
2855 uint32_t gc_heap::fgn_maxgen_percent = 0;
2857 uint32_t gc_heap::fgn_loh_percent = 0;
2859 #ifdef BACKGROUND_GC
2860 BOOL gc_heap::fgn_last_gc_was_concurrent = FALSE;
2861 #endif //BACKGROUND_GC
2863 VOLATILE(bool) gc_heap::full_gc_approach_event_set;
2865 size_t gc_heap::full_gc_counts[gc_type_max];
2867 bool gc_heap::maxgen_size_inc_p = false;
2869 BOOL gc_heap::should_expand_in_full_gc = FALSE;
2871 // Provisional mode related stuff.
2872 bool gc_heap::provisional_mode_triggered = false;
2873 bool gc_heap::pm_trigger_full_gc = false;
2874 size_t gc_heap::provisional_triggered_gc_count = 0;
2875 size_t gc_heap::provisional_off_gc_count = 0;
2876 size_t gc_heap::num_provisional_triggered = 0;
2877 bool gc_heap::pm_stress_on = false;
2880 BOOL gc_heap::heap_analyze_enabled = FALSE;
2881 #endif //HEAP_ANALYZE
2883 #ifndef MULTIPLE_HEAPS
2885 alloc_list gc_heap::loh_alloc_list [NUM_LOH_ALIST-1];
2886 alloc_list gc_heap::gen2_alloc_list[NUM_GEN2_ALIST-1];
2888 dynamic_data gc_heap::dynamic_data_table [NUMBERGENERATIONS+1];
2889 gc_history_per_heap gc_heap::gc_data_per_heap;
2890 size_t gc_heap::maxgen_pinned_compact_before_advance = 0;
2892 uint8_t* gc_heap::alloc_allocated = 0;
2894 size_t gc_heap::allocation_quantum = CLR_SIZE;
2896 GCSpinLock gc_heap::more_space_lock_soh;
2897 GCSpinLock gc_heap::more_space_lock_loh;
2898 VOLATILE(int32_t) gc_heap::loh_alloc_thread_count = 0;
2900 #ifdef SYNCHRONIZATION_STATS
2901 unsigned int gc_heap::good_suspension = 0;
2902 unsigned int gc_heap::bad_suspension = 0;
2903 uint64_t gc_heap::total_msl_acquire = 0;
2904 unsigned int gc_heap::num_msl_acquired = 0;
2905 unsigned int gc_heap::num_high_msl_acquire = 0;
2906 unsigned int gc_heap::num_low_msl_acquire = 0;
2907 #endif //SYNCHRONIZATION_STATS
2909 size_t gc_heap::alloc_contexts_used = 0;
2910 size_t gc_heap::soh_allocation_no_gc = 0;
2911 size_t gc_heap::loh_allocation_no_gc = 0;
2912 bool gc_heap::no_gc_oom_p = false;
2913 heap_segment* gc_heap::saved_loh_segment_no_gc = 0;
2915 #endif //MULTIPLE_HEAPS
2917 #ifndef MULTIPLE_HEAPS
2919 BOOL gc_heap::gen0_bricks_cleared = FALSE;
2922 int gc_heap::gen0_must_clear_bricks = 0;
2923 #endif //FFIND_OBJECT
2925 #ifdef FEATURE_PREMORTEM_FINALIZATION
2926 CFinalize* gc_heap::finalize_queue = 0;
2927 #endif // FEATURE_PREMORTEM_FINALIZATION
2929 generation gc_heap::generation_table [NUMBERGENERATIONS + 1];
2931 size_t gc_heap::interesting_data_per_heap[max_idp_count];
2933 size_t gc_heap::compact_reasons_per_heap[max_compact_reasons_count];
2935 size_t gc_heap::expand_mechanisms_per_heap[max_expand_mechanisms_count];
2937 size_t gc_heap::interesting_mechanism_bits_per_heap[max_gc_mechanism_bits_count];
2939 #endif // MULTIPLE_HEAPS
2941 /* end of per heap static initialization */
2943 /* end of static initialization */
2945 #ifndef DACCESS_COMPILE
2947 void gen_to_condemn_tuning::print (int heap_num)
2950 dprintf (DT_LOG_0, ("condemned reasons (%d %d)", condemn_reasons_gen, condemn_reasons_condition));
2951 dprintf (DT_LOG_0, ("%s", record_condemn_reasons_gen_header));
2952 gc_condemn_reason_gen r_gen;
2953 for (int i = 0; i < gcrg_max; i++)
2955 r_gen = (gc_condemn_reason_gen)(i);
2956 str_reasons_gen[i * 2] = get_gen_char (get_gen (r_gen));
2958 dprintf (DT_LOG_0, ("[%2d]%s", heap_num, str_reasons_gen));
2960 dprintf (DT_LOG_0, ("%s", record_condemn_reasons_condition_header));
2961 gc_condemn_reason_condition r_condition;
2962 for (int i = 0; i < gcrc_max; i++)
2964 r_condition = (gc_condemn_reason_condition)(i);
2965 str_reasons_condition[i * 2] = get_condition_char (get_condition (r_condition));
2968 dprintf (DT_LOG_0, ("[%2d]%s", heap_num, str_reasons_condition));
2970 UNREFERENCED_PARAMETER(heap_num);
2974 void gc_generation_data::print (int heap_num, int gen_num)
2976 #if defined(SIMPLE_DPRINTF) && defined(DT_LOG)
2977 dprintf (DT_LOG_0, ("[%2d]gen%d beg %Id fl %Id fo %Id end %Id fl %Id fo %Id in %Id p %Id np %Id alloc %Id",
2980 free_list_space_before, free_obj_space_before,
2982 free_list_space_after, free_obj_space_after,
2983 in, pinned_surv, npinned_surv,
2986 UNREFERENCED_PARAMETER(heap_num);
2987 UNREFERENCED_PARAMETER(gen_num);
2988 #endif //SIMPLE_DPRINTF && DT_LOG
2991 void gc_history_per_heap::set_mechanism (gc_mechanism_per_heap mechanism_per_heap, uint32_t value)
2993 uint32_t* mechanism = &mechanisms[mechanism_per_heap];
2995 *mechanism |= mechanism_mask;
2996 *mechanism |= (1 << value);
2999 gc_mechanism_descr* descr = &gc_mechanisms_descr[mechanism_per_heap];
3000 dprintf (DT_LOG_0, ("setting %s: %s",
3002 (descr->descr)[value]));
3006 void gc_history_per_heap::print()
3008 #if defined(SIMPLE_DPRINTF) && defined(DT_LOG)
3009 for (int i = 0; i < (sizeof (gen_data)/sizeof (gc_generation_data)); i++)
3011 gen_data[i].print (heap_index, i);
3014 dprintf (DT_LOG_0, ("fla %Id flr %Id esa %Id ca %Id pa %Id paa %Id, rfle %d, ec %Id",
3015 maxgen_size_info.free_list_allocated,
3016 maxgen_size_info.free_list_rejected,
3017 maxgen_size_info.end_seg_allocated,
3018 maxgen_size_info.condemned_allocated,
3019 maxgen_size_info.pinned_allocated,
3020 maxgen_size_info.pinned_allocated_advance,
3021 maxgen_size_info.running_free_list_efficiency,
3022 extra_gen0_committed));
3025 gc_mechanism_descr* descr = 0;
3027 for (int i = 0; i < max_mechanism_per_heap; i++)
3029 mechanism = get_mechanism ((gc_mechanism_per_heap)i);
3033 descr = &gc_mechanisms_descr[(gc_mechanism_per_heap)i];
3034 dprintf (DT_LOG_0, ("[%2d]%s%s",
3037 (descr->descr)[mechanism]));
3040 #endif //SIMPLE_DPRINTF && DT_LOG
3043 void gc_history_global::print()
3046 char str_settings[64];
3047 memset (str_settings, '|', sizeof (char) * 64);
3048 str_settings[max_global_mechanisms_count*2] = 0;
3050 for (int i = 0; i < max_global_mechanisms_count; i++)
3052 str_settings[i * 2] = (get_mechanism_p ((gc_global_mechanism_p)i) ? 'Y' : 'N');
3055 dprintf (DT_LOG_0, ("[hp]|c|p|o|d|b|e|"));
3057 dprintf (DT_LOG_0, ("%4d|%s", num_heaps, str_settings));
3058 dprintf (DT_LOG_0, ("Condemned gen%d(reason: %s; mode: %s), youngest budget %Id(%d), memload %d",
3059 condemned_generation,
3060 str_gc_reasons[reason],
3061 str_gc_pause_modes[pause_mode],
3062 final_youngest_desired,
3063 gen0_reduction_count,
3068 void gc_heap::fire_per_heap_hist_event (gc_history_per_heap* current_gc_data_per_heap, int heap_num)
3070 maxgen_size_increase* maxgen_size_info = &(current_gc_data_per_heap->maxgen_size_info);
3071 FIRE_EVENT(GCPerHeapHistory_V3,
3072 (void *)(maxgen_size_info->free_list_allocated),
3073 (void *)(maxgen_size_info->free_list_rejected),
3074 (void *)(maxgen_size_info->end_seg_allocated),
3075 (void *)(maxgen_size_info->condemned_allocated),
3076 (void *)(maxgen_size_info->pinned_allocated),
3077 (void *)(maxgen_size_info->pinned_allocated_advance),
3078 maxgen_size_info->running_free_list_efficiency,
3079 current_gc_data_per_heap->gen_to_condemn_reasons.get_reasons0(),
3080 current_gc_data_per_heap->gen_to_condemn_reasons.get_reasons1(),
3081 current_gc_data_per_heap->mechanisms[gc_heap_compact],
3082 current_gc_data_per_heap->mechanisms[gc_heap_expand],
3083 current_gc_data_per_heap->heap_index,
3084 (void *)(current_gc_data_per_heap->extra_gen0_committed),
3085 (max_generation + 2),
3086 (uint32_t)(sizeof (gc_generation_data)),
3087 (void *)&(current_gc_data_per_heap->gen_data[0]));
3089 current_gc_data_per_heap->print();
3090 current_gc_data_per_heap->gen_to_condemn_reasons.print (heap_num);
3093 void gc_heap::fire_pevents()
3095 settings.record (&gc_data_global);
3096 gc_data_global.print();
3098 FIRE_EVENT(GCGlobalHeapHistory_V2,
3099 gc_data_global.final_youngest_desired,
3100 gc_data_global.num_heaps,
3101 gc_data_global.condemned_generation,
3102 gc_data_global.gen0_reduction_count,
3103 gc_data_global.reason,
3104 gc_data_global.global_mechanims_p,
3105 gc_data_global.pause_mode,
3106 gc_data_global.mem_pressure);
3108 #ifdef MULTIPLE_HEAPS
3109 for (int i = 0; i < gc_heap::n_heaps; i++)
3111 gc_heap* hp = gc_heap::g_heaps[i];
3112 gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap();
3113 fire_per_heap_hist_event (current_gc_data_per_heap, hp->heap_number);
3116 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
3117 fire_per_heap_hist_event (current_gc_data_per_heap, heap_number);
3122 gc_heap::dt_low_ephemeral_space_p (gc_tuning_point tp)
3128 case tuning_deciding_condemned_gen:
3129 case tuning_deciding_compaction:
3130 case tuning_deciding_expansion:
3131 case tuning_deciding_full_gc:
3133 ret = (!ephemeral_gen_fit_p (tp));
3136 case tuning_deciding_promote_ephemeral:
3138 size_t new_gen0size = approximate_new_allocation();
3139 ptrdiff_t plan_ephemeral_size = total_ephemeral_size;
3141 dprintf (GTC_LOG, ("h%d: plan eph size is %Id, new gen0 is %Id",
3142 heap_number, plan_ephemeral_size, new_gen0size));
3143 // If we were in no_gc_region we could have allocated a larger than normal segment,
3144 // and the next seg we allocate will be a normal sized seg so if we can't fit the new
3145 // ephemeral generations there, do an ephemeral promotion.
3146 ret = ((soh_segment_size - segment_info_size) < (plan_ephemeral_size + new_gen0size));
3157 gc_heap::dt_high_frag_p (gc_tuning_point tp,
3165 case tuning_deciding_condemned_gen:
3167 dynamic_data* dd = dynamic_data_of (gen_number);
3168 float fragmentation_burden = 0;
3172 ret = (dd_fragmentation (dynamic_data_of (max_generation)) >= dd_max_size(dd));
3173 dprintf (GTC_LOG, ("h%d: frag is %Id, max size is %Id",
3174 heap_number, dd_fragmentation (dd), dd_max_size(dd)));
3178 #ifndef MULTIPLE_HEAPS
3179 if (gen_number == max_generation)
3181 float frag_ratio = (float)(dd_fragmentation (dynamic_data_of (max_generation))) / (float)generation_size (max_generation);
3182 if (frag_ratio > 0.65)
3184 dprintf (GTC_LOG, ("g2 FR: %d%%", (int)(frag_ratio*100)));
3188 #endif //!MULTIPLE_HEAPS
3189 size_t fr = generation_unusable_fragmentation (generation_of (gen_number));
3190 ret = (fr > dd_fragmentation_limit(dd));
3193 fragmentation_burden = (float)fr / generation_size (gen_number);
3194 ret = (fragmentation_burden > dd_v_fragmentation_burden_limit (dd));
3196 dprintf (GTC_LOG, ("h%d: gen%d, frag is %Id, alloc effi: %d%%, unusable frag is %Id, ratio is %d",
3197 heap_number, gen_number, dd_fragmentation (dd),
3198 (int)(100*generation_allocator_efficiency (generation_of (gen_number))),
3199 fr, (int)(fragmentation_burden*100)));
3211 gc_heap::dt_estimate_reclaim_space_p (gc_tuning_point tp, int gen_number)
3217 case tuning_deciding_condemned_gen:
3219 if (gen_number == max_generation)
3221 size_t est_maxgen_free = estimated_reclaim (gen_number);
3223 uint32_t num_heaps = 1;
3224 #ifdef MULTIPLE_HEAPS
3225 num_heaps = gc_heap::n_heaps;
3226 #endif //MULTIPLE_HEAPS
3228 size_t min_frag_th = min_reclaim_fragmentation_threshold (num_heaps);
3229 dprintf (GTC_LOG, ("h%d, min frag is %Id", heap_number, min_frag_th));
3230 ret = (est_maxgen_free >= min_frag_th);
3246 // DTREVIEW: Right now we only estimate gen2 fragmentation.
3247 // on 64-bit though we should consider gen1 or even gen0 fragmentatioin as
3250 gc_heap::dt_estimate_high_frag_p (gc_tuning_point tp, int gen_number, uint64_t available_mem)
3256 case tuning_deciding_condemned_gen:
3258 if (gen_number == max_generation)
3260 dynamic_data* dd = dynamic_data_of (gen_number);
3261 float est_frag_ratio = 0;
3262 if (dd_current_size (dd) == 0)
3266 else if ((dd_fragmentation (dd) == 0) || (dd_fragmentation (dd) + dd_current_size (dd) == 0))
3272 est_frag_ratio = (float)dd_fragmentation (dd) / (float)(dd_fragmentation (dd) + dd_current_size (dd));
3275 size_t est_frag = (dd_fragmentation (dd) + (size_t)((dd_desired_allocation (dd) - dd_new_allocation (dd)) * est_frag_ratio));
3276 dprintf (GTC_LOG, ("h%d: gen%d: current_size is %Id, frag is %Id, est_frag_ratio is %d%%, estimated frag is %Id",
3279 dd_current_size (dd),
3280 dd_fragmentation (dd),
3281 (int)(est_frag_ratio*100),
3284 uint32_t num_heaps = 1;
3286 #ifdef MULTIPLE_HEAPS
3287 num_heaps = gc_heap::n_heaps;
3288 #endif //MULTIPLE_HEAPS
3289 uint64_t min_frag_th = min_high_fragmentation_threshold(available_mem, num_heaps);
3290 //dprintf (GTC_LOG, ("h%d, min frag is %I64d", heap_number, min_frag_th));
3291 ret = (est_frag >= min_frag_th);
3308 gc_heap::dt_low_card_table_efficiency_p (gc_tuning_point tp)
3314 case tuning_deciding_condemned_gen:
3316 /* promote into max-generation if the card table has too many
3317 * generation faults besides the n -> 0
3319 ret = (generation_skip_ratio < 30);
3331 in_range_for_segment(uint8_t* add, heap_segment* seg)
3333 return ((add >= heap_segment_mem (seg)) && (add < heap_segment_reserved (seg)));
3336 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
3337 // The array we allocate is organized as follows:
3338 // 0th element is the address of the last array we allocated.
3339 // starting from the 1st element are the segment addresses, that's
3340 // what buckets() returns.
3353 bk* buckets() { return (slots + 1); }
3354 uint8_t*& last_slot (bk* arr) { return arr[0].add; }
3357 static sorted_table* make_sorted_table ();
3358 BOOL insert (uint8_t* add, size_t val);;
3359 size_t lookup (uint8_t*& add);
3360 void remove (uint8_t* add);
3362 void delete_sorted_table();
3363 void delete_old_slots();
3364 void enqueue_old_slot(bk* sl);
3365 BOOL ensure_space_for_insert();
3369 sorted_table::make_sorted_table ()
3373 // allocate one more bk to store the older slot address.
3374 sorted_table* res = (sorted_table*)new char [sizeof (sorted_table) + (size + 1) * sizeof (bk)];
3378 res->slots = (bk*)(res + 1);
3385 sorted_table::delete_sorted_table()
3387 if (slots != (bk*)(this+1))
3395 sorted_table::delete_old_slots()
3397 uint8_t* sl = (uint8_t*)old_slots;
3401 sl = last_slot ((bk*)sl);
3407 sorted_table::enqueue_old_slot(bk* sl)
3409 last_slot (sl) = (uint8_t*)old_slots;
3415 sorted_table::lookup (uint8_t*& add)
3417 ptrdiff_t high = (count-1);
3421 bk* buck = buckets();
3424 mid = ((low + high)/2);
3426 if (buck[ti].add > add)
3428 if ((ti > 0) && (buck[ti-1].add <= add))
3430 add = buck[ti-1].add;
3431 return buck[ti - 1].val;
3437 if (buck[ti+1].add > add)
3440 return buck[ti].val;
3450 sorted_table::ensure_space_for_insert()
3454 size = (size * 3)/2;
3455 assert((size * sizeof (bk)) > 0);
3456 bk* res = (bk*)new (nothrow) char [(size + 1) * sizeof (bk)];
3461 last_slot (res) = 0;
3462 memcpy (((bk*)res + 1), buckets(), count * sizeof (bk));
3463 bk* last_old_slots = slots;
3465 if (last_old_slots != (bk*)(this + 1))
3466 enqueue_old_slot (last_old_slots);
3472 sorted_table::insert (uint8_t* add, size_t val)
3474 //grow if no more room
3475 assert (count < size);
3478 ptrdiff_t high = (count-1);
3482 bk* buck = buckets();
3485 mid = ((low + high)/2);
3487 if (buck[ti].add > add)
3489 if ((ti == 0) || (buck[ti-1].add <= add))
3491 // found insertion point
3492 for (ptrdiff_t k = count; k > ti;k--)
3494 buck [k] = buck [k-1];
3505 if (buck[ti+1].add > add)
3507 //found the insertion point
3508 for (ptrdiff_t k = count; k > ti+1;k--)
3510 buck [k] = buck [k-1];
3512 buck[ti+1].add = add;
3513 buck[ti+1].val = val;
3525 sorted_table::remove (uint8_t* add)
3527 ptrdiff_t high = (count-1);
3531 bk* buck = buckets();
3534 mid = ((low + high)/2);
3536 if (buck[ti].add > add)
3538 if (buck[ti-1].add <= add)
3540 // found the guy to remove
3541 for (ptrdiff_t k = ti; k < count; k++)
3542 buck[k-1] = buck[k];
3550 if (buck[ti+1].add > add)
3552 // found the guy to remove
3553 for (ptrdiff_t k = ti+1; k < count; k++)
3554 buck[k-1] = buck[k];
3565 sorted_table::clear()
3568 buckets()[0].add = MAX_PTR;
3570 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
3572 #ifdef SEG_MAPPING_TABLE
3573 #ifdef GROWABLE_SEG_MAPPING_TABLE
3575 uint8_t* align_on_segment (uint8_t* add)
3577 return (uint8_t*)((size_t)(add + (gc_heap::min_segment_size - 1)) & ~(gc_heap::min_segment_size - 1));
3581 uint8_t* align_lower_segment (uint8_t* add)
3583 return (uint8_t*)((size_t)(add) & ~(gc_heap::min_segment_size - 1));
3586 size_t size_seg_mapping_table_of (uint8_t* from, uint8_t* end)
3588 from = align_lower_segment (from);
3589 end = align_on_segment (end);
3590 dprintf (1, ("from: %Ix, end: %Ix, size: %Ix", from, end, sizeof (seg_mapping)*(((size_t)(end - from) >> gc_heap::min_segment_size_shr))));
3591 return sizeof (seg_mapping)*((size_t)(end - from) >> gc_heap::min_segment_size_shr);
3594 // for seg_mapping_table we want it to start from a pointer sized address.
3596 size_t align_for_seg_mapping_table (size_t size)
3598 return ((size + (sizeof (uint8_t*) - 1)) &~ (sizeof (uint8_t*) - 1));
3602 size_t seg_mapping_word_of (uint8_t* add)
3604 return (size_t)add >> gc_heap::min_segment_size_shr;
3606 #else //GROWABLE_SEG_MAPPING_TABLE
3607 BOOL seg_mapping_table_init()
3610 uint64_t total_address_space = (uint64_t)8*1024*1024*1024*1024;
3612 uint64_t total_address_space = (uint64_t)4*1024*1024*1024;
3615 size_t num_entries = (size_t)(total_address_space >> gc_heap::min_segment_size_shr);
3616 seg_mapping_table = new seg_mapping[num_entries];
3618 if (seg_mapping_table)
3620 memset (seg_mapping_table, 0, num_entries * sizeof (seg_mapping));
3621 dprintf (1, ("created %d entries for heap mapping (%Id bytes)",
3622 num_entries, (num_entries * sizeof (seg_mapping))));
3627 dprintf (1, ("failed to create %d entries for heap mapping (%Id bytes)",
3628 num_entries, (num_entries * sizeof (seg_mapping))));
3632 #endif //GROWABLE_SEG_MAPPING_TABLE
3634 #ifdef FEATURE_BASICFREEZE
3636 size_t ro_seg_begin_index (heap_segment* seg)
3638 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3639 begin_index = max (begin_index, (size_t)g_gc_lowest_address >> gc_heap::min_segment_size_shr);
3644 size_t ro_seg_end_index (heap_segment* seg)
3646 size_t end_index = (size_t)(heap_segment_reserved (seg) - 1) >> gc_heap::min_segment_size_shr;
3647 end_index = min (end_index, (size_t)g_gc_highest_address >> gc_heap::min_segment_size_shr);
3651 void seg_mapping_table_add_ro_segment (heap_segment* seg)
3653 #ifdef GROWABLE_SEG_MAPPING_TABLE
3654 if ((heap_segment_reserved (seg) <= g_gc_lowest_address) || (heap_segment_mem (seg) >= g_gc_highest_address))
3656 #endif //GROWABLE_SEG_MAPPING_TABLE
3658 for (size_t entry_index = ro_seg_begin_index (seg); entry_index <= ro_seg_end_index (seg); entry_index++)
3659 seg_mapping_table[entry_index].seg1 = (heap_segment*)((size_t)seg_mapping_table[entry_index].seg1 | ro_in_entry);
3662 void seg_mapping_table_remove_ro_segment (heap_segment* seg)
3664 UNREFERENCED_PARAMETER(seg);
3666 // POSSIBLE PERF TODO: right now we are not doing anything because we can't simply remove the flag. If it proves
3667 // to be a perf problem, we can search in the current ro segs and see if any lands in this range and only
3668 // remove the flag if none lands in this range.
3672 heap_segment* ro_segment_lookup (uint8_t* o)
3674 uint8_t* ro_seg_start = o;
3675 heap_segment* seg = (heap_segment*)gc_heap::seg_table->lookup (ro_seg_start);
3677 if (ro_seg_start && in_range_for_segment (o, seg))
3683 #endif //FEATURE_BASICFREEZE
3685 void gc_heap::seg_mapping_table_add_segment (heap_segment* seg, gc_heap* hp)
3687 size_t seg_end = (size_t)(heap_segment_reserved (seg) - 1);
3688 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3689 seg_mapping* begin_entry = &seg_mapping_table[begin_index];
3690 size_t end_index = seg_end >> gc_heap::min_segment_size_shr;
3691 seg_mapping* end_entry = &seg_mapping_table[end_index];
3693 dprintf (1, ("adding seg %Ix(%d)-%Ix(%d)",
3694 seg, begin_index, heap_segment_reserved (seg), end_index));
3696 dprintf (1, ("before add: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3697 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3698 end_index, (seg_mapping_table[end_index].boundary + 1)));
3700 #ifdef MULTIPLE_HEAPS
3701 #ifdef SIMPLE_DPRINTF
3702 dprintf (1, ("begin %d: h0: %Ix(%d), h1: %Ix(%d); end %d: h0: %Ix(%d), h1: %Ix(%d)",
3703 begin_index, (uint8_t*)(begin_entry->h0), (begin_entry->h0 ? begin_entry->h0->heap_number : -1),
3704 (uint8_t*)(begin_entry->h1), (begin_entry->h1 ? begin_entry->h1->heap_number : -1),
3705 end_index, (uint8_t*)(end_entry->h0), (end_entry->h0 ? end_entry->h0->heap_number : -1),
3706 (uint8_t*)(end_entry->h1), (end_entry->h1 ? end_entry->h1->heap_number : -1)));
3707 #endif //SIMPLE_DPRINTF
3708 assert (end_entry->boundary == 0);
3709 assert (end_entry->h0 == 0);
3711 assert (begin_entry->h1 == 0);
3712 begin_entry->h1 = hp;
3714 UNREFERENCED_PARAMETER(hp);
3715 #endif //MULTIPLE_HEAPS
3717 end_entry->boundary = (uint8_t*)seg_end;
3719 dprintf (1, ("set entry %d seg1 and %d seg0 to %Ix", begin_index, end_index, seg));
3720 assert ((begin_entry->seg1 == 0) || ((size_t)(begin_entry->seg1) == ro_in_entry));
3721 begin_entry->seg1 = (heap_segment*)((size_t)(begin_entry->seg1) | (size_t)seg);
3722 end_entry->seg0 = seg;
3724 // for every entry inbetween we need to set its heap too.
3725 for (size_t entry_index = (begin_index + 1); entry_index <= (end_index - 1); entry_index++)
3727 assert (seg_mapping_table[entry_index].boundary == 0);
3728 #ifdef MULTIPLE_HEAPS
3729 assert (seg_mapping_table[entry_index].h0 == 0);
3730 seg_mapping_table[entry_index].h1 = hp;
3731 #endif //MULTIPLE_HEAPS
3732 seg_mapping_table[entry_index].seg1 = seg;
3735 dprintf (1, ("after add: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3736 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3737 end_index, (seg_mapping_table[end_index].boundary + 1)));
3738 #if defined(MULTIPLE_HEAPS) && defined(SIMPLE_DPRINTF)
3739 dprintf (1, ("begin %d: h0: %Ix(%d), h1: %Ix(%d); end: %d h0: %Ix(%d), h1: %Ix(%d)",
3740 begin_index, (uint8_t*)(begin_entry->h0), (begin_entry->h0 ? begin_entry->h0->heap_number : -1),
3741 (uint8_t*)(begin_entry->h1), (begin_entry->h1 ? begin_entry->h1->heap_number : -1),
3742 end_index, (uint8_t*)(end_entry->h0), (end_entry->h0 ? end_entry->h0->heap_number : -1),
3743 (uint8_t*)(end_entry->h1), (end_entry->h1 ? end_entry->h1->heap_number : -1)));
3744 #endif //MULTIPLE_HEAPS && SIMPLE_DPRINTF
3747 void gc_heap::seg_mapping_table_remove_segment (heap_segment* seg)
3749 size_t seg_end = (size_t)(heap_segment_reserved (seg) - 1);
3750 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3751 seg_mapping* begin_entry = &seg_mapping_table[begin_index];
3752 size_t end_index = seg_end >> gc_heap::min_segment_size_shr;
3753 seg_mapping* end_entry = &seg_mapping_table[end_index];
3754 dprintf (1, ("removing seg %Ix(%d)-%Ix(%d)",
3755 seg, begin_index, heap_segment_reserved (seg), end_index));
3757 assert (end_entry->boundary == (uint8_t*)seg_end);
3758 end_entry->boundary = 0;
3760 #ifdef MULTIPLE_HEAPS
3761 gc_heap* hp = heap_segment_heap (seg);
3762 assert (end_entry->h0 == hp);
3764 assert (begin_entry->h1 == hp);
3765 begin_entry->h1 = 0;
3766 #endif //MULTIPLE_HEAPS
3768 assert (begin_entry->seg1 != 0);
3769 begin_entry->seg1 = (heap_segment*)((size_t)(begin_entry->seg1) & ro_in_entry);
3770 end_entry->seg0 = 0;
3772 // for every entry inbetween we need to reset its heap too.
3773 for (size_t entry_index = (begin_index + 1); entry_index <= (end_index - 1); entry_index++)
3775 assert (seg_mapping_table[entry_index].boundary == 0);
3776 #ifdef MULTIPLE_HEAPS
3777 assert (seg_mapping_table[entry_index].h0 == 0);
3778 assert (seg_mapping_table[entry_index].h1 == hp);
3779 seg_mapping_table[entry_index].h1 = 0;
3780 #endif //MULTIPLE_HEAPS
3781 seg_mapping_table[entry_index].seg1 = 0;
3784 dprintf (1, ("after remove: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3785 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3786 end_index, (seg_mapping_table[end_index].boundary + 1)));
3787 #ifdef MULTIPLE_HEAPS
3788 dprintf (1, ("begin %d: h0: %Ix, h1: %Ix; end: %d h0: %Ix, h1: %Ix",
3789 begin_index, (uint8_t*)(begin_entry->h0), (uint8_t*)(begin_entry->h1),
3790 end_index, (uint8_t*)(end_entry->h0), (uint8_t*)(end_entry->h1)));
3791 #endif //MULTIPLE_HEAPS
3794 #ifdef MULTIPLE_HEAPS
3796 gc_heap* seg_mapping_table_heap_of_worker (uint8_t* o)
3798 size_t index = (size_t)o >> gc_heap::min_segment_size_shr;
3799 seg_mapping* entry = &seg_mapping_table[index];
3801 gc_heap* hp = ((o > entry->boundary) ? entry->h1 : entry->h0);
3803 dprintf (2, ("checking obj %Ix, index is %Id, entry: boundry: %Ix, h0: %Ix, seg0: %Ix, h1: %Ix, seg1: %Ix",
3804 o, index, (entry->boundary + 1),
3805 (uint8_t*)(entry->h0), (uint8_t*)(entry->seg0),
3806 (uint8_t*)(entry->h1), (uint8_t*)(entry->seg1)));
3809 heap_segment* seg = ((o > entry->boundary) ? entry->seg1 : entry->seg0);
3810 #ifdef FEATURE_BASICFREEZE
3811 if ((size_t)seg & ro_in_entry)
3812 seg = (heap_segment*)((size_t)seg & ~ro_in_entry);
3813 #endif //FEATURE_BASICFREEZE
3817 if (in_range_for_segment (o, seg))
3819 dprintf (2, ("obj %Ix belongs to segment %Ix(-%Ix)", o, seg, (uint8_t*)heap_segment_allocated (seg)));
3823 dprintf (2, ("found seg %Ix(-%Ix) for obj %Ix, but it's not on the seg",
3824 seg, (uint8_t*)heap_segment_allocated (seg), o));
3829 dprintf (2, ("could not find obj %Ix in any existing segments", o));
3836 gc_heap* seg_mapping_table_heap_of (uint8_t* o)
3838 #ifdef GROWABLE_SEG_MAPPING_TABLE
3839 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3841 #endif //GROWABLE_SEG_MAPPING_TABLE
3843 return seg_mapping_table_heap_of_worker (o);
3846 gc_heap* seg_mapping_table_heap_of_gc (uint8_t* o)
3848 #if defined(FEATURE_BASICFREEZE) && defined(GROWABLE_SEG_MAPPING_TABLE)
3849 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3851 #endif //FEATURE_BASICFREEZE || GROWABLE_SEG_MAPPING_TABLE
3853 return seg_mapping_table_heap_of_worker (o);
3855 #endif //MULTIPLE_HEAPS
3857 // Only returns a valid seg if we can actually find o on the seg.
3858 heap_segment* seg_mapping_table_segment_of (uint8_t* o)
3860 #if defined(FEATURE_BASICFREEZE) && defined(GROWABLE_SEG_MAPPING_TABLE)
3861 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3862 #ifdef FEATURE_BASICFREEZE
3863 return ro_segment_lookup (o);
3866 #endif //FEATURE_BASICFREEZE
3867 #endif //FEATURE_BASICFREEZE || GROWABLE_SEG_MAPPING_TABLE
3869 size_t index = (size_t)o >> gc_heap::min_segment_size_shr;
3870 seg_mapping* entry = &seg_mapping_table[index];
3872 dprintf (2, ("checking obj %Ix, index is %Id, entry: boundry: %Ix, seg0: %Ix, seg1: %Ix",
3873 o, index, (entry->boundary + 1),
3874 (uint8_t*)(entry->seg0), (uint8_t*)(entry->seg1)));
3876 heap_segment* seg = ((o > entry->boundary) ? entry->seg1 : entry->seg0);
3877 #ifdef FEATURE_BASICFREEZE
3878 if ((size_t)seg & ro_in_entry)
3879 seg = (heap_segment*)((size_t)seg & ~ro_in_entry);
3880 #endif //FEATURE_BASICFREEZE
3884 if (in_range_for_segment (o, seg))
3886 dprintf (2, ("obj %Ix belongs to segment %Ix(-%Ix)", o, (uint8_t*)heap_segment_mem(seg), (uint8_t*)heap_segment_reserved(seg)));
3890 dprintf (2, ("found seg %Ix(-%Ix) for obj %Ix, but it's not on the seg, setting it to 0",
3891 (uint8_t*)heap_segment_mem(seg), (uint8_t*)heap_segment_reserved(seg), o));
3897 dprintf (2, ("could not find obj %Ix in any existing segments", o));
3900 #ifdef FEATURE_BASICFREEZE
3901 // TODO: This was originally written assuming that the seg_mapping_table would always contain entries for ro
3902 // segments whenever the ro segment falls into the [g_gc_lowest_address,g_gc_highest_address) range. I.e., it had an
3903 // extra "&& (size_t)(entry->seg1) & ro_in_entry" expression. However, at the moment, grow_brick_card_table does
3904 // not correctly go through the ro segments and add them back to the seg_mapping_table when the [lowest,highest)
3905 // range changes. We should probably go ahead and modify grow_brick_card_table and put back the
3906 // "&& (size_t)(entry->seg1) & ro_in_entry" here.
3909 seg = ro_segment_lookup (o);
3910 if (seg && !in_range_for_segment (o, seg))
3913 #endif //FEATURE_BASICFREEZE
3917 #endif //SEG_MAPPING_TABLE
3919 size_t gcard_of ( uint8_t*);
3921 #define memref(i) *(uint8_t**)(i)
3924 #define GC_MARKED (size_t)0x1
3925 #define slot(i, j) ((uint8_t**)(i))[j+1]
3927 #define free_object_base_size (plug_skew + sizeof(ArrayBase))
3929 class CObjectHeader : public Object
3933 #if defined(FEATURE_REDHAWK) || defined(BUILD_AS_STANDALONE)
3934 // The GC expects the following methods that are provided by the Object class in the CLR but not provided
3935 // by Redhawk's version of Object.
3936 uint32_t GetNumComponents()
3938 return ((ArrayBase *)this)->GetNumComponents();
3941 void Validate(BOOL bDeep=TRUE, BOOL bVerifyNextHeader = TRUE)
3943 UNREFERENCED_PARAMETER(bVerifyNextHeader);
3948 MethodTable * pMT = GetMethodTable();
3950 _ASSERTE(pMT->SanityCheck());
3952 bool noRangeChecks =
3953 (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_NO_RANGE_CHECKS) == GCConfig::HEAPVERIFY_NO_RANGE_CHECKS;
3955 BOOL fSmallObjectHeapPtr = FALSE, fLargeObjectHeapPtr = FALSE;
3958 fSmallObjectHeapPtr = g_theGCHeap->IsHeapPointer(this, TRUE);
3959 if (!fSmallObjectHeapPtr)
3960 fLargeObjectHeapPtr = g_theGCHeap->IsHeapPointer(this);
3962 _ASSERTE(fSmallObjectHeapPtr || fLargeObjectHeapPtr);
3965 #ifdef FEATURE_STRUCTALIGN
3966 _ASSERTE(IsStructAligned((uint8_t *)this, GetMethodTable()->GetBaseAlignment()));
3967 #endif // FEATURE_STRUCTALIGN
3969 #ifdef FEATURE_64BIT_ALIGNMENT
3970 if (pMT->RequiresAlign8())
3972 _ASSERTE((((size_t)this) & 0x7) == (pMT->IsValueType() ? 4U : 0U));
3974 #endif // FEATURE_64BIT_ALIGNMENT
3977 if (bDeep && (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC))
3978 g_theGCHeap->ValidateObjectMember(this);
3980 if (fSmallObjectHeapPtr)
3982 #ifdef FEATURE_BASICFREEZE
3983 _ASSERTE(!g_theGCHeap->IsLargeObject(pMT) || g_theGCHeap->IsInFrozenSegment(this));
3985 _ASSERTE(!g_theGCHeap->IsLargeObject(pMT));
3990 void ValidatePromote(ScanContext *sc, uint32_t flags)
3992 UNREFERENCED_PARAMETER(sc);
3993 UNREFERENCED_PARAMETER(flags);
3998 void ValidateHeap(Object *from, BOOL bDeep)
4000 UNREFERENCED_PARAMETER(from);
4002 Validate(bDeep, FALSE);
4005 #endif //FEATURE_REDHAWK || BUILD_AS_STANDALONE
4009 // Header Status Information
4012 MethodTable *GetMethodTable() const
4014 return( (MethodTable *) (((size_t) RawGetMethodTable()) & (~(GC_MARKED))));
4019 RawSetMethodTable((MethodTable *) (((size_t) RawGetMethodTable()) | GC_MARKED));
4022 BOOL IsMarked() const
4024 return !!(((size_t)RawGetMethodTable()) & GC_MARKED);
4029 assert (!(gc_heap::settings.concurrent));
4030 GetHeader()->SetGCBit();
4033 BOOL IsPinned() const
4035 return !!((((CObjectHeader*)this)->GetHeader()->GetBits()) & BIT_SBLK_GC_RESERVE);
4040 RawSetMethodTable( GetMethodTable() );
4043 CGCDesc *GetSlotMap ()
4045 assert (GetMethodTable()->ContainsPointers());
4046 return CGCDesc::GetCGCDescFromMT(GetMethodTable());
4049 void SetFree(size_t size)
4051 assert (size >= free_object_base_size);
4053 assert (g_gc_pFreeObjectMethodTable->GetBaseSize() == free_object_base_size);
4054 assert (g_gc_pFreeObjectMethodTable->RawGetComponentSize() == 1);
4056 RawSetMethodTable( g_gc_pFreeObjectMethodTable );
4058 size_t* numComponentsPtr = (size_t*) &((uint8_t*) this)[ArrayBase::GetOffsetOfNumComponents()];
4059 *numComponentsPtr = size - free_object_base_size;
4061 //This introduces a bug in the free list management.
4062 //((void**) this)[-1] = 0; // clear the sync block,
4063 assert (*numComponentsPtr >= 0);
4064 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
4065 memset (((uint8_t*)this)+sizeof(ArrayBase), 0xcc, *numComponentsPtr);
4066 #endif //VERIFY_HEAP
4071 size_t size = free_object_base_size - plug_skew;
4073 // since we only need to clear 2 ptr size, we do it manually
4074 PTR_PTR m = (PTR_PTR) this;
4075 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
4079 BOOL IsFree () const
4081 return (GetMethodTable() == g_gc_pFreeObjectMethodTable);
4084 #ifdef FEATURE_STRUCTALIGN
4085 int GetRequiredAlignment () const
4087 return GetMethodTable()->GetRequiredAlignment();
4089 #endif // FEATURE_STRUCTALIGN
4091 BOOL ContainsPointers() const
4093 return GetMethodTable()->ContainsPointers();
4096 #ifdef COLLECTIBLE_CLASS
4097 BOOL Collectible() const
4099 return GetMethodTable()->Collectible();
4102 FORCEINLINE BOOL ContainsPointersOrCollectible() const
4104 MethodTable *pMethodTable = GetMethodTable();
4105 return (pMethodTable->ContainsPointers() || pMethodTable->Collectible());
4107 #endif //COLLECTIBLE_CLASS
4109 Object* GetObjectBase() const
4111 return (Object*) this;
4115 #define header(i) ((CObjectHeader*)(i))
4117 #define free_list_slot(x) ((uint8_t**)(x))[2]
4118 #define free_list_undo(x) ((uint8_t**)(x))[-1]
4119 #define UNDO_EMPTY ((uint8_t*)1)
4123 void set_plug_padded (uint8_t* node)
4125 header(node)->SetMarked();
4128 void clear_plug_padded (uint8_t* node)
4130 header(node)->ClearMarked();
4133 BOOL is_plug_padded (uint8_t* node)
4135 return header(node)->IsMarked();
4138 inline void set_plug_padded (uint8_t* node){}
4139 inline void clear_plug_padded (uint8_t* node){}
4141 BOOL is_plug_padded (uint8_t* node){return FALSE;}
4142 #endif //SHORT_PLUGS
4145 inline size_t unused_array_size(uint8_t * p)
4147 assert(((CObjectHeader*)p)->IsFree());
4149 size_t* numComponentsPtr = (size_t*)(p + ArrayBase::GetOffsetOfNumComponents());
4150 return free_object_base_size + *numComponentsPtr;
4153 heap_segment* heap_segment_rw (heap_segment* ns)
4155 if ((ns == 0) || !heap_segment_read_only_p (ns))
4163 ns = heap_segment_next (ns);
4164 } while ((ns != 0) && heap_segment_read_only_p (ns));
4169 //returns the next non ro segment.
4170 heap_segment* heap_segment_next_rw (heap_segment* seg)
4172 heap_segment* ns = heap_segment_next (seg);
4173 return heap_segment_rw (ns);
4176 // returns the segment before seg.
4177 heap_segment* heap_segment_prev_rw (heap_segment* begin, heap_segment* seg)
4179 assert (begin != 0);
4180 heap_segment* prev = begin;
4181 heap_segment* current = heap_segment_next_rw (begin);
4183 while (current && current != seg)
4186 current = heap_segment_next_rw (current);
4199 // returns the segment before seg.
4200 heap_segment* heap_segment_prev (heap_segment* begin, heap_segment* seg)
4202 assert (begin != 0);
4203 heap_segment* prev = begin;
4204 heap_segment* current = heap_segment_next (begin);
4206 while (current && current != seg)
4209 current = heap_segment_next (current);
4222 heap_segment* heap_segment_in_range (heap_segment* ns)
4224 if ((ns == 0) || heap_segment_in_range_p (ns))
4232 ns = heap_segment_next (ns);
4233 } while ((ns != 0) && !heap_segment_in_range_p (ns));
4238 heap_segment* heap_segment_next_in_range (heap_segment* seg)
4240 heap_segment* ns = heap_segment_next (seg);
4241 return heap_segment_in_range (ns);
4246 uint8_t* memory_base;
4251 imemory_data *initial_memory;
4252 imemory_data *initial_normal_heap; // points into initial_memory_array
4253 imemory_data *initial_large_heap; // points into initial_memory_array
4255 size_t block_size_normal;
4256 size_t block_size_large;
4258 size_t block_count; // # of blocks in each
4259 size_t current_block_normal;
4260 size_t current_block_large;
4269 size_t allocation_pattern;
4270 } initial_memory_details;
4272 initial_memory_details memory_details;
4274 BOOL reserve_initial_memory (size_t normal_size, size_t large_size, size_t num_heaps)
4276 BOOL reserve_success = FALSE;
4278 // should only be called once
4279 assert (memory_details.initial_memory == 0);
4281 memory_details.initial_memory = new (nothrow) imemory_data[num_heaps*2];
4282 if (memory_details.initial_memory == 0)
4284 dprintf (2, ("failed to reserve %Id bytes for imemory_data", num_heaps*2*sizeof(imemory_data)));
4288 memory_details.initial_normal_heap = memory_details.initial_memory;
4289 memory_details.initial_large_heap = memory_details.initial_memory + num_heaps;
4290 memory_details.block_size_normal = normal_size;
4291 memory_details.block_size_large = large_size;
4292 memory_details.block_count = num_heaps;
4294 memory_details.current_block_normal = 0;
4295 memory_details.current_block_large = 0;
4297 g_gc_lowest_address = MAX_PTR;
4298 g_gc_highest_address = 0;
4300 if (((size_t)MAX_PTR - large_size) < normal_size)
4302 // we are already overflowing with just one heap.
4303 dprintf (2, ("0x%Ix + 0x%Ix already overflow", normal_size, large_size));
4307 if (((size_t)MAX_PTR / memory_details.block_count) < (normal_size + large_size))
4309 dprintf (2, ("(0x%Ix + 0x%Ix)*0x%Ix overflow", normal_size, large_size, memory_details.block_count));
4313 size_t requestedMemory = memory_details.block_count * (normal_size + large_size);
4315 uint8_t* allatonce_block = (uint8_t*)virtual_alloc (requestedMemory);
4316 if (allatonce_block)
4318 g_gc_lowest_address = allatonce_block;
4319 g_gc_highest_address = allatonce_block + (memory_details.block_count * (large_size + normal_size));
4320 memory_details.allocation_pattern = initial_memory_details::ALLATONCE;
4322 for(size_t i = 0; i < memory_details.block_count; i++)
4324 memory_details.initial_normal_heap[i].memory_base = allatonce_block + (i*normal_size);
4325 memory_details.initial_large_heap[i].memory_base = allatonce_block +
4326 (memory_details.block_count*normal_size) + (i*large_size);
4327 reserve_success = TRUE;
4332 // try to allocate 2 blocks
4335 b1 = (uint8_t*)virtual_alloc (memory_details.block_count * normal_size);
4338 b2 = (uint8_t*)virtual_alloc (memory_details.block_count * large_size);
4341 memory_details.allocation_pattern = initial_memory_details::TWO_STAGE;
4342 g_gc_lowest_address = min(b1,b2);
4343 g_gc_highest_address = max(b1 + memory_details.block_count*normal_size,
4344 b2 + memory_details.block_count*large_size);
4345 for(size_t i = 0; i < memory_details.block_count; i++)
4347 memory_details.initial_normal_heap[i].memory_base = b1 + (i*normal_size);
4348 memory_details.initial_large_heap[i].memory_base = b2 + (i*large_size);
4349 reserve_success = TRUE;
4354 // b2 allocation failed, we'll go on to try allocating each block.
4355 // We could preserve the b1 alloc, but code complexity increases
4356 virtual_free (b1, memory_details.block_count * normal_size);
4360 if ((b2==NULL) && ( memory_details.block_count > 1))
4362 memory_details.allocation_pattern = initial_memory_details::EACH_BLOCK;
4364 imemory_data *current_block = memory_details.initial_memory;
4365 for(size_t i = 0; i < (memory_details.block_count*2); i++, current_block++)
4367 size_t block_size = ((i < memory_details.block_count) ?
4368 memory_details.block_size_normal :
4369 memory_details.block_size_large);
4370 current_block->memory_base =
4371 (uint8_t*)virtual_alloc (block_size);
4372 if (current_block->memory_base == 0)
4374 // Free the blocks that we've allocated so far
4375 current_block = memory_details.initial_memory;
4376 for(size_t j = 0; j < i; j++, current_block++){
4377 if (current_block->memory_base != 0){
4378 block_size = ((j < memory_details.block_count) ?
4379 memory_details.block_size_normal :
4380 memory_details.block_size_large);
4381 virtual_free (current_block->memory_base , block_size);
4384 reserve_success = FALSE;
4389 if (current_block->memory_base < g_gc_lowest_address)
4390 g_gc_lowest_address = current_block->memory_base;
4391 if (((uint8_t *) current_block->memory_base + block_size) > g_gc_highest_address)
4392 g_gc_highest_address = (current_block->memory_base + block_size);
4394 reserve_success = TRUE;
4399 return reserve_success;
4402 void destroy_initial_memory()
4404 if (memory_details.initial_memory != NULL)
4406 if (memory_details.allocation_pattern == initial_memory_details::ALLATONCE)
4408 virtual_free(memory_details.initial_memory[0].memory_base,
4409 memory_details.block_count*(memory_details.block_size_normal +
4410 memory_details.block_size_large));
4412 else if (memory_details.allocation_pattern == initial_memory_details::TWO_STAGE)
4414 virtual_free (memory_details.initial_normal_heap[0].memory_base,
4415 memory_details.block_count*memory_details.block_size_normal);
4417 virtual_free (memory_details.initial_large_heap[0].memory_base,
4418 memory_details.block_count*memory_details.block_size_large);
4422 assert (memory_details.allocation_pattern == initial_memory_details::EACH_BLOCK);
4423 imemory_data *current_block = memory_details.initial_memory;
4424 for(size_t i = 0; i < (memory_details.block_count*2); i++, current_block++)
4426 size_t block_size = (i < memory_details.block_count) ? memory_details.block_size_normal :
4427 memory_details.block_size_large;
4428 if (current_block->memory_base != NULL)
4430 virtual_free (current_block->memory_base, block_size);
4435 delete [] memory_details.initial_memory;
4436 memory_details.initial_memory = NULL;
4437 memory_details.initial_normal_heap = NULL;
4438 memory_details.initial_large_heap = NULL;
4442 void* next_initial_memory (size_t size)
4444 assert ((size == memory_details.block_size_normal) || (size == memory_details.block_size_large));
4447 if ((size != memory_details.block_size_normal) ||
4448 ((memory_details.current_block_normal == memory_details.block_count) &&
4449 (memory_details.block_size_normal == memory_details.block_size_large)))
4451 // If the block sizes are the same, flow block requests from normal to large
4452 assert (memory_details.current_block_large < memory_details.block_count);
4453 assert (memory_details.initial_large_heap != 0);
4455 res = memory_details.initial_large_heap[memory_details.current_block_large].memory_base;
4456 memory_details.current_block_large++;
4460 assert (memory_details.current_block_normal < memory_details.block_count);
4461 assert (memory_details.initial_normal_heap != NULL);
4463 res = memory_details.initial_normal_heap[memory_details.current_block_normal].memory_base;
4464 memory_details.current_block_normal++;
4470 heap_segment* get_initial_segment (size_t size, int h_number)
4472 void* mem = next_initial_memory (size);
4473 heap_segment* res = gc_heap::make_heap_segment ((uint8_t*)mem, size , h_number);
4478 void* virtual_alloc (size_t size)
4480 size_t requested_size = size;
4482 if ((gc_heap::reserved_memory_limit - gc_heap::reserved_memory) < requested_size)
4484 gc_heap::reserved_memory_limit =
4485 GCScan::AskForMoreReservedMemory (gc_heap::reserved_memory_limit, requested_size);
4486 if ((gc_heap::reserved_memory_limit - gc_heap::reserved_memory) < requested_size)
4492 uint32_t flags = VirtualReserveFlags::None;
4493 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
4494 if (virtual_alloc_hardware_write_watch)
4496 flags = VirtualReserveFlags::WriteWatch;
4498 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
4499 void* prgmem = GCToOSInterface::VirtualReserve (requested_size, card_size * card_word_width, flags);
4500 void *aligned_mem = prgmem;
4502 // We don't want (prgmem + size) to be right at the end of the address space
4503 // because we'd have to worry about that everytime we do (address + size).
4504 // We also want to make sure that we leave loh_size_threshold at the end
4505 // so we allocate a small object we don't need to worry about overflow there
4506 // when we do alloc_ptr+size.
4509 uint8_t* end_mem = (uint8_t*)prgmem + requested_size;
4511 if ((end_mem == 0) || ((size_t)(MAX_PTR - end_mem) <= END_SPACE_AFTER_GC))
4513 GCToOSInterface::VirtualRelease (prgmem, requested_size);
4514 dprintf (2, ("Virtual Alloc size %Id returned memory right against 4GB [%Ix, %Ix[ - discarding",
4515 requested_size, (size_t)prgmem, (size_t)((uint8_t*)prgmem+requested_size)));
4523 gc_heap::reserved_memory += requested_size;
4526 dprintf (2, ("Virtual Alloc size %Id: [%Ix, %Ix[",
4527 requested_size, (size_t)prgmem, (size_t)((uint8_t*)prgmem+requested_size)));
4532 void virtual_free (void* add, size_t size)
4534 GCToOSInterface::VirtualRelease (add, size);
4535 gc_heap::reserved_memory -= size;
4536 dprintf (2, ("Virtual Free size %Id: [%Ix, %Ix[",
4537 size, (size_t)add, (size_t)((uint8_t*)add+size)));
4540 static size_t get_valid_segment_size (BOOL large_seg=FALSE)
4542 size_t seg_size, initial_seg_size;
4546 initial_seg_size = INITIAL_ALLOC;
4547 seg_size = static_cast<size_t>(GCConfig::GetSegmentSize());
4551 initial_seg_size = LHEAP_ALLOC;
4552 seg_size = static_cast<size_t>(GCConfig::GetSegmentSize()) / 2;
4555 #ifdef MULTIPLE_HEAPS
4560 if (g_num_processors > 4)
4561 initial_seg_size /= 2;
4562 if (g_num_processors > 8)
4563 initial_seg_size /= 2;
4565 #endif //MULTIPLE_HEAPS
4567 // if seg_size is small but not 0 (0 is default if config not set)
4568 // then set the segment to the minimum size
4569 if (!g_theGCHeap->IsValidSegmentSize(seg_size))
4571 // if requested size is between 1 byte and 4MB, use min
4572 if ((seg_size >> 1) && !(seg_size >> 22))
4573 seg_size = 1024*1024*4;
4575 seg_size = initial_seg_size;
4578 #ifdef SEG_MAPPING_TABLE
4580 seg_size = round_up_power2 (seg_size);
4582 seg_size = round_down_power2 (seg_size);
4584 #endif //SEG_MAPPING_TABLE
4590 gc_heap::compute_new_ephemeral_size()
4592 int eph_gen_max = max_generation - 1 - (settings.promotion ? 1 : 0);
4593 size_t padding_size = 0;
4595 for (int i = 0; i <= eph_gen_max; i++)
4597 dynamic_data* dd = dynamic_data_of (i);
4598 total_ephemeral_size += (dd_survived_size (dd) - dd_pinned_survived_size (dd));
4599 #ifdef RESPECT_LARGE_ALIGNMENT
4600 total_ephemeral_size += dd_num_npinned_plugs (dd) * switch_alignment_size (FALSE);
4601 #endif //RESPECT_LARGE_ALIGNMENT
4602 #ifdef FEATURE_STRUCTALIGN
4603 total_ephemeral_size += dd_num_npinned_plugs (dd) * MAX_STRUCTALIGN;
4604 #endif //FEATURE_STRUCTALIGN
4607 padding_size += dd_padding_size (dd);
4608 #endif //SHORT_PLUGS
4611 total_ephemeral_size += eph_gen_starts_size;
4613 #ifdef RESPECT_LARGE_ALIGNMENT
4614 size_t planned_ephemeral_size = heap_segment_plan_allocated (ephemeral_heap_segment) -
4615 generation_plan_allocation_start (generation_of (max_generation-1));
4616 total_ephemeral_size = min (total_ephemeral_size, planned_ephemeral_size);
4617 #endif //RESPECT_LARGE_ALIGNMENT
4620 total_ephemeral_size = Align ((size_t)((double)total_ephemeral_size * short_plugs_pad_ratio) + 1);
4621 total_ephemeral_size += Align (DESIRED_PLUG_LENGTH);
4622 #endif //SHORT_PLUGS
4624 dprintf (3, ("total ephemeral size is %Ix, padding %Ix(%Ix)",
4625 total_ephemeral_size,
4626 padding_size, (total_ephemeral_size - padding_size)));
4630 #pragma warning(disable:4706) // "assignment within conditional expression" is intentional in this function.
4634 gc_heap::soh_get_segment_to_expand()
4636 size_t size = soh_segment_size;
4638 ordered_plug_indices_init = FALSE;
4639 use_bestfit = FALSE;
4641 //compute the size of the new ephemeral heap segment.
4642 compute_new_ephemeral_size();
4644 if ((settings.pause_mode != pause_low_latency) &&
4645 (settings.pause_mode != pause_no_gc)
4646 #ifdef BACKGROUND_GC
4647 && (!recursive_gc_sync::background_running_p())
4648 #endif //BACKGROUND_GC
4651 allocator* gen_alloc = ((settings.condemned_generation == max_generation) ? 0 :
4652 generation_allocator (generation_of (max_generation)));
4653 dprintf (2, ("(gen%d)soh_get_segment_to_expand", settings.condemned_generation));
4655 // try to find one in the gen 2 segment list, search backwards because the first segments
4656 // tend to be more compact than the later ones.
4657 heap_segment* fseg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
4659 PREFIX_ASSUME(fseg != NULL);
4661 #ifdef SEG_REUSE_STATS
4663 #endif //SEG_REUSE_STATS
4665 heap_segment* seg = ephemeral_heap_segment;
4666 while ((seg = heap_segment_prev_rw (fseg, seg)) && (seg != fseg))
4668 #ifdef SEG_REUSE_STATS
4670 #endif //SEG_REUSE_STATS
4672 if (can_expand_into_p (seg, size/3, total_ephemeral_size, gen_alloc))
4674 get_gc_data_per_heap()->set_mechanism (gc_heap_expand,
4675 (use_bestfit ? expand_reuse_bestfit : expand_reuse_normal));
4676 if (settings.condemned_generation == max_generation)
4680 build_ordered_free_spaces (seg);
4681 dprintf (GTC_LOG, ("can use best fit"));
4684 #ifdef SEG_REUSE_STATS
4685 dprintf (SEG_REUSE_LOG_0, ("(gen%d)soh_get_segment_to_expand: found seg #%d to reuse",
4686 settings.condemned_generation, try_reuse));
4687 #endif //SEG_REUSE_STATS
4688 dprintf (GTC_LOG, ("max_gen: Found existing segment to expand into %Ix", (size_t)seg));
4693 #ifdef SEG_REUSE_STATS
4694 dprintf (SEG_REUSE_LOG_0, ("(gen%d)soh_get_segment_to_expand: found seg #%d to reuse - returning",
4695 settings.condemned_generation, try_reuse));
4696 #endif //SEG_REUSE_STATS
4697 dprintf (GTC_LOG, ("max_gen-1: Found existing segment to expand into %Ix", (size_t)seg));
4699 // If we return 0 here, the allocator will think since we are short on end
4700 // of seg we neeed to trigger a full compacting GC. So if sustained low latency
4701 // is set we should acquire a new seg instead, that way we wouldn't be short.
4702 // The real solution, of course, is to actually implement seg reuse in gen1.
4703 if (settings.pause_mode != pause_sustained_low_latency)
4705 dprintf (GTC_LOG, ("max_gen-1: SustainedLowLatency is set, acquire a new seg"));
4706 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, expand_next_full_gc);
4714 heap_segment* result = get_segment (size, FALSE);
4718 #ifdef BACKGROUND_GC
4719 if (current_c_gc_state == c_gc_state_planning)
4721 // When we expand heap during bgc sweep, we set the seg to be swept so
4722 // we'll always look at cards for objects on the new segment.
4723 result->flags |= heap_segment_flags_swept;
4725 #endif //BACKGROUND_GC
4727 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(result),
4728 (size_t)(heap_segment_reserved (result) - heap_segment_mem(result)),
4729 gc_etw_segment_small_object_heap);
4732 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, (result ? expand_new_seg : expand_no_memory));
4736 dprintf (2, ("h%d: failed to allocate a new segment!", heap_number));
4740 #ifdef MULTIPLE_HEAPS
4741 heap_segment_heap (result) = this;
4742 #endif //MULTIPLE_HEAPS
4745 dprintf (GTC_LOG, ("(gen%d)creating new segment %Ix", settings.condemned_generation, result));
4750 #pragma warning(default:4706)
4753 //returns 0 in case of allocation failure
4755 gc_heap::get_segment (size_t size, BOOL loh_p)
4757 if (heap_hard_limit)
4760 heap_segment* result = 0;
4762 if (segment_standby_list != 0)
4764 result = segment_standby_list;
4765 heap_segment* last = 0;
4768 size_t hs = (size_t)(heap_segment_reserved (result) - (uint8_t*)result);
4769 if ((hs >= size) && ((hs / 2) < size))
4771 dprintf (2, ("Hoarded segment %Ix found", (size_t) result));
4774 heap_segment_next (last) = heap_segment_next (result);
4778 segment_standby_list = heap_segment_next (result);
4785 result = heap_segment_next (result);
4792 init_heap_segment (result);
4793 #ifdef BACKGROUND_GC
4794 if (should_commit_mark_array())
4796 dprintf (GC_TABLE_LOG, ("hoarded seg %Ix, mark_array is %Ix", result, mark_array));
4797 if (!commit_mark_array_new_seg (__this, result))
4799 dprintf (GC_TABLE_LOG, ("failed to commit mark array for hoarded seg"));
4800 // If we can't use it we need to thread it back.
4801 if (segment_standby_list != 0)
4803 heap_segment_next (result) = segment_standby_list;
4804 segment_standby_list = result;
4808 segment_standby_list = result;
4814 #endif //BACKGROUND_GC
4816 #ifdef SEG_MAPPING_TABLE
4818 seg_mapping_table_add_segment (result, __this);
4819 #endif //SEG_MAPPING_TABLE
4824 #ifndef SEG_MAPPING_TABLE
4825 if (!seg_table->ensure_space_for_insert ())
4827 #endif //SEG_MAPPING_TABLE
4828 void* mem = virtual_alloc (size);
4831 fgm_result.set_fgm (fgm_reserve_segment, size, loh_p);
4835 result = gc_heap::make_heap_segment ((uint8_t*)mem, size, heap_number);
4841 if (mem < g_gc_lowest_address)
4843 start = (uint8_t*)mem;
4847 start = (uint8_t*)g_gc_lowest_address;
4850 if (((uint8_t*)mem + size) > g_gc_highest_address)
4852 end = (uint8_t*)mem + size;
4856 end = (uint8_t*)g_gc_highest_address;
4859 if (gc_heap::grow_brick_card_tables (start, end, size, result, __this, loh_p) != 0)
4861 virtual_free (mem, size);
4867 fgm_result.set_fgm (fgm_commit_segment_beg, SEGMENT_INITIAL_COMMIT, loh_p);
4868 virtual_free (mem, size);
4873 #ifdef SEG_MAPPING_TABLE
4874 seg_mapping_table_add_segment (result, __this);
4875 #else //SEG_MAPPING_TABLE
4876 gc_heap::seg_table->insert ((uint8_t*)result, delta);
4877 #endif //SEG_MAPPING_TABLE
4881 #ifdef BACKGROUND_GC
4884 ::record_changed_seg ((uint8_t*)result, heap_segment_reserved (result),
4885 settings.gc_index, current_bgc_state,
4887 bgc_verify_mark_array_cleared (result);
4889 #endif //BACKGROUND_GC
4891 dprintf (GC_TABLE_LOG, ("h%d: new seg: %Ix-%Ix (%Id)", heap_number, result, ((uint8_t*)result + size), size));
4895 void release_segment (heap_segment* sg)
4897 ptrdiff_t delta = 0;
4898 FIRE_EVENT(GCFreeSegment_V1, heap_segment_mem(sg));
4899 virtual_free (sg, (uint8_t*)heap_segment_reserved (sg)-(uint8_t*)sg);
4902 heap_segment* gc_heap::get_segment_for_loh (size_t size
4903 #ifdef MULTIPLE_HEAPS
4905 #endif //MULTIPLE_HEAPS
4908 #ifndef MULTIPLE_HEAPS
4910 #endif //MULTIPLE_HEAPS
4911 heap_segment* res = hp->get_segment (size, TRUE);
4914 #ifdef MULTIPLE_HEAPS
4915 heap_segment_heap (res) = hp;
4916 #endif //MULTIPLE_HEAPS
4917 res->flags |= heap_segment_flags_loh;
4919 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(res), (size_t)(heap_segment_reserved (res) - heap_segment_mem(res)), gc_etw_segment_large_object_heap);
4921 GCToEEInterface::DiagUpdateGenerationBounds();
4923 #ifdef MULTIPLE_HEAPS
4924 hp->thread_loh_segment (res);
4926 thread_loh_segment (res);
4927 #endif //MULTIPLE_HEAPS
4933 void gc_heap::thread_loh_segment (heap_segment* new_seg)
4935 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
4937 while (heap_segment_next_rw (seg))
4938 seg = heap_segment_next_rw (seg);
4939 heap_segment_next (seg) = new_seg;
4943 gc_heap::get_large_segment (size_t size, BOOL* did_full_compact_gc)
4945 *did_full_compact_gc = FALSE;
4946 size_t last_full_compact_gc_count = get_full_compact_gc_count();
4948 //access to get_segment needs to be serialized
4949 add_saved_spinlock_info (true, me_release, mt_get_large_seg);
4950 leave_spin_lock (&more_space_lock_loh);
4951 enter_spin_lock (&gc_heap::gc_lock);
4952 dprintf (SPINLOCK_LOG, ("[%d]Seg: Egc", heap_number));
4953 // if a GC happened between here and before we ask for a segment in
4954 // get_large_segment, we need to count that GC.
4955 size_t current_full_compact_gc_count = get_full_compact_gc_count();
4957 if (current_full_compact_gc_count > last_full_compact_gc_count)
4959 *did_full_compact_gc = TRUE;
4962 heap_segment* res = get_segment_for_loh (size
4963 #ifdef MULTIPLE_HEAPS
4965 #endif //MULTIPLE_HEAPS
4968 dprintf (SPINLOCK_LOG, ("[%d]Seg: A Lgc", heap_number));
4969 leave_spin_lock (&gc_heap::gc_lock);
4970 enter_spin_lock (&more_space_lock_loh);
4971 add_saved_spinlock_info (true, me_acquire, mt_get_large_seg);
4977 BOOL gc_heap::unprotect_segment (heap_segment* seg)
4979 uint8_t* start = align_lower_page (heap_segment_mem (seg));
4980 ptrdiff_t region_size = heap_segment_allocated (seg) - start;
4982 if (region_size != 0 )
4984 dprintf (3, ("unprotecting segment %Ix:", (size_t)seg));
4986 BOOL status = GCToOSInterface::VirtualUnprotect (start, region_size);
4994 #ifdef MULTIPLE_HEAPS
4997 #pragma warning(disable:4035)
4998 static ptrdiff_t get_cycle_count()
5002 #pragma warning(default:4035)
5003 #elif defined(__GNUC__)
5004 static ptrdiff_t get_cycle_count()
5008 __asm__ __volatile__
5009 ("rdtsc":"=a" (cycles), "=d" (cyclesHi));
5013 #error Unknown compiler
5015 #elif defined(_TARGET_AMD64_)
5017 extern "C" uint64_t __rdtsc();
5018 #pragma intrinsic(__rdtsc)
5019 static ptrdiff_t get_cycle_count()
5021 return (ptrdiff_t)__rdtsc();
5023 #elif defined(__GNUC__)
5024 static ptrdiff_t get_cycle_count()
5028 __asm__ __volatile__
5029 ("rdtsc":"=a" (cycles), "=d" (cyclesHi));
5030 return (cyclesHi << 32) | cycles;
5033 extern "C" ptrdiff_t get_cycle_count(void);
5035 #elif defined(_TARGET_ARM_)
5036 static ptrdiff_t get_cycle_count()
5038 // @ARMTODO: cycle counter is not exposed to user mode by CoreARM. For now (until we can show this
5039 // makes a difference on the ARM configurations on which we'll run) just return 0. This will result in
5040 // all buffer access times being reported as equal in access_time().
5043 #elif defined(_TARGET_ARM64_)
5044 static ptrdiff_t get_cycle_count()
5046 // @ARM64TODO: cycle counter is not exposed to user mode by CoreARM. For now (until we can show this
5047 // makes a difference on the ARM configurations on which we'll run) just return 0. This will result in
5048 // all buffer access times being reported as equal in access_time().
5052 #error NYI platform: get_cycle_count
5053 #endif //_TARGET_X86_
5058 static uint8_t* sniff_buffer;
5059 static unsigned n_sniff_buffers;
5060 static unsigned cur_sniff_index;
5062 static uint16_t proc_no_to_heap_no[MAX_SUPPORTED_CPUS];
5063 static uint16_t heap_no_to_proc_no[MAX_SUPPORTED_CPUS];
5064 static uint16_t heap_no_to_numa_node[MAX_SUPPORTED_CPUS];
5065 static uint16_t heap_no_to_cpu_group[MAX_SUPPORTED_CPUS];
5066 static uint16_t heap_no_to_group_proc[MAX_SUPPORTED_CPUS];
5067 static uint16_t numa_node_to_heap_map[MAX_SUPPORTED_CPUS+4];
5069 static int access_time(uint8_t *sniff_buffer, int heap_number, unsigned sniff_index, unsigned n_sniff_buffers)
5071 ptrdiff_t start_cycles = get_cycle_count();
5072 uint8_t sniff = sniff_buffer[(1 + heap_number*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE];
5073 assert (sniff == 0);
5074 ptrdiff_t elapsed_cycles = get_cycle_count() - start_cycles;
5075 // add sniff here just to defeat the optimizer
5076 elapsed_cycles += sniff;
5077 return (int) elapsed_cycles;
5081 static BOOL init(int n_heaps)
5083 assert (sniff_buffer == NULL && n_sniff_buffers == 0);
5084 if (!GCToOSInterface::CanGetCurrentProcessorNumber())
5086 n_sniff_buffers = n_heaps*2+1;
5087 size_t n_cache_lines = 1 + n_heaps * n_sniff_buffers + 1;
5088 size_t sniff_buf_size = n_cache_lines * HS_CACHE_LINE_SIZE;
5089 if (sniff_buf_size / HS_CACHE_LINE_SIZE != n_cache_lines) // check for overlow
5094 sniff_buffer = new (nothrow) uint8_t[sniff_buf_size];
5095 if (sniff_buffer == 0)
5097 memset(sniff_buffer, 0, sniff_buf_size*sizeof(uint8_t));
5100 //can not enable gc numa aware, force all heaps to be in
5101 //one numa node by filling the array with all 0s
5102 if (!GCToOSInterface::CanEnableGCNumaAware())
5103 memset(heap_no_to_numa_node, 0, sizeof (heap_no_to_numa_node));
5108 static void init_cpu_mapping(gc_heap * /*heap*/, int heap_number)
5110 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5112 uint32_t proc_no = GCToOSInterface::GetCurrentProcessorNumber() % gc_heap::n_heaps;
5113 // We can safely cast heap_number to a uint16_t 'cause GetCurrentProcessCpuCount
5114 // only returns up to MAX_SUPPORTED_CPUS procs right now. We only ever create at most
5115 // MAX_SUPPORTED_CPUS GC threads.
5116 proc_no_to_heap_no[proc_no] = (uint16_t)heap_number;
5120 static void mark_heap(int heap_number)
5122 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5125 for (unsigned sniff_index = 0; sniff_index < n_sniff_buffers; sniff_index++)
5126 sniff_buffer[(1 + heap_number*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE] &= 1;
5129 static int select_heap(alloc_context* acontext, int /*hint*/)
5131 UNREFERENCED_PARAMETER(acontext); // only referenced by dprintf
5133 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5134 return proc_no_to_heap_no[GCToOSInterface::GetCurrentProcessorNumber() % gc_heap::n_heaps];
5136 unsigned sniff_index = Interlocked::Increment(&cur_sniff_index);
5137 sniff_index %= n_sniff_buffers;
5140 int best_access_time = 1000*1000*1000;
5141 int second_best_access_time = best_access_time;
5143 uint8_t *l_sniff_buffer = sniff_buffer;
5144 unsigned l_n_sniff_buffers = n_sniff_buffers;
5145 for (int heap_number = 0; heap_number < gc_heap::n_heaps; heap_number++)
5147 int this_access_time = access_time(l_sniff_buffer, heap_number, sniff_index, l_n_sniff_buffers);
5148 if (this_access_time < best_access_time)
5150 second_best_access_time = best_access_time;
5151 best_access_time = this_access_time;
5152 best_heap = heap_number;
5154 else if (this_access_time < second_best_access_time)
5156 second_best_access_time = this_access_time;
5160 if (best_access_time*2 < second_best_access_time)
5162 sniff_buffer[(1 + best_heap*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE] &= 1;
5164 dprintf (3, ("select_heap yields crisp %d for context %p\n", best_heap, (void *)acontext));
5168 dprintf (3, ("select_heap yields vague %d for context %p\n", best_heap, (void *)acontext ));
5174 static bool can_find_heap_fast()
5176 return GCToOSInterface::CanGetCurrentProcessorNumber();
5179 static uint16_t find_proc_no_from_heap_no(int heap_number)
5181 return heap_no_to_proc_no[heap_number];
5184 static void set_proc_no_for_heap(int heap_number, uint16_t proc_no)
5186 heap_no_to_proc_no[heap_number] = proc_no;
5189 static uint16_t find_numa_node_from_heap_no(int heap_number)
5191 return heap_no_to_numa_node[heap_number];
5194 static void set_numa_node_for_heap(int heap_number, uint16_t numa_node)
5196 heap_no_to_numa_node[heap_number] = numa_node;
5199 static uint16_t find_cpu_group_from_heap_no(int heap_number)
5201 return heap_no_to_cpu_group[heap_number];
5204 static void set_cpu_group_for_heap(int heap_number, uint16_t group_number)
5206 heap_no_to_cpu_group[heap_number] = group_number;
5209 static uint16_t find_group_proc_from_heap_no(int heap_number)
5211 return heap_no_to_group_proc[heap_number];
5214 static void set_group_proc_for_heap(int heap_number, uint16_t group_proc)
5216 heap_no_to_group_proc[heap_number] = group_proc;
5219 static void init_numa_node_to_heap_map(int nheaps)
5220 { // called right after GCHeap::Init() for each heap is finished
5221 // when numa is not enabled, heap_no_to_numa_node[] are all filled
5222 // with 0s during initialization, and will be treated as one node
5223 numa_node_to_heap_map[0] = 0;
5226 for (int i=1; i < nheaps; i++)
5228 if (heap_no_to_numa_node[i] != heap_no_to_numa_node[i-1])
5229 numa_node_to_heap_map[node_index++] = (uint16_t)i;
5231 numa_node_to_heap_map[node_index] = (uint16_t)nheaps; //mark the end with nheaps
5234 static void get_heap_range_for_heap(int hn, int* start, int* end)
5235 { // 1-tier/no numa case: heap_no_to_numa_node[] all zeros,
5236 // and treated as in one node. thus: start=0, end=n_heaps
5237 uint16_t numa_node = heap_no_to_numa_node[hn];
5238 *start = (int)numa_node_to_heap_map[numa_node];
5239 *end = (int)(numa_node_to_heap_map[numa_node+1]);
5242 uint8_t* heap_select::sniff_buffer;
5243 unsigned heap_select::n_sniff_buffers;
5244 unsigned heap_select::cur_sniff_index;
5245 uint16_t heap_select::proc_no_to_heap_no[MAX_SUPPORTED_CPUS];
5246 uint16_t heap_select::heap_no_to_proc_no[MAX_SUPPORTED_CPUS];
5247 uint16_t heap_select::heap_no_to_numa_node[MAX_SUPPORTED_CPUS];
5248 uint16_t heap_select::heap_no_to_cpu_group[MAX_SUPPORTED_CPUS];
5249 uint16_t heap_select::heap_no_to_group_proc[MAX_SUPPORTED_CPUS];
5250 uint16_t heap_select::numa_node_to_heap_map[MAX_SUPPORTED_CPUS+4];
5252 BOOL gc_heap::create_thread_support (unsigned number_of_heaps)
5255 if (!gc_start_event.CreateOSManualEventNoThrow (FALSE))
5259 if (!ee_suspend_event.CreateOSAutoEventNoThrow (FALSE))
5263 if (!gc_t_join.init (number_of_heaps, join_flavor_server_gc))
5274 destroy_thread_support();
5280 void gc_heap::destroy_thread_support ()
5282 if (ee_suspend_event.IsValid())
5284 ee_suspend_event.CloseEvent();
5286 if (gc_start_event.IsValid())
5288 gc_start_event.CloseEvent();
5292 void set_thread_group_affinity_for_heap(int heap_number, GCThreadAffinity* affinity)
5294 affinity->Group = GCThreadAffinity::None;
5295 affinity->Processor = GCThreadAffinity::None;
5298 GCToOSInterface::GetGroupForProcessor((uint16_t)heap_number, &gn, &gpn);
5301 for (uintptr_t mask = 1; mask !=0; mask <<=1)
5303 if (bit_number == gpn)
5305 dprintf(3, ("using processor group %d, mask %Ix for heap %d\n", gn, mask, heap_number));
5306 affinity->Processor = gpn;
5307 affinity->Group = gn;
5308 heap_select::set_cpu_group_for_heap(heap_number, gn);
5309 heap_select::set_group_proc_for_heap(heap_number, gpn);
5310 if (GCToOSInterface::CanEnableGCNumaAware())
5312 PROCESSOR_NUMBER proc_no;
5314 proc_no.Number = (uint8_t)gpn;
5315 proc_no.Reserved = 0;
5317 uint16_t node_no = 0;
5318 if (GCToOSInterface::GetNumaProcessorNode(&proc_no, &node_no))
5319 heap_select::set_numa_node_for_heap(heap_number, node_no);
5322 { // no numa setting, each cpu group is treated as a node
5323 heap_select::set_numa_node_for_heap(heap_number, gn);
5331 void set_thread_affinity_mask_for_heap(int heap_number, GCThreadAffinity* affinity)
5333 affinity->Group = GCThreadAffinity::None;
5334 affinity->Processor = GCThreadAffinity::None;
5336 uintptr_t pmask = process_mask;
5338 uint8_t proc_number = 0;
5339 for (uintptr_t mask = 1; mask != 0; mask <<= 1)
5341 if ((mask & pmask) != 0)
5343 if (bit_number == heap_number)
5345 dprintf (3, ("Using processor %d for heap %d", proc_number, heap_number));
5346 affinity->Processor = proc_number;
5347 heap_select::set_proc_no_for_heap(heap_number, proc_number);
5348 if (GCToOSInterface::CanEnableGCNumaAware())
5350 uint16_t node_no = 0;
5351 PROCESSOR_NUMBER proc_no;
5353 proc_no.Number = (uint8_t)proc_number;
5354 proc_no.Reserved = 0;
5355 if (GCToOSInterface::GetNumaProcessorNode(&proc_no, &node_no))
5357 heap_select::set_numa_node_for_heap(heap_number, node_no);
5368 bool gc_heap::create_gc_thread ()
5370 dprintf (3, ("Creating gc thread\n"));
5371 return GCToEEInterface::CreateThread(gc_thread_stub, this, false, ".NET Server GC");
5375 #pragma warning(disable:4715) //IA64 xcompiler recognizes that without the 'break;' the while(1) will never end and therefore not return a value for that code path
5377 void gc_heap::gc_thread_function ()
5379 assert (gc_done_event.IsValid());
5380 assert (gc_start_event.IsValid());
5381 dprintf (3, ("gc thread started"));
5383 heap_select::init_cpu_mapping(this, heap_number);
5387 assert (!gc_t_join.joined());
5389 if (heap_number == 0)
5391 gc_heap::ee_suspend_event.Wait(INFINITE, FALSE);
5393 BEGIN_TIMING(suspend_ee_during_log);
5394 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC);
5395 END_TIMING(suspend_ee_during_log);
5397 proceed_with_gc_p = TRUE;
5399 if (!should_proceed_with_gc())
5401 update_collection_counts_for_no_gc();
5402 proceed_with_gc_p = FALSE;
5406 settings.init_mechanisms();
5407 gc_start_event.Set();
5409 dprintf (3, ("%d gc thread waiting...", heap_number));
5413 gc_start_event.Wait(INFINITE, FALSE);
5414 dprintf (3, ("%d gc thread waiting... Done", heap_number));
5417 assert ((heap_number == 0) || proceed_with_gc_p);
5419 if (proceed_with_gc_p)
5421 garbage_collect (GCHeap::GcCondemnedGeneration);
5423 if (pm_trigger_full_gc)
5425 garbage_collect_pm_full_gc();
5429 if (heap_number == 0)
5431 if (proceed_with_gc_p && (!settings.concurrent))
5436 #ifdef BACKGROUND_GC
5437 recover_bgc_settings();
5438 #endif //BACKGROUND_GC
5440 #ifdef MULTIPLE_HEAPS
5441 for (int i = 0; i < gc_heap::n_heaps; i++)
5443 gc_heap* hp = gc_heap::g_heaps[i];
5444 hp->add_saved_spinlock_info (false, me_release, mt_block_gc);
5445 leave_spin_lock(&hp->more_space_lock_soh);
5447 #endif //MULTIPLE_HEAPS
5449 gc_heap::gc_started = FALSE;
5451 BEGIN_TIMING(restart_ee_during_log);
5452 GCToEEInterface::RestartEE(TRUE);
5453 END_TIMING(restart_ee_during_log);
5454 process_sync_log_stats();
5456 dprintf (SPINLOCK_LOG, ("GC Lgc"));
5457 leave_spin_lock (&gc_heap::gc_lock);
5459 gc_heap::internal_gc_done = true;
5461 if (proceed_with_gc_p)
5465 // If we didn't actually do a GC, it means we didn't wait up the other threads,
5466 // we still need to set the gc_done_event for those threads.
5467 for (int i = 0; i < gc_heap::n_heaps; i++)
5469 gc_heap* hp = gc_heap::g_heaps[i];
5476 int spin_count = 32 * (gc_heap::n_heaps - 1);
5478 // wait until RestartEE has progressed to a stage where we can restart user threads
5479 while (!gc_heap::internal_gc_done && !GCHeap::SafeToRestartManagedThreads())
5481 spin_and_switch (spin_count, (gc_heap::internal_gc_done || GCHeap::SafeToRestartManagedThreads()));
5488 #pragma warning(default:4715) //IA64 xcompiler recognizes that without the 'break;' the while(1) will never end and therefore not return a value for that code path
5491 #endif //MULTIPLE_HEAPS
5493 bool gc_heap::virtual_alloc_commit_for_heap (void* addr, size_t size, int h_number)
5495 #if defined(MULTIPLE_HEAPS) && !defined(FEATURE_REDHAWK)
5496 // Currently there is no way for us to specific the numa node to allocate on via hosting interfaces to
5497 // a host. This will need to be added later.
5498 #if !defined(FEATURE_CORECLR) && !defined(BUILD_AS_STANDALONE)
5499 if (!CLRMemoryHosted())
5502 if (GCToOSInterface::CanEnableGCNumaAware())
5504 uint32_t numa_node = heap_select::find_numa_node_from_heap_no(h_number);
5505 if (GCToOSInterface::VirtualCommit(addr, size, numa_node))
5510 UNREFERENCED_PARAMETER(h_number);
5513 //numa aware not enabled, or call failed --> fallback to VirtualCommit()
5514 return GCToOSInterface::VirtualCommit(addr, size);
5517 bool gc_heap::virtual_commit (void* address, size_t size, int h_number, bool* hard_limit_exceeded_p)
5520 assert (heap_hard_limit == 0);
5523 if (heap_hard_limit)
5525 bool exceeded_p = false;
5527 check_commit_cs.Enter();
5529 if ((current_total_committed + size) > heap_hard_limit)
5531 dprintf (1, ("%Id + %Id = %Id > limit",
5532 current_total_committed, size,
5533 (current_total_committed + size),
5540 current_total_committed += size;
5542 current_total_committed_bookkeeping += size;
5545 check_commit_cs.Leave();
5547 if (hard_limit_exceeded_p)
5548 *hard_limit_exceeded_p = exceeded_p;
5552 dprintf (1, ("can't commit %Ix for %Id bytes > HARD LIMIT %Id", (size_t)address, size, heap_hard_limit));
5557 // If it's a valid heap number it means it's commiting for memory on the GC heap.
5558 bool commit_succeeded_p = ((h_number >= 0) ?
5559 virtual_alloc_commit_for_heap (address, size, h_number) :
5560 GCToOSInterface::VirtualCommit(address, size));
5562 if (!commit_succeeded_p && heap_hard_limit)
5564 check_commit_cs.Enter();
5565 dprintf (1, ("commit failed, updating %Id to %Id",
5566 current_total_committed, (current_total_committed - size)));
5567 current_total_committed -= size;
5569 current_total_committed_bookkeeping -= size;
5571 check_commit_cs.Leave();
5574 return commit_succeeded_p;
5577 bool gc_heap::virtual_decommit (void* address, size_t size, int h_number)
5580 assert (heap_hard_limit == 0);
5583 bool decommit_succeeded_p = GCToOSInterface::VirtualDecommit (address, size);
5585 if (decommit_succeeded_p && heap_hard_limit)
5587 check_commit_cs.Enter();
5588 current_total_committed -= size;
5590 current_total_committed_bookkeeping -= size;
5591 check_commit_cs.Leave();
5594 return decommit_succeeded_p;
5597 #ifndef SEG_MAPPING_TABLE
5599 heap_segment* gc_heap::segment_of (uint8_t* add, ptrdiff_t& delta, BOOL verify_p)
5601 uint8_t* sadd = add;
5602 heap_segment* hs = 0;
5603 heap_segment* hs1 = 0;
5604 if (!((add >= g_gc_lowest_address) && (add < g_gc_highest_address)))
5609 //repeat in case there is a concurrent insertion in the table.
5614 seg_table->lookup (sadd);
5615 hs1 = (heap_segment*)sadd;
5616 } while (hs1 && !in_range_for_segment (add, hs1) && (hs != hs1));
5621 (verify_p && (add > heap_segment_reserved ((heap_segment*)(sadd + delta)))))
5625 #endif //SEG_MAPPING_TABLE
5633 // If we want to save space we can have a pool of plug_and_gap's instead of
5634 // always having 2 allocated for each pinned plug.
5635 gap_reloc_pair saved_pre_plug;
5636 // If we decide to not compact, we need to restore the original values.
5637 gap_reloc_pair saved_pre_plug_reloc;
5639 gap_reloc_pair saved_post_plug;
5641 // Supposedly Pinned objects cannot have references but we are seeing some from pinvoke
5642 // frames. Also if it's an artificially pinned plug created by us, it can certainly
5644 // We know these cases will be rare so we can optimize this to be only allocated on decommand.
5645 gap_reloc_pair saved_post_plug_reloc;
5647 // We need to calculate this after we are done with plan phase and before compact
5648 // phase because compact phase will change the bricks so relocate_address will no
5650 uint8_t* saved_pre_plug_info_reloc_start;
5652 // We need to save this because we will have no way to calculate it, unlike the
5653 // pre plug info start which is right before this plug.
5654 uint8_t* saved_post_plug_info_start;
5657 uint8_t* allocation_context_start_region;
5658 #endif //SHORT_PLUGS
5660 // How the bits in these bytes are organized:
5662 // bit to indicate whether it's a short obj | 3 bits for refs in this short obj | 2 unused bits | bit to indicate if it's collectible | last bit
5663 // last bit indicates if there's pre or post info associated with this plug. If it's not set all other bits will be 0.
5668 // We are seeing this is getting corrupted for a PP with a NP after.
5669 // Save it when we first set it and make sure it doesn't change.
5670 gap_reloc_pair saved_post_plug_debug;
5673 size_t get_max_short_bits()
5675 return (sizeof (gap_reloc_pair) / sizeof (uint8_t*));
5679 size_t get_pre_short_start_bit ()
5681 return (sizeof (saved_pre_p) * 8 - 1 - (sizeof (gap_reloc_pair) / sizeof (uint8_t*)));
5686 return (saved_pre_p & (1 << (sizeof (saved_pre_p) * 8 - 1)));
5689 void set_pre_short()
5691 saved_pre_p |= (1 << (sizeof (saved_pre_p) * 8 - 1));
5694 void set_pre_short_bit (size_t bit)
5696 saved_pre_p |= 1 << (get_pre_short_start_bit() + bit);
5699 BOOL pre_short_bit_p (size_t bit)
5701 return (saved_pre_p & (1 << (get_pre_short_start_bit() + bit)));
5704 #ifdef COLLECTIBLE_CLASS
5705 void set_pre_short_collectible()
5710 BOOL pre_short_collectible_p()
5712 return (saved_pre_p & 2);
5714 #endif //COLLECTIBLE_CLASS
5717 size_t get_post_short_start_bit ()
5719 return (sizeof (saved_post_p) * 8 - 1 - (sizeof (gap_reloc_pair) / sizeof (uint8_t*)));
5724 return (saved_post_p & (1 << (sizeof (saved_post_p) * 8 - 1)));
5727 void set_post_short()
5729 saved_post_p |= (1 << (sizeof (saved_post_p) * 8 - 1));
5732 void set_post_short_bit (size_t bit)
5734 saved_post_p |= 1 << (get_post_short_start_bit() + bit);
5737 BOOL post_short_bit_p (size_t bit)
5739 return (saved_post_p & (1 << (get_post_short_start_bit() + bit)));
5742 #ifdef COLLECTIBLE_CLASS
5743 void set_post_short_collectible()
5748 BOOL post_short_collectible_p()
5750 return (saved_post_p & 2);
5752 #endif //COLLECTIBLE_CLASS
5754 uint8_t* get_plug_address() { return first; }
5756 BOOL has_pre_plug_info() { return saved_pre_p; }
5757 BOOL has_post_plug_info() { return saved_post_p; }
5759 gap_reloc_pair* get_pre_plug_reloc_info() { return &saved_pre_plug_reloc; }
5760 gap_reloc_pair* get_post_plug_reloc_info() { return &saved_post_plug_reloc; }
5761 void set_pre_plug_info_reloc_start (uint8_t* reloc) { saved_pre_plug_info_reloc_start = reloc; }
5762 uint8_t* get_post_plug_info_start() { return saved_post_plug_info_start; }
5764 // We need to temporarily recover the shortened plugs for compact phase so we can
5765 // copy over the whole plug and their related info (mark bits/cards). But we will
5766 // need to set the artificial gap back so compact phase can keep reading the plug info.
5767 // We also need to recover the saved info because we'll need to recover it later.
5769 // So we would call swap_p*_plug_and_saved once to recover the object info; then call
5770 // it again to recover the artificial gap.
5771 void swap_pre_plug_and_saved()
5773 gap_reloc_pair temp;
5774 memcpy (&temp, (first - sizeof (plug_and_gap)), sizeof (temp));
5775 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug_reloc, sizeof (saved_pre_plug_reloc));
5776 saved_pre_plug_reloc = temp;
5779 void swap_post_plug_and_saved()
5781 gap_reloc_pair temp;
5782 memcpy (&temp, saved_post_plug_info_start, sizeof (temp));
5783 memcpy (saved_post_plug_info_start, &saved_post_plug_reloc, sizeof (saved_post_plug_reloc));
5784 saved_post_plug_reloc = temp;
5787 void swap_pre_plug_and_saved_for_profiler()
5789 gap_reloc_pair temp;
5790 memcpy (&temp, (first - sizeof (plug_and_gap)), sizeof (temp));
5791 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug, sizeof (saved_pre_plug));
5792 saved_pre_plug = temp;
5795 void swap_post_plug_and_saved_for_profiler()
5797 gap_reloc_pair temp;
5798 memcpy (&temp, saved_post_plug_info_start, sizeof (temp));
5799 memcpy (saved_post_plug_info_start, &saved_post_plug, sizeof (saved_post_plug));
5800 saved_post_plug = temp;
5803 // We should think about whether it's really necessary to have to copy back the pre plug
5804 // info since it was already copied during compacting plugs. But if a plug doesn't move
5805 // by >= 3 ptr size (the size of gap_reloc_pair), it means we'd have to recover pre plug info.
5806 void recover_plug_info()
5810 if (gc_heap::settings.compaction)
5812 dprintf (3, ("%Ix: REC Pre: %Ix-%Ix",
5814 &saved_pre_plug_reloc,
5815 saved_pre_plug_info_reloc_start));
5816 memcpy (saved_pre_plug_info_reloc_start, &saved_pre_plug_reloc, sizeof (saved_pre_plug_reloc));
5820 dprintf (3, ("%Ix: REC Pre: %Ix-%Ix",
5823 (first - sizeof (plug_and_gap))));
5824 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug, sizeof (saved_pre_plug));
5830 if (gc_heap::settings.compaction)
5832 dprintf (3, ("%Ix: REC Post: %Ix-%Ix",
5834 &saved_post_plug_reloc,
5835 saved_post_plug_info_start));
5836 memcpy (saved_post_plug_info_start, &saved_post_plug_reloc, sizeof (saved_post_plug_reloc));
5840 dprintf (3, ("%Ix: REC Post: %Ix-%Ix",
5843 saved_post_plug_info_start));
5844 memcpy (saved_post_plug_info_start, &saved_post_plug, sizeof (saved_post_plug));
5851 void gc_mechanisms::init_mechanisms()
5853 condemned_generation = 0;
5854 promotion = FALSE;//TRUE;
5856 #ifdef FEATURE_LOH_COMPACTION
5857 loh_compaction = gc_heap::should_compact_loh();
5859 loh_compaction = FALSE;
5860 #endif //FEATURE_LOH_COMPACTION
5861 heap_expansion = FALSE;
5864 elevation_reduced = FALSE;
5865 found_finalizers = FALSE;
5866 #ifdef BACKGROUND_GC
5867 background_p = recursive_gc_sync::background_running_p() != FALSE;
5868 allocations_allowed = TRUE;
5869 #endif //BACKGROUND_GC
5871 entry_memory_load = 0;
5872 exit_memory_load = 0;
5875 stress_induced = FALSE;
5876 #endif // STRESS_HEAP
5879 void gc_mechanisms::first_init()
5882 gen0_reduction_count = 0;
5883 should_lock_elevation = FALSE;
5884 elevation_locked_count = 0;
5885 reason = reason_empty;
5886 #ifdef BACKGROUND_GC
5887 pause_mode = gc_heap::gc_can_use_concurrent ? pause_interactive : pause_batch;
5889 int debug_pause_mode = static_cast<int>(GCConfig::GetLatencyMode());
5890 if (debug_pause_mode >= 0)
5892 assert (debug_pause_mode <= pause_sustained_low_latency);
5893 pause_mode = (gc_pause_mode)debug_pause_mode;
5896 #else //BACKGROUND_GC
5897 pause_mode = pause_batch;
5898 #endif //BACKGROUND_GC
5903 void gc_mechanisms::record (gc_history_global* history)
5905 #ifdef MULTIPLE_HEAPS
5906 history->num_heaps = gc_heap::n_heaps;
5908 history->num_heaps = 1;
5909 #endif //MULTIPLE_HEAPS
5911 history->condemned_generation = condemned_generation;
5912 history->gen0_reduction_count = gen0_reduction_count;
5913 history->reason = reason;
5914 history->pause_mode = (int)pause_mode;
5915 history->mem_pressure = entry_memory_load;
5916 history->global_mechanims_p = 0;
5918 // start setting the boolean values.
5920 history->set_mechanism_p (global_concurrent);
5923 history->set_mechanism_p (global_compaction);
5926 history->set_mechanism_p (global_promotion);
5929 history->set_mechanism_p (global_demotion);
5932 history->set_mechanism_p (global_card_bundles);
5934 if (elevation_reduced)
5935 history->set_mechanism_p (global_elevation);
5938 /**********************************
5939 called at the beginning of GC to fix the allocated size to
5940 what is really allocated, or to turn the free area into an unused object
5941 It needs to be called after all of the other allocation contexts have been
5942 fixed since it relies on alloc_allocated.
5943 ********************************/
5945 //for_gc_p indicates that the work is being done for GC,
5946 //as opposed to concurrent heap verification
5947 void gc_heap::fix_youngest_allocation_area (BOOL for_gc_p)
5949 UNREFERENCED_PARAMETER(for_gc_p);
5951 // The gen 0 alloc context is never used for allocation in the allocator path. It's
5952 // still used in the allocation path during GCs.
5953 assert (generation_allocation_pointer (youngest_generation) == nullptr);
5954 assert (generation_allocation_limit (youngest_generation) == nullptr);
5955 heap_segment_allocated (ephemeral_heap_segment) = alloc_allocated;
5958 void gc_heap::fix_large_allocation_area (BOOL for_gc_p)
5960 UNREFERENCED_PARAMETER(for_gc_p);
5963 alloc_context* acontext =
5965 generation_alloc_context (large_object_generation);
5966 assert (acontext->alloc_ptr == 0);
5967 assert (acontext->alloc_limit == 0);
5969 dprintf (3, ("Large object alloc context: ptr: %Ix, limit %Ix",
5970 (size_t)acontext->alloc_ptr, (size_t)acontext->alloc_limit));
5971 fix_allocation_context (acontext, FALSE, get_alignment_constant (FALSE));
5974 acontext->alloc_ptr = 0;
5975 acontext->alloc_limit = acontext->alloc_ptr;
5980 //for_gc_p indicates that the work is being done for GC,
5981 //as opposed to concurrent heap verification
5982 void gc_heap::fix_allocation_context (alloc_context* acontext, BOOL for_gc_p,
5985 dprintf (3, ("Fixing allocation context %Ix: ptr: %Ix, limit: %Ix",
5987 (size_t)acontext->alloc_ptr, (size_t)acontext->alloc_limit));
5989 if (((size_t)(alloc_allocated - acontext->alloc_limit) > Align (min_obj_size, align_const)) ||
5992 uint8_t* point = acontext->alloc_ptr;
5995 size_t size = (acontext->alloc_limit - acontext->alloc_ptr);
5996 // the allocation area was from the free list
5997 // it was shortened by Align (min_obj_size) to make room for
5998 // at least the shortest unused object
5999 size += Align (min_obj_size, align_const);
6000 assert ((size >= Align (min_obj_size)));
6002 dprintf(3,("Making unused area [%Ix, %Ix[", (size_t)point,
6003 (size_t)point + size ));
6004 make_unused_array (point, size);
6008 generation_free_obj_space (generation_of (0)) += size;
6009 alloc_contexts_used ++;
6015 alloc_allocated = acontext->alloc_ptr;
6016 assert (heap_segment_allocated (ephemeral_heap_segment) <=
6017 heap_segment_committed (ephemeral_heap_segment));
6018 alloc_contexts_used ++;
6023 // We need to update the alloc_bytes to reflect the portion that we have not used
6024 acontext->alloc_bytes -= (acontext->alloc_limit - acontext->alloc_ptr);
6025 acontext->alloc_ptr = 0;
6026 acontext->alloc_limit = acontext->alloc_ptr;
6030 //used by the heap verification for concurrent gc.
6031 //it nulls out the words set by fix_allocation_context for heap_verification
6032 void repair_allocation (gc_alloc_context* acontext, void*)
6034 uint8_t* point = acontext->alloc_ptr;
6038 dprintf (3, ("Clearing [%Ix, %Ix[", (size_t)acontext->alloc_ptr,
6039 (size_t)acontext->alloc_limit+Align(min_obj_size)));
6040 memclr (acontext->alloc_ptr - plug_skew,
6041 (acontext->alloc_limit - acontext->alloc_ptr)+Align (min_obj_size));
6045 void void_allocation (gc_alloc_context* acontext, void*)
6047 uint8_t* point = acontext->alloc_ptr;
6051 dprintf (3, ("Void [%Ix, %Ix[", (size_t)acontext->alloc_ptr,
6052 (size_t)acontext->alloc_limit+Align(min_obj_size)));
6053 acontext->alloc_ptr = 0;
6054 acontext->alloc_limit = acontext->alloc_ptr;
6058 void gc_heap::repair_allocation_contexts (BOOL repair_p)
6060 GCToEEInterface::GcEnumAllocContexts (repair_p ? repair_allocation : void_allocation, NULL);
6063 struct fix_alloc_context_args
6069 void fix_alloc_context (gc_alloc_context* acontext, void* param)
6071 fix_alloc_context_args* args = (fix_alloc_context_args*)param;
6072 g_theGCHeap->FixAllocContext(acontext, (void*)(size_t)(args->for_gc_p), args->heap);
6075 void gc_heap::fix_allocation_contexts (BOOL for_gc_p)
6077 fix_alloc_context_args args;
6078 args.for_gc_p = for_gc_p;
6081 GCToEEInterface::GcEnumAllocContexts(fix_alloc_context, &args);
6082 fix_youngest_allocation_area(for_gc_p);
6083 fix_large_allocation_area(for_gc_p);
6086 void gc_heap::fix_older_allocation_area (generation* older_gen)
6088 heap_segment* older_gen_seg = generation_allocation_segment (older_gen);
6089 if (generation_allocation_limit (older_gen) !=
6090 heap_segment_plan_allocated (older_gen_seg))
6092 uint8_t* point = generation_allocation_pointer (older_gen);
6094 size_t size = (generation_allocation_limit (older_gen) -
6095 generation_allocation_pointer (older_gen));
6098 assert ((size >= Align (min_obj_size)));
6099 dprintf(3,("Making unused area [%Ix, %Ix[", (size_t)point, (size_t)point+size));
6100 make_unused_array (point, size);
6101 if (size >= min_free_list)
6103 generation_allocator (older_gen)->thread_item_front (point, size);
6104 add_gen_free (older_gen->gen_num, size);
6105 generation_free_list_space (older_gen) += size;
6109 generation_free_obj_space (older_gen) += size;
6115 assert (older_gen_seg != ephemeral_heap_segment);
6116 heap_segment_plan_allocated (older_gen_seg) =
6117 generation_allocation_pointer (older_gen);
6118 generation_allocation_limit (older_gen) =
6119 generation_allocation_pointer (older_gen);
6122 generation_allocation_pointer (older_gen) = 0;
6123 generation_allocation_limit (older_gen) = 0;
6126 void gc_heap::set_allocation_heap_segment (generation* gen)
6128 uint8_t* p = generation_allocation_start (gen);
6130 heap_segment* seg = generation_allocation_segment (gen);
6131 if (in_range_for_segment (p, seg))
6134 // try ephemeral heap segment in case of heap expansion
6135 seg = ephemeral_heap_segment;
6136 if (!in_range_for_segment (p, seg))
6138 seg = heap_segment_rw (generation_start_segment (gen));
6140 PREFIX_ASSUME(seg != NULL);
6142 while (!in_range_for_segment (p, seg))
6144 seg = heap_segment_next_rw (seg);
6145 PREFIX_ASSUME(seg != NULL);
6149 generation_allocation_segment (gen) = seg;
6152 void gc_heap::reset_allocation_pointers (generation* gen, uint8_t* start)
6155 assert (Align ((size_t)start) == (size_t)start);
6156 generation_allocation_start (gen) = start;
6157 generation_allocation_pointer (gen) = 0;//start + Align (min_obj_size);
6158 generation_allocation_limit (gen) = 0;//generation_allocation_pointer (gen);
6159 set_allocation_heap_segment (gen);
6162 #ifdef BACKGROUND_GC
6163 //TODO BACKGROUND_GC this is for test only
6165 gc_heap::disallow_new_allocation (int gen_number)
6167 UNREFERENCED_PARAMETER(gen_number);
6168 settings.allocations_allowed = FALSE;
6171 gc_heap::allow_new_allocation (int gen_number)
6173 UNREFERENCED_PARAMETER(gen_number);
6174 settings.allocations_allowed = TRUE;
6177 #endif //BACKGROUND_GC
6179 bool gc_heap::new_allocation_allowed (int gen_number)
6181 #ifdef BACKGROUND_GC
6182 //TODO BACKGROUND_GC this is for test only
6183 if (!settings.allocations_allowed)
6185 dprintf (2, ("new allocation not allowed"));
6188 #endif //BACKGROUND_GC
6190 if (dd_new_allocation (dynamic_data_of (gen_number)) < 0)
6192 if (gen_number != 0)
6194 // For LOH we will give it more budget before we try a GC.
6195 if (settings.concurrent)
6197 dynamic_data* dd2 = dynamic_data_of (max_generation + 1 );
6199 if (dd_new_allocation (dd2) <= (ptrdiff_t)(-2 * dd_desired_allocation (dd2)))
6207 #ifndef MULTIPLE_HEAPS
6208 else if ((settings.pause_mode != pause_no_gc) && (gen_number == 0))
6210 dprintf (3, ("evaluating allocation rate"));
6211 dynamic_data* dd0 = dynamic_data_of (0);
6212 if ((allocation_running_amount - dd_new_allocation (dd0)) >
6215 uint32_t ctime = GCToOSInterface::GetLowPrecisionTimeStamp();
6216 if ((ctime - allocation_running_time) > 1000)
6218 dprintf (2, (">1s since last gen0 gc"));
6223 allocation_running_amount = dd_new_allocation (dd0);
6227 #endif //MULTIPLE_HEAPS
6232 ptrdiff_t gc_heap::get_desired_allocation (int gen_number)
6234 return dd_desired_allocation (dynamic_data_of (gen_number));
6238 ptrdiff_t gc_heap::get_new_allocation (int gen_number)
6240 return dd_new_allocation (dynamic_data_of (gen_number));
6243 //return the amount allocated so far in gen_number
6245 ptrdiff_t gc_heap::get_allocation (int gen_number)
6247 dynamic_data* dd = dynamic_data_of (gen_number);
6249 return dd_desired_allocation (dd) - dd_new_allocation (dd);
6253 BOOL grow_mark_stack (mark*& m, size_t& len, size_t init_len)
6255 size_t new_size = max (init_len, 2*len);
6256 mark* tmp = new (nothrow) mark [new_size];
6259 memcpy (tmp, m, len * sizeof (mark));
6267 dprintf (1, ("Failed to allocate %Id bytes for mark stack", (len * sizeof (mark))));
6273 uint8_t* pinned_plug (mark* m)
6279 size_t& pinned_len (mark* m)
6285 void set_new_pin_info (mark* m, uint8_t* pin_free_space_start)
6287 m->len = pinned_plug (m) - pin_free_space_start;
6289 m->allocation_context_start_region = pin_free_space_start;
6290 #endif //SHORT_PLUGS
6295 uint8_t*& pin_allocation_context_start_region (mark* m)
6297 return m->allocation_context_start_region;
6300 uint8_t* get_plug_start_in_saved (uint8_t* old_loc, mark* pinned_plug_entry)
6302 uint8_t* saved_pre_plug_info = (uint8_t*)(pinned_plug_entry->get_pre_plug_reloc_info());
6303 uint8_t* plug_start_in_saved = saved_pre_plug_info + (old_loc - (pinned_plug (pinned_plug_entry) - sizeof (plug_and_gap)));
6304 //dprintf (1, ("detected a very short plug: %Ix before PP %Ix, pad %Ix",
6305 // old_loc, pinned_plug (pinned_plug_entry), plug_start_in_saved));
6306 dprintf (1, ("EP: %Ix(%Ix), %Ix", old_loc, pinned_plug (pinned_plug_entry), plug_start_in_saved));
6307 return plug_start_in_saved;
6311 void set_padding_in_expand (uint8_t* old_loc,
6312 BOOL set_padding_on_saved_p,
6313 mark* pinned_plug_entry)
6315 if (set_padding_on_saved_p)
6317 set_plug_padded (get_plug_start_in_saved (old_loc, pinned_plug_entry));
6321 set_plug_padded (old_loc);
6326 void clear_padding_in_expand (uint8_t* old_loc,
6327 BOOL set_padding_on_saved_p,
6328 mark* pinned_plug_entry)
6330 if (set_padding_on_saved_p)
6332 clear_plug_padded (get_plug_start_in_saved (old_loc, pinned_plug_entry));
6336 clear_plug_padded (old_loc);
6339 #endif //SHORT_PLUGS
6341 void gc_heap::reset_pinned_queue()
6347 void gc_heap::reset_pinned_queue_bos()
6352 // last_pinned_plug is only for asserting purpose.
6353 void gc_heap::merge_with_last_pinned_plug (uint8_t* last_pinned_plug, size_t plug_size)
6355 if (last_pinned_plug)
6357 mark& last_m = mark_stack_array[mark_stack_tos - 1];
6358 assert (last_pinned_plug == last_m.first);
6359 if (last_m.saved_post_p)
6361 last_m.saved_post_p = FALSE;
6362 dprintf (3, ("setting last plug %Ix post to false", last_m.first));
6363 // We need to recover what the gap has overwritten.
6364 memcpy ((last_m.first + last_m.len - sizeof (plug_and_gap)), &(last_m.saved_post_plug), sizeof (gap_reloc_pair));
6366 last_m.len += plug_size;
6367 dprintf (3, ("recovered the last part of plug %Ix, setting its plug size to %Ix", last_m.first, last_m.len));
6371 void gc_heap::set_allocator_next_pin (uint8_t* alloc_pointer, uint8_t*& alloc_limit)
6373 dprintf (3, ("sanp: ptr: %Ix, limit: %Ix", alloc_pointer, alloc_limit));
6374 dprintf (3, ("oldest %Id: %Ix", mark_stack_bos, pinned_plug (oldest_pin())));
6375 if (!(pinned_plug_que_empty_p()))
6377 mark* oldest_entry = oldest_pin();
6378 uint8_t* plug = pinned_plug (oldest_entry);
6379 if ((plug >= alloc_pointer) && (plug < alloc_limit))
6381 alloc_limit = pinned_plug (oldest_entry);
6382 dprintf (3, ("now setting alloc context: %Ix->%Ix(%Id)",
6383 alloc_pointer, alloc_limit, (alloc_limit - alloc_pointer)));
6388 void gc_heap::set_allocator_next_pin (generation* gen)
6390 dprintf (3, ("SANP: gen%d, ptr; %Ix, limit: %Ix", gen->gen_num, generation_allocation_pointer (gen), generation_allocation_limit (gen)));
6391 if (!(pinned_plug_que_empty_p()))
6393 mark* oldest_entry = oldest_pin();
6394 uint8_t* plug = pinned_plug (oldest_entry);
6395 if ((plug >= generation_allocation_pointer (gen)) &&
6396 (plug < generation_allocation_limit (gen)))
6398 generation_allocation_limit (gen) = pinned_plug (oldest_entry);
6399 dprintf (3, ("SANP: get next pin free space in gen%d for alloc: %Ix->%Ix(%Id)",
6401 generation_allocation_pointer (gen), generation_allocation_limit (gen),
6402 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
6405 assert (!((plug < generation_allocation_pointer (gen)) &&
6406 (plug >= heap_segment_mem (generation_allocation_segment (gen)))));
6410 // After we set the info, we increase tos.
6411 void gc_heap::set_pinned_info (uint8_t* last_pinned_plug, size_t plug_len, uint8_t* alloc_pointer, uint8_t*& alloc_limit)
6413 UNREFERENCED_PARAMETER(last_pinned_plug);
6415 mark& m = mark_stack_array[mark_stack_tos];
6416 assert (m.first == last_pinned_plug);
6420 set_allocator_next_pin (alloc_pointer, alloc_limit);
6423 // After we set the info, we increase tos.
6424 void gc_heap::set_pinned_info (uint8_t* last_pinned_plug, size_t plug_len, generation* gen)
6426 UNREFERENCED_PARAMETER(last_pinned_plug);
6428 mark& m = mark_stack_array[mark_stack_tos];
6429 assert (m.first == last_pinned_plug);
6434 // Why are we checking here? gen is never 0.
6437 set_allocator_next_pin (gen);
6441 size_t gc_heap::deque_pinned_plug ()
6443 dprintf (3, ("dequed: %Id", mark_stack_bos));
6444 size_t m = mark_stack_bos;
6450 mark* gc_heap::pinned_plug_of (size_t bos)
6452 return &mark_stack_array [ bos ];
6456 mark* gc_heap::oldest_pin ()
6458 return pinned_plug_of (mark_stack_bos);
6462 BOOL gc_heap::pinned_plug_que_empty_p ()
6464 return (mark_stack_bos == mark_stack_tos);
6468 mark* gc_heap::before_oldest_pin()
6470 if (mark_stack_bos >= 1)
6471 return pinned_plug_of (mark_stack_bos-1);
6477 BOOL gc_heap::ephemeral_pointer_p (uint8_t* o)
6479 return ((o >= ephemeral_low) && (o < ephemeral_high));
6484 int& gc_heap::mark_stack_busy()
6486 return g_mark_stack_busy [(heap_number+2)*HS_CACHE_LINE_SIZE/sizeof(int)];
6490 void gc_heap::make_mark_stack (mark* arr)
6492 reset_pinned_queue();
6493 mark_stack_array = arr;
6494 mark_stack_array_length = MARK_STACK_INITIAL_LENGTH;
6496 mark_stack_busy() = 0;
6500 #ifdef BACKGROUND_GC
6502 size_t& gc_heap::bpromoted_bytes(int thread)
6504 #ifdef MULTIPLE_HEAPS
6505 return g_bpromoted [thread*16];
6506 #else //MULTIPLE_HEAPS
6507 UNREFERENCED_PARAMETER(thread);
6509 #endif //MULTIPLE_HEAPS
6512 void gc_heap::make_background_mark_stack (uint8_t** arr)
6514 background_mark_stack_array = arr;
6515 background_mark_stack_array_length = MARK_STACK_INITIAL_LENGTH;
6516 background_mark_stack_tos = arr;
6519 void gc_heap::make_c_mark_list (uint8_t** arr)
6522 c_mark_list_index = 0;
6523 c_mark_list_length = 1 + (OS_PAGE_SIZE / MIN_OBJECT_SIZE);
6525 #endif //BACKGROUND_GC
6530 // The card bundle keeps track of groups of card words.
6531 static const size_t card_bundle_word_width = 32;
6533 // How do we express the fact that 32 bits (card_word_width) is one uint32_t?
6534 static const size_t card_bundle_size = (size_t)(GC_PAGE_SIZE / (sizeof(uint32_t)*card_bundle_word_width));
6537 size_t card_bundle_word (size_t cardb)
6539 return cardb / card_bundle_word_width;
6543 uint32_t card_bundle_bit (size_t cardb)
6545 return (uint32_t)(cardb % card_bundle_word_width);
6548 size_t align_cardw_on_bundle (size_t cardw)
6550 return ((size_t)(cardw + card_bundle_size - 1) & ~(card_bundle_size - 1 ));
6553 // Get the card bundle representing a card word
6554 size_t cardw_card_bundle (size_t cardw)
6556 return cardw / card_bundle_size;
6559 // Get the first card word in a card bundle
6560 size_t card_bundle_cardw (size_t cardb)
6562 return cardb * card_bundle_size;
6565 // Clear the specified card bundle
6566 void gc_heap::card_bundle_clear (size_t cardb)
6568 card_bundle_table [card_bundle_word (cardb)] &= ~(1 << card_bundle_bit (cardb));
6569 dprintf (2, ("Cleared card bundle %Ix [%Ix, %Ix[", cardb, (size_t)card_bundle_cardw (cardb),
6570 (size_t)card_bundle_cardw (cardb+1)));
6573 void gc_heap::card_bundle_set (size_t cardb)
6575 if (!card_bundle_set_p (cardb))
6577 card_bundle_table [card_bundle_word (cardb)] |= (1 << card_bundle_bit (cardb));
6581 // Set the card bundle bits between start_cardb and end_cardb
6582 void gc_heap::card_bundles_set (size_t start_cardb, size_t end_cardb)
6584 if (start_cardb == end_cardb)
6586 card_bundle_set(start_cardb);
6590 size_t start_word = card_bundle_word (start_cardb);
6591 size_t end_word = card_bundle_word (end_cardb);
6593 if (start_word < end_word)
6595 // Set the partial words
6596 card_bundle_table [start_word] |= highbits (~0u, card_bundle_bit (start_cardb));
6598 if (card_bundle_bit (end_cardb))
6599 card_bundle_table [end_word] |= lowbits (~0u, card_bundle_bit (end_cardb));
6601 // Set the full words
6602 for (size_t i = start_word + 1; i < end_word; i++)
6603 card_bundle_table [i] = ~0u;
6607 card_bundle_table [start_word] |= (highbits (~0u, card_bundle_bit (start_cardb)) &
6608 lowbits (~0u, card_bundle_bit (end_cardb)));
6612 // Indicates whether the specified bundle is set.
6613 BOOL gc_heap::card_bundle_set_p (size_t cardb)
6615 return (card_bundle_table[card_bundle_word(cardb)] & (1 << card_bundle_bit (cardb)));
6618 // Returns the size (in bytes) of a card bundle representing the region from 'from' to 'end'
6619 size_t size_card_bundle_of (uint8_t* from, uint8_t* end)
6621 // Number of heap bytes represented by a card bundle word
6622 size_t cbw_span = card_size * card_word_width * card_bundle_size * card_bundle_word_width;
6624 // Align the start of the region down
6625 from = (uint8_t*)((size_t)from & ~(cbw_span - 1));
6627 // Align the end of the region up
6628 end = (uint8_t*)((size_t)(end + (cbw_span - 1)) & ~(cbw_span - 1));
6630 // Make sure they're really aligned
6631 assert (((size_t)from & (cbw_span - 1)) == 0);
6632 assert (((size_t)end & (cbw_span - 1)) == 0);
6634 return ((end - from) / cbw_span) * sizeof (uint32_t);
6637 // Takes a pointer to a card bundle table and an address, and returns a pointer that represents
6638 // where a theoretical card bundle table that represents every address (starting from 0) would
6639 // start if the bundle word representing the address were to be located at the pointer passed in.
6640 // The returned 'translated' pointer makes it convenient/fast to calculate where the card bundle
6641 // for a given address is using a simple shift operation on the address.
6642 uint32_t* translate_card_bundle_table (uint32_t* cb, uint8_t* lowest_address)
6644 // The number of bytes of heap memory represented by a card bundle word
6645 const size_t heap_bytes_for_bundle_word = card_size * card_word_width * card_bundle_size * card_bundle_word_width;
6647 // Each card bundle word is 32 bits
6648 return (uint32_t*)((uint8_t*)cb - (((size_t)lowest_address / heap_bytes_for_bundle_word) * sizeof (uint32_t)));
6651 void gc_heap::enable_card_bundles ()
6653 if (can_use_write_watch_for_card_table() && (!card_bundles_enabled()))
6655 dprintf (1, ("Enabling card bundles"));
6657 // We initially set all of the card bundles
6658 card_bundles_set (cardw_card_bundle (card_word (card_of (lowest_address))),
6659 cardw_card_bundle (align_cardw_on_bundle (card_word (card_of (highest_address)))));
6660 settings.card_bundles = TRUE;
6664 BOOL gc_heap::card_bundles_enabled ()
6666 return settings.card_bundles;
6669 #endif // CARD_BUNDLE
6671 #if defined (_TARGET_AMD64_)
6672 #define brick_size ((size_t)4096)
6674 #define brick_size ((size_t)2048)
6675 #endif //_TARGET_AMD64_
6678 size_t gc_heap::brick_of (uint8_t* add)
6680 return (size_t)(add - lowest_address) / brick_size;
6684 uint8_t* gc_heap::brick_address (size_t brick)
6686 return lowest_address + (brick_size * brick);
6690 void gc_heap::clear_brick_table (uint8_t* from, uint8_t* end)
6692 for (size_t i = brick_of (from);i < brick_of (end); i++)
6696 //codes for the brick entries:
6697 //entry == 0 -> not assigned
6698 //entry >0 offset is entry-1
6699 //entry <0 jump back entry bricks
6703 void gc_heap::set_brick (size_t index, ptrdiff_t val)
6709 assert (val < 32767);
6711 brick_table [index] = (short)val+1;
6713 brick_table [index] = (short)val;
6717 int gc_heap::get_brick_entry (size_t index)
6719 #ifdef MULTIPLE_HEAPS
6720 return VolatileLoadWithoutBarrier(&brick_table [index]);
6722 return brick_table[index];
6728 uint8_t* align_on_brick (uint8_t* add)
6730 return (uint8_t*)((size_t)(add + brick_size - 1) & ~(brick_size - 1));
6734 uint8_t* align_lower_brick (uint8_t* add)
6736 return (uint8_t*)(((size_t)add) & ~(brick_size - 1));
6739 size_t size_brick_of (uint8_t* from, uint8_t* end)
6741 assert (((size_t)from & (brick_size-1)) == 0);
6742 assert (((size_t)end & (brick_size-1)) == 0);
6744 return ((end - from) / brick_size) * sizeof (short);
6748 uint8_t* gc_heap::card_address (size_t card)
6750 return (uint8_t*) (card_size * card);
6754 size_t gc_heap::card_of ( uint8_t* object)
6756 return (size_t)(object) / card_size;
6760 size_t gc_heap::card_to_brick (size_t card)
6762 return brick_of (card_address (card));
6766 uint8_t* align_on_card (uint8_t* add)
6768 return (uint8_t*)((size_t)(add + card_size - 1) & ~(card_size - 1 ));
6771 uint8_t* align_on_card_word (uint8_t* add)
6773 return (uint8_t*) ((size_t)(add + (card_size*card_word_width)-1) & ~(card_size*card_word_width - 1));
6777 uint8_t* align_lower_card (uint8_t* add)
6779 return (uint8_t*)((size_t)add & ~(card_size-1));
6783 void gc_heap::clear_card (size_t card)
6785 card_table [card_word (card)] =
6786 (card_table [card_word (card)] & ~(1 << card_bit (card)));
6787 dprintf (3,("Cleared card %Ix [%Ix, %Ix[", card, (size_t)card_address (card),
6788 (size_t)card_address (card+1)));
6792 void gc_heap::set_card (size_t card)
6794 size_t word = card_word (card);
6795 card_table[word] = (card_table [word] | (1 << card_bit (card)));
6797 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
6798 // Also set the card bundle that corresponds to the card
6799 size_t bundle_to_set = cardw_card_bundle(word);
6801 card_bundle_set(bundle_to_set);
6803 dprintf (3,("Set card %Ix [%Ix, %Ix[ and bundle %Ix", card, (size_t)card_address (card), (size_t)card_address (card+1), bundle_to_set));
6804 assert(card_bundle_set_p(bundle_to_set) != 0);
6809 BOOL gc_heap::card_set_p (size_t card)
6811 return ( card_table [ card_word (card) ] & (1 << card_bit (card)));
6814 // Returns the number of DWORDs in the card table that cover the
6815 // range of addresses [from, end[.
6816 size_t count_card_of (uint8_t* from, uint8_t* end)
6818 return card_word (gcard_of (end - 1)) - card_word (gcard_of (from)) + 1;
6821 // Returns the number of bytes to allocate for a card table
6822 // that covers the range of addresses [from, end[.
6823 size_t size_card_of (uint8_t* from, uint8_t* end)
6825 return count_card_of (from, end) * sizeof(uint32_t);
6828 // We don't store seg_mapping_table in card_table_info because there's only always one view.
6829 class card_table_info
6833 uint8_t* lowest_address;
6834 uint8_t* highest_address;
6838 uint32_t* card_bundle_table;
6839 #endif //CARD_BUNDLE
6841 // mark_array is always at the end of the data structure because we
6842 // want to be able to make one commit call for everything before it.
6844 uint32_t* mark_array;
6848 uint32_t* next_card_table;
6851 //These are accessors on untranslated cardtable
6853 unsigned& card_table_refcount (uint32_t* c_table)
6855 return *(unsigned*)((char*)c_table - sizeof (card_table_info));
6859 uint8_t*& card_table_lowest_address (uint32_t* c_table)
6861 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->lowest_address;
6864 uint32_t* translate_card_table (uint32_t* ct)
6866 return (uint32_t*)((uint8_t*)ct - card_word (gcard_of (card_table_lowest_address (ct))) * sizeof(uint32_t));
6870 uint8_t*& card_table_highest_address (uint32_t* c_table)
6872 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->highest_address;
6876 short*& card_table_brick_table (uint32_t* c_table)
6878 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->brick_table;
6883 uint32_t*& card_table_card_bundle_table (uint32_t* c_table)
6885 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->card_bundle_table;
6887 #endif //CARD_BUNDLE
6890 /* Support for mark_array */
6893 uint32_t*& card_table_mark_array (uint32_t* c_table)
6895 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->mark_array;
6899 #define mark_bit_pitch ((size_t)16)
6901 #define mark_bit_pitch ((size_t)8)
6903 #define mark_word_width ((size_t)32)
6904 #define mark_word_size (mark_word_width * mark_bit_pitch)
6907 uint8_t* align_on_mark_bit (uint8_t* add)
6909 return (uint8_t*)((size_t)(add + (mark_bit_pitch - 1)) & ~(mark_bit_pitch - 1));
6913 uint8_t* align_lower_mark_bit (uint8_t* add)
6915 return (uint8_t*)((size_t)(add) & ~(mark_bit_pitch - 1));
6919 BOOL is_aligned_on_mark_word (uint8_t* add)
6921 return ((size_t)add == ((size_t)(add) & ~(mark_word_size - 1)));
6925 uint8_t* align_on_mark_word (uint8_t* add)
6927 return (uint8_t*)((size_t)(add + mark_word_size - 1) & ~(mark_word_size - 1));
6931 uint8_t* align_lower_mark_word (uint8_t* add)
6933 return (uint8_t*)((size_t)(add) & ~(mark_word_size - 1));
6937 size_t mark_bit_of (uint8_t* add)
6939 return ((size_t)add / mark_bit_pitch);
6943 unsigned int mark_bit_bit (size_t mark_bit)
6945 return (unsigned int)(mark_bit % mark_word_width);
6949 size_t mark_bit_word (size_t mark_bit)
6951 return (mark_bit / mark_word_width);
6955 size_t mark_word_of (uint8_t* add)
6957 return ((size_t)add) / mark_word_size;
6960 uint8_t* mark_word_address (size_t wd)
6962 return (uint8_t*)(wd*mark_word_size);
6965 uint8_t* mark_bit_address (size_t mark_bit)
6967 return (uint8_t*)(mark_bit*mark_bit_pitch);
6971 size_t mark_bit_bit_of (uint8_t* add)
6973 return (((size_t)add / mark_bit_pitch) % mark_word_width);
6977 unsigned int gc_heap::mark_array_marked(uint8_t* add)
6979 return mark_array [mark_word_of (add)] & (1 << mark_bit_bit_of (add));
6983 BOOL gc_heap::is_mark_bit_set (uint8_t* add)
6985 return (mark_array [mark_word_of (add)] & (1 << mark_bit_bit_of (add)));
6989 void gc_heap::mark_array_set_marked (uint8_t* add)
6991 size_t index = mark_word_of (add);
6992 uint32_t val = (1 << mark_bit_bit_of (add));
6993 #ifdef MULTIPLE_HEAPS
6994 Interlocked::Or (&(mark_array [index]), val);
6996 mark_array [index] |= val;
7001 void gc_heap::mark_array_clear_marked (uint8_t* add)
7003 mark_array [mark_word_of (add)] &= ~(1 << mark_bit_bit_of (add));
7006 size_t size_mark_array_of (uint8_t* from, uint8_t* end)
7008 assert (((size_t)from & ((mark_word_size)-1)) == 0);
7009 assert (((size_t)end & ((mark_word_size)-1)) == 0);
7010 return sizeof (uint32_t)*(((end - from) / mark_word_size));
7013 //In order to eliminate the lowest_address in the mark array
7014 //computations (mark_word_of, etc) mark_array is offset
7015 // according to the lowest_address.
7016 uint32_t* translate_mark_array (uint32_t* ma)
7018 return (uint32_t*)((uint8_t*)ma - size_mark_array_of (0, g_gc_lowest_address));
7021 // from and end must be page aligned addresses.
7022 void gc_heap::clear_mark_array (uint8_t* from, uint8_t* end, BOOL check_only/*=TRUE*/
7023 #ifdef FEATURE_BASICFREEZE
7024 , BOOL read_only/*=FALSE*/
7025 #endif // FEATURE_BASICFREEZE
7028 if(!gc_can_use_concurrent)
7031 #ifdef FEATURE_BASICFREEZE
7033 #endif // FEATURE_BASICFREEZE
7035 assert (from == align_on_mark_word (from));
7037 assert (end == align_on_mark_word (end));
7039 #ifdef BACKGROUND_GC
7040 uint8_t* current_lowest_address = background_saved_lowest_address;
7041 uint8_t* current_highest_address = background_saved_highest_address;
7043 uint8_t* current_lowest_address = lowest_address;
7044 uint8_t* current_highest_address = highest_address;
7045 #endif //BACKGROUND_GC
7047 //there is a possibility of the addresses to be
7048 //outside of the covered range because of a newly allocated
7049 //large object segment
7050 if ((end <= current_highest_address) && (from >= current_lowest_address))
7052 size_t beg_word = mark_word_of (align_on_mark_word (from));
7053 MAYBE_UNUSED_VAR(beg_word);
7054 //align end word to make sure to cover the address
7055 size_t end_word = mark_word_of (align_on_mark_word (end));
7056 MAYBE_UNUSED_VAR(end_word);
7057 dprintf (3, ("Calling clearing mark array [%Ix, %Ix[ for addresses [%Ix, %Ix[(%s)",
7058 (size_t)mark_word_address (beg_word),
7059 (size_t)mark_word_address (end_word),
7060 (size_t)from, (size_t)end,
7061 (check_only ? "check_only" : "clear")));
7065 while (op < mark_word_address (beg_word))
7067 mark_array_clear_marked (op);
7068 op += mark_bit_pitch;
7071 memset (&mark_array[beg_word], 0, (end_word - beg_word)*sizeof (uint32_t));
7076 //Beware, it is assumed that the mark array word straddling
7077 //start has been cleared before
7078 //verify that the array is empty.
7079 size_t markw = mark_word_of (align_on_mark_word (from));
7080 size_t markw_end = mark_word_of (align_on_mark_word (end));
7081 while (markw < markw_end)
7083 assert (!(mark_array [markw]));
7086 uint8_t* p = mark_word_address (markw_end);
7089 assert (!(mark_array_marked (p)));
7098 //These work on untranslated card tables
7100 uint32_t*& card_table_next (uint32_t* c_table)
7102 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->next_card_table;
7106 size_t& card_table_size (uint32_t* c_table)
7108 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->size;
7111 void own_card_table (uint32_t* c_table)
7113 card_table_refcount (c_table) += 1;
7116 void destroy_card_table (uint32_t* c_table);
7118 void delete_next_card_table (uint32_t* c_table)
7120 uint32_t* n_table = card_table_next (c_table);
7123 if (card_table_next (n_table))
7125 delete_next_card_table (n_table);
7127 if (card_table_refcount (n_table) == 0)
7129 destroy_card_table (n_table);
7130 card_table_next (c_table) = 0;
7135 void release_card_table (uint32_t* c_table)
7137 assert (card_table_refcount (c_table) >0);
7138 card_table_refcount (c_table) -= 1;
7139 if (card_table_refcount (c_table) == 0)
7141 delete_next_card_table (c_table);
7142 if (card_table_next (c_table) == 0)
7144 destroy_card_table (c_table);
7145 // sever the link from the parent
7146 if (&g_gc_card_table[card_word (gcard_of(g_gc_lowest_address))] == c_table)
7148 g_gc_card_table = 0;
7150 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7151 g_gc_card_bundle_table = 0;
7153 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7154 SoftwareWriteWatch::StaticClose();
7155 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7159 uint32_t* p_table = &g_gc_card_table[card_word (gcard_of(g_gc_lowest_address))];
7162 while (p_table && (card_table_next (p_table) != c_table))
7163 p_table = card_table_next (p_table);
7164 card_table_next (p_table) = 0;
7171 void destroy_card_table (uint32_t* c_table)
7173 // delete (uint32_t*)&card_table_refcount(c_table);
7175 GCToOSInterface::VirtualRelease (&card_table_refcount(c_table), card_table_size(c_table));
7176 dprintf (2, ("Table Virtual Free : %Ix", (size_t)&card_table_refcount(c_table)));
7179 uint32_t* gc_heap::make_card_table (uint8_t* start, uint8_t* end)
7181 assert (g_gc_lowest_address == start);
7182 assert (g_gc_highest_address == end);
7184 uint32_t virtual_reserve_flags = VirtualReserveFlags::None;
7186 size_t bs = size_brick_of (start, end);
7187 size_t cs = size_card_of (start, end);
7189 size_t ms = (gc_can_use_concurrent ?
7190 size_mark_array_of (start, end) :
7199 if (can_use_write_watch_for_card_table())
7201 cb = size_card_bundle_of (g_gc_lowest_address, g_gc_highest_address);
7202 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7203 // If we're not manually managing the card bundles, we will need to use OS write
7204 // watch APIs over this region to track changes.
7205 virtual_reserve_flags |= VirtualReserveFlags::WriteWatch;
7208 #endif //CARD_BUNDLE
7211 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7212 size_t sw_ww_table_offset = 0;
7213 if (gc_can_use_concurrent)
7215 size_t sw_ww_size_before_table = sizeof(card_table_info) + cs + bs + cb;
7216 sw_ww_table_offset = SoftwareWriteWatch::GetTableStartByteOffset(sw_ww_size_before_table);
7217 wws = sw_ww_table_offset - sw_ww_size_before_table + SoftwareWriteWatch::GetTableByteSize(start, end);
7219 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7221 #ifdef GROWABLE_SEG_MAPPING_TABLE
7222 size_t st = size_seg_mapping_table_of (g_gc_lowest_address, g_gc_highest_address);
7223 size_t st_table_offset = sizeof(card_table_info) + cs + bs + cb + wws;
7224 size_t st_table_offset_aligned = align_for_seg_mapping_table (st_table_offset);
7226 st += (st_table_offset_aligned - st_table_offset);
7227 #else //GROWABLE_SEG_MAPPING_TABLE
7229 #endif //GROWABLE_SEG_MAPPING_TABLE
7231 // it is impossible for alloc_size to overflow due bounds on each of
7233 size_t alloc_size = sizeof (uint8_t)*(sizeof(card_table_info) + cs + bs + cb + wws + st + ms);
7234 uint8_t* mem = (uint8_t*)GCToOSInterface::VirtualReserve (alloc_size, 0, virtual_reserve_flags);
7239 dprintf (2, ("Init - Card table alloc for %Id bytes: [%Ix, %Ix[",
7240 alloc_size, (size_t)mem, (size_t)(mem+alloc_size)));
7242 // mark array will be committed separately (per segment).
7243 size_t commit_size = alloc_size - ms;
7245 if (!virtual_commit (mem, commit_size))
7247 dprintf (1, ("Card table commit failed"));
7248 GCToOSInterface::VirtualRelease (mem, alloc_size);
7252 // initialize the ref count
7253 uint32_t* ct = (uint32_t*)(mem+sizeof (card_table_info));
7254 card_table_refcount (ct) = 0;
7255 card_table_lowest_address (ct) = start;
7256 card_table_highest_address (ct) = end;
7257 card_table_brick_table (ct) = (short*)((uint8_t*)ct + cs);
7258 card_table_size (ct) = alloc_size;
7259 card_table_next (ct) = 0;
7262 card_table_card_bundle_table (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs);
7264 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7265 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), g_gc_lowest_address);
7268 #endif //CARD_BUNDLE
7270 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7271 if (gc_can_use_concurrent)
7273 SoftwareWriteWatch::InitializeUntranslatedTable(mem + sw_ww_table_offset, start);
7275 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7277 #ifdef GROWABLE_SEG_MAPPING_TABLE
7278 seg_mapping_table = (seg_mapping*)(mem + st_table_offset_aligned);
7279 seg_mapping_table = (seg_mapping*)((uint8_t*)seg_mapping_table -
7280 size_seg_mapping_table_of (0, (align_lower_segment (g_gc_lowest_address))));
7281 #endif //GROWABLE_SEG_MAPPING_TABLE
7284 if (gc_can_use_concurrent)
7285 card_table_mark_array (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs + cb + wws + st);
7287 card_table_mark_array (ct) = NULL;
7290 return translate_card_table(ct);
7293 void gc_heap::set_fgm_result (failure_get_memory f, size_t s, BOOL loh_p)
7295 #ifdef MULTIPLE_HEAPS
7296 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
7298 gc_heap* hp = gc_heap::g_heaps [hn];
7299 hp->fgm_result.set_fgm (f, s, loh_p);
7301 #else //MULTIPLE_HEAPS
7302 fgm_result.set_fgm (f, s, loh_p);
7303 #endif //MULTIPLE_HEAPS
7306 //returns 0 for success, -1 otherwise
7307 // We are doing all the decommitting here because we want to make sure we have
7308 // enough memory to do so - if we do this during copy_brick_card_table and
7309 // and fail to decommit it would make the failure case very complicated to
7310 // handle. This way we can waste some decommit if we call this multiple
7311 // times before the next FGC but it's easier to handle the failure case.
7312 int gc_heap::grow_brick_card_tables (uint8_t* start,
7315 heap_segment* new_seg,
7319 uint8_t* la = g_gc_lowest_address;
7320 uint8_t* ha = g_gc_highest_address;
7321 uint8_t* saved_g_lowest_address = min (start, g_gc_lowest_address);
7322 uint8_t* saved_g_highest_address = max (end, g_gc_highest_address);
7323 seg_mapping* new_seg_mapping_table = nullptr;
7324 #ifdef BACKGROUND_GC
7325 // This value is only for logging purpose - it's not necessarily exactly what we
7326 // would commit for mark array but close enough for diagnostics purpose.
7327 size_t logging_ma_commit_size = size_mark_array_of (0, (uint8_t*)size);
7328 #endif //BACKGROUND_GC
7330 // See if the address is already covered
7331 if ((la != saved_g_lowest_address ) || (ha != saved_g_highest_address))
7334 //modify the higest address so the span covered
7335 //is twice the previous one.
7336 uint8_t* top = (uint8_t*)0 + Align (GCToOSInterface::GetVirtualMemoryLimit());
7337 // On non-Windows systems, we get only an approximate value that can possibly be
7338 // slightly lower than the saved_g_highest_address.
7339 // In such case, we set the top to the saved_g_highest_address so that the
7340 // card and brick tables always cover the whole new range.
7341 if (top < saved_g_highest_address)
7343 top = saved_g_highest_address;
7347 if (ps > (uint64_t)200*1024*1024*1024)
7348 ps += (uint64_t)100*1024*1024*1024;
7353 if (saved_g_lowest_address < g_gc_lowest_address)
7355 if (ps > (size_t)g_gc_lowest_address)
7356 saved_g_lowest_address = (uint8_t*)(size_t)OS_PAGE_SIZE;
7359 assert (((size_t)g_gc_lowest_address - ps) >= OS_PAGE_SIZE);
7360 saved_g_lowest_address = min (saved_g_lowest_address, (g_gc_lowest_address - ps));
7364 if (saved_g_highest_address > g_gc_highest_address)
7366 saved_g_highest_address = max ((saved_g_lowest_address + ps), saved_g_highest_address);
7367 if (saved_g_highest_address > top)
7368 saved_g_highest_address = top;
7371 dprintf (GC_TABLE_LOG, ("Growing card table [%Ix, %Ix[",
7372 (size_t)saved_g_lowest_address,
7373 (size_t)saved_g_highest_address));
7375 bool write_barrier_updated = false;
7376 uint32_t virtual_reserve_flags = VirtualReserveFlags::None;
7377 uint32_t* saved_g_card_table = g_gc_card_table;
7379 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7380 uint32_t* saved_g_card_bundle_table = g_gc_card_bundle_table;
7384 uint32_t* translated_ct = 0;
7387 size_t cs = size_card_of (saved_g_lowest_address, saved_g_highest_address);
7388 size_t bs = size_brick_of (saved_g_lowest_address, saved_g_highest_address);
7391 size_t ms = (gc_heap::gc_can_use_concurrent ?
7392 size_mark_array_of (saved_g_lowest_address, saved_g_highest_address) :
7401 if (can_use_write_watch_for_card_table())
7403 cb = size_card_bundle_of (saved_g_lowest_address, saved_g_highest_address);
7405 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7406 // If we're not manually managing the card bundles, we will need to use OS write
7407 // watch APIs over this region to track changes.
7408 virtual_reserve_flags |= VirtualReserveFlags::WriteWatch;
7411 #endif //CARD_BUNDLE
7414 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7415 size_t sw_ww_table_offset = 0;
7416 if (gc_can_use_concurrent)
7418 size_t sw_ww_size_before_table = sizeof(card_table_info) + cs + bs + cb;
7419 sw_ww_table_offset = SoftwareWriteWatch::GetTableStartByteOffset(sw_ww_size_before_table);
7421 sw_ww_table_offset -
7422 sw_ww_size_before_table +
7423 SoftwareWriteWatch::GetTableByteSize(saved_g_lowest_address, saved_g_highest_address);
7425 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7427 #ifdef GROWABLE_SEG_MAPPING_TABLE
7428 size_t st = size_seg_mapping_table_of (saved_g_lowest_address, saved_g_highest_address);
7429 size_t st_table_offset = sizeof(card_table_info) + cs + bs + cb + wws;
7430 size_t st_table_offset_aligned = align_for_seg_mapping_table (st_table_offset);
7431 st += (st_table_offset_aligned - st_table_offset);
7432 #else //GROWABLE_SEG_MAPPING_TABLE
7434 #endif //GROWABLE_SEG_MAPPING_TABLE
7436 // it is impossible for alloc_size to overflow due bounds on each of
7438 size_t alloc_size = sizeof (uint8_t)*(sizeof(card_table_info) + cs + bs + cb + wws + st + ms);
7439 dprintf (GC_TABLE_LOG, ("card table: %Id; brick table: %Id; card bundle: %Id; sw ww table: %Id; seg table: %Id; mark array: %Id",
7440 cs, bs, cb, wws, st, ms));
7442 uint8_t* mem = (uint8_t*)GCToOSInterface::VirtualReserve (alloc_size, 0, virtual_reserve_flags);
7446 set_fgm_result (fgm_grow_table, alloc_size, loh_p);
7450 dprintf (GC_TABLE_LOG, ("Table alloc for %Id bytes: [%Ix, %Ix[",
7451 alloc_size, (size_t)mem, (size_t)((uint8_t*)mem+alloc_size)));
7454 // mark array will be committed separately (per segment).
7455 size_t commit_size = alloc_size - ms;
7457 if (!virtual_commit (mem, commit_size))
7459 dprintf (GC_TABLE_LOG, ("Table commit failed"));
7460 set_fgm_result (fgm_commit_table, commit_size, loh_p);
7465 ct = (uint32_t*)(mem + sizeof (card_table_info));
7466 card_table_refcount (ct) = 0;
7467 card_table_lowest_address (ct) = saved_g_lowest_address;
7468 card_table_highest_address (ct) = saved_g_highest_address;
7469 card_table_next (ct) = &g_gc_card_table[card_word (gcard_of (la))];
7471 //clear the card table
7473 memclr ((uint8_t*)ct,
7474 (((saved_g_highest_address - saved_g_lowest_address)*sizeof (uint32_t) /
7475 (card_size * card_word_width))
7476 + sizeof (uint32_t)));
7479 bt = (short*)((uint8_t*)ct + cs);
7481 // No initialization needed, will be done in copy_brick_card
7483 card_table_brick_table (ct) = bt;
7486 card_table_card_bundle_table (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs);
7487 //set all bundle to look at all of the cards
7488 memset(card_table_card_bundle_table (ct), 0xFF, cb);
7489 #endif //CARD_BUNDLE
7491 #ifdef GROWABLE_SEG_MAPPING_TABLE
7493 new_seg_mapping_table = (seg_mapping*)(mem + st_table_offset_aligned);
7494 new_seg_mapping_table = (seg_mapping*)((uint8_t*)new_seg_mapping_table -
7495 size_seg_mapping_table_of (0, (align_lower_segment (saved_g_lowest_address))));
7496 memcpy(&new_seg_mapping_table[seg_mapping_word_of(g_gc_lowest_address)],
7497 &seg_mapping_table[seg_mapping_word_of(g_gc_lowest_address)],
7498 size_seg_mapping_table_of(g_gc_lowest_address, g_gc_highest_address));
7500 // new_seg_mapping_table gets assigned to seg_mapping_table at the bottom of this function,
7501 // not here. The reason for this is that, if we fail at mark array committing (OOM) and we've
7502 // already switched seg_mapping_table to point to the new mapping table, we'll decommit it and
7503 // run into trouble. By not assigning here, we're making sure that we will not change seg_mapping_table
7504 // if an OOM occurs.
7506 #endif //GROWABLE_SEG_MAPPING_TABLE
7509 if(gc_can_use_concurrent)
7510 card_table_mark_array (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs + cb + wws + st);
7512 card_table_mark_array (ct) = NULL;
7515 translated_ct = translate_card_table (ct);
7517 dprintf (GC_TABLE_LOG, ("card table: %Ix(translated: %Ix), seg map: %Ix, mark array: %Ix",
7518 (size_t)ct, (size_t)translated_ct, (size_t)new_seg_mapping_table, (size_t)card_table_mark_array (ct)));
7520 #ifdef BACKGROUND_GC
7521 if (hp->should_commit_mark_array())
7523 dprintf (GC_TABLE_LOG, ("new low: %Ix, new high: %Ix, latest mark array is %Ix(translate: %Ix)",
7524 saved_g_lowest_address, saved_g_highest_address,
7525 card_table_mark_array (ct),
7526 translate_mark_array (card_table_mark_array (ct))));
7527 uint32_t* new_mark_array = (uint32_t*)((uint8_t*)card_table_mark_array (ct) - size_mark_array_of (0, saved_g_lowest_address));
7528 if (!commit_new_mark_array_global (new_mark_array))
7530 dprintf (GC_TABLE_LOG, ("failed to commit portions in the mark array for existing segments"));
7531 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7535 if (!commit_mark_array_new_seg (hp, new_seg, translated_ct, saved_g_lowest_address))
7537 dprintf (GC_TABLE_LOG, ("failed to commit mark array for the new seg"));
7538 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7544 clear_commit_flag_global();
7546 #endif //BACKGROUND_GC
7548 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7549 if (gc_can_use_concurrent)
7551 // The current design of software write watch requires that the runtime is suspended during resize. Suspending
7552 // on resize is preferred because it is a far less frequent operation than GetWriteWatch() / ResetWriteWatch().
7553 // Suspending here allows copying dirty state from the old table into the new table, and not have to merge old
7554 // table info lazily as done for card tables.
7556 // Either this thread was the thread that did the suspension which means we are suspended; or this is called
7557 // from a GC thread which means we are in a blocking GC and also suspended.
7558 bool is_runtime_suspended = GCToEEInterface::IsGCThread();
7559 if (!is_runtime_suspended)
7561 // Note on points where the runtime is suspended anywhere in this function. Upon an attempt to suspend the
7562 // runtime, a different thread may suspend first, causing this thread to block at the point of the suspend call.
7563 // So, at any suspend point, externally visible state needs to be consistent, as code that depends on that state
7564 // may run while this thread is blocked. This includes updates to g_gc_card_table, g_gc_lowest_address, and
7565 // g_gc_highest_address.
7569 g_gc_card_table = translated_ct;
7571 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7572 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), saved_g_lowest_address);
7575 SoftwareWriteWatch::SetResizedUntranslatedTable(
7576 mem + sw_ww_table_offset,
7577 saved_g_lowest_address,
7578 saved_g_highest_address);
7580 seg_mapping_table = new_seg_mapping_table;
7582 // Since the runtime is already suspended, update the write barrier here as well.
7583 // This passes a bool telling whether we need to switch to the post
7584 // grow version of the write barrier. This test tells us if the new
7585 // segment was allocated at a lower address than the old, requiring
7586 // that we start doing an upper bounds check in the write barrier.
7587 g_gc_lowest_address = saved_g_lowest_address;
7588 g_gc_highest_address = saved_g_highest_address;
7589 stomp_write_barrier_resize(true, la != saved_g_lowest_address);
7590 write_barrier_updated = true;
7592 if (!is_runtime_suspended)
7598 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7600 g_gc_card_table = translated_ct;
7602 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7603 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), saved_g_lowest_address);
7607 if (!write_barrier_updated)
7609 seg_mapping_table = new_seg_mapping_table;
7610 GCToOSInterface::FlushProcessWriteBuffers();
7611 g_gc_lowest_address = saved_g_lowest_address;
7612 g_gc_highest_address = saved_g_highest_address;
7614 // This passes a bool telling whether we need to switch to the post
7615 // grow version of the write barrier. This test tells us if the new
7616 // segment was allocated at a lower address than the old, requiring
7617 // that we start doing an upper bounds check in the write barrier.
7618 // This will also suspend the runtime if the write barrier type needs
7619 // to be changed, so we are doing this after all global state has
7620 // been updated. See the comment above suspend_EE() above for more
7622 stomp_write_barrier_resize(GCToEEInterface::IsGCThread(), la != saved_g_lowest_address);
7628 //cleanup mess and return -1;
7632 assert(g_gc_card_table == saved_g_card_table);
7634 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7635 assert(g_gc_card_bundle_table == saved_g_card_bundle_table);
7638 //delete (uint32_t*)((uint8_t*)ct - sizeof(card_table_info));
7639 if (!GCToOSInterface::VirtualRelease (mem, alloc_size))
7641 dprintf (GC_TABLE_LOG, ("GCToOSInterface::VirtualRelease failed"));
7642 assert (!"release failed");
7650 #ifdef BACKGROUND_GC
7651 if (hp->should_commit_mark_array())
7653 dprintf (GC_TABLE_LOG, ("in range new seg %Ix, mark_array is %Ix", new_seg, hp->mark_array));
7654 if (!commit_mark_array_new_seg (hp, new_seg))
7656 dprintf (GC_TABLE_LOG, ("failed to commit mark array for the new seg in range"));
7657 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7661 #endif //BACKGROUND_GC
7667 //copy all of the arrays managed by the card table for a page aligned range
7668 void gc_heap::copy_brick_card_range (uint8_t* la, uint32_t* old_card_table,
7669 short* old_brick_table,
7671 uint8_t* start, uint8_t* end)
7673 ptrdiff_t brick_offset = brick_of (start) - brick_of (la);
7676 dprintf (2, ("copying tables for range [%Ix %Ix[", (size_t)start, (size_t)end));
7679 short* brick_start = &brick_table [brick_of (start)];
7680 if (old_brick_table)
7682 // segments are always on page boundaries
7683 memcpy (brick_start, &old_brick_table[brick_offset],
7684 size_brick_of (start, end));
7689 // This is a large heap, just clear the brick table
7692 uint32_t* old_ct = &old_card_table[card_word (card_of (la))];
7694 #ifdef BACKGROUND_GC
7695 UNREFERENCED_PARAMETER(seg);
7696 if (recursive_gc_sync::background_running_p())
7698 uint32_t* old_mark_array = card_table_mark_array (old_ct);
7700 // We don't need to go through all the card tables here because
7701 // we only need to copy from the GC version of the mark array - when we
7702 // mark (even in allocate_large_object) we always use that mark array.
7703 if ((card_table_highest_address (old_ct) >= start) &&
7704 (card_table_lowest_address (old_ct) <= end))
7706 if ((background_saved_highest_address >= start) &&
7707 (background_saved_lowest_address <= end))
7709 //copy the mark bits
7710 // segments are always on page boundaries
7711 uint8_t* m_start = max (background_saved_lowest_address, start);
7712 uint8_t* m_end = min (background_saved_highest_address, end);
7713 memcpy (&mark_array[mark_word_of (m_start)],
7714 &old_mark_array[mark_word_of (m_start) - mark_word_of (la)],
7715 size_mark_array_of (m_start, m_end));
7720 //only large segments can be out of range
7721 assert (old_brick_table == 0);
7724 #else //BACKGROUND_GC
7726 clear_mark_array (start, heap_segment_committed(seg));
7727 #endif //BACKGROUND_GC
7730 // n way merge with all of the card table ever used in between
7731 uint32_t* ct = card_table_next (&card_table[card_word (card_of(lowest_address))]);
7734 while (card_table_next (old_ct) != ct)
7736 //copy if old card table contained [start, end[
7737 if ((card_table_highest_address (ct) >= end) &&
7738 (card_table_lowest_address (ct) <= start))
7740 // or the card_tables
7742 size_t start_word = card_word (card_of (start));
7744 uint32_t* dest = &card_table[start_word];
7745 uint32_t* src = &((translate_card_table (ct))[start_word]);
7746 ptrdiff_t count = count_card_of (start, end);
7747 for (int x = 0; x < count; x++)
7751 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7754 card_bundle_set(cardw_card_bundle(start_word+x));
7762 ct = card_table_next (ct);
7766 //initialize all of the arrays managed by the card table for a page aligned range when an existing ro segment becomes in range
7767 void gc_heap::init_brick_card_range (heap_segment* seg)
7769 dprintf (2, ("initialising tables for range [%Ix %Ix[",
7770 (size_t)heap_segment_mem (seg),
7771 (size_t)heap_segment_allocated (seg)));
7773 // initialize the brick table
7774 for (size_t b = brick_of (heap_segment_mem (seg));
7775 b < brick_of (align_on_brick (heap_segment_allocated (seg)));
7782 if (recursive_gc_sync::background_running_p() && (seg->flags & heap_segment_flags_ma_committed))
7785 clear_mark_array (heap_segment_mem (seg), heap_segment_committed(seg));
7789 clear_card_for_addresses (heap_segment_mem (seg),
7790 heap_segment_allocated (seg));
7793 void gc_heap::copy_brick_card_table()
7795 uint8_t* la = lowest_address;
7796 uint8_t* ha = highest_address;
7797 MAYBE_UNUSED_VAR(ha);
7798 uint32_t* old_card_table = card_table;
7799 short* old_brick_table = brick_table;
7801 assert (la == card_table_lowest_address (&old_card_table[card_word (card_of (la))]));
7802 assert (ha == card_table_highest_address (&old_card_table[card_word (card_of (la))]));
7804 /* todo: Need a global lock for this */
7805 uint32_t* ct = &g_gc_card_table[card_word (gcard_of (g_gc_lowest_address))];
7806 own_card_table (ct);
7807 card_table = translate_card_table (ct);
7808 /* End of global lock */
7809 highest_address = card_table_highest_address (ct);
7810 lowest_address = card_table_lowest_address (ct);
7812 brick_table = card_table_brick_table (ct);
7815 if (gc_can_use_concurrent)
7817 mark_array = translate_mark_array (card_table_mark_array (ct));
7818 assert (mark_word_of (g_gc_highest_address) ==
7819 mark_word_of (align_on_mark_word (g_gc_highest_address)));
7826 #if defined(MARK_ARRAY) && defined(_DEBUG)
7827 size_t cb_end = (size_t)((uint8_t*)card_table_card_bundle_table (ct) + size_card_bundle_of (g_gc_lowest_address, g_gc_highest_address));
7828 #ifdef GROWABLE_SEG_MAPPING_TABLE
7829 size_t st = size_seg_mapping_table_of (g_gc_lowest_address, g_gc_highest_address);
7830 size_t cb_end_aligned = align_for_seg_mapping_table (cb_end);
7831 st += (cb_end_aligned - cb_end);
7832 #else //GROWABLE_SEG_MAPPING_TABLE
7834 #endif //GROWABLE_SEG_MAPPING_TABLE
7835 #endif //MARK_ARRAY && _DEBUG
7836 card_bundle_table = translate_card_bundle_table (card_table_card_bundle_table (ct), g_gc_lowest_address);
7838 // Ensure that the word that represents g_gc_lowest_address in the translated table is located at the
7839 // start of the untranslated table.
7840 assert (&card_bundle_table [card_bundle_word (cardw_card_bundle (card_word (card_of (g_gc_lowest_address))))] ==
7841 card_table_card_bundle_table (ct));
7843 //set the card table if we are in a heap growth scenario
7844 if (card_bundles_enabled())
7846 card_bundles_set (cardw_card_bundle (card_word (card_of (lowest_address))),
7847 cardw_card_bundle (align_cardw_on_bundle (card_word (card_of (highest_address)))));
7849 //check if we need to turn on card_bundles.
7850 #ifdef MULTIPLE_HEAPS
7851 // use INT64 arithmetic here because of possible overflow on 32p
7852 uint64_t th = (uint64_t)MH_TH_CARD_BUNDLE*gc_heap::n_heaps;
7854 // use INT64 arithmetic here because of possible overflow on 32p
7855 uint64_t th = (uint64_t)SH_TH_CARD_BUNDLE;
7856 #endif //MULTIPLE_HEAPS
7857 if (reserved_memory >= th)
7859 enable_card_bundles();
7862 #endif //CARD_BUNDLE
7864 // for each of the segments and heaps, copy the brick table and
7865 // or the card table
7866 heap_segment* seg = generation_start_segment (generation_of (max_generation));
7869 if (heap_segment_read_only_p (seg) && !heap_segment_in_range_p (seg))
7871 //check if it became in range
7872 if ((heap_segment_reserved (seg) > lowest_address) &&
7873 (heap_segment_mem (seg) < highest_address))
7875 set_ro_segment_in_range (seg);
7881 uint8_t* end = align_on_page (heap_segment_allocated (seg));
7882 copy_brick_card_range (la, old_card_table,
7885 align_lower_page (heap_segment_mem (seg)),
7888 seg = heap_segment_next (seg);
7891 seg = generation_start_segment (large_object_generation);
7894 if (heap_segment_read_only_p (seg) && !heap_segment_in_range_p (seg))
7896 //check if it became in range
7897 if ((heap_segment_reserved (seg) > lowest_address) &&
7898 (heap_segment_mem (seg) < highest_address))
7900 set_ro_segment_in_range (seg);
7905 uint8_t* end = align_on_page (heap_segment_allocated (seg));
7906 copy_brick_card_range (la, old_card_table,
7909 align_lower_page (heap_segment_mem (seg)),
7912 seg = heap_segment_next (seg);
7915 release_card_table (&old_card_table[card_word (card_of(la))]);
7918 #ifdef FEATURE_BASICFREEZE
7919 BOOL gc_heap::insert_ro_segment (heap_segment* seg)
7921 enter_spin_lock (&gc_heap::gc_lock);
7923 if (!gc_heap::seg_table->ensure_space_for_insert ()
7924 || (should_commit_mark_array() && !commit_mark_array_new_seg(__this, seg)))
7926 leave_spin_lock(&gc_heap::gc_lock);
7930 //insert at the head of the segment list
7931 generation* gen2 = generation_of (max_generation);
7932 heap_segment* oldhead = generation_start_segment (gen2);
7933 heap_segment_next (seg) = oldhead;
7934 generation_start_segment (gen2) = seg;
7936 seg_table->insert (heap_segment_mem(seg), (size_t)seg);
7938 #ifdef SEG_MAPPING_TABLE
7939 seg_mapping_table_add_ro_segment (seg);
7940 #endif //SEG_MAPPING_TABLE
7943 if ((heap_segment_reserved (seg) > lowest_address) &&
7944 (heap_segment_mem (seg) < highest_address))
7946 set_ro_segment_in_range (seg);
7949 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(seg), (size_t)(heap_segment_reserved (seg) - heap_segment_mem(seg)), gc_etw_segment_read_only_heap);
7951 leave_spin_lock (&gc_heap::gc_lock);
7955 // No one is calling this function right now. If this is getting called we need
7956 // to take care of decommitting the mark array for it - we will need to remember
7957 // which portion of the mark array was committed and only decommit that.
7958 void gc_heap::remove_ro_segment (heap_segment* seg)
7960 //clear the mark bits so a new segment allocated in its place will have a clear mark bits
7962 if (gc_can_use_concurrent)
7964 clear_mark_array (align_lower_mark_word (max (heap_segment_mem (seg), lowest_address)),
7965 align_on_card_word (min (heap_segment_allocated (seg), highest_address)),
7966 false); // read_only segments need the mark clear
7970 enter_spin_lock (&gc_heap::gc_lock);
7972 seg_table->remove ((uint8_t*)seg);
7974 #ifdef SEG_MAPPING_TABLE
7975 seg_mapping_table_remove_ro_segment (seg);
7976 #endif //SEG_MAPPING_TABLE
7978 // Locate segment (and previous segment) in the list.
7979 generation* gen2 = generation_of (max_generation);
7980 heap_segment* curr_seg = generation_start_segment (gen2);
7981 heap_segment* prev_seg = NULL;
7983 while (curr_seg && curr_seg != seg)
7985 prev_seg = curr_seg;
7986 curr_seg = heap_segment_next (curr_seg);
7988 assert (curr_seg == seg);
7990 // Patch previous segment (or list head if there is none) to skip the removed segment.
7992 heap_segment_next (prev_seg) = heap_segment_next (curr_seg);
7994 generation_start_segment (gen2) = heap_segment_next (curr_seg);
7996 leave_spin_lock (&gc_heap::gc_lock);
7998 #endif //FEATURE_BASICFREEZE
8000 BOOL gc_heap::set_ro_segment_in_range (heap_segment* seg)
8003 seg->flags |= heap_segment_flags_inrange;
8004 // init_brick_card_range (seg);
8005 ro_segments_in_range = TRUE;
8006 //right now, segments aren't protected
8007 //unprotect_segment (seg);
8013 uint8_t** make_mark_list (size_t size)
8015 uint8_t** mark_list = new (nothrow) uint8_t* [size];
8019 #define swap(a,b){uint8_t* t; t = a; a = b; b = t;}
8021 void verify_qsort_array (uint8_t* *low, uint8_t* *high)
8025 for (i = low+1; i <= high; i++)
8034 #ifndef USE_INTROSORT
8035 void qsort1( uint8_t* *low, uint8_t* *high, unsigned int depth)
8037 if (((low + 16) >= high) || (depth > 100))
8041 for (i = low+1; i <= high; i++)
8044 for (j=i;j >low && val<*(j-1);j--)
8053 uint8_t *pivot, **left, **right;
8055 //sort low middle and high
8056 if (*(low+((high-low)/2)) < *low)
8057 swap (*(low+((high-low)/2)), *low);
8060 if (*high < *(low+((high-low)/2)))
8061 swap (*(low+((high-low)/2)), *high);
8063 swap (*(low+((high-low)/2)), *(high-1));
8065 left = low; right = high-1;
8067 while (*(--right) > pivot);
8068 while (*(++left) < pivot);
8071 swap(*left, *right);
8076 swap (*left, *(high-1));
8077 qsort1(low, left-1, depth+1);
8078 qsort1(left+1, high, depth+1);
8081 #endif //USE_INTROSORT
8082 void rqsort1( uint8_t* *low, uint8_t* *high)
8084 if ((low + 16) >= high)
8088 for (i = low+1; i <= high; i++)
8091 for (j=i;j >low && val>*(j-1);j--)
8100 uint8_t *pivot, **left, **right;
8102 //sort low middle and high
8103 if (*(low+((high-low)/2)) > *low)
8104 swap (*(low+((high-low)/2)), *low);
8107 if (*high > *(low+((high-low)/2)))
8108 swap (*(low+((high-low)/2)), *high);
8110 swap (*(low+((high-low)/2)), *(high-1));
8112 left = low; right = high-1;
8114 while (*(--right) < pivot);
8115 while (*(++left) > pivot);
8118 swap(*left, *right);
8123 swap (*left, *(high-1));
8124 rqsort1(low, left-1);
8125 rqsort1(left+1, high);
8129 #ifdef USE_INTROSORT
8134 static const int size_threshold = 64;
8135 static const int max_depth = 100;
8138 inline static void swap_elements(uint8_t** i,uint8_t** j)
8146 static void sort (uint8_t** begin, uint8_t** end, int ignored)
8149 introsort_loop (begin, end, max_depth);
8150 insertionsort (begin, end);
8155 static void introsort_loop (uint8_t** lo, uint8_t** hi, int depth_limit)
8157 while (hi-lo >= size_threshold)
8159 if (depth_limit == 0)
8164 uint8_t** p=median_partition (lo, hi);
8165 depth_limit=depth_limit-1;
8166 introsort_loop (p, hi, depth_limit);
8171 static uint8_t** median_partition (uint8_t** low, uint8_t** high)
8173 uint8_t *pivot, **left, **right;
8175 //sort low middle and high
8176 if (*(low+((high-low)/2)) < *low)
8177 swap_elements ((low+((high-low)/2)), low);
8179 swap_elements (low, high);
8180 if (*high < *(low+((high-low)/2)))
8181 swap_elements ((low+((high-low)/2)), high);
8183 swap_elements ((low+((high-low)/2)), (high-1));
8185 left = low; right = high-1;
8187 while (*(--right) > pivot);
8188 while (*(++left) < pivot);
8191 swap_elements(left, right);
8196 swap_elements (left, (high-1));
8201 static void insertionsort (uint8_t** lo, uint8_t** hi)
8203 for (uint8_t** i=lo+1; i <= hi; i++)
8207 while((j > lo) && (t <*(j-1)))
8216 static void heapsort (uint8_t** lo, uint8_t** hi)
8218 size_t n = hi - lo + 1;
8219 for (size_t i=n / 2; i >= 1; i--)
8223 for (size_t i = n; i > 1; i--)
8225 swap_elements (lo, lo + i - 1);
8226 downheap(1, i - 1, lo);
8230 static void downheap (size_t i, size_t n, uint8_t** lo)
8232 uint8_t* d = *(lo + i - 1);
8237 if (child < n && *(lo + child - 1)<(*(lo + child)))
8241 if (!(d<*(lo + child - 1)))
8245 *(lo + i - 1) = *(lo + child - 1);
8253 #endif //USE_INTROSORT
8255 #ifdef MULTIPLE_HEAPS
8256 #ifdef PARALLEL_MARK_LIST_SORT
8257 void gc_heap::sort_mark_list()
8259 // if this heap had a mark list overflow, we don't do anything
8260 if (mark_list_index > mark_list_end)
8262 // printf("sort_mark_list: overflow on heap %d\n", heap_number);
8266 // if any other heap had a mark list overflow, we fake one too,
8267 // so we don't use an incomplete mark list by mistake
8268 for (int i = 0; i < n_heaps; i++)
8270 if (g_heaps[i]->mark_list_index > g_heaps[i]->mark_list_end)
8272 mark_list_index = mark_list_end + 1;
8273 // printf("sort_mark_list: overflow on heap %d\n", i);
8278 // unsigned long start = GetCycleCount32();
8280 dprintf (3, ("Sorting mark lists"));
8281 if (mark_list_index > mark_list)
8282 _sort (mark_list, mark_list_index - 1, 0);
8284 // printf("first phase of sort_mark_list for heap %d took %u cycles to sort %u entries\n", this->heap_number, GetCycleCount32() - start, mark_list_index - mark_list);
8285 // start = GetCycleCount32();
8287 // first set the pieces for all heaps to empty
8289 for (heap_num = 0; heap_num < n_heaps; heap_num++)
8291 mark_list_piece_start[heap_num] = NULL;
8292 mark_list_piece_end[heap_num] = NULL;
8295 uint8_t** x = mark_list;
8297 // predicate means: x is still within the mark list, and within the bounds of this heap
8298 #define predicate(x) (((x) < mark_list_index) && (*(x) < heap->ephemeral_high))
8301 while (x < mark_list_index)
8304 // find the heap x points into - searching cyclically from the last heap,
8305 // because in many cases the right heap is the next one or comes soon after
8306 int last_heap_num = heap_num;
8307 MAYBE_UNUSED_VAR(last_heap_num);
8311 if (heap_num >= n_heaps)
8313 assert(heap_num != last_heap_num); // we should always find the heap - infinite loop if not!
8314 heap = g_heaps[heap_num];
8316 while (!(*x >= heap->ephemeral_low && *x < heap->ephemeral_high));
8318 // x is the start of the mark list piece for this heap
8319 mark_list_piece_start[heap_num] = x;
8321 // to find the end of the mark list piece for this heap, find the first x
8322 // that has !predicate(x), i.e. that is either not in this heap, or beyond the end of the list
8325 // let's see if we get lucky and the whole rest belongs to this piece
8326 if (predicate(mark_list_index-1))
8328 x = mark_list_index;
8329 mark_list_piece_end[heap_num] = x;
8333 // we play a variant of binary search to find the point sooner.
8334 // the first loop advances by increasing steps until the predicate turns false.
8335 // then we retreat the last step, and the second loop advances by decreasing steps, keeping the predicate true.
8340 uint8_t** temp_x = x;
8347 while (predicate(x));
8348 // we know that only the last step was wrong, so we undo it
8352 // loop invariant - predicate holds at x, but not x + inc
8353 assert (predicate(x) && !(((x + inc) > x) && predicate(x + inc)));
8355 if (((x + inc) > x) && predicate(x + inc))
8361 // the termination condition and the loop invariant together imply this:
8362 assert(predicate(x) && !predicate(x + inc) && (inc == 1));
8363 // so the spot we're looking for is one further
8366 mark_list_piece_end[heap_num] = x;
8371 // printf("second phase of sort_mark_list for heap %d took %u cycles\n", this->heap_number, GetCycleCount32() - start);
8374 void gc_heap::append_to_mark_list(uint8_t **start, uint8_t **end)
8376 size_t slots_needed = end - start;
8377 size_t slots_available = mark_list_end + 1 - mark_list_index;
8378 size_t slots_to_copy = min(slots_needed, slots_available);
8379 memcpy(mark_list_index, start, slots_to_copy*sizeof(*start));
8380 mark_list_index += slots_to_copy;
8381 // printf("heap %d: appended %Id slots to mark_list\n", heap_number, slots_to_copy);
8384 void gc_heap::merge_mark_lists()
8386 uint8_t** source[MAX_SUPPORTED_CPUS];
8387 uint8_t** source_end[MAX_SUPPORTED_CPUS];
8388 int source_heap[MAX_SUPPORTED_CPUS];
8389 int source_count = 0;
8391 // in case of mark list overflow, don't bother
8392 if (mark_list_index > mark_list_end)
8394 // printf("merge_mark_lists: overflow\n");
8398 dprintf(3, ("merge_mark_lists: heap_number = %d starts out with %Id entries", heap_number, mark_list_index - mark_list));
8399 // unsigned long start = GetCycleCount32();
8400 for (int i = 0; i < n_heaps; i++)
8402 gc_heap* heap = g_heaps[i];
8403 if (heap->mark_list_piece_start[heap_number] < heap->mark_list_piece_end[heap_number])
8405 source[source_count] = heap->mark_list_piece_start[heap_number];
8406 source_end[source_count] = heap->mark_list_piece_end[heap_number];
8407 source_heap[source_count] = i;
8408 if (source_count < MAX_SUPPORTED_CPUS)
8412 // printf("first phase of merge_mark_lists for heap %d took %u cycles\n", heap_number, GetCycleCount32() - start);
8414 dprintf(3, ("heap_number = %d has %d sources\n", heap_number, source_count));
8415 #if defined(_DEBUG) || defined(TRACE_GC)
8416 for (int j = 0; j < source_count; j++)
8418 dprintf(3, ("heap_number = %d ", heap_number));
8419 dprintf(3, (" source from heap %d = %Ix .. %Ix (%Id entries)",
8420 (size_t)(source_heap[j]), (size_t)(source[j][0]), (size_t)(source_end[j][-1]), (size_t)(source_end[j] - source[j])));
8421 // the sources should all be sorted
8422 for (uint8_t **x = source[j]; x < source_end[j] - 1; x++)
8426 dprintf(3, ("oops, mark_list from source %d for heap %d isn't sorted\n", j, heap_number));
8431 #endif //_DEBUG || TRACE_GC
8433 // start = GetCycleCount32();
8435 mark_list = &g_mark_list_copy [heap_number*mark_list_size];
8436 mark_list_index = mark_list;
8437 mark_list_end = &mark_list [mark_list_size-1];
8438 int piece_count = 0;
8439 if (source_count == 0)
8443 else if (source_count == 1)
8445 mark_list = source[0];
8446 mark_list_index = source_end[0];
8447 mark_list_end = mark_list_index;
8452 while (source_count > 1)
8454 // find the lowest and second lowest value in the sources we're merging from
8455 int lowest_source = 0;
8456 uint8_t *lowest = *source[0];
8457 uint8_t *second_lowest = *source[1];
8458 for (int i = 1; i < source_count; i++)
8460 if (lowest > *source[i])
8462 second_lowest = lowest;
8463 lowest = *source[i];
8466 else if (second_lowest > *source[i])
8468 second_lowest = *source[i];
8472 // find the point in the lowest source where it either runs out or is not <= second_lowest anymore
8474 // let's first try to get lucky and see if the whole source is <= second_lowest -- this is actually quite common
8476 if (source_end[lowest_source][-1] <= second_lowest)
8477 x = source_end[lowest_source];
8480 // use linear search to find the end -- could also use binary search as in sort_mark_list,
8481 // but saw no improvement doing that
8482 for (x = source[lowest_source]; x < source_end[lowest_source] && *x <= second_lowest; x++)
8486 // blast this piece to the mark list
8487 append_to_mark_list(source[lowest_source], x);
8490 source[lowest_source] = x;
8492 // check whether this source is now exhausted
8493 if (x >= source_end[lowest_source])
8495 // if it's not the source with the highest index, copy the source with the highest index
8496 // over it so the non-empty sources are always at the beginning
8497 if (lowest_source < source_count-1)
8499 source[lowest_source] = source[source_count-1];
8500 source_end[lowest_source] = source_end[source_count-1];
8505 // we're left with just one source that we copy
8506 append_to_mark_list(source[0], source_end[0]);
8510 // printf("second phase of merge_mark_lists for heap %d took %u cycles to merge %d pieces\n", heap_number, GetCycleCount32() - start, piece_count);
8512 #if defined(_DEBUG) || defined(TRACE_GC)
8513 // the final mark list must be sorted
8514 for (uint8_t **x = mark_list; x < mark_list_index - 1; x++)
8518 dprintf(3, ("oops, mark_list for heap %d isn't sorted at the end of merge_mark_lists", heap_number));
8522 #endif //defined(_DEBUG) || defined(TRACE_GC)
8524 #else //PARALLEL_MARK_LIST_SORT
8525 void gc_heap::combine_mark_lists()
8527 dprintf (3, ("Combining mark lists"));
8528 //verify if a heap has overflowed its mark list
8529 BOOL use_mark_list = TRUE;
8530 for (int i = 0; i < n_heaps; i++)
8532 if (g_heaps [i]->mark_list_index > g_heaps [i]->mark_list_end)
8534 use_mark_list = FALSE;
8541 dprintf (3, ("Using mark list"));
8542 //compact the gaps out of the mark list
8544 uint8_t** current_gap = g_heaps [gn]->mark_list_index;
8545 uint8_t** current_gap_end = g_heaps[gn]->mark_list_end + 1;
8546 uint8_t** dst_last = current_gap-1;
8548 int srcn = n_heaps-1;
8549 gc_heap* srch = g_heaps [srcn];
8550 uint8_t** src = srch->mark_list_index - 1;
8551 uint8_t** src_beg = srch->mark_list;
8553 while (current_gap <= src)
8555 while ((gn < n_heaps-1) && (current_gap >= current_gap_end))
8557 //go to the next gap
8559 dprintf (3, ("Going to the next gap %d", gn));
8560 assert (gn < n_heaps);
8561 current_gap = g_heaps [gn]->mark_list_index;
8562 current_gap_end = g_heaps[gn]->mark_list_end + 1;
8563 assert ((gn == (n_heaps-1)) || (current_gap_end == g_heaps[gn+1]->mark_list));
8565 while ((srcn > 0) && (src < src_beg))
8567 //go to the previous source
8569 dprintf (3, ("going to the previous source %d", srcn));
8571 gc_heap* srch = g_heaps [srcn];
8572 src = srch->mark_list_index - 1;
8573 src_beg = srch->mark_list;
8575 if (current_gap < src)
8577 dst_last = current_gap;
8578 *current_gap++ = *src--;
8581 dprintf (3, ("src: %Ix dst_last: %Ix", (size_t)src, (size_t)dst_last));
8583 uint8_t** end_of_list = max (src, dst_last);
8585 //sort the resulting compacted list
8586 assert (end_of_list < &g_mark_list [n_heaps*mark_list_size]);
8587 if (end_of_list > &g_mark_list[0])
8588 _sort (&g_mark_list[0], end_of_list, 0);
8589 //adjust the mark_list to the begining of the resulting mark list.
8590 for (int i = 0; i < n_heaps; i++)
8592 g_heaps [i]->mark_list = g_mark_list;
8593 g_heaps [i]->mark_list_index = end_of_list + 1;
8594 g_heaps [i]->mark_list_end = end_of_list + 1;
8599 uint8_t** end_of_list = g_mark_list;
8600 //adjust the mark_list to the begining of the resulting mark list.
8601 //put the index beyond the end to turn off mark list processing
8602 for (int i = 0; i < n_heaps; i++)
8604 g_heaps [i]->mark_list = g_mark_list;
8605 g_heaps [i]->mark_list_index = end_of_list + 1;
8606 g_heaps [i]->mark_list_end = end_of_list;
8610 #endif // PARALLEL_MARK_LIST_SORT
8611 #endif //MULTIPLE_HEAPS
8614 class seg_free_spaces
8616 struct seg_free_space
8622 struct free_space_bucket
8624 seg_free_space* free_space;
8625 ptrdiff_t count_add; // Assigned when we first contruct the array.
8626 ptrdiff_t count_fit; // How many items left when we are fitting plugs.
8629 void move_bucket (int old_power2, int new_power2)
8631 // PREFAST warning 22015: old_power2 could be negative
8632 assert (old_power2 >= 0);
8633 assert (old_power2 >= new_power2);
8635 if (old_power2 == new_power2)
8640 seg_free_space* src_index = free_space_buckets[old_power2].free_space;
8641 for (int i = old_power2; i > new_power2; i--)
8643 seg_free_space** dest = &(free_space_buckets[i].free_space);
8646 seg_free_space* dest_index = free_space_buckets[i - 1].free_space;
8647 if (i > (new_power2 + 1))
8649 seg_free_space temp = *src_index;
8650 *src_index = *dest_index;
8653 src_index = dest_index;
8656 free_space_buckets[old_power2].count_fit--;
8657 free_space_buckets[new_power2].count_fit++;
8662 void dump_free_space (seg_free_space* item)
8669 mark* m = (mark*)(item->start);
8670 len = pinned_len (m);
8671 addr = pinned_plug (m) - len;
8675 heap_segment* seg = (heap_segment*)(item->start);
8676 addr = heap_segment_plan_allocated (seg);
8677 len = heap_segment_committed (seg) - addr;
8680 dprintf (SEG_REUSE_LOG_1, ("[%d]0x%Ix %Id", heap_num, addr, len));
8685 seg_free_space* item = NULL;
8688 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------\nnow the free spaces look like:", heap_num));
8689 for (i = 0; i < (free_space_bucket_count - 1); i++)
8691 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces for 2^%d bucket:", heap_num, (base_power2 + i)));
8692 dprintf (SEG_REUSE_LOG_1, ("[%d]%s %s", heap_num, "start", "len"));
8693 item = free_space_buckets[i].free_space;
8694 while (item < free_space_buckets[i + 1].free_space)
8696 dump_free_space (item);
8699 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------", heap_num));
8702 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces for 2^%d bucket:", heap_num, (base_power2 + i)));
8703 dprintf (SEG_REUSE_LOG_1, ("[%d]%s %s", heap_num, "start", "len"));
8704 item = free_space_buckets[i].free_space;
8706 while (item <= &seg_free_space_array[free_space_item_count - 1])
8708 dump_free_space (item);
8711 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------", heap_num));
8716 free_space_bucket* free_space_buckets;
8717 seg_free_space* seg_free_space_array;
8718 ptrdiff_t free_space_bucket_count;
8719 ptrdiff_t free_space_item_count;
8723 BOOL has_end_of_seg;
8728 seg_free_spaces (int h_number)
8730 heap_num = h_number;
8735 size_t total_prealloc_size =
8736 MAX_NUM_BUCKETS * sizeof (free_space_bucket) +
8737 MAX_NUM_FREE_SPACES * sizeof (seg_free_space);
8739 free_space_buckets = (free_space_bucket*) new (nothrow) uint8_t[total_prealloc_size];
8741 return (!!free_space_buckets);
8744 // We take the ordered free space array we got from the 1st pass,
8745 // and feed the portion that we decided to use to this method, ie,
8746 // the largest item_count free spaces.
8747 void add_buckets (int base, size_t* ordered_free_spaces, int bucket_count, size_t item_count)
8749 assert (free_space_buckets);
8750 assert (item_count <= (size_t)MAX_PTR);
8752 free_space_bucket_count = bucket_count;
8753 free_space_item_count = item_count;
8756 has_end_of_seg = FALSE;
8759 ptrdiff_t total_item_count = 0;
8762 seg_free_space_array = (seg_free_space*)(free_space_buckets + free_space_bucket_count);
8764 for (i = 0; i < (ptrdiff_t)item_count; i++)
8766 seg_free_space_array[i].start = 0;
8767 seg_free_space_array[i].is_plug = FALSE;
8770 for (i = 0; i < bucket_count; i++)
8772 free_space_buckets[i].count_add = ordered_free_spaces[i];
8773 free_space_buckets[i].count_fit = ordered_free_spaces[i];
8774 free_space_buckets[i].free_space = &seg_free_space_array[total_item_count];
8775 total_item_count += free_space_buckets[i].count_add;
8778 assert (total_item_count == (ptrdiff_t)item_count);
8781 // If we are adding a free space before a plug we pass the
8782 // mark stack position so we can update the length; we could
8783 // also be adding the free space after the last plug in which
8784 // case start is the segment which we'll need to update the
8785 // heap_segment_plan_allocated.
8786 void add (void* start, BOOL plug_p, BOOL first_p)
8788 size_t size = (plug_p ?
8789 pinned_len ((mark*)start) :
8790 (heap_segment_committed ((heap_segment*)start) -
8791 heap_segment_plan_allocated ((heap_segment*)start)));
8795 dprintf (SEG_REUSE_LOG_1, ("[%d]Adding a free space before plug: %Id", heap_num, size));
8799 dprintf (SEG_REUSE_LOG_1, ("[%d]Adding a free space at end of seg: %Id", heap_num, size));
8801 has_end_of_seg = TRUE;
8807 size_t eph_gen_starts = gc_heap::eph_gen_starts_size;
8808 size -= eph_gen_starts;
8811 mark* m = (mark*)(start);
8812 pinned_len (m) -= eph_gen_starts;
8816 heap_segment* seg = (heap_segment*)start;
8817 heap_segment_plan_allocated (seg) += eph_gen_starts;
8821 int bucket_power2 = index_of_highest_set_bit (size);
8822 if (bucket_power2 < base_power2)
8827 free_space_bucket* bucket = &free_space_buckets[bucket_power2 - base_power2];
8829 seg_free_space* bucket_free_space = bucket->free_space;
8830 assert (plug_p || (!plug_p && bucket->count_add));
8832 if (bucket->count_add == 0)
8834 dprintf (SEG_REUSE_LOG_1, ("[%d]Already have enough of 2^%d", heap_num, bucket_power2));
8838 ptrdiff_t index = bucket->count_add - 1;
8840 dprintf (SEG_REUSE_LOG_1, ("[%d]Building free spaces: adding %Ix; len: %Id (2^%d)",
8843 (pinned_plug ((mark*)start) - pinned_len ((mark*)start)) :
8844 heap_segment_plan_allocated ((heap_segment*)start)),
8850 bucket_free_space[index].is_plug = TRUE;
8853 bucket_free_space[index].start = start;
8854 bucket->count_add--;
8859 // Do a consistency check after all free spaces are added.
8863 int end_of_seg_count = 0;
8865 for (i = 0; i < free_space_item_count; i++)
8867 assert (seg_free_space_array[i].start);
8868 if (!(seg_free_space_array[i].is_plug))
8876 assert (end_of_seg_count == 1);
8880 assert (end_of_seg_count == 0);
8883 for (i = 0; i < free_space_bucket_count; i++)
8885 assert (free_space_buckets[i].count_add == 0);
8891 uint8_t* fit (uint8_t* old_loc,
8893 BOOL set_padding_on_saved_p,
8894 mark* pinned_plug_entry,
8895 #endif //SHORT_PLUGS
8897 REQD_ALIGN_AND_OFFSET_DCL)
8902 assert (!is_plug_padded (old_loc));
8903 #endif //SHORT_PLUGS
8904 assert (!node_realigned (old_loc));
8907 size_t saved_plug_size = plug_size;
8909 #ifdef FEATURE_STRUCTALIGN
8910 // BARTOKTODO (4841): this code path is disabled (see can_fit_all_blocks_p) until we take alignment requirements into account
8911 _ASSERTE(requiredAlignment == DATA_ALIGNMENT && false);
8912 #endif // FEATURE_STRUCTALIGN
8913 // TODO: this is also not large alignment ready. We would need to consider alignment when chosing the
8916 size_t plug_size_to_fit = plug_size;
8918 // best fit is only done for gen1 to gen2 and we do not pad in gen2.
8919 int pad_in_front = 0;
8922 plug_size_to_fit += (pad_in_front ? Align(min_obj_size) : 0);
8923 #endif //SHORT_PLUGS
8925 int plug_power2 = index_of_highest_set_bit (round_up_power2 (plug_size_to_fit + Align(min_obj_size)));
8927 uint8_t* new_address = 0;
8929 if (plug_power2 < base_power2)
8931 plug_power2 = base_power2;
8934 int chosen_power2 = plug_power2 - base_power2;
8936 for (i = chosen_power2; i < free_space_bucket_count; i++)
8938 if (free_space_buckets[i].count_fit != 0)
8945 dprintf (SEG_REUSE_LOG_1, ("[%d]Fitting plug len %Id (2^%d) using 2^%d free space",
8949 (chosen_power2 + base_power2)));
8951 assert (i < free_space_bucket_count);
8953 seg_free_space* bucket_free_space = free_space_buckets[chosen_power2].free_space;
8954 ptrdiff_t free_space_count = free_space_buckets[chosen_power2].count_fit;
8955 size_t new_free_space_size = 0;
8956 BOOL can_fit = FALSE;
8959 for (i = 0; i < free_space_count; i++)
8961 size_t free_space_size = 0;
8964 BOOL short_plugs_padding_p = FALSE;
8965 #endif //SHORT_PLUGS
8966 BOOL realign_padding_p = FALSE;
8968 if (bucket_free_space[i].is_plug)
8970 mark* m = (mark*)(bucket_free_space[i].start);
8971 uint8_t* plug_free_space_start = pinned_plug (m) - pinned_len (m);
8974 if ((pad_in_front & USE_PADDING_FRONT) &&
8975 (((plug_free_space_start - pin_allocation_context_start_region (m))==0) ||
8976 ((plug_free_space_start - pin_allocation_context_start_region (m))>=DESIRED_PLUG_LENGTH)))
8978 pad = Align (min_obj_size);
8979 short_plugs_padding_p = TRUE;
8981 #endif //SHORT_PLUGS
8983 if (!((old_loc == 0) || same_large_alignment_p (old_loc, plug_free_space_start+pad)))
8985 pad += switch_alignment_size (pad != 0);
8986 realign_padding_p = TRUE;
8989 plug_size = saved_plug_size + pad;
8991 free_space_size = pinned_len (m);
8992 new_address = pinned_plug (m) - pinned_len (m);
8994 if (free_space_size >= (plug_size + Align (min_obj_size)) ||
8995 free_space_size == plug_size)
8997 new_free_space_size = free_space_size - plug_size;
8998 pinned_len (m) = new_free_space_size;
8999 #ifdef SIMPLE_DPRINTF
9000 dprintf (SEG_REUSE_LOG_0, ("[%d]FP: 0x%Ix->0x%Ix(%Ix)(%Ix), [0x%Ix (2^%d) -> [0x%Ix (2^%d)",
9007 index_of_highest_set_bit (free_space_size),
9008 (pinned_plug (m) - pinned_len (m)),
9009 index_of_highest_set_bit (new_free_space_size)));
9010 #endif //SIMPLE_DPRINTF
9013 if (short_plugs_padding_p)
9015 pin_allocation_context_start_region (m) = plug_free_space_start;
9016 set_padding_in_expand (old_loc, set_padding_on_saved_p, pinned_plug_entry);
9018 #endif //SHORT_PLUGS
9020 if (realign_padding_p)
9022 set_node_realigned (old_loc);
9030 heap_segment* seg = (heap_segment*)(bucket_free_space[i].start);
9031 free_space_size = heap_segment_committed (seg) - heap_segment_plan_allocated (seg);
9033 if (!((old_loc == 0) || same_large_alignment_p (old_loc, heap_segment_plan_allocated (seg))))
9035 pad = switch_alignment_size (FALSE);
9036 realign_padding_p = TRUE;
9039 plug_size = saved_plug_size + pad;
9041 if (free_space_size >= (plug_size + Align (min_obj_size)) ||
9042 free_space_size == plug_size)
9044 new_address = heap_segment_plan_allocated (seg);
9045 new_free_space_size = free_space_size - plug_size;
9046 heap_segment_plan_allocated (seg) = new_address + plug_size;
9047 #ifdef SIMPLE_DPRINTF
9048 dprintf (SEG_REUSE_LOG_0, ("[%d]FS: 0x%Ix-> 0x%Ix(%Ix) (2^%d) -> 0x%Ix (2^%d)",
9053 index_of_highest_set_bit (free_space_size),
9054 heap_segment_plan_allocated (seg),
9055 index_of_highest_set_bit (new_free_space_size)));
9056 #endif //SIMPLE_DPRINTF
9058 if (realign_padding_p)
9059 set_node_realigned (old_loc);
9073 assert (chosen_power2 == 0);
9083 assert ((chosen_power2 && (i == 0)) ||
9084 ((!chosen_power2) && (i < free_space_count)));
9087 int new_bucket_power2 = index_of_highest_set_bit (new_free_space_size);
9089 if (new_bucket_power2 < base_power2)
9091 new_bucket_power2 = base_power2;
9094 move_bucket (chosen_power2, new_bucket_power2 - base_power2);
9103 if (free_space_buckets)
9105 delete [] free_space_buckets;
9107 if (seg_free_space_array)
9109 delete [] seg_free_space_array;
9115 #define marked(i) header(i)->IsMarked()
9116 #define set_marked(i) header(i)->SetMarked()
9117 #define clear_marked(i) header(i)->ClearMarked()
9118 #define pinned(i) header(i)->IsPinned()
9119 #define set_pinned(i) header(i)->SetPinned()
9120 #define clear_pinned(i) header(i)->GetHeader()->ClrGCBit();
9122 inline size_t my_get_size (Object* ob)
9124 MethodTable* mT = header(ob)->GetMethodTable();
9125 return (mT->GetBaseSize() +
9126 (mT->HasComponentSize() ?
9127 ((size_t)((CObjectHeader*)ob)->GetNumComponents() * mT->RawGetComponentSize()) : 0));
9130 //#define size(i) header(i)->GetSize()
9131 #define size(i) my_get_size (header(i))
9133 #define contain_pointers(i) header(i)->ContainsPointers()
9134 #ifdef COLLECTIBLE_CLASS
9135 #define contain_pointers_or_collectible(i) header(i)->ContainsPointersOrCollectible()
9137 #define get_class_object(i) GCToEEInterface::GetLoaderAllocatorObjectForGC((Object *)i)
9138 #define is_collectible(i) method_table(i)->Collectible()
9139 #else //COLLECTIBLE_CLASS
9140 #define contain_pointers_or_collectible(i) header(i)->ContainsPointers()
9141 #endif //COLLECTIBLE_CLASS
9143 #if defined (MARK_ARRAY) && defined (BACKGROUND_GC)
9145 void gc_heap::seg_clear_mark_array_bits_soh (heap_segment* seg)
9147 uint8_t* range_beg = 0;
9148 uint8_t* range_end = 0;
9149 if (bgc_mark_array_range (seg, FALSE, &range_beg, &range_end))
9151 clear_mark_array (range_beg, align_on_mark_word (range_end), FALSE
9152 #ifdef FEATURE_BASICFREEZE
9154 #endif // FEATURE_BASICFREEZE
9159 void gc_heap::clear_batch_mark_array_bits (uint8_t* start, uint8_t* end)
9161 if ((start < background_saved_highest_address) &&
9162 (end > background_saved_lowest_address))
9164 start = max (start, background_saved_lowest_address);
9165 end = min (end, background_saved_highest_address);
9167 size_t start_mark_bit = mark_bit_of (start);
9168 size_t end_mark_bit = mark_bit_of (end);
9169 unsigned int startbit = mark_bit_bit (start_mark_bit);
9170 unsigned int endbit = mark_bit_bit (end_mark_bit);
9171 size_t startwrd = mark_bit_word (start_mark_bit);
9172 size_t endwrd = mark_bit_word (end_mark_bit);
9174 dprintf (3, ("Clearing all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
9175 (size_t)start, (size_t)start_mark_bit,
9176 (size_t)end, (size_t)end_mark_bit));
9178 unsigned int firstwrd = lowbits (~0, startbit);
9179 unsigned int lastwrd = highbits (~0, endbit);
9181 if (startwrd == endwrd)
9183 unsigned int wrd = firstwrd | lastwrd;
9184 mark_array[startwrd] &= wrd;
9188 // clear the first mark word.
9191 mark_array[startwrd] &= firstwrd;
9195 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
9197 mark_array[wrdtmp] = 0;
9200 // clear the last mark word.
9203 mark_array[endwrd] &= lastwrd;
9208 void gc_heap::bgc_clear_batch_mark_array_bits (uint8_t* start, uint8_t* end)
9210 if ((start < background_saved_highest_address) &&
9211 (end > background_saved_lowest_address))
9213 start = max (start, background_saved_lowest_address);
9214 end = min (end, background_saved_highest_address);
9216 clear_batch_mark_array_bits (start, end);
9220 void gc_heap::clear_mark_array_by_objects (uint8_t* from, uint8_t* end, BOOL loh_p)
9222 dprintf (3, ("clearing mark array bits by objects for addr [%Ix,[%Ix",
9224 int align_const = get_alignment_constant (!loh_p);
9230 uint8_t* next_o = o + Align (size (o), align_const);
9232 if (background_object_marked (o, TRUE))
9234 dprintf (3, ("%Ix was marked by bgc, is now cleared", o));
9240 #endif //MARK_ARRAY && BACKGROUND_GC
9243 BOOL gc_heap::is_mark_set (uint8_t* o)
9248 #if defined (_MSC_VER) && defined (_TARGET_X86_)
9249 #pragma optimize("y", on) // Small critical routines, don't put in EBP frame
9250 #endif //_MSC_VER && _TARGET_X86_
9252 // return the generation number of an object.
9253 // It is assumed that the object is valid.
9254 //Note that this will return max_generation for a LOH object
9255 int gc_heap::object_gennum (uint8_t* o)
9257 if (in_range_for_segment (o, ephemeral_heap_segment) &&
9258 (o >= generation_allocation_start (generation_of (max_generation-1))))
9260 // in an ephemeral generation.
9261 for ( int i = 0; i < max_generation-1; i++)
9263 if ((o >= generation_allocation_start (generation_of (i))))
9266 return max_generation-1;
9270 return max_generation;
9274 int gc_heap::object_gennum_plan (uint8_t* o)
9276 if (in_range_for_segment (o, ephemeral_heap_segment))
9278 for (int i = 0; i <= max_generation-1; i++)
9280 uint8_t* plan_start = generation_plan_allocation_start (generation_of (i));
9281 if (plan_start && (o >= plan_start))
9287 return max_generation;
9290 #if defined(_MSC_VER) && defined(_TARGET_X86_)
9291 #pragma optimize("", on) // Go back to command line default optimizations
9292 #endif //_MSC_VER && _TARGET_X86_
9294 heap_segment* gc_heap::make_heap_segment (uint8_t* new_pages, size_t size, int h_number)
9296 size_t initial_commit = SEGMENT_INITIAL_COMMIT;
9298 //Commit the first page
9299 if (!virtual_commit (new_pages, initial_commit, h_number))
9304 //overlay the heap_segment
9305 heap_segment* new_segment = (heap_segment*)new_pages;
9307 uint8_t* start = new_pages + segment_info_size;
9308 heap_segment_mem (new_segment) = start;
9309 heap_segment_used (new_segment) = start;
9310 heap_segment_reserved (new_segment) = new_pages + size;
9311 heap_segment_committed (new_segment) = new_pages + initial_commit;
9312 init_heap_segment (new_segment);
9313 dprintf (2, ("Creating heap segment %Ix", (size_t)new_segment));
9317 void gc_heap::init_heap_segment (heap_segment* seg)
9320 heap_segment_next (seg) = 0;
9321 heap_segment_plan_allocated (seg) = heap_segment_mem (seg);
9322 heap_segment_allocated (seg) = heap_segment_mem (seg);
9323 #ifdef BACKGROUND_GC
9324 heap_segment_background_allocated (seg) = 0;
9325 heap_segment_saved_bg_allocated (seg) = 0;
9326 #endif //BACKGROUND_GC
9329 //Releases the segment to the OS.
9330 // this is always called on one thread only so calling seg_table->remove is fine.
9331 void gc_heap::delete_heap_segment (heap_segment* seg, BOOL consider_hoarding)
9333 if (!heap_segment_loh_p (seg))
9335 //cleanup the brick table back to the empty value
9336 clear_brick_table (heap_segment_mem (seg), heap_segment_reserved (seg));
9339 if (consider_hoarding)
9341 assert ((heap_segment_mem (seg) - (uint8_t*)seg) <= ptrdiff_t(2*OS_PAGE_SIZE));
9342 size_t ss = (size_t) (heap_segment_reserved (seg) - (uint8_t*)seg);
9343 //Don't keep the big ones.
9344 if (ss <= INITIAL_ALLOC)
9346 dprintf (2, ("Hoarding segment %Ix", (size_t)seg));
9347 #ifdef BACKGROUND_GC
9348 // We don't need to clear the decommitted flag because when this segment is used
9349 // for a new segment the flags will be cleared.
9350 if (!heap_segment_decommitted_p (seg))
9351 #endif //BACKGROUND_GC
9353 decommit_heap_segment (seg);
9356 #ifdef SEG_MAPPING_TABLE
9357 seg_mapping_table_remove_segment (seg);
9358 #endif //SEG_MAPPING_TABLE
9360 heap_segment_next (seg) = segment_standby_list;
9361 segment_standby_list = seg;
9368 dprintf (2, ("h%d: del seg: [%Ix, %Ix[",
9369 heap_number, (size_t)seg,
9370 (size_t)(heap_segment_reserved (seg))));
9372 #ifdef BACKGROUND_GC
9373 ::record_changed_seg ((uint8_t*)seg, heap_segment_reserved (seg),
9374 settings.gc_index, current_bgc_state,
9376 decommit_mark_array_by_seg (seg);
9377 #endif //BACKGROUND_GC
9379 #ifdef SEG_MAPPING_TABLE
9380 seg_mapping_table_remove_segment (seg);
9381 #else //SEG_MAPPING_TABLE
9382 seg_table->remove ((uint8_t*)seg);
9383 #endif //SEG_MAPPING_TABLE
9385 release_segment (seg);
9389 //resets the pages beyond allocates size so they won't be swapped out and back in
9391 void gc_heap::reset_heap_segment_pages (heap_segment* seg)
9393 size_t page_start = align_on_page ((size_t)heap_segment_allocated (seg));
9394 size_t size = (size_t)heap_segment_committed (seg) - page_start;
9396 GCToOSInterface::VirtualReset((void*)page_start, size, false /* unlock */);
9399 void gc_heap::decommit_heap_segment_pages (heap_segment* seg,
9402 uint8_t* page_start = align_on_page (heap_segment_allocated(seg));
9403 size_t size = heap_segment_committed (seg) - page_start;
9404 extra_space = align_on_page (extra_space);
9405 if (size >= max ((extra_space + 2*OS_PAGE_SIZE), 100*OS_PAGE_SIZE))
9407 page_start += max(extra_space, 32*OS_PAGE_SIZE);
9408 size -= max (extra_space, 32*OS_PAGE_SIZE);
9410 virtual_decommit (page_start, size, heap_number);
9411 dprintf (3, ("Decommitting heap segment [%Ix, %Ix[(%d)",
9413 (size_t)(page_start + size),
9415 heap_segment_committed (seg) = page_start;
9416 if (heap_segment_used (seg) > heap_segment_committed (seg))
9418 heap_segment_used (seg) = heap_segment_committed (seg);
9423 //decommit all pages except one or 2
9424 void gc_heap::decommit_heap_segment (heap_segment* seg)
9426 uint8_t* page_start = align_on_page (heap_segment_mem (seg));
9428 dprintf (3, ("Decommitting heap segment %Ix", (size_t)seg));
9430 #ifdef BACKGROUND_GC
9431 page_start += OS_PAGE_SIZE;
9432 #endif //BACKGROUND_GC
9434 size_t size = heap_segment_committed (seg) - page_start;
9435 virtual_decommit (page_start, size, heap_number);
9437 //re-init the segment object
9438 heap_segment_committed (seg) = page_start;
9439 if (heap_segment_used (seg) > heap_segment_committed (seg))
9441 heap_segment_used (seg) = heap_segment_committed (seg);
9445 void gc_heap::clear_gen0_bricks()
9447 if (!gen0_bricks_cleared)
9449 gen0_bricks_cleared = TRUE;
9450 //initialize brick table for gen 0
9451 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
9452 b < brick_of (align_on_brick
9453 (heap_segment_allocated (ephemeral_heap_segment)));
9461 #ifdef BACKGROUND_GC
9462 void gc_heap::rearrange_small_heap_segments()
9464 heap_segment* seg = freeable_small_heap_segment;
9467 heap_segment* next_seg = heap_segment_next (seg);
9468 // TODO: we need to consider hoarding here.
9469 delete_heap_segment (seg, FALSE);
9472 freeable_small_heap_segment = 0;
9474 #endif //BACKGROUND_GC
9476 void gc_heap::rearrange_large_heap_segments()
9478 dprintf (2, ("deleting empty large segments"));
9479 heap_segment* seg = freeable_large_heap_segment;
9482 heap_segment* next_seg = heap_segment_next (seg);
9483 delete_heap_segment (seg, GCConfig::GetRetainVM());
9486 freeable_large_heap_segment = 0;
9489 void gc_heap::rearrange_heap_segments(BOOL compacting)
9492 generation_start_segment (generation_of (max_generation));
9494 heap_segment* prev_seg = 0;
9495 heap_segment* next_seg = 0;
9498 next_seg = heap_segment_next (seg);
9500 //link ephemeral segment when expanding
9501 if ((next_seg == 0) && (seg != ephemeral_heap_segment))
9503 seg->next = ephemeral_heap_segment;
9504 next_seg = heap_segment_next (seg);
9507 //re-used expanded heap segment
9508 if ((seg == ephemeral_heap_segment) && next_seg)
9510 heap_segment_next (prev_seg) = next_seg;
9511 heap_segment_next (seg) = 0;
9515 uint8_t* end_segment = (compacting ?
9516 heap_segment_plan_allocated (seg) :
9517 heap_segment_allocated (seg));
9518 // check if the segment was reached by allocation
9519 if ((end_segment == heap_segment_mem (seg))&&
9520 !heap_segment_read_only_p (seg))
9522 //if not, unthread and delete
9524 assert (seg != ephemeral_heap_segment);
9525 heap_segment_next (prev_seg) = next_seg;
9526 delete_heap_segment (seg, GCConfig::GetRetainVM());
9528 dprintf (2, ("Deleting heap segment %Ix", (size_t)seg));
9532 if (!heap_segment_read_only_p (seg))
9536 heap_segment_allocated (seg) =
9537 heap_segment_plan_allocated (seg);
9540 // reset the pages between allocated and committed.
9541 if (seg != ephemeral_heap_segment)
9543 decommit_heap_segment_pages (seg, 0);
9557 uint8_t* g_addresses [array_size+2]; // to get around the bug in GetWriteWatch
9559 #ifdef TIME_WRITE_WATCH
9560 static unsigned int tot_cycles = 0;
9561 #endif //TIME_WRITE_WATCH
9565 inline void gc_heap::verify_card_bundle_bits_set(size_t first_card_word, size_t last_card_word)
9568 for (size_t x = cardw_card_bundle (first_card_word); x < cardw_card_bundle (last_card_word); x++)
9570 if (!card_bundle_set_p (x))
9572 assert (!"Card bundle not set");
9573 dprintf (3, ("Card bundle %Ix not set", x));
9579 // Verifies that any bundles that are not set represent only cards that are not set.
9580 inline void gc_heap::verify_card_bundles()
9583 size_t lowest_card = card_word (card_of (lowest_address));
9584 size_t highest_card = card_word (card_of (highest_address));
9585 size_t cardb = cardw_card_bundle (lowest_card);
9586 size_t end_cardb = cardw_card_bundle (align_cardw_on_bundle (highest_card));
9588 while (cardb < end_cardb)
9590 uint32_t* card_word = &card_table[max(card_bundle_cardw (cardb), lowest_card)];
9591 uint32_t* card_word_end = &card_table[min(card_bundle_cardw (cardb+1), highest_card)];
9593 if (card_bundle_set_p (cardb) == 0)
9595 // Verify that no card is set
9596 while (card_word < card_word_end)
9598 if (*card_word != 0)
9600 dprintf (3, ("gc: %d, Card word %Ix for address %Ix set, card_bundle %Ix clear",
9601 dd_collection_count (dynamic_data_of (0)),
9602 (size_t)(card_word-&card_table[0]),
9603 (size_t)(card_address ((size_t)(card_word-&card_table[0]) * card_word_width)), cardb));
9606 assert((*card_word)==0);
9616 // If card bundles are enabled, use write watch to find pages in the card table that have
9617 // been dirtied, and set the corresponding card bundle bits.
9618 void gc_heap::update_card_table_bundle()
9620 if (card_bundles_enabled())
9622 // The address of the card word containing the card representing the lowest heap address
9623 uint8_t* base_address = (uint8_t*)(&card_table[card_word (card_of (lowest_address))]);
9625 // The address of the card word containing the card representing the highest heap address
9626 uint8_t* high_address = (uint8_t*)(&card_table[card_word (card_of (highest_address))]);
9628 uint8_t* saved_base_address = base_address;
9629 uintptr_t bcount = array_size;
9630 size_t saved_region_size = align_on_page (high_address) - saved_base_address;
9634 size_t region_size = align_on_page (high_address) - base_address;
9636 dprintf (3,("Probing card table pages [%Ix, %Ix[", (size_t)base_address, (size_t)base_address+region_size));
9637 bool success = GCToOSInterface::GetWriteWatch(false /* resetState */,
9640 (void**)g_addresses,
9642 assert (success && "GetWriteWatch failed!");
9644 dprintf (3,("Found %d pages written", bcount));
9645 for (unsigned i = 0; i < bcount; i++)
9647 // Offset of the dirty page from the start of the card table (clamped to base_address)
9648 size_t bcardw = (uint32_t*)(max(g_addresses[i],base_address)) - &card_table[0];
9650 // Offset of the end of the page from the start of the card table (clamped to high addr)
9651 size_t ecardw = (uint32_t*)(min(g_addresses[i]+OS_PAGE_SIZE, high_address)) - &card_table[0];
9652 assert (bcardw >= card_word (card_of (g_gc_lowest_address)));
9654 // Set the card bundle bits representing the dirty card table page
9655 card_bundles_set (cardw_card_bundle (bcardw), cardw_card_bundle (align_cardw_on_bundle (ecardw)));
9656 dprintf (3,("Set Card bundle [%Ix, %Ix[", cardw_card_bundle (bcardw), cardw_card_bundle (align_cardw_on_bundle (ecardw))));
9658 verify_card_bundle_bits_set(bcardw, ecardw);
9661 if (bcount >= array_size)
9663 base_address = g_addresses [array_size-1] + OS_PAGE_SIZE;
9664 bcount = array_size;
9667 } while ((bcount >= array_size) && (base_address < high_address));
9669 // Now that we've updated the card bundle bits, reset the write-tracking state.
9670 GCToOSInterface::ResetWriteWatch (saved_base_address, saved_region_size);
9673 #endif //CARD_BUNDLE
9676 void gc_heap::reset_write_watch_for_gc_heap(void* base_address, size_t region_size)
9678 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9679 SoftwareWriteWatch::ClearDirty(base_address, region_size);
9680 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9681 GCToOSInterface::ResetWriteWatch(base_address, region_size);
9682 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9686 void gc_heap::get_write_watch_for_gc_heap(bool reset, void *base_address, size_t region_size, void** dirty_pages, uintptr_t* dirty_page_count_ref, bool is_runtime_suspended)
9688 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9689 SoftwareWriteWatch::GetDirty(base_address, region_size, dirty_pages, dirty_page_count_ref, reset, is_runtime_suspended);
9690 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9691 UNREFERENCED_PARAMETER(is_runtime_suspended);
9692 bool success = GCToOSInterface::GetWriteWatch(reset, base_address, region_size, dirty_pages, dirty_page_count_ref);
9694 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9697 const size_t ww_reset_quantum = 128*1024*1024;
9700 void gc_heap::switch_one_quantum()
9702 enable_preemptive ();
9703 GCToOSInterface::Sleep (1);
9704 disable_preemptive (true);
9707 void gc_heap::reset_ww_by_chunk (uint8_t* start_address, size_t total_reset_size)
9709 size_t reset_size = 0;
9710 size_t remaining_reset_size = 0;
9711 size_t next_reset_size = 0;
9713 while (reset_size != total_reset_size)
9715 remaining_reset_size = total_reset_size - reset_size;
9716 next_reset_size = ((remaining_reset_size >= ww_reset_quantum) ? ww_reset_quantum : remaining_reset_size);
9717 if (next_reset_size)
9719 reset_write_watch_for_gc_heap(start_address, next_reset_size);
9720 reset_size += next_reset_size;
9722 switch_one_quantum();
9726 assert (reset_size == total_reset_size);
9729 // This does a Sleep(1) for every reset ww_reset_quantum bytes of reset
9730 // we do concurrently.
9731 void gc_heap::switch_on_reset (BOOL concurrent_p, size_t* current_total_reset_size, size_t last_reset_size)
9735 *current_total_reset_size += last_reset_size;
9737 dprintf (2, ("reset %Id bytes so far", *current_total_reset_size));
9739 if (*current_total_reset_size > ww_reset_quantum)
9741 switch_one_quantum();
9743 *current_total_reset_size = 0;
9748 void gc_heap::reset_write_watch (BOOL concurrent_p)
9750 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9751 // Software write watch currently requires the runtime to be suspended during reset. See SoftwareWriteWatch::ClearDirty().
9752 assert(!concurrent_p);
9753 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9755 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
9757 PREFIX_ASSUME(seg != NULL);
9759 size_t reset_size = 0;
9760 size_t region_size = 0;
9762 dprintf (2, ("bgc lowest: %Ix, bgc highest: %Ix", background_saved_lowest_address, background_saved_highest_address));
9766 uint8_t* base_address = align_lower_page (heap_segment_mem (seg));
9767 base_address = max (base_address, background_saved_lowest_address);
9769 uint8_t* high_address = 0;
9770 high_address = ((seg == ephemeral_heap_segment) ? alloc_allocated : heap_segment_allocated (seg));
9771 high_address = min (high_address, background_saved_highest_address);
9773 if (base_address < high_address)
9775 region_size = high_address - base_address;
9777 #ifdef TIME_WRITE_WATCH
9778 unsigned int time_start = GetCycleCount32();
9779 #endif //TIME_WRITE_WATCH
9780 dprintf (3, ("h%d: soh ww: [%Ix(%Id)", heap_number, (size_t)base_address, region_size));
9781 //reset_ww_by_chunk (base_address, region_size);
9782 reset_write_watch_for_gc_heap(base_address, region_size);
9784 #ifdef TIME_WRITE_WATCH
9785 unsigned int time_stop = GetCycleCount32();
9786 tot_cycles += time_stop - time_start;
9787 printf ("ResetWriteWatch Duration: %d, total: %d\n",
9788 time_stop - time_start, tot_cycles);
9789 #endif //TIME_WRITE_WATCH
9791 switch_on_reset (concurrent_p, &reset_size, region_size);
9794 seg = heap_segment_next_rw (seg);
9796 concurrent_print_time_delta ("CRWW soh");
9799 //concurrent_print_time_delta ("CRW soh");
9801 seg = heap_segment_rw (generation_start_segment (large_object_generation));
9803 PREFIX_ASSUME(seg != NULL);
9807 uint8_t* base_address = align_lower_page (heap_segment_mem (seg));
9808 uint8_t* high_address = heap_segment_allocated (seg);
9810 base_address = max (base_address, background_saved_lowest_address);
9811 high_address = min (high_address, background_saved_highest_address);
9813 if (base_address < high_address)
9815 region_size = high_address - base_address;
9817 #ifdef TIME_WRITE_WATCH
9818 unsigned int time_start = GetCycleCount32();
9819 #endif //TIME_WRITE_WATCH
9820 dprintf (3, ("h%d: loh ww: [%Ix(%Id)", heap_number, (size_t)base_address, region_size));
9821 //reset_ww_by_chunk (base_address, region_size);
9822 reset_write_watch_for_gc_heap(base_address, region_size);
9824 #ifdef TIME_WRITE_WATCH
9825 unsigned int time_stop = GetCycleCount32();
9826 tot_cycles += time_stop - time_start;
9827 printf ("ResetWriteWatch Duration: %d, total: %d\n",
9828 time_stop - time_start, tot_cycles);
9829 #endif //TIME_WRITE_WATCH
9831 switch_on_reset (concurrent_p, &reset_size, region_size);
9834 seg = heap_segment_next_rw (seg);
9836 concurrent_print_time_delta ("CRWW loh");
9839 #ifdef DEBUG_WRITE_WATCH
9840 debug_write_watch = (uint8_t**)~0;
9841 #endif //DEBUG_WRITE_WATCH
9844 #endif //WRITE_WATCH
9846 #ifdef BACKGROUND_GC
9847 void gc_heap::restart_vm()
9849 //assert (generation_allocation_pointer (youngest_generation) == 0);
9850 dprintf (3, ("Restarting EE"));
9851 STRESS_LOG0(LF_GC, LL_INFO10000, "Concurrent GC: Retarting EE\n");
9852 ee_proceed_event.Set();
9856 void fire_alloc_wait_event (alloc_wait_reason awr, BOOL begin_p)
9858 if (awr != awr_ignored)
9862 FIRE_EVENT(BGCAllocWaitBegin, awr);
9866 FIRE_EVENT(BGCAllocWaitEnd, awr);
9872 void gc_heap::fire_alloc_wait_event_begin (alloc_wait_reason awr)
9874 fire_alloc_wait_event (awr, TRUE);
9878 void gc_heap::fire_alloc_wait_event_end (alloc_wait_reason awr)
9880 fire_alloc_wait_event (awr, FALSE);
9882 #endif //BACKGROUND_GC
9883 void gc_heap::make_generation (generation& gen, heap_segment* seg, uint8_t* start, uint8_t* pointer)
9885 gen.allocation_start = start;
9886 gen.allocation_context.alloc_ptr = pointer;
9887 gen.allocation_context.alloc_limit = pointer;
9888 gen.allocation_context.alloc_bytes = 0;
9889 gen.allocation_context.alloc_bytes_loh = 0;
9890 gen.allocation_context_start_region = pointer;
9891 gen.start_segment = seg;
9892 gen.allocation_segment = seg;
9893 gen.plan_allocation_start = 0;
9894 gen.free_list_space = 0;
9895 gen.pinned_allocated = 0;
9896 gen.free_list_allocated = 0;
9897 gen.end_seg_allocated = 0;
9898 gen.condemned_allocated = 0;
9899 gen.free_obj_space = 0;
9900 gen.allocation_size = 0;
9901 gen.pinned_allocation_sweep_size = 0;
9902 gen.pinned_allocation_compact_size = 0;
9903 gen.allocate_end_seg_p = FALSE;
9904 gen.free_list_allocator.clear();
9906 #ifdef FREE_USAGE_STATS
9907 memset (gen.gen_free_spaces, 0, sizeof (gen.gen_free_spaces));
9908 memset (gen.gen_current_pinned_free_spaces, 0, sizeof (gen.gen_current_pinned_free_spaces));
9909 memset (gen.gen_plugs, 0, sizeof (gen.gen_plugs));
9910 #endif //FREE_USAGE_STATS
9913 void gc_heap::adjust_ephemeral_limits ()
9915 ephemeral_low = generation_allocation_start (generation_of (max_generation - 1));
9916 ephemeral_high = heap_segment_reserved (ephemeral_heap_segment);
9918 dprintf (3, ("new ephemeral low: %Ix new ephemeral high: %Ix",
9919 (size_t)ephemeral_low, (size_t)ephemeral_high))
9921 #ifndef MULTIPLE_HEAPS
9922 // This updates the write barrier helpers with the new info.
9923 stomp_write_barrier_ephemeral(ephemeral_low, ephemeral_high);
9924 #endif // MULTIPLE_HEAPS
9927 #if defined(TRACE_GC) || defined(GC_CONFIG_DRIVEN)
9928 FILE* CreateLogFile(const GCConfigStringHolder& temp_logfile_name, bool is_config)
9932 if (!temp_logfile_name.Get())
9937 char logfile_name[MAX_LONGPATH+1];
9938 uint32_t pid = GCToOSInterface::GetCurrentProcessId();
9939 const char* suffix = is_config ? ".config.log" : ".log";
9940 _snprintf_s(logfile_name, MAX_LONGPATH+1, _TRUNCATE, "%s.%d%s", temp_logfile_name.Get(), pid, suffix);
9941 logFile = fopen(logfile_name, "wb");
9944 #endif //TRACE_GC || GC_CONFIG_DRIVEN
9946 size_t gc_heap::get_segment_size_hard_limit (uint32_t* num_heaps, bool should_adjust_num_heaps)
9948 assert (heap_hard_limit);
9949 size_t aligned_hard_limit = ((heap_hard_limit + min_segment_size_hard_limit - 1) & ~(min_segment_size_hard_limit - 1));
9950 if (should_adjust_num_heaps)
9952 uint32_t max_num_heaps = (uint32_t)(aligned_hard_limit / min_segment_size_hard_limit);
9953 if (*num_heaps > max_num_heaps)
9955 *num_heaps = max_num_heaps;
9959 size_t seg_size = aligned_hard_limit / *num_heaps;
9960 size_t aligned_seg_size = round_up_power2 (seg_size);
9962 assert (g_theGCHeap->IsValidSegmentSize (aligned_seg_size));
9964 size_t seg_size_from_config = (size_t)GCConfig::GetSegmentSize();
9965 if (seg_size_from_config)
9967 size_t aligned_seg_size_config = round_up_power2 (seg_size_from_config);
9969 aligned_seg_size = max (aligned_seg_size, aligned_seg_size_config);
9972 //printf ("limit: %Idmb, aligned: %Idmb, %d heaps, seg size from config: %Idmb, seg size %Idmb",
9973 // (heap_hard_limit / 1024 / 1024),
9974 // (aligned_hard_limit / 1024 / 1024),
9976 // (seg_size_from_config / 1024 / 1024),
9977 // (aligned_seg_size / 1024 / 1024));
9978 return aligned_seg_size;
9981 HRESULT gc_heap::initialize_gc (size_t segment_size,
9983 #ifdef MULTIPLE_HEAPS
9984 ,unsigned number_of_heaps
9985 #endif //MULTIPLE_HEAPS
9989 if (GCConfig::GetLogEnabled())
9991 gc_log = CreateLogFile(GCConfig::GetLogFile(), false);
9996 // GCLogFileSize in MBs.
9997 gc_log_file_size = static_cast<size_t>(GCConfig::GetLogFileSize());
9999 if (gc_log_file_size <= 0 || gc_log_file_size > 500)
10005 gc_log_lock.Initialize();
10006 gc_log_buffer = new (nothrow) uint8_t [gc_log_buffer_size];
10007 if (!gc_log_buffer)
10013 memset (gc_log_buffer, '*', gc_log_buffer_size);
10015 max_gc_buffers = gc_log_file_size * 1024 * 1024 / gc_log_buffer_size;
10019 #ifdef GC_CONFIG_DRIVEN
10020 if (GCConfig::GetConfigLogEnabled())
10022 gc_config_log = CreateLogFile(GCConfig::GetConfigLogFile(), true);
10024 if (gc_config_log == NULL)
10027 gc_config_log_buffer = new (nothrow) uint8_t [gc_config_log_buffer_size];
10028 if (!gc_config_log_buffer)
10030 fclose(gc_config_log);
10034 compact_ratio = static_cast<int>(GCConfig::GetCompactRatio());
10036 // h# | GC | gen | C | EX | NF | BF | ML | DM || PreS | PostS | Merge | Conv | Pre | Post | PrPo | PreP | PostP |
10037 cprintf (("%2s | %6s | %1s | %1s | %2s | %2s | %2s | %2s | %2s || %5s | %5s | %5s | %5s | %5s | %5s | %5s | %5s | %5s |",
10038 "h#", // heap index
10041 "C", // compaction (empty means sweeping), 'M' means it was mandatory, 'W' means it was not
10042 "EX", // heap expansion
10043 "NF", // normal fit
10044 "BF", // best fit (if it indicates neither NF nor BF it means it had to acquire a new seg.
10047 "PreS", // short object before pinned plug
10048 "PostS", // short object after pinned plug
10049 "Merge", // merged pinned plugs
10050 "Conv", // converted to pinned plug
10051 "Pre", // plug before pinned plug but not after
10052 "Post", // plug after pinned plug but not before
10053 "PrPo", // plug both before and after pinned plug
10054 "PreP", // pre short object padded
10055 "PostP" // post short object padded
10058 #endif //GC_CONFIG_DRIVEN
10061 GCConfigStringHolder logFileName = GCConfig::GetMixLogFile();
10062 if (logFileName.Get() != nullptr)
10064 GCStatistics::logFileName = _strdup(logFileName.Get());
10065 GCStatistics::logFile = fopen(GCStatistics::logFileName, "a");
10066 if (!GCStatistics::logFile)
10073 HRESULT hres = S_OK;
10076 hardware_write_watch_api_supported();
10077 #ifdef BACKGROUND_GC
10078 if (can_use_write_watch_for_gc_heap() && GCConfig::GetConcurrentGC())
10080 gc_can_use_concurrent = true;
10081 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
10082 virtual_alloc_hardware_write_watch = true;
10083 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
10087 gc_can_use_concurrent = false;
10089 #endif //BACKGROUND_GC
10090 #endif //WRITE_WATCH
10092 #ifdef BACKGROUND_GC
10093 // leave the first page to contain only segment info
10094 // because otherwise we could need to revisit the first page frequently in
10096 segment_info_size = OS_PAGE_SIZE;
10098 segment_info_size = Align (sizeof (heap_segment), get_alignment_constant (FALSE));
10099 #endif //BACKGROUND_GC
10101 reserved_memory = 0;
10102 unsigned block_count;
10103 #ifdef MULTIPLE_HEAPS
10104 reserved_memory_limit = (segment_size + heap_size) * number_of_heaps;
10105 block_count = number_of_heaps;
10106 #else //MULTIPLE_HEAPS
10107 reserved_memory_limit = segment_size + heap_size;
10109 #endif //MULTIPLE_HEAPS
10111 if (heap_hard_limit)
10113 check_commit_cs.Initialize();
10116 if (!reserve_initial_memory(segment_size,heap_size,block_count))
10117 return E_OUTOFMEMORY;
10120 //check if we need to turn on card_bundles.
10121 #ifdef MULTIPLE_HEAPS
10122 // use INT64 arithmetic here because of possible overflow on 32p
10123 uint64_t th = (uint64_t)MH_TH_CARD_BUNDLE*number_of_heaps;
10125 // use INT64 arithmetic here because of possible overflow on 32p
10126 uint64_t th = (uint64_t)SH_TH_CARD_BUNDLE;
10127 #endif //MULTIPLE_HEAPS
10129 if (can_use_write_watch_for_card_table() && reserved_memory >= th)
10131 settings.card_bundles = TRUE;
10135 settings.card_bundles = FALSE;
10137 #endif //CARD_BUNDLE
10139 settings.first_init();
10141 int latency_level_from_config = static_cast<int>(GCConfig::GetLatencyLevel());
10142 if (latency_level_from_config >= latency_level_first && latency_level_from_config <= latency_level_last)
10144 gc_heap::latency_level = static_cast<gc_latency_level>(latency_level_from_config);
10147 init_static_data();
10149 g_gc_card_table = make_card_table (g_gc_lowest_address, g_gc_highest_address);
10151 if (!g_gc_card_table)
10152 return E_OUTOFMEMORY;
10154 gc_started = FALSE;
10156 #ifdef MULTIPLE_HEAPS
10157 g_heaps = new (nothrow) gc_heap* [number_of_heaps];
10159 return E_OUTOFMEMORY;
10162 #pragma warning(push)
10163 #pragma warning(disable:22011) // Suppress PREFast warning about integer underflow/overflow
10164 #endif // _PREFAST_
10165 g_promoted = new (nothrow) size_t [number_of_heaps*16];
10166 g_bpromoted = new (nothrow) size_t [number_of_heaps*16];
10168 g_mark_stack_busy = new (nothrow) int[(number_of_heaps+2)*HS_CACHE_LINE_SIZE/sizeof(int)];
10169 #endif //MH_SC_MARK
10171 #pragma warning(pop)
10172 #endif // _PREFAST_
10173 if (!g_promoted || !g_bpromoted)
10174 return E_OUTOFMEMORY;
10177 if (!g_mark_stack_busy)
10178 return E_OUTOFMEMORY;
10179 #endif //MH_SC_MARK
10181 if (!create_thread_support (number_of_heaps))
10182 return E_OUTOFMEMORY;
10184 if (!heap_select::init (number_of_heaps))
10185 return E_OUTOFMEMORY;
10187 #endif //MULTIPLE_HEAPS
10189 #ifdef MULTIPLE_HEAPS
10190 yp_spin_count_unit = 32 * number_of_heaps;
10192 yp_spin_count_unit = 32 * g_num_processors;
10193 #endif //MULTIPLE_HEAPS
10195 if (!init_semi_shared())
10203 //Initializes PER_HEAP_ISOLATED data members.
10205 gc_heap::init_semi_shared()
10209 // This is used for heap expansion - it's to fix exactly the start for gen 0
10210 // through (max_generation-1). When we expand the heap we allocate all these
10211 // gen starts at the beginning of the new ephemeral seg.
10212 eph_gen_starts_size = (Align (min_obj_size)) * max_generation;
10215 #ifdef MULTIPLE_HEAPS
10216 mark_list_size = min (150*1024, max (8192, soh_segment_size/(2*10*32)));
10217 g_mark_list = make_mark_list (mark_list_size*n_heaps);
10219 min_balance_threshold = alloc_quantum_balance_units * CLR_SIZE * 2;
10220 #ifdef PARALLEL_MARK_LIST_SORT
10221 g_mark_list_copy = make_mark_list (mark_list_size*n_heaps);
10222 if (!g_mark_list_copy)
10226 #endif //PARALLEL_MARK_LIST_SORT
10228 #else //MULTIPLE_HEAPS
10230 mark_list_size = max (8192, soh_segment_size/(64*32));
10231 g_mark_list = make_mark_list (mark_list_size);
10233 #endif //MULTIPLE_HEAPS
10235 dprintf (3, ("mark_list_size: %d", mark_list_size));
10243 #if defined(SEG_MAPPING_TABLE) && !defined(GROWABLE_SEG_MAPPING_TABLE)
10244 if (!seg_mapping_table_init())
10246 #endif //SEG_MAPPING_TABLE && !GROWABLE_SEG_MAPPING_TABLE
10248 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
10249 seg_table = sorted_table::make_sorted_table();
10253 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
10255 segment_standby_list = 0;
10257 if (!full_gc_approach_event.CreateManualEventNoThrow(FALSE))
10261 if (!full_gc_end_event.CreateManualEventNoThrow(FALSE))
10266 fgn_maxgen_percent = 0;
10267 fgn_loh_percent = 0;
10268 full_gc_approach_event_set = false;
10270 memset (full_gc_counts, 0, sizeof (full_gc_counts));
10273 should_expand_in_full_gc = FALSE;
10275 #ifdef FEATURE_LOH_COMPACTION
10276 loh_compaction_always_p = GCConfig::GetLOHCompactionMode() != 0;
10277 loh_compaction_mode = loh_compaction_default;
10278 #endif //FEATURE_LOH_COMPACTION
10280 loh_size_threshold = (size_t)GCConfig::GetLOHThreshold();
10281 assert (loh_size_threshold >= LARGE_OBJECT_SIZE);
10283 #ifdef BACKGROUND_GC
10284 memset (ephemeral_fgc_counts, 0, sizeof (ephemeral_fgc_counts));
10285 bgc_alloc_spin_count = static_cast<uint32_t>(GCConfig::GetBGCSpinCount());
10286 bgc_alloc_spin = static_cast<uint32_t>(GCConfig::GetBGCSpin());
10289 int number_bgc_threads = 1;
10290 #ifdef MULTIPLE_HEAPS
10291 number_bgc_threads = n_heaps;
10292 #endif //MULTIPLE_HEAPS
10293 if (!create_bgc_threads_support (number_bgc_threads))
10298 #endif //BACKGROUND_GC
10300 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
10302 #ifdef GC_CONFIG_DRIVEN
10303 compact_or_sweep_gcs[0] = 0;
10304 compact_or_sweep_gcs[1] = 0;
10305 #endif //GC_CONFIG_DRIVEN
10308 short_plugs_pad_ratio = (double)DESIRED_PLUG_LENGTH / (double)(DESIRED_PLUG_LENGTH - Align (min_obj_size));
10309 #endif //SHORT_PLUGS
10317 if (full_gc_approach_event.IsValid())
10319 full_gc_approach_event.CloseEvent();
10321 if (full_gc_end_event.IsValid())
10323 full_gc_end_event.CloseEvent();
10330 gc_heap* gc_heap::make_gc_heap (
10331 #ifdef MULTIPLE_HEAPS
10334 #endif //MULTIPLE_HEAPS
10339 #ifdef MULTIPLE_HEAPS
10340 res = new (nothrow) gc_heap;
10344 res->vm_heap = vm_hp;
10345 res->alloc_context_count = 0;
10348 #ifdef PARALLEL_MARK_LIST_SORT
10349 res->mark_list_piece_start = new (nothrow) uint8_t**[n_heaps];
10350 if (!res->mark_list_piece_start)
10354 #pragma warning(push)
10355 #pragma warning(disable:22011) // Suppress PREFast warning about integer underflow/overflow
10356 #endif // _PREFAST_
10357 res->mark_list_piece_end = new (nothrow) uint8_t**[n_heaps + 32]; // +32 is padding to reduce false sharing
10359 #pragma warning(pop)
10360 #endif // _PREFAST_
10362 if (!res->mark_list_piece_end)
10364 #endif //PARALLEL_MARK_LIST_SORT
10368 #endif //MULTIPLE_HEAPS
10370 if (res->init_gc_heap (
10371 #ifdef MULTIPLE_HEAPS
10373 #else //MULTIPLE_HEAPS
10375 #endif //MULTIPLE_HEAPS
10381 #ifdef MULTIPLE_HEAPS
10384 return (gc_heap*)1;
10385 #endif //MULTIPLE_HEAPS
10389 gc_heap::wait_for_gc_done(int32_t timeOut)
10391 bool cooperative_mode = enable_preemptive ();
10393 uint32_t dwWaitResult = NOERROR;
10395 gc_heap* wait_heap = NULL;
10396 while (gc_heap::gc_started)
10398 #ifdef MULTIPLE_HEAPS
10399 wait_heap = GCHeap::GetHeap(heap_select::select_heap(NULL, 0))->pGenGCHeap;
10400 dprintf(2, ("waiting for the gc_done_event on heap %d", wait_heap->heap_number));
10401 #endif // MULTIPLE_HEAPS
10404 PREFIX_ASSUME(wait_heap != NULL);
10405 #endif // _PREFAST_
10407 dwWaitResult = wait_heap->gc_done_event.Wait(timeOut, FALSE);
10409 disable_preemptive (cooperative_mode);
10411 return dwWaitResult;
10415 gc_heap::set_gc_done()
10417 enter_gc_done_event_lock();
10418 if (!gc_done_event_set)
10420 gc_done_event_set = true;
10421 dprintf (2, ("heap %d: setting gc_done_event", heap_number));
10422 gc_done_event.Set();
10424 exit_gc_done_event_lock();
10428 gc_heap::reset_gc_done()
10430 enter_gc_done_event_lock();
10431 if (gc_done_event_set)
10433 gc_done_event_set = false;
10434 dprintf (2, ("heap %d: resetting gc_done_event", heap_number));
10435 gc_done_event.Reset();
10437 exit_gc_done_event_lock();
10441 gc_heap::enter_gc_done_event_lock()
10443 uint32_t dwSwitchCount = 0;
10446 if (Interlocked::CompareExchange(&gc_done_event_lock, 0, -1) >= 0)
10448 while (gc_done_event_lock >= 0)
10450 if (g_num_processors > 1)
10452 int spin_count = yp_spin_count_unit;
10453 for (int j = 0; j < spin_count; j++)
10455 if (gc_done_event_lock < 0)
10457 YieldProcessor(); // indicate to the processor that we are spinning
10459 if (gc_done_event_lock >= 0)
10460 GCToOSInterface::YieldThread(++dwSwitchCount);
10463 GCToOSInterface::YieldThread(++dwSwitchCount);
10470 gc_heap::exit_gc_done_event_lock()
10472 gc_done_event_lock = -1;
10475 #ifndef MULTIPLE_HEAPS
10477 #ifdef RECORD_LOH_STATE
10478 int gc_heap::loh_state_index = 0;
10479 gc_heap::loh_state_info gc_heap::last_loh_states[max_saved_loh_states];
10480 #endif //RECORD_LOH_STATE
10482 VOLATILE(int32_t) gc_heap::gc_done_event_lock;
10483 VOLATILE(bool) gc_heap::gc_done_event_set;
10484 GCEvent gc_heap::gc_done_event;
10485 #endif //!MULTIPLE_HEAPS
10486 VOLATILE(bool) gc_heap::internal_gc_done;
10488 void gc_heap::add_saved_spinlock_info (
10490 msl_enter_state enter_state,
10491 msl_take_state take_state)
10494 #ifdef SPINLOCK_HISTORY
10495 spinlock_info* current = &last_spinlock_info[spinlock_info_index];
10497 current->enter_state = enter_state;
10498 current->take_state = take_state;
10499 current->thread_id.SetToCurrentThread();
10500 current->loh_p = loh_p;
10501 dprintf (SPINLOCK_LOG, ("[%d]%s %s %s",
10503 (loh_p ? "loh" : "soh"),
10504 ((enter_state == me_acquire) ? "E" : "L"),
10505 msl_take_state_str[take_state]));
10507 spinlock_info_index++;
10509 assert (spinlock_info_index <= max_saved_spinlock_info);
10511 if (spinlock_info_index >= max_saved_spinlock_info)
10513 spinlock_info_index = 0;
10516 MAYBE_UNUSED_VAR(enter_state);
10517 MAYBE_UNUSED_VAR(take_state);
10518 #endif //SPINLOCK_HISTORY
10522 gc_heap::init_gc_heap (int h_number)
10524 #ifdef MULTIPLE_HEAPS
10528 allocated_since_last_gc = 0;
10530 #ifdef SPINLOCK_HISTORY
10531 spinlock_info_index = 0;
10532 memset (last_spinlock_info, 0, sizeof(last_spinlock_info));
10533 #endif //SPINLOCK_HISTORY
10535 // initialize per heap members.
10536 ephemeral_low = (uint8_t*)1;
10538 ephemeral_high = MAX_PTR;
10540 ephemeral_heap_segment = 0;
10542 freeable_large_heap_segment = 0;
10544 condemned_generation_num = 0;
10546 blocking_collection = FALSE;
10548 generation_skip_ratio = 100;
10550 mark_stack_tos = 0;
10552 mark_stack_bos = 0;
10554 mark_stack_array_length = 0;
10556 mark_stack_array = 0;
10558 #if defined (_DEBUG) && defined (VERIFY_HEAP)
10559 verify_pinned_queue_p = FALSE;
10560 #endif // _DEBUG && VERIFY_HEAP
10562 loh_pinned_queue_tos = 0;
10564 loh_pinned_queue_bos = 0;
10566 loh_pinned_queue_length = 0;
10568 loh_pinned_queue_decay = LOH_PIN_DECAY;
10570 loh_pinned_queue = 0;
10572 min_overflow_address = MAX_PTR;
10574 max_overflow_address = 0;
10576 gen0_bricks_cleared = FALSE;
10578 gen0_must_clear_bricks = 0;
10580 allocation_quantum = CLR_SIZE;
10582 more_space_lock_soh = gc_lock;
10584 more_space_lock_loh = gc_lock;
10586 ro_segments_in_range = FALSE;
10588 loh_alloc_since_cg = 0;
10590 new_heap_segment = NULL;
10592 gen0_allocated_after_gc_p = false;
10594 #ifdef RECORD_LOH_STATE
10595 loh_state_index = 0;
10596 #endif //RECORD_LOH_STATE
10597 #endif //MULTIPLE_HEAPS
10599 #ifdef MULTIPLE_HEAPS
10600 if (h_number > n_heaps)
10602 assert (!"Number of heaps exceeded");
10606 heap_number = h_number;
10607 #endif //MULTIPLE_HEAPS
10609 memset (&oom_info, 0, sizeof (oom_info));
10610 memset (&fgm_result, 0, sizeof (fgm_result));
10611 if (!gc_done_event.CreateManualEventNoThrow(FALSE))
10615 gc_done_event_lock = -1;
10616 gc_done_event_set = false;
10618 #ifndef SEG_MAPPING_TABLE
10619 if (!gc_heap::seg_table->ensure_space_for_insert ())
10623 #endif //!SEG_MAPPING_TABLE
10625 heap_segment* seg = get_initial_segment (soh_segment_size, h_number);
10629 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(seg),
10630 (size_t)(heap_segment_reserved (seg) - heap_segment_mem(seg)),
10631 gc_etw_segment_small_object_heap);
10633 #ifdef SEG_MAPPING_TABLE
10634 seg_mapping_table_add_segment (seg, __this);
10635 #else //SEG_MAPPING_TABLE
10636 seg_table->insert ((uint8_t*)seg, sdelta);
10637 #endif //SEG_MAPPING_TABLE
10639 #ifdef MULTIPLE_HEAPS
10640 heap_segment_heap (seg) = this;
10641 #endif //MULTIPLE_HEAPS
10643 /* todo: Need a global lock for this */
10644 uint32_t* ct = &g_gc_card_table [card_word (card_of (g_gc_lowest_address))];
10645 own_card_table (ct);
10646 card_table = translate_card_table (ct);
10647 /* End of global lock */
10649 brick_table = card_table_brick_table (ct);
10650 highest_address = card_table_highest_address (ct);
10651 lowest_address = card_table_lowest_address (ct);
10654 card_bundle_table = translate_card_bundle_table (card_table_card_bundle_table (ct), g_gc_lowest_address);
10655 assert (&card_bundle_table [card_bundle_word (cardw_card_bundle (card_word (card_of (g_gc_lowest_address))))] ==
10656 card_table_card_bundle_table (ct));
10657 #endif //CARD_BUNDLE
10660 if (gc_can_use_concurrent)
10661 mark_array = translate_mark_array (card_table_mark_array (&g_gc_card_table[card_word (card_of (g_gc_lowest_address))]));
10664 #endif //MARK_ARRAY
10666 uint8_t* start = heap_segment_mem (seg);
10668 for (int i = 0; i < 1 + max_generation; i++)
10670 make_generation (generation_table [ (max_generation - i) ],
10672 generation_table [(max_generation - i)].gen_num = max_generation - i;
10673 start += Align (min_obj_size);
10676 heap_segment_allocated (seg) = start;
10677 alloc_allocated = start;
10678 heap_segment_used (seg) = start - plug_skew;
10680 ephemeral_heap_segment = seg;
10682 #ifndef SEG_MAPPING_TABLE
10683 if (!gc_heap::seg_table->ensure_space_for_insert ())
10687 #endif //!SEG_MAPPING_TABLE
10688 //Create the large segment generation
10689 heap_segment* lseg = get_initial_segment(min_loh_segment_size, h_number);
10692 lseg->flags |= heap_segment_flags_loh;
10694 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(lseg),
10695 (size_t)(heap_segment_reserved (lseg) - heap_segment_mem(lseg)),
10696 gc_etw_segment_large_object_heap);
10698 #ifdef SEG_MAPPING_TABLE
10699 seg_mapping_table_add_segment (lseg, __this);
10700 #else //SEG_MAPPING_TABLE
10701 seg_table->insert ((uint8_t*)lseg, sdelta);
10702 #endif //SEG_MAPPING_TABLE
10704 generation_table [max_generation].free_list_allocator = allocator(NUM_GEN2_ALIST, BASE_GEN2_ALIST, gen2_alloc_list);
10705 //assign the alloc_list for the large generation
10706 generation_table [max_generation+1].free_list_allocator = allocator(NUM_LOH_ALIST, BASE_LOH_ALIST, loh_alloc_list);
10707 generation_table [max_generation+1].gen_num = max_generation+1;
10708 make_generation (generation_table [max_generation+1],lseg, heap_segment_mem (lseg), 0);
10709 heap_segment_allocated (lseg) = heap_segment_mem (lseg) + Align (min_obj_size, get_alignment_constant (FALSE));
10710 heap_segment_used (lseg) = heap_segment_allocated (lseg) - plug_skew;
10712 for (int gen_num = 0; gen_num <= 1 + max_generation; gen_num++)
10714 generation* gen = generation_of (gen_num);
10715 make_unused_array (generation_allocation_start (gen), Align (min_obj_size));
10718 #ifdef MULTIPLE_HEAPS
10719 heap_segment_heap (lseg) = this;
10721 //initialize the alloc context heap
10722 generation_alloc_context (generation_of (0))->set_alloc_heap(vm_heap);
10724 //initialize the alloc context heap
10725 generation_alloc_context (generation_of (max_generation+1))->set_alloc_heap(vm_heap);
10727 #endif //MULTIPLE_HEAPS
10729 //Do this only once
10730 #ifdef MULTIPLE_HEAPS
10732 #endif //MULTIPLE_HEAPS
10734 #ifndef INTERIOR_POINTERS
10735 //set the brick_table for large objects
10736 //but default value is clearded
10737 //clear_brick_table ((uint8_t*)heap_segment_mem (lseg),
10738 // (uint8_t*)heap_segment_reserved (lseg));
10740 #else //INTERIOR_POINTERS
10742 //Because of the interior pointer business, we have to clear
10743 //the whole brick table
10744 //but the default value is cleared
10745 // clear_brick_table (lowest_address, highest_address);
10746 #endif //INTERIOR_POINTERS
10749 if (!init_dynamic_data())
10754 etw_allocation_running_amount[0] = 0;
10755 etw_allocation_running_amount[1] = 0;
10757 //needs to be done after the dynamic data has been initialized
10758 #ifndef MULTIPLE_HEAPS
10759 allocation_running_amount = dd_min_size (dynamic_data_of (0));
10760 #endif //!MULTIPLE_HEAPS
10762 fgn_last_alloc = dd_min_size (dynamic_data_of (0));
10764 mark* arr = new (nothrow) (mark [MARK_STACK_INITIAL_LENGTH]);
10768 make_mark_stack(arr);
10770 #ifdef BACKGROUND_GC
10771 freeable_small_heap_segment = 0;
10772 gchist_index_per_heap = 0;
10773 uint8_t** b_arr = new (nothrow) (uint8_t* [MARK_STACK_INITIAL_LENGTH]);
10777 make_background_mark_stack (b_arr);
10778 #endif //BACKGROUND_GC
10780 ephemeral_low = generation_allocation_start(generation_of(max_generation - 1));
10781 ephemeral_high = heap_segment_reserved(ephemeral_heap_segment);
10782 if (heap_number == 0)
10784 stomp_write_barrier_initialize(
10785 #ifdef MULTIPLE_HEAPS
10786 reinterpret_cast<uint8_t*>(1), reinterpret_cast<uint8_t*>(~0)
10788 ephemeral_low, ephemeral_high
10789 #endif //!MULTIPLE_HEAPS
10794 // why would we clear the mark array for this page? it should be cleared..
10795 // clear the first committed page
10796 //if(gc_can_use_concurrent)
10798 // clear_mark_array (align_lower_page (heap_segment_mem (seg)), heap_segment_committed (seg));
10800 #endif //MARK_ARRAY
10802 #ifdef MULTIPLE_HEAPS
10803 //register the heap in the heaps array
10805 if (!create_gc_thread ())
10808 g_heaps [heap_number] = this;
10810 #endif //MULTIPLE_HEAPS
10812 #ifdef FEATURE_PREMORTEM_FINALIZATION
10813 HRESULT hr = AllocateCFinalize(&finalize_queue);
10816 #endif // FEATURE_PREMORTEM_FINALIZATION
10818 max_free_space_items = MAX_NUM_FREE_SPACES;
10820 bestfit_seg = new (nothrow) seg_free_spaces (heap_number);
10827 if (!bestfit_seg->alloc())
10832 last_gc_before_oom = FALSE;
10834 sufficient_gen0_space_p = FALSE;
10836 #ifdef MULTIPLE_HEAPS
10838 #ifdef HEAP_ANALYZE
10840 heap_analyze_success = TRUE;
10842 internal_root_array = 0;
10844 internal_root_array_index = 0;
10846 internal_root_array_length = initial_internal_roots;
10850 current_obj_size = 0;
10852 #endif //HEAP_ANALYZE
10854 #endif // MULTIPLE_HEAPS
10856 #ifdef BACKGROUND_GC
10857 bgc_thread_id.Clear();
10859 if (!create_bgc_thread_support())
10864 bgc_alloc_lock = new (nothrow) exclusive_sync;
10865 if (!bgc_alloc_lock)
10870 bgc_alloc_lock->init();
10874 if (!recursive_gc_sync::init())
10878 bgc_thread_running = 0;
10880 bgc_threads_timeout_cs.Initialize();
10881 expanded_in_fgc = 0;
10882 current_bgc_state = bgc_not_in_process;
10883 background_soh_alloc_count = 0;
10884 background_loh_alloc_count = 0;
10885 bgc_overflow_count = 0;
10886 end_loh_size = dd_min_size (dynamic_data_of (max_generation + 1));
10887 #endif //BACKGROUND_GC
10889 #ifdef GC_CONFIG_DRIVEN
10890 memset (interesting_data_per_heap, 0, sizeof (interesting_data_per_heap));
10891 memset(compact_reasons_per_heap, 0, sizeof (compact_reasons_per_heap));
10892 memset(expand_mechanisms_per_heap, 0, sizeof (expand_mechanisms_per_heap));
10893 memset(interesting_mechanism_bits_per_heap, 0, sizeof (interesting_mechanism_bits_per_heap));
10894 #endif //GC_CONFIG_DRIVEN
10900 gc_heap::destroy_semi_shared()
10902 //TODO: will need to move this to per heap
10903 //#ifdef BACKGROUND_GC
10904 // if (c_mark_list)
10905 // delete c_mark_list;
10906 //#endif //BACKGROUND_GC
10910 delete g_mark_list;
10913 #if defined(SEG_MAPPING_TABLE) && !defined(GROWABLE_SEG_MAPPING_TABLE)
10914 if (seg_mapping_table)
10915 delete seg_mapping_table;
10916 #endif //SEG_MAPPING_TABLE && !GROWABLE_SEG_MAPPING_TABLE
10918 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
10919 //destroy the segment map
10920 seg_table->delete_sorted_table();
10921 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
10925 gc_heap::self_destroy()
10927 #ifdef BACKGROUND_GC
10929 #endif //BACKGROUND_GC
10931 if (gc_done_event.IsValid())
10933 gc_done_event.CloseEvent();
10936 // destroy every segment.
10937 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
10939 PREFIX_ASSUME(seg != NULL);
10941 heap_segment* next_seg;
10944 next_seg = heap_segment_next_rw (seg);
10945 delete_heap_segment (seg);
10949 seg = heap_segment_rw (generation_start_segment (generation_of (max_generation+1)));
10951 PREFIX_ASSUME(seg != NULL);
10955 next_seg = heap_segment_next_rw (seg);
10956 delete_heap_segment (seg);
10960 // get rid of the card table
10961 release_card_table (card_table);
10963 // destroy the mark stack
10964 delete mark_stack_array;
10966 #ifdef FEATURE_PREMORTEM_FINALIZATION
10967 if (finalize_queue)
10968 delete finalize_queue;
10969 #endif // FEATURE_PREMORTEM_FINALIZATION
10973 gc_heap::destroy_gc_heap(gc_heap* heap)
10975 heap->self_destroy();
10979 // Destroys resources owned by gc. It is assumed that a last GC has been performed and that
10980 // the finalizer queue has been drained.
10981 void gc_heap::shutdown_gc()
10983 destroy_semi_shared();
10985 #ifdef MULTIPLE_HEAPS
10986 //delete the heaps array
10988 destroy_thread_support();
10990 #endif //MULTIPLE_HEAPS
10991 //destroy seg_manager
10993 destroy_initial_memory();
10995 GCToOSInterface::Shutdown();
10999 BOOL gc_heap::size_fit_p (size_t size REQD_ALIGN_AND_OFFSET_DCL, uint8_t* alloc_pointer, uint8_t* alloc_limit,
11000 uint8_t* old_loc, int use_padding)
11002 BOOL already_padded = FALSE;
11004 if ((old_loc != 0) && (use_padding & USE_PADDING_FRONT))
11006 alloc_pointer = alloc_pointer + Align (min_obj_size);
11007 already_padded = TRUE;
11009 #endif //SHORT_PLUGS
11011 if (!((old_loc == 0) || same_large_alignment_p (old_loc, alloc_pointer)))
11012 size = size + switch_alignment_size (already_padded);
11014 #ifdef FEATURE_STRUCTALIGN
11015 alloc_pointer = StructAlign(alloc_pointer, requiredAlignment, alignmentOffset);
11016 #endif // FEATURE_STRUCTALIGN
11018 // in allocate_in_condemned_generation we can have this when we
11019 // set the alloc_limit to plan_allocated which could be less than
11021 if (alloc_limit < alloc_pointer)
11028 return (((size_t)(alloc_limit - alloc_pointer) >= (size + ((use_padding & USE_PADDING_TAIL)? Align(min_obj_size) : 0)))
11030 ||((!(use_padding & USE_PADDING_FRONT)) && ((alloc_pointer + size) == alloc_limit))
11031 #else //SHORT_PLUGS
11032 ||((alloc_pointer + size) == alloc_limit)
11033 #endif //SHORT_PLUGS
11038 assert (size == Align (min_obj_size));
11039 return ((size_t)(alloc_limit - alloc_pointer) >= size);
11044 BOOL gc_heap::a_size_fit_p (size_t size, uint8_t* alloc_pointer, uint8_t* alloc_limit,
11047 // We could have run into cases where this is true when alloc_allocated is the
11048 // the same as the seg committed.
11049 if (alloc_limit < alloc_pointer)
11054 return ((size_t)(alloc_limit - alloc_pointer) >= (size + Align(min_obj_size, align_const)));
11057 // Grow by committing more pages
11058 BOOL gc_heap::grow_heap_segment (heap_segment* seg, uint8_t* high_address, bool* hard_limit_exceeded_p)
11060 assert (high_address <= heap_segment_reserved (seg));
11062 if (hard_limit_exceeded_p)
11063 *hard_limit_exceeded_p = false;
11065 //return 0 if we are at the end of the segment.
11066 if (align_on_page (high_address) > heap_segment_reserved (seg))
11069 if (high_address <= heap_segment_committed (seg))
11072 size_t c_size = align_on_page ((size_t)(high_address - heap_segment_committed (seg)));
11073 c_size = max (c_size, commit_min_th);
11074 c_size = min (c_size, (size_t)(heap_segment_reserved (seg) - heap_segment_committed (seg)));
11079 STRESS_LOG2(LF_GC, LL_INFO10000,
11080 "Growing heap_segment: %Ix high address: %Ix\n",
11081 (size_t)seg, (size_t)high_address);
11083 bool ret = virtual_commit (heap_segment_committed (seg), c_size, heap_number, hard_limit_exceeded_p);
11087 #ifndef BACKGROUND_GC
11088 clear_mark_array (heap_segment_committed (seg),
11089 heap_segment_committed (seg)+c_size, TRUE);
11090 #endif //BACKGROUND_GC
11091 #endif //MARK_ARRAY
11092 heap_segment_committed (seg) += c_size;
11094 STRESS_LOG1(LF_GC, LL_INFO10000, "New commit: %Ix",
11095 (size_t)heap_segment_committed (seg));
11097 assert (heap_segment_committed (seg) <= heap_segment_reserved (seg));
11098 assert (high_address <= heap_segment_committed (seg));
11105 int gc_heap::grow_heap_segment (heap_segment* seg, uint8_t* allocated, uint8_t* old_loc, size_t size, BOOL pad_front_p REQD_ALIGN_AND_OFFSET_DCL)
11108 if ((old_loc != 0) && pad_front_p)
11110 allocated = allocated + Align (min_obj_size);
11112 #endif //SHORT_PLUGS
11114 if (!((old_loc == 0) || same_large_alignment_p (old_loc, allocated)))
11115 size = size + switch_alignment_size (FALSE);
11116 #ifdef FEATURE_STRUCTALIGN
11117 size_t pad = ComputeStructAlignPad(allocated, requiredAlignment, alignmentOffset);
11118 return grow_heap_segment (seg, allocated + pad + size);
11119 #else // FEATURE_STRUCTALIGN
11120 return grow_heap_segment (seg, allocated + size);
11121 #endif // FEATURE_STRUCTALIGN
11124 //used only in older generation allocation (i.e during gc).
11125 void gc_heap::adjust_limit (uint8_t* start, size_t limit_size, generation* gen,
11128 UNREFERENCED_PARAMETER(gennum);
11129 dprintf (3, ("gc Expanding segment allocation"));
11130 heap_segment* seg = generation_allocation_segment (gen);
11131 if ((generation_allocation_limit (gen) != start) || (start != heap_segment_plan_allocated (seg)))
11133 if (generation_allocation_limit (gen) == heap_segment_plan_allocated (seg))
11135 assert (generation_allocation_pointer (gen) >= heap_segment_mem (seg));
11136 assert (generation_allocation_pointer (gen) <= heap_segment_committed (seg));
11137 heap_segment_plan_allocated (generation_allocation_segment (gen)) = generation_allocation_pointer (gen);
11141 uint8_t* hole = generation_allocation_pointer (gen);
11142 size_t size = (generation_allocation_limit (gen) - generation_allocation_pointer (gen));
11146 dprintf (3, ("filling up hole: %Ix, size %Ix", hole, size));
11147 size_t allocated_size = generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen);
11148 if (size >= Align (min_free_list))
11150 if (allocated_size < min_free_list)
11152 if (size >= (Align (min_free_list) + Align (min_obj_size)))
11154 //split hole into min obj + threadable free item
11155 make_unused_array (hole, min_obj_size);
11156 generation_free_obj_space (gen) += Align (min_obj_size);
11157 make_unused_array (hole + Align (min_obj_size), size - Align (min_obj_size));
11158 generation_free_list_space (gen) += size - Align (min_obj_size);
11159 generation_allocator(gen)->thread_item_front (hole + Align (min_obj_size),
11160 size - Align (min_obj_size));
11161 add_gen_free (gen->gen_num, (size - Align (min_obj_size)));
11165 dprintf (3, ("allocated size too small, can't put back rest on free list %Ix", allocated_size));
11166 make_unused_array (hole, size);
11167 generation_free_obj_space (gen) += size;
11172 dprintf (3, ("threading hole in front of free list"));
11173 make_unused_array (hole, size);
11174 generation_free_list_space (gen) += size;
11175 generation_allocator(gen)->thread_item_front (hole, size);
11176 add_gen_free (gen->gen_num, size);
11181 make_unused_array (hole, size);
11182 generation_free_obj_space (gen) += size;
11186 generation_allocation_pointer (gen) = start;
11187 generation_allocation_context_start_region (gen) = start;
11189 generation_allocation_limit (gen) = (start + limit_size);
11192 void verify_mem_cleared (uint8_t* start, size_t size)
11194 if (!Aligned (size))
11199 PTR_PTR curr_ptr = (PTR_PTR) start;
11200 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
11202 if (*(curr_ptr++) != 0)
11209 #if defined (VERIFY_HEAP) && defined (BACKGROUND_GC)
11210 void gc_heap::set_batch_mark_array_bits (uint8_t* start, uint8_t* end)
11212 size_t start_mark_bit = mark_bit_of (start);
11213 size_t end_mark_bit = mark_bit_of (end);
11214 unsigned int startbit = mark_bit_bit (start_mark_bit);
11215 unsigned int endbit = mark_bit_bit (end_mark_bit);
11216 size_t startwrd = mark_bit_word (start_mark_bit);
11217 size_t endwrd = mark_bit_word (end_mark_bit);
11219 dprintf (3, ("Setting all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
11220 (size_t)start, (size_t)start_mark_bit,
11221 (size_t)end, (size_t)end_mark_bit));
11223 unsigned int firstwrd = ~(lowbits (~0, startbit));
11224 unsigned int lastwrd = ~(highbits (~0, endbit));
11226 if (startwrd == endwrd)
11228 unsigned int wrd = firstwrd & lastwrd;
11229 mark_array[startwrd] |= wrd;
11233 // set the first mark word.
11236 mark_array[startwrd] |= firstwrd;
11240 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
11242 mark_array[wrdtmp] = ~(unsigned int)0;
11245 // set the last mark word.
11248 mark_array[endwrd] |= lastwrd;
11252 // makes sure that the mark array bits between start and end are 0.
11253 void gc_heap::check_batch_mark_array_bits (uint8_t* start, uint8_t* end)
11255 size_t start_mark_bit = mark_bit_of (start);
11256 size_t end_mark_bit = mark_bit_of (end);
11257 unsigned int startbit = mark_bit_bit (start_mark_bit);
11258 unsigned int endbit = mark_bit_bit (end_mark_bit);
11259 size_t startwrd = mark_bit_word (start_mark_bit);
11260 size_t endwrd = mark_bit_word (end_mark_bit);
11262 //dprintf (3, ("Setting all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
11263 // (size_t)start, (size_t)start_mark_bit,
11264 // (size_t)end, (size_t)end_mark_bit));
11266 unsigned int firstwrd = ~(lowbits (~0, startbit));
11267 unsigned int lastwrd = ~(highbits (~0, endbit));
11269 if (startwrd == endwrd)
11271 unsigned int wrd = firstwrd & lastwrd;
11272 if (mark_array[startwrd] & wrd)
11274 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11276 mark_array [startwrd], mark_word_address (startwrd)));
11282 // set the first mark word.
11285 if (mark_array[startwrd] & firstwrd)
11287 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11288 firstwrd, startwrd,
11289 mark_array [startwrd], mark_word_address (startwrd)));
11296 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
11298 if (mark_array[wrdtmp])
11300 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11302 mark_array [wrdtmp], mark_word_address (wrdtmp)));
11307 // set the last mark word.
11310 if (mark_array[endwrd] & lastwrd)
11312 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11314 mark_array [lastwrd], mark_word_address (lastwrd)));
11319 #endif //VERIFY_HEAP && BACKGROUND_GC
11321 allocator::allocator (unsigned int num_b, size_t fbs, alloc_list* b)
11323 assert (num_b < MAX_BUCKET_COUNT);
11324 num_buckets = num_b;
11325 frst_bucket_size = fbs;
11329 alloc_list& allocator::alloc_list_of (unsigned int bn)
11331 assert (bn < num_buckets);
11333 return first_bucket;
11335 return buckets [bn-1];
11338 size_t& allocator::alloc_list_damage_count_of (unsigned int bn)
11340 assert (bn < num_buckets);
11342 return first_bucket.alloc_list_damage_count();
11344 return buckets [bn-1].alloc_list_damage_count();
11347 void allocator::unlink_item (unsigned int bn, uint8_t* item, uint8_t* prev_item, BOOL use_undo_p)
11349 //unlink the free_item
11350 alloc_list* al = &alloc_list_of (bn);
11353 if (use_undo_p && (free_list_undo (prev_item) == UNDO_EMPTY))
11355 assert (item == free_list_slot (prev_item));
11356 free_list_undo (prev_item) = item;
11357 alloc_list_damage_count_of (bn)++;
11359 free_list_slot (prev_item) = free_list_slot(item);
11363 al->alloc_list_head() = (uint8_t*)free_list_slot(item);
11365 if (al->alloc_list_tail() == item)
11367 al->alloc_list_tail() = prev_item;
11371 void allocator::clear()
11373 for (unsigned int i = 0; i < num_buckets; i++)
11375 alloc_list_head_of (i) = 0;
11376 alloc_list_tail_of (i) = 0;
11380 //always thread to the end.
11381 void allocator::thread_free_item (uint8_t* item, uint8_t*& head, uint8_t*& tail)
11383 free_list_slot (item) = 0;
11384 free_list_undo (item) = UNDO_EMPTY;
11385 assert (item != head);
11391 //TODO: This shouldn't happen anymore - verify that's the case.
11392 //the following is necessary because the last free element
11393 //may have been truncated, and tail isn't updated.
11394 else if (free_list_slot (head) == 0)
11396 free_list_slot (head) = item;
11400 assert (item != tail);
11401 assert (free_list_slot(tail) == 0);
11402 free_list_slot (tail) = item;
11407 void allocator::thread_item (uint8_t* item, size_t size)
11409 size_t sz = frst_bucket_size;
11410 unsigned int a_l_number = 0;
11412 for (; a_l_number < (num_buckets-1); a_l_number++)
11420 alloc_list* al = &alloc_list_of (a_l_number);
11421 thread_free_item (item,
11422 al->alloc_list_head(),
11423 al->alloc_list_tail());
11426 void allocator::thread_item_front (uint8_t* item, size_t size)
11428 //find right free list
11429 size_t sz = frst_bucket_size;
11430 unsigned int a_l_number = 0;
11431 for (; a_l_number < (num_buckets-1); a_l_number++)
11439 alloc_list* al = &alloc_list_of (a_l_number);
11440 free_list_slot (item) = al->alloc_list_head();
11441 free_list_undo (item) = UNDO_EMPTY;
11443 if (al->alloc_list_tail() == 0)
11445 al->alloc_list_tail() = al->alloc_list_head();
11447 al->alloc_list_head() = item;
11448 if (al->alloc_list_tail() == 0)
11450 al->alloc_list_tail() = item;
11454 void allocator::copy_to_alloc_list (alloc_list* toalist)
11456 for (unsigned int i = 0; i < num_buckets; i++)
11458 toalist [i] = alloc_list_of (i);
11459 #ifdef FL_VERIFICATION
11460 uint8_t* free_item = alloc_list_head_of (i);
11465 free_item = free_list_slot (free_item);
11468 toalist[i].item_count = count;
11469 #endif //FL_VERIFICATION
11473 void allocator::copy_from_alloc_list (alloc_list* fromalist)
11475 BOOL repair_list = !discard_if_no_fit_p ();
11476 for (unsigned int i = 0; i < num_buckets; i++)
11478 size_t count = alloc_list_damage_count_of (i);
11479 alloc_list_of (i) = fromalist [i];
11480 assert (alloc_list_damage_count_of (i) == 0);
11484 //repair the the list
11485 //new items may have been added during the plan phase
11486 //items may have been unlinked.
11487 uint8_t* free_item = alloc_list_head_of (i);
11488 while (free_item && count)
11490 assert (((CObjectHeader*)free_item)->IsFree());
11491 if ((free_list_undo (free_item) != UNDO_EMPTY))
11494 free_list_slot (free_item) = free_list_undo (free_item);
11495 free_list_undo (free_item) = UNDO_EMPTY;
11498 free_item = free_list_slot (free_item);
11501 #ifdef FL_VERIFICATION
11502 free_item = alloc_list_head_of (i);
11503 size_t item_count = 0;
11507 free_item = free_list_slot (free_item);
11510 assert (item_count == alloc_list_of (i).item_count);
11511 #endif //FL_VERIFICATION
11514 uint8_t* tail_item = alloc_list_tail_of (i);
11515 assert ((tail_item == 0) || (free_list_slot (tail_item) == 0));
11520 void allocator::commit_alloc_list_changes()
11522 BOOL repair_list = !discard_if_no_fit_p ();
11525 for (unsigned int i = 0; i < num_buckets; i++)
11527 //remove the undo info from list.
11528 uint8_t* free_item = alloc_list_head_of (i);
11529 size_t count = alloc_list_damage_count_of (i);
11530 while (free_item && count)
11532 assert (((CObjectHeader*)free_item)->IsFree());
11534 if (free_list_undo (free_item) != UNDO_EMPTY)
11536 free_list_undo (free_item) = UNDO_EMPTY;
11540 free_item = free_list_slot (free_item);
11543 alloc_list_damage_count_of (i) = 0;
11548 void gc_heap::adjust_limit_clr (uint8_t* start, size_t limit_size,
11549 alloc_context* acontext, heap_segment* seg,
11550 int align_const, int gen_number)
11552 bool loh_p = (gen_number > 0);
11553 GCSpinLock* msl = loh_p ? &more_space_lock_loh : &more_space_lock_soh;
11555 size_t aligned_min_obj_size = Align(min_obj_size, align_const);
11559 assert (heap_segment_used (seg) <= heap_segment_committed (seg));
11562 #ifdef MULTIPLE_HEAPS
11563 if (gen_number == 0)
11565 if (!gen0_allocated_after_gc_p)
11567 gen0_allocated_after_gc_p = true;
11570 #endif //MULTIPLE_HEAPS
11572 dprintf (3, ("Expanding segment allocation [%Ix, %Ix[", (size_t)start,
11573 (size_t)start + limit_size - aligned_min_obj_size));
11575 if ((acontext->alloc_limit != start) &&
11576 (acontext->alloc_limit + aligned_min_obj_size)!= start)
11578 uint8_t* hole = acontext->alloc_ptr;
11581 size_t size = (acontext->alloc_limit - acontext->alloc_ptr);
11582 dprintf (3, ("filling up hole [%Ix, %Ix[", (size_t)hole, (size_t)hole + size + Align (min_obj_size, align_const)));
11583 // when we are finishing an allocation from a free list
11584 // we know that the free area was Align(min_obj_size) larger
11585 acontext->alloc_bytes -= size;
11586 size_t free_obj_size = size + aligned_min_obj_size;
11587 make_unused_array (hole, free_obj_size);
11588 generation_free_obj_space (generation_of (gen_number)) += free_obj_size;
11590 acontext->alloc_ptr = start;
11594 if (gen_number == 0)
11596 size_t pad_size = Align (min_obj_size, align_const);
11597 make_unused_array (acontext->alloc_ptr, pad_size);
11598 dprintf (3, ("contigous ac: making min obj gap %Ix->%Ix(%Id)",
11599 acontext->alloc_ptr, (acontext->alloc_ptr + pad_size), pad_size));
11600 acontext->alloc_ptr += pad_size;
11603 acontext->alloc_limit = (start + limit_size - aligned_min_obj_size);
11604 acontext->alloc_bytes += limit_size - ((gen_number < max_generation + 1) ? aligned_min_obj_size : 0);
11606 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
11607 if (g_fEnableAppDomainMonitoring)
11609 GCToEEInterface::RecordAllocatedBytesForHeap(limit_size, heap_number);
11611 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
11613 uint8_t* saved_used = 0;
11617 saved_used = heap_segment_used (seg);
11620 if (seg == ephemeral_heap_segment)
11622 //Sometimes the allocated size is advanced without clearing the
11623 //memory. Let's catch up here
11624 if (heap_segment_used (seg) < (alloc_allocated - plug_skew))
11627 #ifndef BACKGROUND_GC
11628 clear_mark_array (heap_segment_used (seg) + plug_skew, alloc_allocated);
11629 #endif //BACKGROUND_GC
11630 #endif //MARK_ARRAY
11631 heap_segment_used (seg) = alloc_allocated - plug_skew;
11634 #ifdef BACKGROUND_GC
11637 uint8_t* old_allocated = heap_segment_allocated (seg) - plug_skew - limit_size;
11638 #ifdef FEATURE_LOH_COMPACTION
11639 old_allocated -= Align (loh_padding_obj_size, align_const);
11640 #endif //FEATURE_LOH_COMPACTION
11642 assert (heap_segment_used (seg) >= old_allocated);
11644 #endif //BACKGROUND_GC
11646 (start - plug_skew + limit_size) <= heap_segment_used (seg))
11648 add_saved_spinlock_info (loh_p, me_release, mt_clr_mem);
11649 leave_spin_lock (msl);
11650 dprintf (3, ("clearing memory at %Ix for %d bytes", (start - plug_skew), limit_size));
11651 memclr (start - plug_skew, limit_size);
11655 uint8_t* used = heap_segment_used (seg);
11656 heap_segment_used (seg) = start + limit_size - plug_skew;
11658 add_saved_spinlock_info (loh_p, me_release, mt_clr_mem);
11659 leave_spin_lock (msl);
11661 if ((start - plug_skew) < used)
11663 if (used != saved_used)
11668 dprintf (2, ("clearing memory before used at %Ix for %Id bytes",
11669 (start - plug_skew), (plug_skew + used - start)));
11670 memclr (start - plug_skew, used - (start - plug_skew));
11674 //this portion can be done after we release the lock
11675 if (seg == ephemeral_heap_segment)
11677 #ifdef FFIND_OBJECT
11678 if (gen0_must_clear_bricks > 0)
11680 //set the brick table to speed up find_object
11681 size_t b = brick_of (acontext->alloc_ptr);
11682 set_brick (b, acontext->alloc_ptr - brick_address (b));
11684 dprintf (3, ("Allocation Clearing bricks [%Ix, %Ix[",
11685 b, brick_of (align_on_brick (start + limit_size))));
11686 volatile short* x = &brick_table [b];
11687 short* end_x = &brick_table [brick_of (align_on_brick (start + limit_size))];
11689 for (;x < end_x;x++)
11693 #endif //FFIND_OBJECT
11695 gen0_bricks_cleared = FALSE;
11699 // verifying the memory is completely cleared.
11700 //verify_mem_cleared (start - plug_skew, limit_size);
11703 size_t gc_heap::new_allocation_limit (size_t size, size_t physical_limit, int gen_number)
11705 dynamic_data* dd = dynamic_data_of (gen_number);
11706 ptrdiff_t new_alloc = dd_new_allocation (dd);
11707 assert (new_alloc == (ptrdiff_t)Align (new_alloc,
11708 get_alignment_constant (!(gen_number == (max_generation+1)))));
11710 ptrdiff_t logical_limit = max (new_alloc, (ptrdiff_t)size);
11711 size_t limit = min (logical_limit, (ptrdiff_t)physical_limit);
11712 assert (limit == Align (limit, get_alignment_constant (!(gen_number == (max_generation+1)))));
11713 dd_new_allocation (dd) = (new_alloc - limit);
11717 size_t gc_heap::limit_from_size (size_t size, size_t physical_limit, int gen_number,
11720 size_t padded_size = size + Align (min_obj_size, align_const);
11721 // for LOH this is not true...we could select a physical_limit that's exactly the same
11723 assert ((gen_number != 0) || (physical_limit >= padded_size));
11724 size_t min_size_to_allocate = ((gen_number == 0) ? allocation_quantum : 0);
11726 // For SOH if the size asked for is very small, we want to allocate more than
11727 // just what's asked for if possible.
11728 size_t desired_size_to_allocate = max (padded_size, min_size_to_allocate);
11729 size_t new_physical_limit = min (physical_limit, desired_size_to_allocate);
11731 size_t new_limit = new_allocation_limit (padded_size,
11732 new_physical_limit,
11734 assert (new_limit >= (size + Align (min_obj_size, align_const)));
11735 dprintf (100, ("requested to allocate %Id bytes, actual size is %Id", size, new_limit));
11739 void gc_heap::handle_oom (int heap_num, oom_reason reason, size_t alloc_size,
11740 uint8_t* allocated, uint8_t* reserved)
11742 UNREFERENCED_PARAMETER(heap_num);
11744 if (reason == oom_budget)
11746 alloc_size = dd_min_size (dynamic_data_of (0)) / 2;
11749 if ((reason == oom_budget) && ((!fgm_result.loh_p) && (fgm_result.fgm != fgm_no_failure)))
11751 // This means during the last GC we needed to reserve and/or commit more memory
11752 // but we couldn't. We proceeded with the GC and ended up not having enough
11753 // memory at the end. This is a legitimate OOM situtation. Otherwise we
11754 // probably made a mistake and didn't expand the heap when we should have.
11755 reason = oom_low_mem;
11758 oom_info.reason = reason;
11759 oom_info.allocated = allocated;
11760 oom_info.reserved = reserved;
11761 oom_info.alloc_size = alloc_size;
11762 oom_info.gc_index = settings.gc_index;
11763 oom_info.fgm = fgm_result.fgm;
11764 oom_info.size = fgm_result.size;
11765 oom_info.available_pagefile_mb = fgm_result.available_pagefile_mb;
11766 oom_info.loh_p = fgm_result.loh_p;
11768 fgm_result.fgm = fgm_no_failure;
11770 // Break early - before the more_space_lock is release so no other threads
11771 // could have allocated on the same heap when OOM happened.
11772 if (GCConfig::GetBreakOnOOM())
11774 GCToOSInterface::DebugBreak();
11778 #ifdef BACKGROUND_GC
11779 BOOL gc_heap::background_allowed_p()
11781 return ( gc_can_use_concurrent && ((settings.pause_mode == pause_interactive) || (settings.pause_mode == pause_sustained_low_latency)) );
11783 #endif //BACKGROUND_GC
11785 void gc_heap::check_for_full_gc (int gen_num, size_t size)
11787 BOOL should_notify = FALSE;
11788 // if we detect full gc because of the allocation budget specified this is TRUE;
11789 // it's FALSE if it's due to other factors.
11790 BOOL alloc_factor = TRUE;
11793 int n_initial = gen_num;
11794 BOOL local_blocking_collection = FALSE;
11795 BOOL local_elevation_requested = FALSE;
11796 int new_alloc_remain_percent = 0;
11798 if (full_gc_approach_event_set)
11803 if (gen_num != (max_generation + 1))
11805 gen_num = max_generation;
11808 dynamic_data* dd_full = dynamic_data_of (gen_num);
11809 ptrdiff_t new_alloc_remain = 0;
11810 uint32_t pct = ((gen_num == (max_generation + 1)) ? fgn_loh_percent : fgn_maxgen_percent);
11812 for (int gen_index = 0; gen_index <= (max_generation + 1); gen_index++)
11814 dprintf (2, ("FGN: h#%d: gen%d: %Id(%Id)",
11815 heap_number, gen_index,
11816 dd_new_allocation (dynamic_data_of (gen_index)),
11817 dd_desired_allocation (dynamic_data_of (gen_index))));
11820 // For small object allocations we only check every fgn_check_quantum bytes.
11821 if (n_initial == 0)
11823 dprintf (2, ("FGN: gen0 last recorded alloc: %Id", fgn_last_alloc));
11824 dynamic_data* dd_0 = dynamic_data_of (n_initial);
11825 if (((fgn_last_alloc - dd_new_allocation (dd_0)) < fgn_check_quantum) &&
11826 (dd_new_allocation (dd_0) >= 0))
11832 fgn_last_alloc = dd_new_allocation (dd_0);
11833 dprintf (2, ("FGN: gen0 last recorded alloc is now: %Id", fgn_last_alloc));
11836 // We don't consider the size that came from soh 'cause it doesn't contribute to the
11841 for (i = n+1; i <= max_generation; i++)
11843 if (get_new_allocation (i) <= 0)
11845 n = min (i, max_generation);
11851 dprintf (2, ("FGN: h#%d: gen%d budget exceeded", heap_number, n));
11852 if (gen_num == max_generation)
11854 // If it's small object heap we should first see if we will even be looking at gen2 budget
11855 // in the next GC or not. If not we should go directly to checking other factors.
11856 if (n < (max_generation - 1))
11858 goto check_other_factors;
11862 new_alloc_remain = dd_new_allocation (dd_full) - size;
11864 new_alloc_remain_percent = (int)(((float)(new_alloc_remain) / (float)dd_desired_allocation (dd_full)) * 100);
11866 dprintf (2, ("FGN: alloc threshold for gen%d is %d%%, current threshold is %d%%",
11867 gen_num, pct, new_alloc_remain_percent));
11869 if (new_alloc_remain_percent <= (int)pct)
11871 #ifdef BACKGROUND_GC
11872 // If background GC is enabled, we still want to check whether this will
11873 // be a blocking GC or not because we only want to notify when it's a
11874 // blocking full GC.
11875 if (background_allowed_p())
11877 goto check_other_factors;
11879 #endif //BACKGROUND_GC
11881 should_notify = TRUE;
11885 check_other_factors:
11887 dprintf (2, ("FGC: checking other factors"));
11888 n = generation_to_condemn (n,
11889 &local_blocking_collection,
11890 &local_elevation_requested,
11893 if (local_elevation_requested && (n == max_generation))
11895 if (settings.should_lock_elevation)
11897 int local_elevation_locked_count = settings.elevation_locked_count + 1;
11898 if (local_elevation_locked_count != 6)
11900 dprintf (2, ("FGN: lock count is %d - Condemning max_generation-1",
11901 local_elevation_locked_count));
11902 n = max_generation - 1;
11907 dprintf (2, ("FGN: we estimate gen%d will be collected", n));
11909 #ifdef BACKGROUND_GC
11910 // When background GC is enabled it decreases the accuracy of our predictability -
11911 // by the time the GC happens, we may not be under BGC anymore. If we try to
11912 // predict often enough it should be ok.
11913 if ((n == max_generation) &&
11914 (recursive_gc_sync::background_running_p()))
11916 n = max_generation - 1;
11917 dprintf (2, ("FGN: bgc - 1 instead of 2"));
11920 if ((n == max_generation) && !local_blocking_collection)
11922 if (!background_allowed_p())
11924 local_blocking_collection = TRUE;
11927 #endif //BACKGROUND_GC
11929 dprintf (2, ("FGN: we estimate gen%d will be collected: %s",
11931 (local_blocking_collection ? "blocking" : "background")));
11933 if ((n == max_generation) && local_blocking_collection)
11935 alloc_factor = FALSE;
11936 should_notify = TRUE;
11944 dprintf (2, ("FGN: gen%d detecting full GC approaching(%s) (GC#%d) (%Id%% left in gen%d)",
11946 (alloc_factor ? "alloc" : "other"),
11947 dd_collection_count (dynamic_data_of (0)),
11948 new_alloc_remain_percent,
11951 send_full_gc_notification (n_initial, alloc_factor);
11955 void gc_heap::send_full_gc_notification (int gen_num, BOOL due_to_alloc_p)
11957 if (!full_gc_approach_event_set)
11959 assert (full_gc_approach_event.IsValid());
11960 FIRE_EVENT(GCFullNotify_V1, gen_num, due_to_alloc_p);
11962 full_gc_end_event.Reset();
11963 full_gc_approach_event.Set();
11964 full_gc_approach_event_set = true;
11968 wait_full_gc_status gc_heap::full_gc_wait (GCEvent *event, int time_out_ms)
11970 if (fgn_maxgen_percent == 0)
11972 return wait_full_gc_na;
11975 uint32_t wait_result = user_thread_wait(event, FALSE, time_out_ms);
11977 if ((wait_result == WAIT_OBJECT_0) || (wait_result == WAIT_TIMEOUT))
11979 if (fgn_maxgen_percent == 0)
11981 return wait_full_gc_cancelled;
11984 if (wait_result == WAIT_OBJECT_0)
11986 #ifdef BACKGROUND_GC
11987 if (fgn_last_gc_was_concurrent)
11989 fgn_last_gc_was_concurrent = FALSE;
11990 return wait_full_gc_na;
11993 #endif //BACKGROUND_GC
11995 return wait_full_gc_success;
12000 return wait_full_gc_timeout;
12005 return wait_full_gc_failed;
12009 size_t gc_heap::get_full_compact_gc_count()
12011 return full_gc_counts[gc_type_compacting];
12014 // DTREVIEW - we should check this in dt_low_ephemeral_space_p
12017 BOOL gc_heap::short_on_end_of_seg (int gen_number,
12021 UNREFERENCED_PARAMETER(gen_number);
12022 uint8_t* allocated = heap_segment_allocated(seg);
12024 BOOL sufficient_p = sufficient_space_end_seg (allocated,
12025 heap_segment_reserved (seg),
12026 end_space_after_gc(),
12027 tuning_deciding_short_on_seg);
12030 if (sufficient_gen0_space_p)
12032 dprintf (GTC_LOG, ("gen0 has enough free space"));
12035 sufficient_p = sufficient_gen0_space_p;
12038 return !sufficient_p;
12042 #pragma warning(disable:4706) // "assignment within conditional expression" is intentional in this function.
12046 BOOL gc_heap::a_fit_free_list_p (int gen_number,
12048 alloc_context* acontext,
12051 BOOL can_fit = FALSE;
12052 generation* gen = generation_of (gen_number);
12053 allocator* gen_allocator = generation_allocator (gen);
12054 size_t sz_list = gen_allocator->first_bucket_size();
12055 for (unsigned int a_l_idx = 0; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
12057 if ((size < sz_list) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
12059 uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
12060 uint8_t* prev_free_item = 0;
12062 while (free_list != 0)
12064 dprintf (3, ("considering free list %Ix", (size_t)free_list));
12065 size_t free_list_size = unused_array_size (free_list);
12066 if ((size + Align (min_obj_size, align_const)) <= free_list_size)
12068 dprintf (3, ("Found adequate unused area: [%Ix, size: %Id",
12069 (size_t)free_list, free_list_size));
12071 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
12072 // We ask for more Align (min_obj_size)
12073 // to make sure that we can insert a free object
12074 // in adjust_limit will set the limit lower
12075 size_t limit = limit_from_size (size, free_list_size, gen_number, align_const);
12077 uint8_t* remain = (free_list + limit);
12078 size_t remain_size = (free_list_size - limit);
12079 if (remain_size >= Align(min_free_list, align_const))
12081 make_unused_array (remain, remain_size);
12082 gen_allocator->thread_item_front (remain, remain_size);
12083 assert (remain_size >= Align (min_obj_size, align_const));
12087 //absorb the entire free list
12088 limit += remain_size;
12090 generation_free_list_space (gen) -= limit;
12092 adjust_limit_clr (free_list, limit, acontext, 0, align_const, gen_number);
12097 else if (gen_allocator->discard_if_no_fit_p())
12099 assert (prev_free_item == 0);
12100 dprintf (3, ("couldn't use this free area, discarding"));
12101 generation_free_obj_space (gen) += free_list_size;
12103 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
12104 generation_free_list_space (gen) -= free_list_size;
12108 prev_free_item = free_list;
12110 free_list = free_list_slot (free_list);
12113 sz_list = sz_list * 2;
12120 #ifdef BACKGROUND_GC
12121 void gc_heap::bgc_loh_alloc_clr (uint8_t* alloc_start,
12123 alloc_context* acontext,
12129 make_unused_array (alloc_start, size);
12131 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
12132 if (g_fEnableAppDomainMonitoring)
12134 GCToEEInterface::RecordAllocatedBytesForHeap(size, heap_number);
12136 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
12138 size_t size_of_array_base = sizeof(ArrayBase);
12140 bgc_alloc_lock->loh_alloc_done_with_index (lock_index);
12142 // clear memory while not holding the lock.
12143 size_t size_to_skip = size_of_array_base;
12144 size_t size_to_clear = size - size_to_skip - plug_skew;
12145 size_t saved_size_to_clear = size_to_clear;
12148 uint8_t* end = alloc_start + size - plug_skew;
12149 uint8_t* used = heap_segment_used (seg);
12152 if ((alloc_start + size_to_skip) < used)
12154 size_to_clear = used - (alloc_start + size_to_skip);
12160 dprintf (2, ("bgc loh: setting used to %Ix", end));
12161 heap_segment_used (seg) = end;
12164 dprintf (2, ("bgc loh: used: %Ix, alloc: %Ix, end of alloc: %Ix, clear %Id bytes",
12165 used, alloc_start, end, size_to_clear));
12169 dprintf (2, ("bgc loh: [%Ix-[%Ix(%Id)", alloc_start, alloc_start+size, size));
12173 // since we filled in 0xcc for free object when we verify heap,
12174 // we need to make sure we clear those bytes.
12175 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
12177 if (size_to_clear < saved_size_to_clear)
12179 size_to_clear = saved_size_to_clear;
12182 #endif //VERIFY_HEAP
12184 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to clear large obj", heap_number));
12185 add_saved_spinlock_info (true, me_release, mt_clr_large_mem);
12186 leave_spin_lock (&more_space_lock_loh);
12187 memclr (alloc_start + size_to_skip, size_to_clear);
12189 bgc_alloc_lock->loh_alloc_set (alloc_start);
12191 acontext->alloc_ptr = alloc_start;
12192 acontext->alloc_limit = (alloc_start + size - Align (min_obj_size, align_const));
12194 // need to clear the rest of the object before we hand it out.
12195 clear_unused_array(alloc_start, size);
12197 #endif //BACKGROUND_GC
12199 BOOL gc_heap::a_fit_free_list_large_p (size_t size,
12200 alloc_context* acontext,
12203 BOOL can_fit = FALSE;
12204 int gen_number = max_generation + 1;
12205 generation* gen = generation_of (gen_number);
12206 allocator* loh_allocator = generation_allocator (gen);
12208 #ifdef FEATURE_LOH_COMPACTION
12209 size_t loh_pad = Align (loh_padding_obj_size, align_const);
12210 #endif //FEATURE_LOH_COMPACTION
12212 #ifdef BACKGROUND_GC
12214 #endif //BACKGROUND_GC
12215 size_t sz_list = loh_allocator->first_bucket_size();
12216 for (unsigned int a_l_idx = 0; a_l_idx < loh_allocator->number_of_buckets(); a_l_idx++)
12218 if ((size < sz_list) || (a_l_idx == (loh_allocator->number_of_buckets()-1)))
12220 uint8_t* free_list = loh_allocator->alloc_list_head_of (a_l_idx);
12221 uint8_t* prev_free_item = 0;
12222 while (free_list != 0)
12224 dprintf (3, ("considering free list %Ix", (size_t)free_list));
12226 size_t free_list_size = unused_array_size(free_list);
12228 #ifdef FEATURE_LOH_COMPACTION
12229 if ((size + loh_pad) <= free_list_size)
12231 if (((size + Align (min_obj_size, align_const)) <= free_list_size)||
12232 (size == free_list_size))
12233 #endif //FEATURE_LOH_COMPACTION
12235 #ifdef BACKGROUND_GC
12236 cookie = bgc_alloc_lock->loh_alloc_set (free_list);
12237 bgc_track_loh_alloc();
12238 #endif //BACKGROUND_GC
12240 //unlink the free_item
12241 loh_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
12243 // Substract min obj size because limit_from_size adds it. Not needed for LOH
12244 size_t limit = limit_from_size (size - Align(min_obj_size, align_const), free_list_size,
12245 gen_number, align_const);
12247 #ifdef FEATURE_LOH_COMPACTION
12248 make_unused_array (free_list, loh_pad);
12250 free_list += loh_pad;
12251 free_list_size -= loh_pad;
12252 #endif //FEATURE_LOH_COMPACTION
12254 uint8_t* remain = (free_list + limit);
12255 size_t remain_size = (free_list_size - limit);
12256 if (remain_size != 0)
12258 assert (remain_size >= Align (min_obj_size, align_const));
12259 make_unused_array (remain, remain_size);
12261 if (remain_size >= Align(min_free_list, align_const))
12263 loh_thread_gap_front (remain, remain_size, gen);
12264 assert (remain_size >= Align (min_obj_size, align_const));
12268 generation_free_obj_space (gen) += remain_size;
12270 generation_free_list_space (gen) -= free_list_size;
12271 dprintf (3, ("found fit on loh at %Ix", free_list));
12272 #ifdef BACKGROUND_GC
12275 bgc_loh_alloc_clr (free_list, limit, acontext, align_const, cookie, FALSE, 0);
12278 #endif //BACKGROUND_GC
12280 adjust_limit_clr (free_list, limit, acontext, 0, align_const, gen_number);
12283 //fix the limit to compensate for adjust_limit_clr making it too short
12284 acontext->alloc_limit += Align (min_obj_size, align_const);
12288 prev_free_item = free_list;
12289 free_list = free_list_slot (free_list);
12292 sz_list = sz_list * 2;
12299 #pragma warning(default:4706)
12302 BOOL gc_heap::a_fit_segment_end_p (int gen_number,
12305 alloc_context* acontext,
12307 BOOL* commit_failed_p)
12309 *commit_failed_p = FALSE;
12311 bool hard_limit_short_seg_end_p = false;
12312 #ifdef BACKGROUND_GC
12314 #endif //BACKGROUND_GC
12316 uint8_t*& allocated = ((gen_number == 0) ?
12318 heap_segment_allocated(seg));
12320 size_t pad = Align (min_obj_size, align_const);
12322 #ifdef FEATURE_LOH_COMPACTION
12323 size_t loh_pad = Align (loh_padding_obj_size, align_const);
12324 if (gen_number == (max_generation + 1))
12328 #endif //FEATURE_LOH_COMPACTION
12330 uint8_t* end = heap_segment_committed (seg) - pad;
12332 if (a_size_fit_p (size, allocated, end, align_const))
12334 limit = limit_from_size (size,
12336 gen_number, align_const);
12340 end = heap_segment_reserved (seg) - pad;
12342 if (a_size_fit_p (size, allocated, end, align_const))
12344 limit = limit_from_size (size,
12346 gen_number, align_const);
12348 if (grow_heap_segment (seg, (allocated + limit), &hard_limit_short_seg_end_p))
12354 if (!hard_limit_short_seg_end_p)
12356 dprintf (2, ("can't grow segment, doing a full gc"));
12357 *commit_failed_p = TRUE;
12361 assert (heap_hard_limit);
12370 #ifdef BACKGROUND_GC
12371 if (gen_number != 0)
12373 cookie = bgc_alloc_lock->loh_alloc_set (allocated);
12374 bgc_track_loh_alloc();
12376 #endif //BACKGROUND_GC
12378 uint8_t* old_alloc;
12379 old_alloc = allocated;
12380 #ifdef FEATURE_LOH_COMPACTION
12381 if (gen_number == (max_generation + 1))
12383 make_unused_array (old_alloc, loh_pad);
12384 old_alloc += loh_pad;
12385 allocated += loh_pad;
12388 #endif //FEATURE_LOH_COMPACTION
12390 #if defined (VERIFY_HEAP) && defined (_DEBUG)
12391 ((void**) allocated)[-1] = 0; //clear the sync block
12392 #endif //VERIFY_HEAP && _DEBUG
12393 allocated += limit;
12395 dprintf (3, ("found fit at end of seg: %Ix", old_alloc));
12397 #ifdef BACKGROUND_GC
12400 bgc_loh_alloc_clr (old_alloc, limit, acontext, align_const, cookie, TRUE, seg);
12403 #endif //BACKGROUND_GC
12405 adjust_limit_clr (old_alloc, limit, acontext, seg, align_const, gen_number);
12415 BOOL gc_heap::loh_a_fit_segment_end_p (int gen_number,
12417 alloc_context* acontext,
12419 BOOL* commit_failed_p,
12422 *commit_failed_p = FALSE;
12423 heap_segment* seg = generation_allocation_segment (generation_of (gen_number));
12424 BOOL can_allocate_p = FALSE;
12428 #ifdef BACKGROUND_GC
12429 if (seg->flags & heap_segment_flags_loh_delete)
12431 dprintf (3, ("h%d skipping seg %Ix to be deleted", heap_number, (size_t)seg));
12434 #endif //BACKGROUND_GC
12436 if (a_fit_segment_end_p (gen_number, seg, (size - Align (min_obj_size, align_const)),
12437 acontext, align_const, commit_failed_p))
12439 acontext->alloc_limit += Align (min_obj_size, align_const);
12440 can_allocate_p = TRUE;
12444 if (*commit_failed_p)
12446 *oom_r = oom_cant_commit;
12451 seg = heap_segment_next_rw (seg);
12454 return can_allocate_p;
12457 #ifdef BACKGROUND_GC
12459 void gc_heap::wait_for_background (alloc_wait_reason awr, bool loh_p)
12461 GCSpinLock* msl = loh_p ? &more_space_lock_loh : &more_space_lock_soh;
12463 dprintf (2, ("BGC is already in progress, waiting for it to finish"));
12464 add_saved_spinlock_info (loh_p, me_release, mt_wait_bgc);
12465 leave_spin_lock (msl);
12466 background_gc_wait (awr);
12467 enter_spin_lock (msl);
12468 add_saved_spinlock_info (loh_p, me_acquire, mt_wait_bgc);
12471 void gc_heap::wait_for_bgc_high_memory (alloc_wait_reason awr, bool loh_p)
12473 if (recursive_gc_sync::background_running_p())
12475 uint32_t memory_load;
12476 get_memory_info (&memory_load);
12477 if (memory_load >= m_high_memory_load_th)
12479 dprintf (GTC_LOG, ("high mem - wait for BGC to finish, wait reason: %d", awr));
12480 wait_for_background (awr, loh_p);
12485 #endif //BACKGROUND_GC
12487 // We request to trigger an ephemeral GC but we may get a full compacting GC.
12488 // return TRUE if that's the case.
12489 BOOL gc_heap::trigger_ephemeral_gc (gc_reason gr)
12491 #ifdef BACKGROUND_GC
12492 wait_for_bgc_high_memory (awr_loh_oos_bgc, false);
12493 #endif //BACKGROUND_GC
12495 BOOL did_full_compact_gc = FALSE;
12497 dprintf (2, ("triggering a gen1 GC"));
12498 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12499 vm_heap->GarbageCollectGeneration(max_generation - 1, gr);
12501 #ifdef MULTIPLE_HEAPS
12502 enter_spin_lock (&more_space_lock_soh);
12503 add_saved_spinlock_info (false, me_acquire, mt_t_eph_gc);
12504 #endif //MULTIPLE_HEAPS
12506 size_t current_full_compact_gc_count = get_full_compact_gc_count();
12508 if (current_full_compact_gc_count > last_full_compact_gc_count)
12510 dprintf (2, ("attempted to trigger an ephemeral GC and got a full compacting GC"));
12511 did_full_compact_gc = TRUE;
12514 return did_full_compact_gc;
12517 BOOL gc_heap::soh_try_fit (int gen_number,
12519 alloc_context* acontext,
12521 BOOL* commit_failed_p,
12522 BOOL* short_seg_end_p)
12524 BOOL can_allocate = TRUE;
12525 if (short_seg_end_p)
12527 *short_seg_end_p = FALSE;
12530 can_allocate = a_fit_free_list_p (gen_number, size, acontext, align_const);
12533 if (short_seg_end_p)
12535 *short_seg_end_p = short_on_end_of_seg (gen_number, ephemeral_heap_segment, align_const);
12537 // If the caller doesn't care, we always try to fit at the end of seg;
12538 // otherwise we would only try if we are actually not short at end of seg.
12539 if (!short_seg_end_p || !(*short_seg_end_p))
12541 can_allocate = a_fit_segment_end_p (gen_number, ephemeral_heap_segment, size,
12542 acontext, align_const, commit_failed_p);
12546 return can_allocate;
12549 allocation_state gc_heap::allocate_small (int gen_number,
12551 alloc_context* acontext,
12554 #if defined (BACKGROUND_GC) && !defined (MULTIPLE_HEAPS)
12555 if (recursive_gc_sync::background_running_p())
12557 background_soh_alloc_count++;
12558 if ((background_soh_alloc_count % bgc_alloc_spin_count) == 0)
12560 add_saved_spinlock_info (false, me_release, mt_alloc_small);
12561 leave_spin_lock (&more_space_lock_soh);
12562 bool cooperative_mode = enable_preemptive();
12563 GCToOSInterface::Sleep (bgc_alloc_spin);
12564 disable_preemptive (cooperative_mode);
12565 enter_spin_lock (&more_space_lock_soh);
12566 add_saved_spinlock_info (false, me_acquire, mt_alloc_small);
12570 //GCToOSInterface::YieldThread (0);
12573 #endif //BACKGROUND_GC && !MULTIPLE_HEAPS
12575 gc_reason gr = reason_oos_soh;
12576 oom_reason oom_r = oom_no_failure;
12578 // No variable values should be "carried over" from one state to the other.
12579 // That's why there are local variable for each state
12581 allocation_state soh_alloc_state = a_state_start;
12583 // If we can get a new seg it means allocation will succeed.
12586 dprintf (3, ("[h%d]soh state is %s", heap_number, allocation_state_str[soh_alloc_state]));
12588 switch (soh_alloc_state)
12590 case a_state_can_allocate:
12591 case a_state_cant_allocate:
12595 case a_state_start:
12597 soh_alloc_state = a_state_try_fit;
12600 case a_state_try_fit:
12602 BOOL commit_failed_p = FALSE;
12603 BOOL can_use_existing_p = FALSE;
12605 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12606 align_const, &commit_failed_p,
12608 soh_alloc_state = (can_use_existing_p ?
12609 a_state_can_allocate :
12611 a_state_trigger_full_compact_gc :
12612 a_state_trigger_ephemeral_gc));
12615 case a_state_try_fit_after_bgc:
12617 BOOL commit_failed_p = FALSE;
12618 BOOL can_use_existing_p = FALSE;
12619 BOOL short_seg_end_p = FALSE;
12621 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12622 align_const, &commit_failed_p,
12624 soh_alloc_state = (can_use_existing_p ?
12625 a_state_can_allocate :
12627 a_state_trigger_2nd_ephemeral_gc :
12628 a_state_trigger_full_compact_gc));
12631 case a_state_try_fit_after_cg:
12633 BOOL commit_failed_p = FALSE;
12634 BOOL can_use_existing_p = FALSE;
12635 BOOL short_seg_end_p = FALSE;
12637 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12638 align_const, &commit_failed_p,
12641 if (can_use_existing_p)
12643 soh_alloc_state = a_state_can_allocate;
12645 #ifdef MULTIPLE_HEAPS
12646 else if (gen0_allocated_after_gc_p)
12648 // some other threads already grabbed the more space lock and allocated
12649 // so we should attempt an ephemeral GC again.
12650 soh_alloc_state = a_state_trigger_ephemeral_gc;
12652 #endif //MULTIPLE_HEAPS
12653 else if (short_seg_end_p)
12655 soh_alloc_state = a_state_cant_allocate;
12656 oom_r = oom_budget;
12660 assert (commit_failed_p);
12661 soh_alloc_state = a_state_cant_allocate;
12662 oom_r = oom_cant_commit;
12666 case a_state_check_and_wait_for_bgc:
12668 BOOL bgc_in_progress_p = FALSE;
12669 BOOL did_full_compacting_gc = FALSE;
12671 bgc_in_progress_p = check_and_wait_for_bgc (awr_gen0_oos_bgc, &did_full_compacting_gc, false);
12672 soh_alloc_state = (did_full_compacting_gc ?
12673 a_state_try_fit_after_cg :
12674 a_state_try_fit_after_bgc);
12677 case a_state_trigger_ephemeral_gc:
12679 BOOL commit_failed_p = FALSE;
12680 BOOL can_use_existing_p = FALSE;
12681 BOOL short_seg_end_p = FALSE;
12682 BOOL bgc_in_progress_p = FALSE;
12683 BOOL did_full_compacting_gc = FALSE;
12685 did_full_compacting_gc = trigger_ephemeral_gc (gr);
12686 if (did_full_compacting_gc)
12688 soh_alloc_state = a_state_try_fit_after_cg;
12692 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12693 align_const, &commit_failed_p,
12695 #ifdef BACKGROUND_GC
12696 bgc_in_progress_p = recursive_gc_sync::background_running_p();
12697 #endif //BACKGROUND_GC
12699 if (can_use_existing_p)
12701 soh_alloc_state = a_state_can_allocate;
12705 if (short_seg_end_p)
12707 if (should_expand_in_full_gc)
12709 dprintf (2, ("gen1 GC wanted to expand!"));
12710 soh_alloc_state = a_state_trigger_full_compact_gc;
12714 soh_alloc_state = (bgc_in_progress_p ?
12715 a_state_check_and_wait_for_bgc :
12716 a_state_trigger_full_compact_gc);
12719 else if (commit_failed_p)
12721 soh_alloc_state = a_state_trigger_full_compact_gc;
12725 #ifdef MULTIPLE_HEAPS
12726 // some other threads already grabbed the more space lock and allocated
12727 // so we should attempt an ephemeral GC again.
12728 assert (gen0_allocated_after_gc_p);
12729 soh_alloc_state = a_state_trigger_ephemeral_gc;
12730 #else //MULTIPLE_HEAPS
12731 assert (!"shouldn't get here");
12732 #endif //MULTIPLE_HEAPS
12738 case a_state_trigger_2nd_ephemeral_gc:
12740 BOOL commit_failed_p = FALSE;
12741 BOOL can_use_existing_p = FALSE;
12742 BOOL short_seg_end_p = FALSE;
12743 BOOL did_full_compacting_gc = FALSE;
12746 did_full_compacting_gc = trigger_ephemeral_gc (gr);
12748 if (did_full_compacting_gc)
12750 soh_alloc_state = a_state_try_fit_after_cg;
12754 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12755 align_const, &commit_failed_p,
12757 if (short_seg_end_p || commit_failed_p)
12759 soh_alloc_state = a_state_trigger_full_compact_gc;
12763 assert (can_use_existing_p);
12764 soh_alloc_state = a_state_can_allocate;
12769 case a_state_trigger_full_compact_gc:
12771 if (fgn_maxgen_percent)
12773 dprintf (2, ("FGN: SOH doing last GC before we throw OOM"));
12774 send_full_gc_notification (max_generation, FALSE);
12777 BOOL got_full_compacting_gc = FALSE;
12779 got_full_compacting_gc = trigger_full_compact_gc (gr, &oom_r, false);
12780 soh_alloc_state = (got_full_compacting_gc ? a_state_try_fit_after_cg : a_state_cant_allocate);
12785 assert (!"Invalid state!");
12792 if (soh_alloc_state == a_state_cant_allocate)
12794 assert (oom_r != oom_no_failure);
12795 handle_oom (heap_number,
12798 heap_segment_allocated (ephemeral_heap_segment),
12799 heap_segment_reserved (ephemeral_heap_segment));
12801 add_saved_spinlock_info (false, me_release, mt_alloc_small_cant);
12802 leave_spin_lock (&more_space_lock_soh);
12805 assert ((soh_alloc_state == a_state_can_allocate) ||
12806 (soh_alloc_state == a_state_cant_allocate) ||
12807 (soh_alloc_state == a_state_retry_allocate));
12809 return soh_alloc_state;
12812 #ifdef BACKGROUND_GC
12814 void gc_heap::bgc_track_loh_alloc()
12816 if (current_c_gc_state == c_gc_state_planning)
12818 Interlocked::Increment (&loh_alloc_thread_count);
12819 dprintf (3, ("h%d: inc lc: %d", heap_number, (int32_t)loh_alloc_thread_count));
12824 void gc_heap::bgc_untrack_loh_alloc()
12826 if (current_c_gc_state == c_gc_state_planning)
12828 Interlocked::Decrement (&loh_alloc_thread_count);
12829 dprintf (3, ("h%d: dec lc: %d", heap_number, (int32_t)loh_alloc_thread_count));
12833 BOOL gc_heap::bgc_loh_should_allocate()
12835 size_t min_gc_size = dd_min_size (dynamic_data_of (max_generation + 1));
12837 if ((bgc_begin_loh_size + bgc_loh_size_increased) < (min_gc_size * 10))
12842 if (((bgc_begin_loh_size / end_loh_size) >= 2) || (bgc_loh_size_increased >= bgc_begin_loh_size))
12844 if ((bgc_begin_loh_size / end_loh_size) > 2)
12846 dprintf (3, ("alloc-ed too much before bgc started"));
12850 dprintf (3, ("alloc-ed too much after bgc started"));
12856 bgc_alloc_spin_loh = (uint32_t)(((float)bgc_loh_size_increased / (float)bgc_begin_loh_size) * 10);
12860 #endif //BACKGROUND_GC
12862 size_t gc_heap::get_large_seg_size (size_t size)
12864 size_t default_seg_size = min_loh_segment_size;
12865 #ifdef SEG_MAPPING_TABLE
12866 size_t align_size = default_seg_size;
12867 #else //SEG_MAPPING_TABLE
12868 size_t align_size = default_seg_size / 2;
12869 #endif //SEG_MAPPING_TABLE
12870 int align_const = get_alignment_constant (FALSE);
12871 size_t large_seg_size = align_on_page (
12872 max (default_seg_size,
12873 ((size + 2 * Align(min_obj_size, align_const) + OS_PAGE_SIZE +
12874 align_size) / align_size * align_size)));
12875 return large_seg_size;
12878 BOOL gc_heap::loh_get_new_seg (generation* gen,
12881 BOOL* did_full_compact_gc,
12884 UNREFERENCED_PARAMETER(gen);
12885 UNREFERENCED_PARAMETER(align_const);
12887 *did_full_compact_gc = FALSE;
12889 size_t seg_size = get_large_seg_size (size);
12891 heap_segment* new_seg = get_large_segment (seg_size, did_full_compact_gc);
12895 loh_alloc_since_cg += seg_size;
12902 return (new_seg != 0);
12905 // PERF TODO: this is too aggressive; and in hard limit we should
12906 // count the actual allocated bytes instead of only updating it during
12907 // getting a new seg.
12908 BOOL gc_heap::retry_full_compact_gc (size_t size)
12910 size_t seg_size = get_large_seg_size (size);
12912 if (loh_alloc_since_cg >= (2 * (uint64_t)seg_size))
12917 #ifdef MULTIPLE_HEAPS
12918 uint64_t total_alloc_size = 0;
12919 for (int i = 0; i < n_heaps; i++)
12921 total_alloc_size += g_heaps[i]->loh_alloc_since_cg;
12924 if (total_alloc_size >= (2 * (uint64_t)seg_size))
12928 #endif //MULTIPLE_HEAPS
12933 BOOL gc_heap::check_and_wait_for_bgc (alloc_wait_reason awr,
12934 BOOL* did_full_compact_gc,
12937 BOOL bgc_in_progress = FALSE;
12938 *did_full_compact_gc = FALSE;
12939 #ifdef BACKGROUND_GC
12940 if (recursive_gc_sync::background_running_p())
12942 bgc_in_progress = TRUE;
12943 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12944 wait_for_background (awr, loh_p);
12945 size_t current_full_compact_gc_count = get_full_compact_gc_count();
12946 if (current_full_compact_gc_count > last_full_compact_gc_count)
12948 *did_full_compact_gc = TRUE;
12951 #endif //BACKGROUND_GC
12953 return bgc_in_progress;
12956 BOOL gc_heap::loh_try_fit (int gen_number,
12958 alloc_context* acontext,
12960 BOOL* commit_failed_p,
12963 BOOL can_allocate = TRUE;
12965 if (!a_fit_free_list_large_p (size, acontext, align_const))
12967 can_allocate = loh_a_fit_segment_end_p (gen_number, size,
12968 acontext, align_const,
12969 commit_failed_p, oom_r);
12971 #ifdef BACKGROUND_GC
12972 if (can_allocate && recursive_gc_sync::background_running_p())
12974 bgc_loh_size_increased += size;
12976 #endif //BACKGROUND_GC
12978 #ifdef BACKGROUND_GC
12981 if (recursive_gc_sync::background_running_p())
12983 bgc_loh_allocated_in_free += size;
12986 #endif //BACKGROUND_GC
12988 return can_allocate;
12991 BOOL gc_heap::trigger_full_compact_gc (gc_reason gr,
12995 BOOL did_full_compact_gc = FALSE;
12997 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12999 // Set this so the next GC will be a full compacting GC.
13000 if (!last_gc_before_oom)
13002 last_gc_before_oom = TRUE;
13005 #ifdef BACKGROUND_GC
13006 if (recursive_gc_sync::background_running_p())
13008 wait_for_background (((gr == reason_oos_soh) ? awr_gen0_oos_bgc : awr_loh_oos_bgc), loh_p);
13009 dprintf (2, ("waited for BGC - done"));
13011 #endif //BACKGROUND_GC
13013 GCSpinLock* msl = loh_p ? &more_space_lock_loh : &more_space_lock_soh;
13014 size_t current_full_compact_gc_count = get_full_compact_gc_count();
13015 if (current_full_compact_gc_count > last_full_compact_gc_count)
13017 dprintf (3, ("a full compacting GC triggered while waiting for BGC (%d->%d)", last_full_compact_gc_count, current_full_compact_gc_count));
13018 assert (current_full_compact_gc_count > last_full_compact_gc_count);
13019 did_full_compact_gc = TRUE;
13023 dprintf (3, ("h%d full GC", heap_number));
13025 trigger_gc_for_alloc (max_generation, gr, msl, loh_p, mt_t_full_gc);
13027 current_full_compact_gc_count = get_full_compact_gc_count();
13029 if (current_full_compact_gc_count == last_full_compact_gc_count)
13031 dprintf (2, ("attempted to trigger a full compacting GC but didn't get it"));
13032 // We requested a full GC but didn't get because of the elevation logic
13033 // which means we should fail.
13034 *oom_r = oom_unproductive_full_gc;
13038 dprintf (3, ("h%d: T full compacting GC (%d->%d)",
13040 last_full_compact_gc_count,
13041 current_full_compact_gc_count));
13043 assert (current_full_compact_gc_count > last_full_compact_gc_count);
13044 did_full_compact_gc = TRUE;
13048 return did_full_compact_gc;
13051 #ifdef RECORD_LOH_STATE
13052 void gc_heap::add_saved_loh_state (allocation_state loh_state_to_save, EEThreadId thread_id)
13054 // When the state is can_allocate we already have released the more
13055 // space lock. So we are not logging states here since this code
13056 // is not thread safe.
13057 if (loh_state_to_save != a_state_can_allocate)
13059 last_loh_states[loh_state_index].alloc_state = loh_state_to_save;
13060 last_loh_states[loh_state_index].thread_id = thread_id;
13063 if (loh_state_index == max_saved_loh_states)
13065 loh_state_index = 0;
13068 assert (loh_state_index < max_saved_loh_states);
13071 #endif //RECORD_LOH_STATE
13073 bool gc_heap::should_retry_other_heap (size_t size)
13075 #ifdef MULTIPLE_HEAPS
13076 if (heap_hard_limit)
13078 size_t total_heap_committed_recorded =
13079 current_total_committed - current_total_committed_bookkeeping;
13080 size_t min_size = dd_min_size (g_heaps[0]->dynamic_data_of (max_generation + 1));
13081 size_t slack_space = max (commit_min_th, min_size);
13082 bool retry_p = ((total_heap_committed_recorded + size) < (heap_hard_limit - slack_space));
13083 dprintf (1, ("%Id - %Id - total committed %Id - size %Id = %Id, %s",
13084 heap_hard_limit, slack_space, total_heap_committed_recorded, size,
13085 (heap_hard_limit - slack_space - total_heap_committed_recorded - size),
13086 (retry_p ? "retry" : "no retry")));
13090 #endif //MULTIPLE_HEAPS
13096 allocation_state gc_heap::allocate_large (int gen_number,
13098 alloc_context* acontext,
13101 #ifdef BACKGROUND_GC
13102 if (recursive_gc_sync::background_running_p())
13104 background_loh_alloc_count++;
13105 //if ((background_loh_alloc_count % bgc_alloc_spin_count_loh) == 0)
13107 if (bgc_loh_should_allocate())
13109 if (!bgc_alloc_spin_loh)
13111 add_saved_spinlock_info (true, me_release, mt_alloc_large);
13112 leave_spin_lock (&more_space_lock_loh);
13113 bool cooperative_mode = enable_preemptive();
13114 GCToOSInterface::YieldThread (bgc_alloc_spin_loh);
13115 disable_preemptive (cooperative_mode);
13116 enter_spin_lock (&more_space_lock_loh);
13117 add_saved_spinlock_info (true, me_acquire, mt_alloc_large);
13118 dprintf (SPINLOCK_LOG, ("[%d]spin Emsl loh", heap_number));
13123 wait_for_background (awr_loh_alloc_during_bgc, true);
13127 #endif //BACKGROUND_GC
13129 gc_reason gr = reason_oos_loh;
13130 generation* gen = generation_of (gen_number);
13131 oom_reason oom_r = oom_no_failure;
13132 size_t current_full_compact_gc_count = 0;
13134 // No variable values should be "carried over" from one state to the other.
13135 // That's why there are local variable for each state
13136 allocation_state loh_alloc_state = a_state_start;
13137 #ifdef RECORD_LOH_STATE
13138 EEThreadId current_thread_id;
13139 current_thread_id.SetToCurrentThread();
13140 #endif //RECORD_LOH_STATE
13142 // If we can get a new seg it means allocation will succeed.
13145 dprintf (3, ("[h%d]loh state is %s", heap_number, allocation_state_str[loh_alloc_state]));
13147 #ifdef RECORD_LOH_STATE
13148 add_saved_loh_state (loh_alloc_state, current_thread_id);
13149 #endif //RECORD_LOH_STATE
13150 switch (loh_alloc_state)
13152 case a_state_can_allocate:
13153 case a_state_cant_allocate:
13157 case a_state_start:
13159 loh_alloc_state = a_state_try_fit;
13162 case a_state_try_fit:
13164 BOOL commit_failed_p = FALSE;
13165 BOOL can_use_existing_p = FALSE;
13167 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
13168 align_const, &commit_failed_p, &oom_r);
13169 loh_alloc_state = (can_use_existing_p ?
13170 a_state_can_allocate :
13172 a_state_trigger_full_compact_gc :
13173 a_state_acquire_seg));
13174 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
13177 case a_state_try_fit_new_seg:
13179 BOOL commit_failed_p = FALSE;
13180 BOOL can_use_existing_p = FALSE;
13182 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
13183 align_const, &commit_failed_p, &oom_r);
13184 // Even after we got a new seg it doesn't necessarily mean we can allocate,
13185 // another LOH allocating thread could have beat us to acquire the msl so
13186 // we need to try again.
13187 loh_alloc_state = (can_use_existing_p ? a_state_can_allocate : a_state_try_fit);
13188 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
13191 case a_state_try_fit_after_cg:
13193 BOOL commit_failed_p = FALSE;
13194 BOOL can_use_existing_p = FALSE;
13196 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
13197 align_const, &commit_failed_p, &oom_r);
13198 // If we failed to commit, we bail right away 'cause we already did a
13199 // full compacting GC.
13200 loh_alloc_state = (can_use_existing_p ?
13201 a_state_can_allocate :
13203 a_state_cant_allocate :
13204 a_state_acquire_seg_after_cg));
13205 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
13208 case a_state_try_fit_after_bgc:
13210 BOOL commit_failed_p = FALSE;
13211 BOOL can_use_existing_p = FALSE;
13213 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
13214 align_const, &commit_failed_p, &oom_r);
13215 loh_alloc_state = (can_use_existing_p ?
13216 a_state_can_allocate :
13218 a_state_trigger_full_compact_gc :
13219 a_state_acquire_seg_after_bgc));
13220 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
13223 case a_state_acquire_seg:
13225 BOOL can_get_new_seg_p = FALSE;
13226 BOOL did_full_compacting_gc = FALSE;
13228 current_full_compact_gc_count = get_full_compact_gc_count();
13230 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
13231 loh_alloc_state = (can_get_new_seg_p ?
13232 a_state_try_fit_new_seg :
13233 (did_full_compacting_gc ?
13234 a_state_check_retry_seg :
13235 a_state_check_and_wait_for_bgc));
13238 case a_state_acquire_seg_after_cg:
13240 BOOL can_get_new_seg_p = FALSE;
13241 BOOL did_full_compacting_gc = FALSE;
13243 current_full_compact_gc_count = get_full_compact_gc_count();
13245 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
13246 // Since we release the msl before we try to allocate a seg, other
13247 // threads could have allocated a bunch of segments before us so
13248 // we might need to retry.
13249 loh_alloc_state = (can_get_new_seg_p ?
13250 a_state_try_fit_after_cg :
13251 a_state_check_retry_seg);
13254 case a_state_acquire_seg_after_bgc:
13256 BOOL can_get_new_seg_p = FALSE;
13257 BOOL did_full_compacting_gc = FALSE;
13259 current_full_compact_gc_count = get_full_compact_gc_count();
13261 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
13262 loh_alloc_state = (can_get_new_seg_p ?
13263 a_state_try_fit_new_seg :
13264 (did_full_compacting_gc ?
13265 a_state_check_retry_seg :
13266 a_state_trigger_full_compact_gc));
13267 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13270 case a_state_check_and_wait_for_bgc:
13272 BOOL bgc_in_progress_p = FALSE;
13273 BOOL did_full_compacting_gc = FALSE;
13275 bgc_in_progress_p = check_and_wait_for_bgc (awr_loh_oos_bgc, &did_full_compacting_gc, true);
13276 loh_alloc_state = (!bgc_in_progress_p ?
13277 a_state_trigger_full_compact_gc :
13278 (did_full_compacting_gc ?
13279 a_state_try_fit_after_cg :
13280 a_state_try_fit_after_bgc));
13283 case a_state_trigger_full_compact_gc:
13285 if (fgn_maxgen_percent)
13287 dprintf (2, ("FGN: LOH doing last GC before we throw OOM"));
13288 send_full_gc_notification (max_generation, FALSE);
13291 BOOL got_full_compacting_gc = FALSE;
13293 got_full_compacting_gc = trigger_full_compact_gc (gr, &oom_r, true);
13294 loh_alloc_state = (got_full_compacting_gc ? a_state_try_fit_after_cg : a_state_cant_allocate);
13295 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13298 case a_state_check_retry_seg:
13300 BOOL should_retry_gc = retry_full_compact_gc (size);
13301 BOOL should_retry_get_seg = FALSE;
13302 if (!should_retry_gc)
13304 size_t last_full_compact_gc_count = current_full_compact_gc_count;
13305 current_full_compact_gc_count = get_full_compact_gc_count();
13306 if (current_full_compact_gc_count > last_full_compact_gc_count)
13308 should_retry_get_seg = TRUE;
13312 loh_alloc_state = (should_retry_gc ?
13313 a_state_trigger_full_compact_gc :
13314 (should_retry_get_seg ?
13315 a_state_try_fit_after_cg :
13316 a_state_cant_allocate));
13317 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13322 assert (!"Invalid state!");
13329 if (loh_alloc_state == a_state_cant_allocate)
13331 assert (oom_r != oom_no_failure);
13332 if (should_retry_other_heap (size))
13334 loh_alloc_state = a_state_retry_allocate;
13338 handle_oom (heap_number,
13344 add_saved_spinlock_info (true, me_release, mt_alloc_large_cant);
13345 leave_spin_lock (&more_space_lock_loh);
13348 assert ((loh_alloc_state == a_state_can_allocate) ||
13349 (loh_alloc_state == a_state_cant_allocate) ||
13350 (loh_alloc_state == a_state_retry_allocate));
13351 return loh_alloc_state;
13354 // BGC's final mark phase will acquire the msl, so release it here and re-acquire.
13355 void gc_heap::trigger_gc_for_alloc (int gen_number, gc_reason gr,
13356 GCSpinLock* msl, bool loh_p,
13357 msl_take_state take_state)
13359 #ifdef BACKGROUND_GC
13362 add_saved_spinlock_info (loh_p, me_release, take_state);
13363 leave_spin_lock (msl);
13365 #endif //BACKGROUND_GC
13367 vm_heap->GarbageCollectGeneration (gen_number, gr);
13369 #ifdef MULTIPLE_HEAPS
13372 enter_spin_lock (msl);
13373 add_saved_spinlock_info (loh_p, me_acquire, take_state);
13375 #endif //MULTIPLE_HEAPS
13377 #ifdef BACKGROUND_GC
13380 enter_spin_lock (msl);
13381 add_saved_spinlock_info (loh_p, me_acquire, take_state);
13383 #endif //BACKGROUND_GC
13386 allocation_state gc_heap::try_allocate_more_space (alloc_context* acontext, size_t size,
13389 if (gc_heap::gc_started)
13391 wait_for_gc_done();
13392 return a_state_retry_allocate;
13395 bool loh_p = (gen_number > 0);
13396 GCSpinLock* msl = loh_p ? &more_space_lock_loh : &more_space_lock_soh;
13398 #ifdef SYNCHRONIZATION_STATS
13399 int64_t msl_acquire_start = GCToOSInterface::QueryPerformanceCounter();
13400 #endif //SYNCHRONIZATION_STATS
13401 enter_spin_lock (msl);
13402 add_saved_spinlock_info (loh_p, me_acquire, mt_try_alloc);
13403 dprintf (SPINLOCK_LOG, ("[%d]Emsl for alloc", heap_number));
13404 #ifdef SYNCHRONIZATION_STATS
13405 int64_t msl_acquire = GCToOSInterface::QueryPerformanceCounter() - msl_acquire_start;
13406 total_msl_acquire += msl_acquire;
13407 num_msl_acquired++;
13408 if (msl_acquire > 200)
13410 num_high_msl_acquire++;
13414 num_low_msl_acquire++;
13416 #endif //SYNCHRONIZATION_STATS
13419 // We are commenting this out 'cause we don't see the point - we already
13420 // have checked gc_started when we were acquiring the msl - no need to check
13421 // again. This complicates the logic in bgc_suspend_EE 'cause that one would
13422 // need to release msl which causes all sorts of trouble.
13423 if (gc_heap::gc_started)
13425 #ifdef SYNCHRONIZATION_STATS
13427 #endif //SYNCHRONIZATION_STATS
13428 BOOL fStress = (g_pConfig->GetGCStressLevel() & GCConfig::GCSTRESS_TRANSITION) != 0;
13431 //Rendez vous early (MP scaling issue)
13432 //dprintf (1, ("[%d]waiting for gc", heap_number));
13433 wait_for_gc_done();
13434 #ifdef MULTIPLE_HEAPS
13436 #endif //MULTIPLE_HEAPS
13441 dprintf (3, ("requested to allocate %d bytes on gen%d", size, gen_number));
13443 int align_const = get_alignment_constant (gen_number != (max_generation+1));
13445 if (fgn_maxgen_percent)
13447 check_for_full_gc (gen_number, size);
13450 if (!(new_allocation_allowed (gen_number)))
13452 if (fgn_maxgen_percent && (gen_number == 0))
13454 // We only check gen0 every so often, so take this opportunity to check again.
13455 check_for_full_gc (gen_number, size);
13458 #ifdef BACKGROUND_GC
13459 wait_for_bgc_high_memory (awr_gen0_alloc, loh_p);
13460 #endif //BACKGROUND_GC
13462 #ifdef SYNCHRONIZATION_STATS
13464 #endif //SYNCHRONIZATION_STATS
13465 dprintf (/*100*/ 2, ("running out of budget on gen%d, gc", gen_number));
13467 if (!settings.concurrent || (gen_number == 0))
13469 trigger_gc_for_alloc (0, ((gen_number == 0) ? reason_alloc_soh : reason_alloc_loh),
13470 msl, loh_p, mt_try_budget);
13474 allocation_state can_allocate = ((gen_number == 0) ?
13475 allocate_small (gen_number, size, acontext, align_const) :
13476 allocate_large (gen_number, size, acontext, align_const));
13478 if (can_allocate == a_state_can_allocate)
13480 size_t alloc_context_bytes = acontext->alloc_limit + Align (min_obj_size, align_const) - acontext->alloc_ptr;
13481 int etw_allocation_index = ((gen_number == 0) ? 0 : 1);
13483 etw_allocation_running_amount[etw_allocation_index] += alloc_context_bytes;
13485 allocated_since_last_gc += alloc_context_bytes;
13487 if (etw_allocation_running_amount[etw_allocation_index] > etw_allocation_tick)
13489 #ifdef FEATURE_REDHAWK
13490 FIRE_EVENT(GCAllocationTick_V1, (uint32_t)etw_allocation_running_amount[etw_allocation_index],
13491 (gen_number == 0) ? gc_etw_alloc_soh : gc_etw_alloc_loh);
13493 // Unfortunately some of the ETW macros do not check whether the ETW feature is enabled.
13494 // The ones that do are much less efficient.
13495 #if defined(FEATURE_EVENT_TRACE)
13496 if (EVENT_ENABLED(GCAllocationTick_V3))
13498 fire_etw_allocation_event (etw_allocation_running_amount[etw_allocation_index], gen_number, acontext->alloc_ptr);
13500 #endif //FEATURE_EVENT_TRACE
13501 #endif //FEATURE_REDHAWK
13502 etw_allocation_running_amount[etw_allocation_index] = 0;
13506 return can_allocate;
13509 #ifdef MULTIPLE_HEAPS
13510 void gc_heap::balance_heaps (alloc_context* acontext)
13512 if (acontext->alloc_count < 4)
13514 if (acontext->alloc_count == 0)
13516 acontext->set_home_heap(GCHeap::GetHeap( heap_select::select_heap(acontext, 0) ));
13517 gc_heap* hp = acontext->get_home_heap()->pGenGCHeap;
13518 dprintf (3, ("First allocation for context %Ix on heap %d\n", (size_t)acontext, (size_t)hp->heap_number));
13519 acontext->set_alloc_heap(acontext->get_home_heap());
13520 hp->alloc_context_count++;
13525 BOOL set_home_heap = FALSE;
13528 if (heap_select::can_find_heap_fast())
13530 if (acontext->get_home_heap() != NULL)
13531 hint = acontext->get_home_heap()->pGenGCHeap->heap_number;
13532 if (acontext->get_home_heap() != GCHeap::GetHeap(hint = heap_select::select_heap(acontext, hint)) || ((acontext->alloc_count & 15) == 0))
13534 set_home_heap = TRUE;
13540 if ((acontext->alloc_count & 3) == 0)
13541 set_home_heap = TRUE;
13547 // Since we are balancing up to MAX_SUPPORTED_CPUS, no need for this.
13548 if (n_heaps > MAX_SUPPORTED_CPUS)
13550 // on machines with many processors cache affinity is really king, so don't even try
13551 // to balance on these.
13552 acontext->home_heap = GCHeap::GetHeap( heap_select::select_heap(acontext, hint) );
13553 acontext->alloc_heap = acontext->home_heap;
13558 gc_heap* org_hp = acontext->get_alloc_heap()->pGenGCHeap;
13560 dynamic_data* dd = org_hp->dynamic_data_of (0);
13561 ptrdiff_t org_size = dd_new_allocation (dd);
13562 int org_alloc_context_count;
13563 int max_alloc_context_count;
13565 ptrdiff_t max_size;
13566 size_t delta = dd_min_size (dd)/4;
13568 int start, end, finish;
13569 heap_select::get_heap_range_for_heap(org_hp->heap_number, &start, &end);
13570 finish = start + n_heaps;
13576 max_size = org_size + delta;
13577 acontext->set_home_heap(GCHeap::GetHeap( heap_select::select_heap(acontext, hint) ));
13579 if (org_hp == acontext->get_home_heap()->pGenGCHeap)
13580 max_size = max_size + delta;
13582 org_alloc_context_count = org_hp->alloc_context_count;
13583 max_alloc_context_count = org_alloc_context_count;
13584 if (max_alloc_context_count > 1)
13585 max_size /= max_alloc_context_count;
13587 for (int i = start; i < end; i++)
13589 gc_heap* hp = GCHeap::GetHeap(i%n_heaps)->pGenGCHeap;
13590 dd = hp->dynamic_data_of (0);
13591 ptrdiff_t size = dd_new_allocation (dd);
13592 if (hp == acontext->get_home_heap()->pGenGCHeap)
13593 size = size + delta;
13594 int hp_alloc_context_count = hp->alloc_context_count;
13595 if (hp_alloc_context_count > 0)
13596 size /= (hp_alloc_context_count + 1);
13597 if (size > max_size)
13601 max_alloc_context_count = hp_alloc_context_count;
13605 while (org_alloc_context_count != org_hp->alloc_context_count ||
13606 max_alloc_context_count != max_hp->alloc_context_count);
13608 if ((max_hp == org_hp) && (end < finish))
13610 start = end; end = finish;
13611 delta = dd_min_size(dd)/2; // Make it twice as hard to balance to remote nodes on NUMA.
13615 if (max_hp != org_hp)
13617 org_hp->alloc_context_count--;
13618 max_hp->alloc_context_count++;
13619 acontext->set_alloc_heap(GCHeap::GetHeap(max_hp->heap_number));
13620 if (!gc_thread_no_affinitize_p)
13622 if (GCToOSInterface::CanEnableGCCPUGroups())
13623 { //only set ideal processor when max_hp and org_hp are in the same cpu
13624 //group. DO NOT MOVE THREADS ACROSS CPU GROUPS
13625 uint16_t org_gn = heap_select::find_cpu_group_from_heap_no(org_hp->heap_number);
13626 uint16_t max_gn = heap_select::find_cpu_group_from_heap_no(max_hp->heap_number);
13627 if (org_gn == max_gn) //only set within CPU group, so SetThreadIdealProcessor is enough
13629 uint16_t group_proc_no = heap_select::find_group_proc_from_heap_no(max_hp->heap_number);
13631 GCThreadAffinity affinity;
13632 affinity.Processor = group_proc_no;
13633 affinity.Group = org_gn;
13634 if (!GCToOSInterface::SetCurrentThreadIdealAffinity(&affinity))
13636 dprintf (3, ("Failed to set the ideal processor and group for heap %d.",
13637 org_hp->heap_number));
13643 uint16_t proc_no = heap_select::find_proc_no_from_heap_no(max_hp->heap_number);
13645 GCThreadAffinity affinity;
13646 affinity.Processor = proc_no;
13647 affinity.Group = GCThreadAffinity::None;
13649 if (!GCToOSInterface::SetCurrentThreadIdealAffinity(&affinity))
13651 dprintf (3, ("Failed to set the ideal processor for heap %d.",
13652 org_hp->heap_number));
13656 dprintf (3, ("Switching context %p (home heap %d) ",
13658 acontext->get_home_heap()->pGenGCHeap->heap_number));
13659 dprintf (3, (" from heap %d (%Id free bytes, %d contexts) ",
13660 org_hp->heap_number,
13662 org_alloc_context_count));
13663 dprintf (3, (" to heap %d (%Id free bytes, %d contexts)\n",
13664 max_hp->heap_number,
13665 dd_new_allocation(max_hp->dynamic_data_of(0)),
13666 max_alloc_context_count));
13671 acontext->alloc_count++;
13674 gc_heap* gc_heap::balance_heaps_loh (alloc_context* acontext, size_t alloc_size)
13676 gc_heap* org_hp = acontext->get_alloc_heap()->pGenGCHeap;
13677 dprintf (3, ("[h%d] LA: %Id", org_hp->heap_number, alloc_size));
13679 //if (size > 128*1024)
13682 dynamic_data* dd = org_hp->dynamic_data_of (max_generation + 1);
13684 ptrdiff_t org_size = dd_new_allocation (dd);
13686 ptrdiff_t max_size;
13687 size_t delta = dd_min_size (dd) * 4;
13689 int start, end, finish;
13690 heap_select::get_heap_range_for_heap(org_hp->heap_number, &start, &end);
13691 finish = start + n_heaps;
13696 max_size = org_size + delta;
13697 dprintf (3, ("orig hp: %d, max size: %d",
13698 org_hp->heap_number,
13701 for (int i = start; i < end; i++)
13703 gc_heap* hp = GCHeap::GetHeap(i%n_heaps)->pGenGCHeap;
13704 dd = hp->dynamic_data_of (max_generation + 1);
13705 ptrdiff_t size = dd_new_allocation (dd);
13706 dprintf (3, ("hp: %d, size: %d",
13709 if (size > max_size)
13713 dprintf (3, ("max hp: %d, max size: %d",
13714 max_hp->heap_number,
13720 if ((max_hp == org_hp) && (end < finish))
13722 start = end; end = finish;
13723 delta = dd_min_size(dd) * 4; // Need to tuning delta
13727 if (max_hp != org_hp)
13729 dprintf (3, ("loh: %d(%Id)->%d(%Id)",
13730 org_hp->heap_number, dd_new_allocation (org_hp->dynamic_data_of (max_generation + 1)),
13731 max_hp->heap_number, dd_new_allocation (max_hp->dynamic_data_of (max_generation + 1))));
13741 #endif //MULTIPLE_HEAPS
13743 BOOL gc_heap::allocate_more_space(alloc_context* acontext, size_t size,
13744 int alloc_generation_number)
13746 allocation_state status;
13749 #ifdef MULTIPLE_HEAPS
13750 if (alloc_generation_number == 0)
13752 balance_heaps (acontext);
13753 status = acontext->get_alloc_heap()->pGenGCHeap->try_allocate_more_space (acontext, size, alloc_generation_number);
13757 gc_heap* alloc_heap = balance_heaps_loh (acontext, size);
13758 status = alloc_heap->try_allocate_more_space (acontext, size, alloc_generation_number);
13759 if (status == a_state_retry_allocate)
13761 dprintf (3, ("LOH h%d alloc retry!", alloc_heap->heap_number));
13765 status = try_allocate_more_space (acontext, size, alloc_generation_number);
13766 #endif //MULTIPLE_HEAPS
13768 while (status == a_state_retry_allocate);
13770 return (status == a_state_can_allocate);
13774 CObjectHeader* gc_heap::allocate (size_t jsize, alloc_context* acontext)
13776 size_t size = Align (jsize);
13777 assert (size >= Align (min_obj_size));
13780 uint8_t* result = acontext->alloc_ptr;
13781 acontext->alloc_ptr+=size;
13782 if (acontext->alloc_ptr <= acontext->alloc_limit)
13784 CObjectHeader* obj = (CObjectHeader*)result;
13790 acontext->alloc_ptr -= size;
13793 #pragma inline_depth(0)
13796 if (! allocate_more_space (acontext, size, 0))
13800 #pragma inline_depth(20)
13808 void gc_heap::leave_allocation_segment (generation* gen)
13810 adjust_limit (0, 0, gen, max_generation);
13813 void gc_heap::init_free_and_plug()
13815 #ifdef FREE_USAGE_STATS
13816 for (int i = 0; i <= settings.condemned_generation; i++)
13818 generation* gen = generation_of (i);
13819 memset (gen->gen_free_spaces, 0, sizeof (gen->gen_free_spaces));
13820 memset (gen->gen_plugs, 0, sizeof (gen->gen_plugs));
13821 memset (gen->gen_current_pinned_free_spaces, 0, sizeof (gen->gen_current_pinned_free_spaces));
13824 if (settings.condemned_generation != max_generation)
13826 for (int i = (settings.condemned_generation + 1); i <= max_generation; i++)
13828 generation* gen = generation_of (i);
13829 memset (gen->gen_plugs, 0, sizeof (gen->gen_plugs));
13832 #endif //FREE_USAGE_STATS
13835 void gc_heap::print_free_and_plug (const char* msg)
13837 #if defined(FREE_USAGE_STATS) && defined(SIMPLE_DPRINTF)
13838 int older_gen = ((settings.condemned_generation == max_generation) ? max_generation : (settings.condemned_generation + 1));
13839 for (int i = 0; i <= older_gen; i++)
13841 generation* gen = generation_of (i);
13842 for (int j = 0; j < NUM_GEN_POWER2; j++)
13844 if ((gen->gen_free_spaces[j] != 0) || (gen->gen_plugs[j] != 0))
13846 dprintf (2, ("[%s][h%d][%s#%d]gen%d: 2^%d: F: %Id, P: %Id",
13849 (settings.concurrent ? "BGC" : "GC"),
13852 (j + 9), gen->gen_free_spaces[j], gen->gen_plugs[j]));
13857 UNREFERENCED_PARAMETER(msg);
13858 #endif //FREE_USAGE_STATS && SIMPLE_DPRINTF
13861 void gc_heap::add_gen_plug (int gen_number, size_t plug_size)
13863 #ifdef FREE_USAGE_STATS
13864 dprintf (3, ("adding plug size %Id to gen%d", plug_size, gen_number));
13865 generation* gen = generation_of (gen_number);
13866 size_t sz = BASE_GEN_SIZE;
13869 for (; i < NUM_GEN_POWER2; i++)
13871 if (plug_size < sz)
13878 (gen->gen_plugs[i])++;
13880 UNREFERENCED_PARAMETER(gen_number);
13881 UNREFERENCED_PARAMETER(plug_size);
13882 #endif //FREE_USAGE_STATS
13885 void gc_heap::add_item_to_current_pinned_free (int gen_number, size_t free_size)
13887 #ifdef FREE_USAGE_STATS
13888 generation* gen = generation_of (gen_number);
13889 size_t sz = BASE_GEN_SIZE;
13892 for (; i < NUM_GEN_POWER2; i++)
13894 if (free_size < sz)
13901 (gen->gen_current_pinned_free_spaces[i])++;
13902 generation_pinned_free_obj_space (gen) += free_size;
13903 dprintf (3, ("left pin free %Id(2^%d) to gen%d, total %Id bytes (%Id)",
13904 free_size, (i + 10), gen_number,
13905 generation_pinned_free_obj_space (gen),
13906 gen->gen_current_pinned_free_spaces[i]));
13908 UNREFERENCED_PARAMETER(gen_number);
13909 UNREFERENCED_PARAMETER(free_size);
13910 #endif //FREE_USAGE_STATS
13913 void gc_heap::add_gen_free (int gen_number, size_t free_size)
13915 #ifdef FREE_USAGE_STATS
13916 dprintf (3, ("adding free size %Id to gen%d", free_size, gen_number));
13917 generation* gen = generation_of (gen_number);
13918 size_t sz = BASE_GEN_SIZE;
13921 for (; i < NUM_GEN_POWER2; i++)
13923 if (free_size < sz)
13930 (gen->gen_free_spaces[i])++;
13932 UNREFERENCED_PARAMETER(gen_number);
13933 UNREFERENCED_PARAMETER(free_size);
13934 #endif //FREE_USAGE_STATS
13937 void gc_heap::remove_gen_free (int gen_number, size_t free_size)
13939 #ifdef FREE_USAGE_STATS
13940 dprintf (3, ("removing free %Id from gen%d", free_size, gen_number));
13941 generation* gen = generation_of (gen_number);
13942 size_t sz = BASE_GEN_SIZE;
13945 for (; i < NUM_GEN_POWER2; i++)
13947 if (free_size < sz)
13954 (gen->gen_free_spaces[i])--;
13956 UNREFERENCED_PARAMETER(gen_number);
13957 UNREFERENCED_PARAMETER(free_size);
13958 #endif //FREE_USAGE_STATS
13961 uint8_t* gc_heap::allocate_in_older_generation (generation* gen, size_t size,
13962 int from_gen_number,
13963 uint8_t* old_loc REQD_ALIGN_AND_OFFSET_DCL)
13965 size = Align (size);
13966 assert (size >= Align (min_obj_size));
13967 assert (from_gen_number < max_generation);
13968 assert (from_gen_number >= 0);
13969 assert (generation_of (from_gen_number + 1) == gen);
13971 allocator* gen_allocator = generation_allocator (gen);
13972 BOOL discard_p = gen_allocator->discard_if_no_fit_p ();
13973 int pad_in_front = ((old_loc != 0) && ((from_gen_number+1) != max_generation)) ? USE_PADDING_FRONT : 0;
13975 size_t real_size = size + Align (min_obj_size);
13977 real_size += Align (min_obj_size);
13979 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
13980 generation_allocation_limit (gen), old_loc, USE_PADDING_TAIL | pad_in_front)))
13982 size_t sz_list = gen_allocator->first_bucket_size();
13983 for (unsigned int a_l_idx = 0; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
13985 if ((real_size < (sz_list / 2)) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
13987 uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
13988 uint8_t* prev_free_item = 0;
13989 while (free_list != 0)
13991 dprintf (3, ("considering free list %Ix", (size_t)free_list));
13993 size_t free_list_size = unused_array_size (free_list);
13995 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, free_list, (free_list + free_list_size),
13996 old_loc, USE_PADDING_TAIL | pad_in_front))
13998 dprintf (4, ("F:%Ix-%Id",
13999 (size_t)free_list, free_list_size));
14001 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, !discard_p);
14002 generation_free_list_space (gen) -= free_list_size;
14003 remove_gen_free (gen->gen_num, free_list_size);
14005 adjust_limit (free_list, free_list_size, gen, from_gen_number+1);
14006 generation_allocate_end_seg_p (gen) = FALSE;
14009 // We do first fit on bucket 0 because we are not guaranteed to find a fit there.
14010 else if (discard_p || (a_l_idx == 0))
14012 dprintf (3, ("couldn't use this free area, discarding"));
14013 generation_free_obj_space (gen) += free_list_size;
14015 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
14016 generation_free_list_space (gen) -= free_list_size;
14017 remove_gen_free (gen->gen_num, free_list_size);
14021 prev_free_item = free_list;
14023 free_list = free_list_slot (free_list);
14026 sz_list = sz_list * 2;
14028 //go back to the beginning of the segment list
14029 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
14030 if (seg != generation_allocation_segment (gen))
14032 leave_allocation_segment (gen);
14033 generation_allocation_segment (gen) = seg;
14035 while (seg != ephemeral_heap_segment)
14037 if (size_fit_p(size REQD_ALIGN_AND_OFFSET_ARG, heap_segment_plan_allocated (seg),
14038 heap_segment_committed (seg), old_loc, USE_PADDING_TAIL | pad_in_front))
14040 dprintf (3, ("using what's left in committed"));
14041 adjust_limit (heap_segment_plan_allocated (seg),
14042 heap_segment_committed (seg) -
14043 heap_segment_plan_allocated (seg),
14044 gen, from_gen_number+1);
14045 generation_allocate_end_seg_p (gen) = TRUE;
14046 // dformat (t, 3, "Expanding segment allocation");
14047 heap_segment_plan_allocated (seg) =
14048 heap_segment_committed (seg);
14053 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, heap_segment_plan_allocated (seg),
14054 heap_segment_reserved (seg), old_loc, USE_PADDING_TAIL | pad_in_front) &&
14055 grow_heap_segment (seg, heap_segment_plan_allocated (seg), old_loc, size, pad_in_front REQD_ALIGN_AND_OFFSET_ARG))
14057 dprintf (3, ("using what's left in reserved"));
14058 adjust_limit (heap_segment_plan_allocated (seg),
14059 heap_segment_committed (seg) -
14060 heap_segment_plan_allocated (seg),
14061 gen, from_gen_number+1);
14062 generation_allocate_end_seg_p (gen) = TRUE;
14063 heap_segment_plan_allocated (seg) =
14064 heap_segment_committed (seg);
14070 leave_allocation_segment (gen);
14071 heap_segment* next_seg = heap_segment_next_rw (seg);
14074 dprintf (3, ("getting next segment"));
14075 generation_allocation_segment (gen) = next_seg;
14076 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
14077 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
14086 seg = generation_allocation_segment (gen);
14088 //No need to fix the last region. Will be done later
14099 uint8_t* result = generation_allocation_pointer (gen);
14103 if ((pad_in_front & USE_PADDING_FRONT) &&
14104 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
14105 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
14107 pad = Align (min_obj_size);
14108 set_plug_padded (old_loc);
14110 #endif //SHORT_PLUGS
14112 #ifdef FEATURE_STRUCTALIGN
14113 _ASSERTE(!old_loc || alignmentOffset != 0);
14114 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
14117 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
14118 set_node_aligninfo (old_loc, requiredAlignment, pad1);
14121 #else // FEATURE_STRUCTALIGN
14122 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
14124 pad += switch_alignment_size (is_plug_padded (old_loc));
14125 set_node_realigned (old_loc);
14126 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
14127 (size_t)old_loc, (size_t)(result+pad)));
14128 assert (same_large_alignment_p (result + pad, old_loc));
14130 #endif // FEATURE_STRUCTALIGN
14131 dprintf (3, ("Allocate %Id bytes", size));
14133 if ((old_loc == 0) || (pad != 0))
14135 //allocating a non plug or a gap, so reset the start region
14136 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14139 generation_allocation_pointer (gen) += size + pad;
14140 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
14141 if (generation_allocate_end_seg_p (gen))
14143 generation_end_seg_allocated (gen) += size;
14147 generation_free_list_allocated (gen) += size;
14149 generation_allocation_size (gen) += size;
14151 dprintf (3, ("aio: ptr: %Ix, limit: %Ix, sr: %Ix",
14152 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14153 generation_allocation_context_start_region (gen)));
14155 return result + pad;;
14159 void gc_heap::repair_allocation_in_expanded_heap (generation* consing_gen)
14161 //make sure that every generation has a planned allocation start
14162 int gen_number = max_generation - 1;
14163 while (gen_number>= 0)
14165 generation* gen = generation_of (gen_number);
14166 if (0 == generation_plan_allocation_start (gen))
14168 realloc_plan_generation_start (gen, consing_gen);
14170 assert (generation_plan_allocation_start (gen));
14175 // now we know the planned allocation size
14176 size_t size = (generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
14177 heap_segment* seg = generation_allocation_segment (consing_gen);
14178 if (generation_allocation_limit (consing_gen) == heap_segment_plan_allocated (seg))
14182 heap_segment_plan_allocated (seg) = generation_allocation_pointer (consing_gen);
14187 assert (settings.condemned_generation == max_generation);
14188 uint8_t* first_address = generation_allocation_limit (consing_gen);
14189 //look through the pinned plugs for relevant ones.
14190 //Look for the right pinned plug to start from.
14193 while (mi != mark_stack_tos)
14195 m = pinned_plug_of (mi);
14196 if ((pinned_plug (m) == first_address))
14201 assert (mi != mark_stack_tos);
14202 pinned_len (m) = size;
14206 //tododefrag optimize for new segment (plan_allocated == mem)
14207 uint8_t* gc_heap::allocate_in_expanded_heap (generation* gen,
14212 BOOL set_padding_on_saved_p,
14213 mark* pinned_plug_entry,
14214 #endif //SHORT_PLUGS
14215 BOOL consider_bestfit,
14216 int active_new_gen_number
14217 REQD_ALIGN_AND_OFFSET_DCL)
14219 UNREFERENCED_PARAMETER(active_new_gen_number);
14220 dprintf (3, ("aie: P: %Ix, size: %Ix", old_loc, size));
14222 size = Align (size);
14223 assert (size >= Align (min_obj_size));
14224 int pad_in_front = ((old_loc != 0) && (active_new_gen_number != max_generation)) ? USE_PADDING_FRONT : 0;
14226 if (consider_bestfit && use_bestfit)
14228 assert (bestfit_seg);
14229 dprintf (SEG_REUSE_LOG_1, ("reallocating 0x%Ix in expanded heap, size: %Id",
14231 return bestfit_seg->fit (old_loc,
14233 set_padding_on_saved_p,
14235 #endif //SHORT_PLUGS
14236 size REQD_ALIGN_AND_OFFSET_ARG);
14239 heap_segment* seg = generation_allocation_segment (gen);
14241 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14242 generation_allocation_limit (gen), old_loc,
14243 ((generation_allocation_limit (gen) !=
14244 heap_segment_plan_allocated (seg))? USE_PADDING_TAIL : 0) | pad_in_front)))
14246 dprintf (3, ("aie: can't fit: ptr: %Ix, limit: %Ix", generation_allocation_pointer (gen),
14247 generation_allocation_limit (gen)));
14250 uint8_t* first_address = (generation_allocation_limit (gen) ?
14251 generation_allocation_limit (gen) :
14252 heap_segment_mem (seg));
14253 assert (in_range_for_segment (first_address, seg));
14255 uint8_t* end_address = heap_segment_reserved (seg);
14257 dprintf (3, ("aie: first_addr: %Ix, gen alloc limit: %Ix, end_address: %Ix",
14258 first_address, generation_allocation_limit (gen), end_address));
14263 if (heap_segment_allocated (seg) != heap_segment_mem (seg))
14265 assert (settings.condemned_generation == max_generation);
14266 //look through the pinned plugs for relevant ones.
14267 //Look for the right pinned plug to start from.
14268 while (mi != mark_stack_tos)
14270 m = pinned_plug_of (mi);
14271 if ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address))
14273 dprintf (3, ("aie: found pin: %Ix", pinned_plug (m)));
14279 if (mi != mark_stack_tos)
14281 //fix old free list.
14282 size_t hsize = (generation_allocation_limit (gen) - generation_allocation_pointer (gen));
14284 dprintf(3,("gc filling up hole"));
14285 ptrdiff_t mi1 = (ptrdiff_t)mi;
14286 while ((mi1 >= 0) &&
14287 (pinned_plug (pinned_plug_of(mi1)) != generation_allocation_limit (gen)))
14289 dprintf (3, ("aie: checking pin %Ix", pinned_plug (pinned_plug_of(mi1))));
14294 size_t saved_pinned_len = pinned_len (pinned_plug_of(mi1));
14295 pinned_len (pinned_plug_of(mi1)) = hsize;
14296 dprintf (3, ("changing %Ix len %Ix->%Ix",
14297 pinned_plug (pinned_plug_of(mi1)),
14298 saved_pinned_len, pinned_len (pinned_plug_of(mi1))));
14305 assert (generation_allocation_limit (gen) ==
14306 generation_allocation_pointer (gen));
14307 mi = mark_stack_tos;
14310 while ((mi != mark_stack_tos) && in_range_for_segment (pinned_plug (m), seg))
14312 size_t len = pinned_len (m);
14313 uint8_t* free_list = (pinned_plug (m) - len);
14314 dprintf (3, ("aie: testing free item: %Ix->%Ix(%Ix)",
14315 free_list, (free_list + len), len));
14316 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, free_list, (free_list + len), old_loc, USE_PADDING_TAIL | pad_in_front))
14318 dprintf (3, ("aie: Found adequate unused area: %Ix, size: %Id",
14319 (size_t)free_list, len));
14321 generation_allocation_pointer (gen) = free_list;
14322 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14323 generation_allocation_limit (gen) = (free_list + len);
14325 goto allocate_in_free;
14328 m = pinned_plug_of (mi);
14331 //switch to the end of the segment.
14332 generation_allocation_pointer (gen) = heap_segment_plan_allocated (seg);
14333 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14334 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14335 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14336 dprintf (3, ("aie: switching to end of seg: %Ix->%Ix(%Ix)",
14337 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14338 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
14340 if (!size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14341 generation_allocation_limit (gen), old_loc, USE_PADDING_TAIL | pad_in_front))
14343 dprintf (3, ("aie: ptr: %Ix, limit: %Ix, can't alloc", generation_allocation_pointer (gen),
14344 generation_allocation_limit (gen)));
14345 assert (!"Can't allocate if no free space");
14356 uint8_t* result = generation_allocation_pointer (gen);
14360 if ((pad_in_front & USE_PADDING_FRONT) &&
14361 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
14362 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
14365 pad = Align (min_obj_size);
14366 set_padding_in_expand (old_loc, set_padding_on_saved_p, pinned_plug_entry);
14368 #endif //SHORT_PLUGS
14370 #ifdef FEATURE_STRUCTALIGN
14371 _ASSERTE(!old_loc || alignmentOffset != 0);
14372 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
14375 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
14376 set_node_aligninfo (old_loc, requiredAlignment, pad1);
14380 #else // FEATURE_STRUCTALIGN
14381 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
14383 pad += switch_alignment_size (is_plug_padded (old_loc));
14384 set_node_realigned (old_loc);
14385 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
14386 (size_t)old_loc, (size_t)(result+pad)));
14387 assert (same_large_alignment_p (result + pad, old_loc));
14390 #endif // FEATURE_STRUCTALIGN
14392 if ((old_loc == 0) || (pad != 0))
14394 //allocating a non plug or a gap, so reset the start region
14395 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14398 generation_allocation_pointer (gen) += size + pad;
14399 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
14400 dprintf (3, ("Allocated in expanded heap %Ix:%Id", (size_t)(result+pad), size));
14402 dprintf (3, ("aie: ptr: %Ix, limit: %Ix, sr: %Ix",
14403 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14404 generation_allocation_context_start_region (gen)));
14406 return result + pad;
14410 generation* gc_heap::ensure_ephemeral_heap_segment (generation* consing_gen)
14412 heap_segment* seg = generation_allocation_segment (consing_gen);
14413 if (seg != ephemeral_heap_segment)
14415 assert (generation_allocation_pointer (consing_gen)>= heap_segment_mem (seg));
14416 assert (generation_allocation_pointer (consing_gen)<= heap_segment_committed (seg));
14418 //fix the allocated size of the segment.
14419 heap_segment_plan_allocated (seg) = generation_allocation_pointer (consing_gen);
14421 generation* new_consing_gen = generation_of (max_generation - 1);
14422 generation_allocation_pointer (new_consing_gen) =
14423 heap_segment_mem (ephemeral_heap_segment);
14424 generation_allocation_limit (new_consing_gen) =
14425 generation_allocation_pointer (new_consing_gen);
14426 generation_allocation_context_start_region (new_consing_gen) =
14427 generation_allocation_pointer (new_consing_gen);
14428 generation_allocation_segment (new_consing_gen) = ephemeral_heap_segment;
14430 return new_consing_gen;
14433 return consing_gen;
14436 uint8_t* gc_heap::allocate_in_condemned_generations (generation* gen,
14438 int from_gen_number,
14440 BOOL* convert_to_pinned_p,
14441 uint8_t* next_pinned_plug,
14442 heap_segment* current_seg,
14443 #endif //SHORT_PLUGS
14445 REQD_ALIGN_AND_OFFSET_DCL)
14447 // Make sure that the youngest generation gap hasn't been allocated
14448 if (settings.promotion)
14450 assert (generation_plan_allocation_start (youngest_generation) == 0);
14453 size = Align (size);
14454 assert (size >= Align (min_obj_size));
14455 int to_gen_number = from_gen_number;
14456 if (from_gen_number != (int)max_generation)
14458 to_gen_number = from_gen_number + (settings.promotion ? 1 : 0);
14461 dprintf (3, ("aic gen%d: s: %Id", gen->gen_num, size));
14463 int pad_in_front = ((old_loc != 0) && (to_gen_number != max_generation)) ? USE_PADDING_FRONT : 0;
14465 if ((from_gen_number != -1) && (from_gen_number != (int)max_generation) && settings.promotion)
14467 generation_condemned_allocated (generation_of (from_gen_number + (settings.promotion ? 1 : 0))) += size;
14468 generation_allocation_size (generation_of (from_gen_number + (settings.promotion ? 1 : 0))) += size;
14472 heap_segment* seg = generation_allocation_segment (gen);
14473 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14474 generation_allocation_limit (gen), old_loc,
14475 ((generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))?USE_PADDING_TAIL:0)|pad_in_front)))
14477 if ((! (pinned_plug_que_empty_p()) &&
14478 (generation_allocation_limit (gen) ==
14479 pinned_plug (oldest_pin()))))
14481 size_t entry = deque_pinned_plug();
14482 mark* pinned_plug_entry = pinned_plug_of (entry);
14483 size_t len = pinned_len (pinned_plug_entry);
14484 uint8_t* plug = pinned_plug (pinned_plug_entry);
14485 set_new_pin_info (pinned_plug_entry, generation_allocation_pointer (gen));
14487 #ifdef FREE_USAGE_STATS
14488 generation_allocated_in_pinned_free (gen) += generation_allocated_since_last_pin (gen);
14489 dprintf (3, ("allocated %Id so far within pin %Ix, total->%Id",
14490 generation_allocated_since_last_pin (gen),
14492 generation_allocated_in_pinned_free (gen)));
14493 generation_allocated_since_last_pin (gen) = 0;
14495 add_item_to_current_pinned_free (gen->gen_num, pinned_len (pinned_plug_of (entry)));
14496 #endif //FREE_USAGE_STATS
14498 dprintf (3, ("mark stack bos: %Id, tos: %Id, aic: p %Ix len: %Ix->%Ix",
14499 mark_stack_bos, mark_stack_tos, plug, len, pinned_len (pinned_plug_of (entry))));
14501 assert(mark_stack_array[entry].len == 0 ||
14502 mark_stack_array[entry].len >= Align(min_obj_size));
14503 generation_allocation_pointer (gen) = plug + len;
14504 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14505 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14506 set_allocator_next_pin (gen);
14508 //Add the size of the pinned plug to the right pinned allocations
14509 //find out which gen this pinned plug came from
14510 int frgn = object_gennum (plug);
14511 if ((frgn != (int)max_generation) && settings.promotion)
14513 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
14514 int togn = object_gennum_plan (plug);
14517 generation_pinned_allocation_compact_size (generation_of (togn)) += len;
14523 if (generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))
14525 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14526 dprintf (3, ("changed limit to plan alloc: %Ix", generation_allocation_limit (gen)));
14530 if (heap_segment_plan_allocated (seg) != heap_segment_committed (seg))
14532 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14533 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14534 dprintf (3, ("changed limit to commit: %Ix", generation_allocation_limit (gen)));
14538 #ifndef RESPECT_LARGE_ALIGNMENT
14539 assert (gen != youngest_generation);
14540 #endif //RESPECT_LARGE_ALIGNMENT
14542 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14543 heap_segment_reserved (seg), old_loc, USE_PADDING_TAIL | pad_in_front) &&
14544 (grow_heap_segment (seg, generation_allocation_pointer (gen), old_loc,
14545 size, pad_in_front REQD_ALIGN_AND_OFFSET_ARG)))
14547 dprintf (3, ("Expanded segment allocation by committing more memory"));
14548 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14549 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14553 heap_segment* next_seg = heap_segment_next (seg);
14554 assert (generation_allocation_pointer (gen)>=
14555 heap_segment_mem (seg));
14556 // Verify that all pinned plugs for this segment are consumed
14557 if (!pinned_plug_que_empty_p() &&
14558 ((pinned_plug (oldest_pin()) <
14559 heap_segment_allocated (seg)) &&
14560 (pinned_plug (oldest_pin()) >=
14561 generation_allocation_pointer (gen))))
14563 LOG((LF_GC, LL_INFO10, "remaining pinned plug %Ix while leaving segment on allocation",
14564 pinned_plug (oldest_pin())));
14567 assert (generation_allocation_pointer (gen)>=
14568 heap_segment_mem (seg));
14569 assert (generation_allocation_pointer (gen)<=
14570 heap_segment_committed (seg));
14571 heap_segment_plan_allocated (seg) = generation_allocation_pointer (gen);
14575 generation_allocation_segment (gen) = next_seg;
14576 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
14577 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
14578 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14582 return 0; //should only happen during allocation of generation 0 gap
14583 // in that case we are going to grow the heap anyway
14588 set_allocator_next_pin (gen);
14595 assert (generation_allocation_pointer (gen)>=
14596 heap_segment_mem (generation_allocation_segment (gen)));
14597 uint8_t* result = generation_allocation_pointer (gen);
14600 if ((pad_in_front & USE_PADDING_FRONT) &&
14601 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
14602 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
14604 ptrdiff_t dist = old_loc - result;
14607 dprintf (3, ("old alloc: %Ix, same as new alloc, not padding", old_loc));
14612 if ((dist > 0) && (dist < (ptrdiff_t)Align (min_obj_size)))
14614 dprintf (3, ("old alloc: %Ix, only %d bytes > new alloc! Shouldn't happen", old_loc, dist));
14618 pad = Align (min_obj_size);
14619 set_plug_padded (old_loc);
14622 #endif //SHORT_PLUGS
14623 #ifdef FEATURE_STRUCTALIGN
14624 _ASSERTE(!old_loc || alignmentOffset != 0);
14625 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
14626 if ((old_loc != 0))
14628 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
14629 set_node_aligninfo (old_loc, requiredAlignment, pad1);
14632 #else // FEATURE_STRUCTALIGN
14633 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
14635 pad += switch_alignment_size (is_plug_padded (old_loc));
14636 set_node_realigned(old_loc);
14637 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
14638 (size_t)old_loc, (size_t)(result+pad)));
14639 assert (same_large_alignment_p (result + pad, old_loc));
14641 #endif // FEATURE_STRUCTALIGN
14644 if ((next_pinned_plug != 0) && (pad != 0) && (generation_allocation_segment (gen) == current_seg))
14646 assert (old_loc != 0);
14647 ptrdiff_t dist_to_next_pin = (ptrdiff_t)(next_pinned_plug - (generation_allocation_pointer (gen) + size + pad));
14648 assert (dist_to_next_pin >= 0);
14650 if ((dist_to_next_pin >= 0) && (dist_to_next_pin < (ptrdiff_t)Align (min_obj_size)))
14652 dprintf (3, ("%Ix->(%Ix,%Ix),%Ix(%Ix)(%Ix),NP->PP",
14654 generation_allocation_pointer (gen),
14655 generation_allocation_limit (gen),
14658 dist_to_next_pin));
14659 clear_plug_padded (old_loc);
14661 *convert_to_pinned_p = TRUE;
14662 record_interesting_data_point (idp_converted_pin);
14667 #endif //SHORT_PLUGS
14669 if ((old_loc == 0) || (pad != 0))
14671 //allocating a non plug or a gap, so reset the start region
14672 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14675 generation_allocation_pointer (gen) += size + pad;
14676 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
14678 #ifdef FREE_USAGE_STATS
14679 generation_allocated_since_last_pin (gen) += size;
14680 #endif //FREE_USAGE_STATS
14682 dprintf (3, ("aic: ptr: %Ix, limit: %Ix, sr: %Ix",
14683 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14684 generation_allocation_context_start_region (gen)));
14686 assert (result + pad);
14687 return result + pad;
14691 inline int power (int x, int y)
14694 for (int i = 0; i < y; i++)
14701 int gc_heap::joined_generation_to_condemn (BOOL should_evaluate_elevation,
14704 BOOL* blocking_collection_p
14705 STRESS_HEAP_ARG(int n_original))
14707 int n = current_gen;
14708 #ifdef MULTIPLE_HEAPS
14709 BOOL joined_last_gc_before_oom = FALSE;
14710 for (int i = 0; i < n_heaps; i++)
14712 if (g_heaps[i]->last_gc_before_oom)
14714 dprintf (GTC_LOG, ("h%d is setting blocking to TRUE", i));
14715 joined_last_gc_before_oom = TRUE;
14720 BOOL joined_last_gc_before_oom = last_gc_before_oom;
14721 #endif //MULTIPLE_HEAPS
14723 if (joined_last_gc_before_oom && settings.pause_mode != pause_low_latency)
14725 assert (*blocking_collection_p);
14728 if (should_evaluate_elevation && (n == max_generation))
14730 dprintf (GTC_LOG, ("lock: %d(%d)",
14731 (settings.should_lock_elevation ? 1 : 0),
14732 settings.elevation_locked_count));
14734 if (settings.should_lock_elevation)
14736 settings.elevation_locked_count++;
14737 if (settings.elevation_locked_count == 6)
14739 settings.elevation_locked_count = 0;
14743 n = max_generation - 1;
14744 settings.elevation_reduced = TRUE;
14749 settings.elevation_locked_count = 0;
14754 settings.should_lock_elevation = FALSE;
14755 settings.elevation_locked_count = 0;
14758 if (provisional_mode_triggered && (n == max_generation))
14760 // There are a few cases where we should not reduce the generation.
14761 if ((initial_gen == max_generation) || (settings.reason == reason_alloc_loh))
14763 // If we are doing a full GC in the provisional mode, we always
14764 // make it blocking because we don't want to get into a situation
14765 // where foreground GCs are asking for a compacting full GC right away
14766 // and not getting it.
14767 dprintf (GTC_LOG, ("full GC induced, not reducing gen"));
14768 *blocking_collection_p = TRUE;
14770 else if (should_expand_in_full_gc || joined_last_gc_before_oom)
14772 dprintf (GTC_LOG, ("need full blocking GCs to expand heap or avoid OOM, not reducing gen"));
14773 assert (*blocking_collection_p);
14777 dprintf (GTC_LOG, ("reducing gen in PM: %d->%d->%d", initial_gen, n, (max_generation - 1)));
14778 n = max_generation - 1;
14782 if (should_expand_in_full_gc)
14784 should_expand_in_full_gc = FALSE;
14787 if (heap_hard_limit)
14789 // If we have already consumed 90% of the limit, we should check to see if we should compact LOH.
14790 // TODO: should unify this with gen2.
14791 dprintf (GTC_LOG, ("committed %Id is %d%% of limit %Id",
14792 current_total_committed, (int)((float)current_total_committed * 100.0 / (float)heap_hard_limit),
14794 if ((current_total_committed * 10) >= (heap_hard_limit * 9))
14796 bool full_compact_gc_p = false;
14798 size_t loh_frag = get_total_gen_fragmentation (max_generation + 1);
14800 // If the LOH frag is >= 1/8 it's worth compacting it
14801 if ((loh_frag * 8) >= heap_hard_limit)
14803 dprintf (GTC_LOG, ("loh frag: %Id > 1/8 of limit %Id", loh_frag, (heap_hard_limit / 8)));
14804 full_compact_gc_p = true;
14808 // If there's not much fragmentation but it looks like it'll be productive to
14809 // collect LOH, do that.
14810 size_t est_loh_reclaim = get_total_gen_estimated_reclaim (max_generation + 1);
14811 full_compact_gc_p = ((est_loh_reclaim * 8) >= heap_hard_limit);
14812 dprintf (GTC_LOG, ("loh est reclaim: %Id, 1/8 of limit %Id", est_loh_reclaim, (heap_hard_limit / 8)));
14815 if (full_compact_gc_p)
14817 n = max_generation;
14818 *blocking_collection_p = TRUE;
14819 settings.loh_compaction = TRUE;
14820 dprintf (GTC_LOG, ("compacting LOH due to hard limit"));
14825 if ((n == max_generation) && (*blocking_collection_p == FALSE))
14827 // If we are doing a gen2 we should reset elevation regardless and let the gen2
14828 // decide if we should lock again or in the bgc case by design we will not retract
14830 settings.should_lock_elevation = FALSE;
14831 settings.elevation_locked_count = 0;
14832 dprintf (1, ("doing bgc, reset elevation"));
14836 #ifdef BACKGROUND_GC
14837 // We can only do Concurrent GC Stress if the caller did not explicitly ask for all
14838 // generations to be collected,
14840 // [LOCALGC TODO] STRESS_HEAP is not defined for a standalone GC so there are multiple
14841 // things that need to be fixed in this code block.
14842 if (n_original != max_generation &&
14843 g_pConfig->GetGCStressLevel() && gc_can_use_concurrent)
14845 #ifndef FEATURE_REDHAWK
14846 // for the GC stress mix mode throttle down gen2 collections
14847 if (g_pConfig->IsGCStressMix())
14849 size_t current_gc_count = 0;
14851 #ifdef MULTIPLE_HEAPS
14852 current_gc_count = (size_t)dd_collection_count (g_heaps[0]->dynamic_data_of (0));
14854 current_gc_count = (size_t)dd_collection_count (dynamic_data_of (0));
14855 #endif //MULTIPLE_HEAPS
14856 // in gc stress, only escalate every 10th non-gen2 collection to a gen2...
14857 if ((current_gc_count % 10) == 0)
14859 n = max_generation;
14862 // for traditional GC stress
14864 #endif // !FEATURE_REDHAWK
14865 if (*blocking_collection_p)
14867 // We call StressHeap() a lot for Concurrent GC Stress. However,
14868 // if we can not do a concurrent collection, no need to stress anymore.
14869 // @TODO: Enable stress when the memory pressure goes down again
14870 GCStressPolicy::GlobalDisable();
14874 n = max_generation;
14877 #endif //BACKGROUND_GC
14878 #endif //STRESS_HEAP
14884 size_t get_survived_size (gc_history_per_heap* hist)
14886 size_t surv_size = 0;
14887 gc_generation_data* gen_data;
14889 for (int gen_number = 0; gen_number <= (max_generation + 1); gen_number++)
14891 gen_data = &(hist->gen_data[gen_number]);
14892 surv_size += (gen_data->size_after -
14893 gen_data->free_list_space_after -
14894 gen_data->free_obj_space_after);
14900 size_t gc_heap::get_total_survived_size()
14902 size_t total_surv_size = 0;
14903 #ifdef MULTIPLE_HEAPS
14904 for (int i = 0; i < gc_heap::n_heaps; i++)
14906 gc_heap* hp = gc_heap::g_heaps[i];
14907 gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap();
14908 total_surv_size += get_survived_size (current_gc_data_per_heap);
14911 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
14912 total_surv_size = get_survived_size (current_gc_data_per_heap);
14913 #endif //MULTIPLE_HEAPS
14914 return total_surv_size;
14917 size_t gc_heap::get_total_allocated_since_last_gc()
14919 size_t total_allocated_size = 0;
14920 #ifdef MULTIPLE_HEAPS
14921 for (int i = 0; i < gc_heap::n_heaps; i++)
14923 gc_heap* hp = gc_heap::g_heaps[i];
14924 total_allocated_size += hp->allocated_since_last_gc;
14925 hp->allocated_since_last_gc = 0;
14928 total_allocated_size = allocated_since_last_gc;
14929 allocated_since_last_gc = 0;
14930 #endif //MULTIPLE_HEAPS
14931 return total_allocated_size;
14934 // Gets what's allocated on both SOH and LOH that hasn't been collected.
14935 size_t gc_heap::get_current_allocated()
14937 dynamic_data* dd = dynamic_data_of (0);
14938 size_t current_alloc = dd_desired_allocation (dd) - dd_new_allocation (dd);
14939 dd = dynamic_data_of (max_generation + 1);
14940 current_alloc += dd_desired_allocation (dd) - dd_new_allocation (dd);
14942 return current_alloc;
14945 size_t gc_heap::get_total_allocated()
14947 size_t total_current_allocated = 0;
14948 #ifdef MULTIPLE_HEAPS
14949 for (int i = 0; i < gc_heap::n_heaps; i++)
14951 gc_heap* hp = gc_heap::g_heaps[i];
14952 total_current_allocated += hp->get_current_allocated();
14955 total_current_allocated = get_current_allocated();
14956 #endif //MULTIPLE_HEAPS
14957 return total_current_allocated;
14960 size_t gc_heap::current_generation_size (int gen_number)
14962 dynamic_data* dd = dynamic_data_of (gen_number);
14963 size_t gen_size = (dd_current_size (dd) + dd_desired_allocation (dd)
14964 - dd_new_allocation (dd));
14970 #pragma warning(push)
14971 #pragma warning(disable:6326) // "Potential comparison of a constant with another constant" is intentional in this function.
14975 This is called by when we are actually doing a GC, or when we are just checking whether
14976 we would do a full blocking GC, in which case check_only_p is TRUE.
14978 The difference between calling this with check_only_p TRUE and FALSE is that when it's
14980 settings.reason is ignored
14981 budgets are not checked (since they are checked before this is called)
14982 it doesn't change anything non local like generation_skip_ratio
14984 int gc_heap::generation_to_condemn (int n_initial,
14985 BOOL* blocking_collection_p,
14986 BOOL* elevation_requested_p,
14989 gc_mechanisms temp_settings = settings;
14990 gen_to_condemn_tuning temp_condemn_reasons;
14991 gc_mechanisms* local_settings = (check_only_p ? &temp_settings : &settings);
14992 gen_to_condemn_tuning* local_condemn_reasons = (check_only_p ? &temp_condemn_reasons : &gen_to_condemn_reasons);
14995 if ((local_settings->reason == reason_oos_soh) || (local_settings->reason == reason_oos_loh))
14997 assert (n_initial >= 1);
15000 assert (settings.reason != reason_empty);
15003 local_condemn_reasons->init();
15007 if (heap_number == 0)
15009 dprintf (GTC_LOG, ("init: %d(%d)", n_initial, settings.reason));
15013 BOOL low_memory_detected = g_low_memory_status;
15014 uint32_t memory_load = 0;
15015 uint64_t available_physical = 0;
15016 uint64_t available_page_file = 0;
15017 BOOL check_memory = FALSE;
15018 BOOL high_fragmentation = FALSE;
15019 BOOL v_high_memory_load = FALSE;
15020 BOOL high_memory_load = FALSE;
15021 BOOL low_ephemeral_space = FALSE;
15022 BOOL evaluate_elevation = TRUE;
15023 *elevation_requested_p = FALSE;
15024 *blocking_collection_p = FALSE;
15026 BOOL check_max_gen_alloc = TRUE;
15030 #endif //STRESS_HEAP
15034 dd_fragmentation (dynamic_data_of (0)) =
15035 generation_free_list_space (youngest_generation) +
15036 generation_free_obj_space (youngest_generation);
15038 dd_fragmentation (dynamic_data_of (max_generation + 1)) =
15039 generation_free_list_space (large_object_generation) +
15040 generation_free_obj_space (large_object_generation);
15042 //save new_allocation
15043 for (i = 0; i <= max_generation+1; i++)
15045 dynamic_data* dd = dynamic_data_of (i);
15046 dprintf (GTC_LOG, ("h%d: g%d: l: %Id (%Id)",
15048 dd_new_allocation (dd),
15049 dd_desired_allocation (dd)));
15050 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
15053 local_condemn_reasons->set_gen (gen_initial, n);
15056 #ifdef BACKGROUND_GC
15057 if (recursive_gc_sync::background_running_p())
15059 dprintf (GTC_LOG, ("bgc in prog, 1"));
15060 check_max_gen_alloc = FALSE;
15062 #endif //BACKGROUND_GC
15064 if (check_max_gen_alloc)
15066 //figure out if large objects need to be collected.
15067 if (get_new_allocation (max_generation+1) <= 0)
15069 n = max_generation;
15070 local_condemn_reasons->set_gen (gen_alloc_budget, n);
15074 //figure out which generation ran out of allocation
15075 for (i = n+1; i <= (check_max_gen_alloc ? max_generation : (max_generation - 1)); i++)
15077 if (get_new_allocation (i) <= 0)
15088 local_condemn_reasons->set_gen (gen_alloc_budget, n);
15091 dprintf (GTC_LOG, ("h%d: g%d budget", heap_number, ((get_new_allocation (max_generation+1) <= 0) ? 3 : n)));
15095 #if defined(BACKGROUND_GC) && !defined(MULTIPLE_HEAPS)
15096 //time based tuning
15097 // if enough time has elapsed since the last gc
15098 // and the number of gc is too low (1/10 of lower gen) then collect
15099 // This should also be enabled if we have memory concerns
15100 int n_time_max = max_generation;
15104 if (recursive_gc_sync::background_running_p())
15106 n_time_max = max_generation - 1;
15110 if ((local_settings->pause_mode == pause_interactive) ||
15111 (local_settings->pause_mode == pause_sustained_low_latency))
15113 dynamic_data* dd0 = dynamic_data_of (0);
15114 size_t now = GetHighPrecisionTimeStamp();
15116 for (i = (temp_gen+1); i <= n_time_max; i++)
15118 dynamic_data* dd = dynamic_data_of (i);
15119 if ((now > dd_time_clock(dd) + dd_time_clock_interval(dd)) &&
15120 (dd_gc_clock (dd0) > (dd_gc_clock (dd) + dd_gc_clock_interval(dd))) &&
15121 ((n < max_generation) || ((dd_current_size (dd) < dd_max_size (dd0)))))
15123 n = min (i, n_time_max);
15124 dprintf (GTC_LOG, ("time %d", n));
15129 local_condemn_reasons->set_gen (gen_time_tuning, n);
15135 dprintf (GTC_LOG, ("Condemning %d based on time tuning and fragmentation", n));
15137 #endif //BACKGROUND_GC && !MULTIPLE_HEAPS
15139 if (n < (max_generation - 1))
15141 if (dt_low_card_table_efficiency_p (tuning_deciding_condemned_gen))
15143 n = max (n, max_generation - 1);
15144 local_settings->promotion = TRUE;
15145 dprintf (GTC_LOG, ("h%d: skip %d, c %d",
15146 heap_number, generation_skip_ratio, n));
15147 local_condemn_reasons->set_condition (gen_low_card_p);
15153 generation_skip_ratio = 100;
15156 if (dt_low_ephemeral_space_p (check_only_p ?
15157 tuning_deciding_full_gc :
15158 tuning_deciding_condemned_gen))
15160 low_ephemeral_space = TRUE;
15162 n = max (n, max_generation - 1);
15163 local_condemn_reasons->set_condition (gen_low_ephemeral_p);
15164 dprintf (GTC_LOG, ("h%d: low eph", heap_number));
15166 if (!provisional_mode_triggered)
15168 #ifdef BACKGROUND_GC
15169 if (!gc_can_use_concurrent || (generation_free_list_space (generation_of (max_generation)) == 0))
15170 #endif //BACKGROUND_GC
15172 //It is better to defragment first if we are running out of space for
15173 //the ephemeral generation but we have enough fragmentation to make up for it
15174 //in the non ephemeral generation. Essentially we are trading a gen2 for
15175 // having to expand heap in ephemeral collections.
15176 if (dt_high_frag_p (tuning_deciding_condemned_gen,
15177 max_generation - 1,
15180 high_fragmentation = TRUE;
15181 local_condemn_reasons->set_condition (gen_max_high_frag_e_p);
15182 dprintf (GTC_LOG, ("heap%d: gen1 frag", heap_number));
15188 //figure out which ephemeral generation is too fragramented
15190 for (i = n+1; i < max_generation; i++)
15192 if (dt_high_frag_p (tuning_deciding_condemned_gen, i))
15194 dprintf (GTC_LOG, ("h%d g%d too frag", heap_number, i));
15201 if (low_ephemeral_space)
15204 local_settings->promotion = TRUE;
15209 local_condemn_reasons->set_condition (gen_eph_high_frag_p);
15214 if (settings.pause_mode == pause_low_latency)
15216 if (!is_induced (settings.reason))
15218 n = min (n, max_generation - 1);
15219 dprintf (GTC_LOG, ("low latency mode is enabled, condemning %d", n));
15220 evaluate_elevation = FALSE;
15226 // It's hard to catch when we get to the point that the memory load is so high
15227 // we get an induced GC from the finalizer thread so we are checking the memory load
15228 // for every gen0 GC.
15229 check_memory = (check_only_p ?
15231 ((n >= 1) || low_memory_detected));
15235 //find out if we are short on memory
15236 get_memory_info (&memory_load, &available_physical, &available_page_file);
15237 if (heap_number == 0)
15239 dprintf (GTC_LOG, ("ml: %d", memory_load));
15242 // Need to get it early enough for all heaps to use.
15243 entry_available_physical_mem = available_physical;
15244 local_settings->entry_memory_load = memory_load;
15246 // @TODO: Force compaction more often under GCSTRESS
15247 if (memory_load >= high_memory_load_th || low_memory_detected)
15249 #ifdef SIMPLE_DPRINTF
15250 // stress log can't handle any parameter that's bigger than a void*.
15251 if (heap_number == 0)
15253 dprintf (GTC_LOG, ("tp: %I64d, ap: %I64d", total_physical_mem, available_physical));
15255 #endif //SIMPLE_DPRINTF
15257 high_memory_load = TRUE;
15259 if (memory_load >= v_high_memory_load_th || low_memory_detected)
15261 // TODO: Perhaps in 64-bit we should be estimating gen1's fragmentation as well since
15262 // gen1/gen0 may take a lot more memory than gen2.
15263 if (!high_fragmentation)
15265 high_fragmentation = dt_estimate_reclaim_space_p (tuning_deciding_condemned_gen, max_generation);
15267 v_high_memory_load = TRUE;
15271 if (!high_fragmentation)
15273 high_fragmentation = dt_estimate_high_frag_p (tuning_deciding_condemned_gen, max_generation, available_physical);
15277 if (high_fragmentation)
15279 if (high_memory_load)
15281 local_condemn_reasons->set_condition (gen_max_high_frag_m_p);
15283 else if (v_high_memory_load)
15285 local_condemn_reasons->set_condition (gen_max_high_frag_vm_p);
15291 dprintf (GTC_LOG, ("h%d: le: %d, hm: %d, vm: %d, f: %d",
15292 heap_number, low_ephemeral_space, high_memory_load, v_high_memory_load,
15293 high_fragmentation));
15295 if (should_expand_in_full_gc)
15297 dprintf (GTC_LOG, ("h%d: expand_in_full - BLOCK", heap_number));
15298 *blocking_collection_p = TRUE;
15299 evaluate_elevation = FALSE;
15300 n = max_generation;
15301 local_condemn_reasons->set_condition (gen_expand_fullgc_p);
15304 if (last_gc_before_oom)
15306 dprintf (GTC_LOG, ("h%d: alloc full - BLOCK", heap_number));
15307 n = max_generation;
15308 *blocking_collection_p = TRUE;
15310 if ((local_settings->reason == reason_oos_loh) ||
15311 (local_settings->reason == reason_alloc_loh))
15313 evaluate_elevation = FALSE;
15316 local_condemn_reasons->set_condition (gen_before_oom);
15321 if (is_induced_blocking (settings.reason) &&
15322 n_initial == max_generation
15323 IN_STRESS_HEAP( && !settings.stress_induced ))
15325 if (heap_number == 0)
15327 dprintf (GTC_LOG, ("induced - BLOCK"));
15330 *blocking_collection_p = TRUE;
15331 local_condemn_reasons->set_condition (gen_induced_fullgc_p);
15332 evaluate_elevation = FALSE;
15335 if (settings.reason == reason_induced_noforce)
15337 local_condemn_reasons->set_condition (gen_induced_noforce_p);
15338 evaluate_elevation = FALSE;
15342 if (!provisional_mode_triggered && evaluate_elevation && (low_ephemeral_space || high_memory_load || v_high_memory_load))
15344 *elevation_requested_p = TRUE;
15346 // if we are in high memory load and have consumed 10% of the gen2 budget, do a gen2 now.
15347 if (high_memory_load || v_high_memory_load)
15349 dynamic_data* dd_max = dynamic_data_of (max_generation);
15350 if (((float)dd_new_allocation (dd_max) / (float)dd_desired_allocation (dd_max)) < 0.9)
15352 dprintf (GTC_LOG, ("%Id left in gen2 alloc (%Id)",
15353 dd_new_allocation (dd_max), dd_desired_allocation (dd_max)));
15354 n = max_generation;
15355 local_condemn_reasons->set_condition (gen_almost_max_alloc);
15359 if (n <= max_generation)
15362 if (high_fragmentation)
15364 //elevate to max_generation
15365 n = max_generation;
15366 dprintf (GTC_LOG, ("h%d: f full", heap_number));
15368 #ifdef BACKGROUND_GC
15369 if (high_memory_load || v_high_memory_load)
15371 // For background GC we want to do blocking collections more eagerly because we don't
15372 // want to get into the situation where the memory load becomes high while we are in
15373 // a background GC and we'd have to wait for the background GC to finish to start
15374 // a blocking collection (right now the implemenation doesn't handle converting
15375 // a background GC to a blocking collection midway.
15376 dprintf (GTC_LOG, ("h%d: bgc - BLOCK", heap_number));
15377 *blocking_collection_p = TRUE;
15380 if (v_high_memory_load)
15382 dprintf (GTC_LOG, ("h%d: - BLOCK", heap_number));
15383 *blocking_collection_p = TRUE;
15385 #endif //BACKGROUND_GC
15389 n = max (n, max_generation - 1);
15390 dprintf (GTC_LOG, ("h%d: nf c %d", heap_number, n));
15397 if (!provisional_mode_triggered && (n == (max_generation - 1)) && (n_alloc < (max_generation -1)))
15399 dprintf (GTC_LOG, ("h%d: budget %d, check 2",
15400 heap_number, n_alloc));
15401 if (get_new_allocation (max_generation) <= 0)
15403 dprintf (GTC_LOG, ("h%d: budget alloc", heap_number));
15404 n = max_generation;
15405 local_condemn_reasons->set_condition (gen_max_gen1);
15409 //figure out if max_generation is too fragmented -> blocking collection
15410 if (!provisional_mode_triggered && (n == max_generation))
15412 if (dt_high_frag_p (tuning_deciding_condemned_gen, n))
15414 dprintf (GTC_LOG, ("h%d: g%d too frag", heap_number, n));
15415 local_condemn_reasons->set_condition (gen_max_high_frag_p);
15416 if (local_settings->pause_mode != pause_sustained_low_latency)
15418 *blocking_collection_p = TRUE;
15423 #ifdef BACKGROUND_GC
15424 if (n == max_generation)
15426 if (heap_number == 0)
15428 BOOL bgc_heap_too_small = TRUE;
15429 size_t gen2size = 0;
15430 size_t gen3size = 0;
15431 #ifdef MULTIPLE_HEAPS
15432 for (int i = 0; i < n_heaps; i++)
15434 if (((g_heaps[i]->current_generation_size (max_generation)) > bgc_min_per_heap) ||
15435 ((g_heaps[i]->current_generation_size (max_generation + 1)) > bgc_min_per_heap))
15437 bgc_heap_too_small = FALSE;
15441 #else //MULTIPLE_HEAPS
15442 if ((current_generation_size (max_generation) > bgc_min_per_heap) ||
15443 (current_generation_size (max_generation + 1) > bgc_min_per_heap))
15445 bgc_heap_too_small = FALSE;
15447 #endif //MULTIPLE_HEAPS
15449 if (bgc_heap_too_small)
15451 dprintf (GTC_LOG, ("gen2 and gen3 too small"));
15454 // do not turn stress-induced collections into blocking GCs
15455 if (!settings.stress_induced)
15456 #endif //STRESS_HEAP
15458 *blocking_collection_p = TRUE;
15461 local_condemn_reasons->set_condition (gen_gen2_too_small);
15465 #endif //BACKGROUND_GC
15471 #ifdef BACKGROUND_GC
15472 // We can only do Concurrent GC Stress if the caller did not explicitly ask for all
15473 // generations to be collected,
15475 if (orig_gen != max_generation &&
15476 g_pConfig->GetGCStressLevel() && gc_can_use_concurrent)
15478 *elevation_requested_p = FALSE;
15480 #endif //BACKGROUND_GC
15481 #endif //STRESS_HEAP
15485 fgm_result.available_pagefile_mb = (size_t)(available_page_file / (1024 * 1024));
15488 local_condemn_reasons->set_gen (gen_final_per_heap, n);
15489 get_gc_data_per_heap()->gen_to_condemn_reasons.init (local_condemn_reasons);
15492 local_condemn_reasons->print (heap_number);
15495 if ((local_settings->reason == reason_oos_soh) ||
15496 (local_settings->reason == reason_oos_loh))
15506 #pragma warning(pop)
15510 size_t gc_heap::min_reclaim_fragmentation_threshold (uint32_t num_heaps)
15512 // if the memory load is higher, the threshold we'd want to collect gets lower.
15513 size_t min_mem_based_on_available =
15514 (500 - (settings.entry_memory_load - high_memory_load_th) * 40) * 1024 * 1024 / num_heaps;
15516 size_t ten_percent_size = (size_t)((float)generation_size (max_generation) * 0.10);
15517 uint64_t three_percent_mem = mem_one_percent * 3 / num_heaps;
15519 #ifdef SIMPLE_DPRINTF
15520 dprintf (GTC_LOG, ("min av: %Id, 10%% gen2: %Id, 3%% mem: %I64d",
15521 min_mem_based_on_available, ten_percent_size, three_percent_mem));
15522 #endif //SIMPLE_DPRINTF
15523 return (size_t)(min (min_mem_based_on_available, min (ten_percent_size, three_percent_mem)));
15527 uint64_t gc_heap::min_high_fragmentation_threshold(uint64_t available_mem, uint32_t num_heaps)
15529 return min (available_mem, (256*1024*1024)) / num_heaps;
15533 CORINFO_EXCEPTION_GC = 0xE0004743 // 'GC'
15537 #ifdef BACKGROUND_GC
15538 void gc_heap::init_background_gc ()
15540 //reset the allocation so foreground gc can allocate into older (max_generation) generation
15541 generation* gen = generation_of (max_generation);
15542 generation_allocation_pointer (gen)= 0;
15543 generation_allocation_limit (gen) = 0;
15544 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
15546 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
15548 //reset the plan allocation for each segment
15549 for (heap_segment* seg = generation_allocation_segment (gen); seg != ephemeral_heap_segment;
15550 seg = heap_segment_next_rw (seg))
15552 heap_segment_plan_allocated (seg) = heap_segment_allocated (seg);
15555 if (heap_number == 0)
15557 dprintf (2, ("heap%d: bgc lowest: %Ix, highest: %Ix",
15559 background_saved_lowest_address,
15560 background_saved_highest_address));
15563 gc_lh_block_event.Reset();
15566 #endif //BACKGROUND_GC
15569 void fire_drain_mark_list_event (size_t mark_list_objects)
15571 FIRE_EVENT(BGCDrainMark, mark_list_objects);
15575 void fire_revisit_event (size_t dirtied_pages,
15576 size_t marked_objects,
15577 BOOL large_objects_p)
15579 FIRE_EVENT(BGCRevisit, dirtied_pages, marked_objects, large_objects_p);
15583 void fire_overflow_event (uint8_t* overflow_min,
15584 uint8_t* overflow_max,
15585 size_t marked_objects,
15586 int large_objects_p)
15588 FIRE_EVENT(BGCOverflow, (uint64_t)overflow_min, (uint64_t)overflow_max, marked_objects, large_objects_p);
15591 void gc_heap::concurrent_print_time_delta (const char* msg)
15594 size_t current_time = GetHighPrecisionTimeStamp();
15595 size_t elapsed_time = current_time - time_bgc_last;
15596 time_bgc_last = current_time;
15598 dprintf (2, ("h%d: %s T %Id ms", heap_number, msg, elapsed_time));
15600 UNREFERENCED_PARAMETER(msg);
15604 void gc_heap::free_list_info (int gen_num, const char* msg)
15606 UNREFERENCED_PARAMETER(gen_num);
15607 #if defined (BACKGROUND_GC) && defined (TRACE_GC)
15608 dprintf (3, ("h%d: %s", heap_number, msg));
15609 for (int i = 0; i <= (max_generation + 1); i++)
15611 generation* gen = generation_of (i);
15612 if ((generation_allocation_size (gen) == 0) &&
15613 (generation_free_list_space (gen) == 0) &&
15614 (generation_free_obj_space (gen) == 0))
15616 // don't print if everything is 0.
15620 dprintf (3, ("h%d: g%d: a-%Id, fl-%Id, fo-%Id",
15622 generation_allocation_size (gen),
15623 generation_free_list_space (gen),
15624 generation_free_obj_space (gen)));
15628 UNREFERENCED_PARAMETER(msg);
15629 #endif // BACKGROUND_GC && TRACE_GC
15632 void gc_heap::update_collection_counts_for_no_gc()
15634 assert (settings.pause_mode == pause_no_gc);
15636 settings.condemned_generation = max_generation;
15637 #ifdef MULTIPLE_HEAPS
15638 for (int i = 0; i < n_heaps; i++)
15639 g_heaps[i]->update_collection_counts();
15640 #else //MULTIPLE_HEAPS
15641 update_collection_counts();
15642 #endif //MULTIPLE_HEAPS
15644 full_gc_counts[gc_type_blocking]++;
15647 BOOL gc_heap::should_proceed_with_gc()
15649 if (gc_heap::settings.pause_mode == pause_no_gc)
15651 if (current_no_gc_region_info.started)
15653 // The no_gc mode was already in progress yet we triggered another GC,
15654 // this effectively exits the no_gc mode.
15655 restore_data_for_no_gc();
15658 return should_proceed_for_no_gc();
15664 //internal part of gc used by the serial and concurrent version
15665 void gc_heap::gc1()
15667 #ifdef BACKGROUND_GC
15668 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15669 #endif //BACKGROUND_GC
15672 mark_time = plan_time = reloc_time = compact_time = sweep_time = 0;
15675 verify_soh_segment_list();
15677 int n = settings.condemned_generation;
15679 if (settings.reason == reason_pm_full_gc)
15681 assert (n == max_generation);
15684 gen_to_condemn_tuning* local_condemn_reasons = &(get_gc_data_per_heap()->gen_to_condemn_reasons);
15685 local_condemn_reasons->init();
15686 local_condemn_reasons->set_gen (gen_initial, n);
15687 local_condemn_reasons->set_gen (gen_final_per_heap, n);
15690 update_collection_counts ();
15692 #ifdef BACKGROUND_GC
15693 bgc_alloc_lock->check();
15694 #endif //BACKGROUND_GC
15696 free_list_info (max_generation, "beginning");
15698 vm_heap->GcCondemnedGeneration = settings.condemned_generation;
15700 assert (g_gc_card_table == card_table);
15702 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
15703 assert (g_gc_card_bundle_table == card_bundle_table);
15707 if (n == max_generation)
15709 gc_low = lowest_address;
15710 gc_high = highest_address;
15714 gc_low = generation_allocation_start (generation_of (n));
15715 gc_high = heap_segment_reserved (ephemeral_heap_segment);
15717 #ifdef BACKGROUND_GC
15718 if (settings.concurrent)
15721 time_bgc_last = GetHighPrecisionTimeStamp();
15724 FIRE_EVENT(BGCBegin);
15726 concurrent_print_time_delta ("BGC");
15728 //#ifdef WRITE_WATCH
15729 //reset_write_watch (FALSE);
15730 //#endif //WRITE_WATCH
15732 concurrent_print_time_delta ("RW");
15733 background_mark_phase();
15734 free_list_info (max_generation, "after mark phase");
15736 background_sweep();
15737 free_list_info (max_generation, "after sweep phase");
15740 #endif //BACKGROUND_GC
15742 mark_phase (n, FALSE);
15744 GCScan::GcRuntimeStructuresValid (FALSE);
15746 GCScan::GcRuntimeStructuresValid (TRUE);
15750 size_t end_gc_time = GetHighPrecisionTimeStamp();
15751 // printf ("generation: %d, elapsed time: %Id\n", n, end_gc_time - dd_time_clock (dynamic_data_of (0)));
15753 //adjust the allocation size from the pinned quantities.
15754 for (int gen_number = 0; gen_number <= min (max_generation,n+1); gen_number++)
15756 generation* gn = generation_of (gen_number);
15757 if (settings.compaction)
15759 generation_pinned_allocated (gn) += generation_pinned_allocation_compact_size (gn);
15760 generation_allocation_size (generation_of (gen_number)) += generation_pinned_allocation_compact_size (gn);
15764 generation_pinned_allocated (gn) += generation_pinned_allocation_sweep_size (gn);
15765 generation_allocation_size (generation_of (gen_number)) += generation_pinned_allocation_sweep_size (gn);
15767 generation_pinned_allocation_sweep_size (gn) = 0;
15768 generation_pinned_allocation_compact_size (gn) = 0;
15771 #ifdef BACKGROUND_GC
15772 if (settings.concurrent)
15774 dynamic_data* dd = dynamic_data_of (n);
15775 dd_gc_elapsed_time (dd) = end_gc_time - dd_time_clock (dd);
15777 free_list_info (max_generation, "after computing new dynamic data");
15779 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
15781 for (int gen_number = 0; gen_number < max_generation; gen_number++)
15783 dprintf (2, ("end of BGC: gen%d new_alloc: %Id",
15784 gen_number, dd_desired_allocation (dynamic_data_of (gen_number))));
15785 current_gc_data_per_heap->gen_data[gen_number].size_after = generation_size (gen_number);
15786 current_gc_data_per_heap->gen_data[gen_number].free_list_space_after = generation_free_list_space (generation_of (gen_number));
15787 current_gc_data_per_heap->gen_data[gen_number].free_obj_space_after = generation_free_obj_space (generation_of (gen_number));
15791 #endif //BACKGROUND_GC
15793 free_list_info (max_generation, "end");
15794 for (int gen_number = 0; gen_number <= n; gen_number++)
15796 dynamic_data* dd = dynamic_data_of (gen_number);
15797 dd_gc_elapsed_time (dd) = end_gc_time - dd_time_clock (dd);
15798 compute_new_dynamic_data (gen_number);
15801 if (n != max_generation)
15803 int gen_num_for_data = ((n < (max_generation - 1)) ? (n + 1) : (max_generation + 1));
15804 for (int gen_number = (n + 1); gen_number <= gen_num_for_data; gen_number++)
15806 get_gc_data_per_heap()->gen_data[gen_number].size_after = generation_size (gen_number);
15807 get_gc_data_per_heap()->gen_data[gen_number].free_list_space_after = generation_free_list_space (generation_of (gen_number));
15808 get_gc_data_per_heap()->gen_data[gen_number].free_obj_space_after = generation_free_obj_space (generation_of (gen_number));
15812 get_gc_data_per_heap()->maxgen_size_info.running_free_list_efficiency = (uint32_t)(generation_allocator_efficiency (generation_of (max_generation)) * 100);
15814 free_list_info (max_generation, "after computing new dynamic data");
15816 if (heap_number == 0)
15818 dprintf (GTC_LOG, ("GC#%d(gen%d) took %Idms",
15819 dd_collection_count (dynamic_data_of (0)),
15820 settings.condemned_generation,
15821 dd_gc_elapsed_time (dynamic_data_of (0))));
15824 for (int gen_number = 0; gen_number <= (max_generation + 1); gen_number++)
15826 dprintf (2, ("end of FGC/NGC: gen%d new_alloc: %Id",
15827 gen_number, dd_desired_allocation (dynamic_data_of (gen_number))));
15831 if (n < max_generation)
15833 compute_promoted_allocation (1 + n);
15835 dynamic_data* dd = dynamic_data_of (1 + n);
15836 size_t new_fragmentation = generation_free_list_space (generation_of (1 + n)) +
15837 generation_free_obj_space (generation_of (1 + n));
15839 #ifdef BACKGROUND_GC
15840 if (current_c_gc_state != c_gc_state_planning)
15841 #endif //BACKGROUND_GC
15843 if (settings.promotion)
15845 dd_fragmentation (dd) = new_fragmentation;
15849 //assert (dd_fragmentation (dd) == new_fragmentation);
15854 #ifdef BACKGROUND_GC
15855 if (!settings.concurrent)
15856 #endif //BACKGROUND_GC
15858 #ifndef FEATURE_REDHAWK
15859 // GCToEEInterface::IsGCThread() always returns false on CoreRT, but this assert is useful in CoreCLR.
15860 assert(GCToEEInterface::IsGCThread());
15861 #endif // FEATURE_REDHAWK
15862 adjust_ephemeral_limits();
15865 #ifdef BACKGROUND_GC
15866 assert (ephemeral_low == generation_allocation_start (generation_of ( max_generation -1)));
15867 assert (ephemeral_high == heap_segment_reserved (ephemeral_heap_segment));
15868 #endif //BACKGROUND_GC
15870 if (fgn_maxgen_percent)
15872 if (settings.condemned_generation == (max_generation - 1))
15874 check_for_full_gc (max_generation - 1, 0);
15876 else if (settings.condemned_generation == max_generation)
15878 if (full_gc_approach_event_set
15879 #ifdef MULTIPLE_HEAPS
15880 && (heap_number == 0)
15881 #endif //MULTIPLE_HEAPS
15884 dprintf (2, ("FGN-GC: setting gen2 end event"));
15886 full_gc_approach_event.Reset();
15887 #ifdef BACKGROUND_GC
15888 // By definition WaitForFullGCComplete only succeeds if it's full, *blocking* GC, otherwise need to return N/A
15889 fgn_last_gc_was_concurrent = settings.concurrent ? TRUE : FALSE;
15890 #endif //BACKGROUND_GC
15891 full_gc_end_event.Set();
15892 full_gc_approach_event_set = false;
15897 #ifdef BACKGROUND_GC
15898 if (!settings.concurrent)
15899 #endif //BACKGROUND_GC
15901 //decide on the next allocation quantum
15902 if (alloc_contexts_used >= 1)
15904 allocation_quantum = Align (min ((size_t)CLR_SIZE,
15905 (size_t)max (1024, get_new_allocation (0) / (2 * alloc_contexts_used))),
15906 get_alignment_constant(FALSE));
15907 dprintf (3, ("New allocation quantum: %d(0x%Ix)", allocation_quantum, allocation_quantum));
15911 descr_generations (FALSE);
15913 verify_soh_segment_list();
15915 #ifdef BACKGROUND_GC
15916 add_to_history_per_heap();
15917 if (heap_number == 0)
15921 #endif // BACKGROUND_GC
15924 if (GCStatistics::Enabled() && heap_number == 0)
15925 g_GCStatistics.AddGCStats(settings,
15926 dd_gc_elapsed_time(dynamic_data_of(settings.condemned_generation)));
15930 fprintf (stdout, "%d,%d,%d,%d,%d,%d\n",
15931 n, mark_time, plan_time, reloc_time, compact_time, sweep_time);
15934 #ifdef BACKGROUND_GC
15935 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15936 #endif //BACKGROUND_GC
15938 #if defined(VERIFY_HEAP) || (defined (FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC))
15941 // Note that right now g_pConfig->GetHeapVerifyLevel always returns the same
15942 // value. If we ever allow randomly adjusting this as the process runs,
15943 // we cannot call it this way as joins need to match - we must have the same
15944 // value for all heaps like we do with bgc_heap_walk_for_etw_p.
15945 || (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
15947 #if defined(FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC)
15948 || (bgc_heap_walk_for_etw_p && settings.concurrent)
15952 #ifdef BACKGROUND_GC
15953 bool cooperative_mode = true;
15955 if (settings.concurrent)
15957 cooperative_mode = enable_preemptive ();
15959 #ifdef MULTIPLE_HEAPS
15960 bgc_t_join.join(this, gc_join_suspend_ee_verify);
15961 if (bgc_t_join.joined())
15963 bgc_threads_sync_event.Reset();
15965 dprintf(2, ("Joining BGC threads to suspend EE for verify heap"));
15966 bgc_t_join.restart();
15968 if (heap_number == 0)
15971 bgc_threads_sync_event.Set();
15975 bgc_threads_sync_event.Wait(INFINITE, FALSE);
15976 dprintf (2, ("bgc_threads_sync_event is signalled"));
15980 #endif //MULTIPLE_HEAPS
15982 //fix the allocation area so verify_heap can proceed.
15983 fix_allocation_contexts (FALSE);
15985 #endif //BACKGROUND_GC
15987 #ifdef BACKGROUND_GC
15988 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15989 #ifdef FEATURE_EVENT_TRACE
15990 if (bgc_heap_walk_for_etw_p && settings.concurrent)
15992 GCToEEInterface::DiagWalkBGCSurvivors(__this);
15994 #ifdef MULTIPLE_HEAPS
15995 bgc_t_join.join(this, gc_join_after_profiler_heap_walk);
15996 if (bgc_t_join.joined())
15998 bgc_t_join.restart();
16000 #endif // MULTIPLE_HEAPS
16002 #endif // FEATURE_EVENT_TRACE
16003 #endif //BACKGROUND_GC
16006 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
16007 verify_heap (FALSE);
16008 #endif // VERIFY_HEAP
16010 #ifdef BACKGROUND_GC
16011 if (settings.concurrent)
16013 repair_allocation_contexts (TRUE);
16015 #ifdef MULTIPLE_HEAPS
16016 bgc_t_join.join(this, gc_join_restart_ee_verify);
16017 if (bgc_t_join.joined())
16019 bgc_threads_sync_event.Reset();
16021 dprintf(2, ("Joining BGC threads to restart EE after verify heap"));
16022 bgc_t_join.restart();
16024 if (heap_number == 0)
16027 bgc_threads_sync_event.Set();
16031 bgc_threads_sync_event.Wait(INFINITE, FALSE);
16032 dprintf (2, ("bgc_threads_sync_event is signalled"));
16036 #endif //MULTIPLE_HEAPS
16038 disable_preemptive (cooperative_mode);
16040 #endif //BACKGROUND_GC
16042 #endif // defined(VERIFY_HEAP) || (defined(FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC))
16044 #ifdef MULTIPLE_HEAPS
16045 if (!settings.concurrent)
16047 gc_t_join.join(this, gc_join_done);
16048 if (gc_t_join.joined ())
16050 gc_heap::internal_gc_done = false;
16052 //equalize the new desired size of the generations
16053 int limit = settings.condemned_generation;
16054 if (limit == max_generation)
16056 limit = max_generation+1;
16058 for (int gen = 0; gen <= limit; gen++)
16060 size_t total_desired = 0;
16062 for (int i = 0; i < gc_heap::n_heaps; i++)
16064 gc_heap* hp = gc_heap::g_heaps[i];
16065 dynamic_data* dd = hp->dynamic_data_of (gen);
16066 size_t temp_total_desired = total_desired + dd_desired_allocation (dd);
16067 if (temp_total_desired < total_desired)
16070 total_desired = (size_t)MAX_PTR;
16073 total_desired = temp_total_desired;
16076 size_t desired_per_heap = Align (total_desired/gc_heap::n_heaps,
16077 get_alignment_constant ((gen != (max_generation+1))));
16081 #if 1 //subsumed by the linear allocation model
16082 // to avoid spikes in mem usage due to short terms fluctuations in survivorship,
16083 // apply some smoothing.
16084 static size_t smoothed_desired_per_heap = 0;
16085 size_t smoothing = 3; // exponential smoothing factor
16086 if (smoothing > VolatileLoad(&settings.gc_index))
16087 smoothing = VolatileLoad(&settings.gc_index);
16088 smoothed_desired_per_heap = desired_per_heap / smoothing + ((smoothed_desired_per_heap / smoothing) * (smoothing-1));
16089 dprintf (1, ("sn = %Id n = %Id", smoothed_desired_per_heap, desired_per_heap));
16090 desired_per_heap = Align(smoothed_desired_per_heap, get_alignment_constant (true));
16093 if (!heap_hard_limit)
16095 // if desired_per_heap is close to min_gc_size, trim it
16096 // down to min_gc_size to stay in the cache
16097 gc_heap* hp = gc_heap::g_heaps[0];
16098 dynamic_data* dd = hp->dynamic_data_of (gen);
16099 size_t min_gc_size = dd_min_size(dd);
16100 // if min GC size larger than true on die cache, then don't bother
16101 // limiting the desired size
16102 if ((min_gc_size <= GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE)) &&
16103 desired_per_heap <= 2*min_gc_size)
16105 desired_per_heap = min_gc_size;
16109 desired_per_heap = joined_youngest_desired (desired_per_heap);
16110 dprintf (2, ("final gen0 new_alloc: %Id", desired_per_heap));
16112 gc_data_global.final_youngest_desired = desired_per_heap;
16114 #if 1 //subsumed by the linear allocation model
16115 if (gen == (max_generation + 1))
16117 // to avoid spikes in mem usage due to short terms fluctuations in survivorship,
16118 // apply some smoothing.
16119 static size_t smoothed_desired_per_heap_loh = 0;
16120 size_t smoothing = 3; // exponential smoothing factor
16121 size_t loh_count = dd_collection_count (dynamic_data_of (max_generation));
16122 if (smoothing > loh_count)
16123 smoothing = loh_count;
16124 smoothed_desired_per_heap_loh = desired_per_heap / smoothing + ((smoothed_desired_per_heap_loh / smoothing) * (smoothing-1));
16125 dprintf (2, ("smoothed_desired_per_heap_loh = %Id desired_per_heap = %Id", smoothed_desired_per_heap_loh, desired_per_heap));
16126 desired_per_heap = Align(smoothed_desired_per_heap_loh, get_alignment_constant (false));
16129 for (int i = 0; i < gc_heap::n_heaps; i++)
16131 gc_heap* hp = gc_heap::g_heaps[i];
16132 dynamic_data* dd = hp->dynamic_data_of (gen);
16133 dd_desired_allocation (dd) = desired_per_heap;
16134 dd_gc_new_allocation (dd) = desired_per_heap;
16135 dd_new_allocation (dd) = desired_per_heap;
16139 hp->fgn_last_alloc = desired_per_heap;
16144 #ifdef FEATURE_LOH_COMPACTION
16145 BOOL all_heaps_compacted_p = TRUE;
16146 #endif //FEATURE_LOH_COMPACTION
16147 for (int i = 0; i < gc_heap::n_heaps; i++)
16149 gc_heap* hp = gc_heap::g_heaps[i];
16150 hp->decommit_ephemeral_segment_pages();
16151 hp->rearrange_large_heap_segments();
16152 #ifdef FEATURE_LOH_COMPACTION
16153 all_heaps_compacted_p &= hp->loh_compacted_p;
16154 #endif //FEATURE_LOH_COMPACTION
16157 #ifdef FEATURE_LOH_COMPACTION
16158 check_loh_compact_mode (all_heaps_compacted_p);
16159 #endif //FEATURE_LOH_COMPACTION
16162 pm_full_gc_init_or_clear();
16164 gc_t_join.restart();
16166 alloc_context_count = 0;
16167 heap_select::mark_heap (heap_number);
16171 gc_data_global.final_youngest_desired =
16172 dd_desired_allocation (dynamic_data_of (0));
16174 check_loh_compact_mode (loh_compacted_p);
16176 decommit_ephemeral_segment_pages();
16179 if (!(settings.concurrent))
16181 rearrange_large_heap_segments();
16185 pm_full_gc_init_or_clear();
16187 #ifdef BACKGROUND_GC
16188 recover_bgc_settings();
16189 #endif //BACKGROUND_GC
16190 #endif //MULTIPLE_HEAPS
16193 void gc_heap::save_data_for_no_gc()
16195 current_no_gc_region_info.saved_pause_mode = settings.pause_mode;
16196 #ifdef MULTIPLE_HEAPS
16197 // This is to affect heap balancing.
16198 for (int i = 0; i < n_heaps; i++)
16200 current_no_gc_region_info.saved_gen0_min_size = dd_min_size (g_heaps[i]->dynamic_data_of (0));
16201 dd_min_size (g_heaps[i]->dynamic_data_of (0)) = min_balance_threshold;
16202 current_no_gc_region_info.saved_gen3_min_size = dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1));
16203 dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1)) = 0;
16205 #endif //MULTIPLE_HEAPS
16208 void gc_heap::restore_data_for_no_gc()
16210 gc_heap::settings.pause_mode = current_no_gc_region_info.saved_pause_mode;
16211 #ifdef MULTIPLE_HEAPS
16212 for (int i = 0; i < n_heaps; i++)
16214 dd_min_size (g_heaps[i]->dynamic_data_of (0)) = current_no_gc_region_info.saved_gen0_min_size;
16215 dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1)) = current_no_gc_region_info.saved_gen3_min_size;
16217 #endif //MULTIPLE_HEAPS
16220 start_no_gc_region_status gc_heap::prepare_for_no_gc_region (uint64_t total_size,
16221 BOOL loh_size_known,
16223 BOOL disallow_full_blocking)
16225 if (current_no_gc_region_info.started)
16227 return start_no_gc_in_progress;
16230 start_no_gc_region_status status = start_no_gc_success;
16232 save_data_for_no_gc();
16233 settings.pause_mode = pause_no_gc;
16234 current_no_gc_region_info.start_status = start_no_gc_success;
16236 uint64_t allocation_no_gc_loh = 0;
16237 uint64_t allocation_no_gc_soh = 0;
16238 assert(total_size != 0);
16239 if (loh_size_known)
16241 assert(loh_size != 0);
16242 assert(loh_size <= total_size);
16243 allocation_no_gc_loh = loh_size;
16244 allocation_no_gc_soh = total_size - loh_size;
16248 allocation_no_gc_soh = total_size;
16249 allocation_no_gc_loh = total_size;
16252 int soh_align_const = get_alignment_constant (TRUE);
16253 size_t max_soh_allocated = soh_segment_size - segment_info_size - eph_gen_starts_size;
16254 size_t size_per_heap = 0;
16255 const double scale_factor = 1.05;
16258 #ifdef MULTIPLE_HEAPS
16259 num_heaps = n_heaps;
16260 #endif // MULTIPLE_HEAPS
16262 uint64_t total_allowed_soh_allocation = max_soh_allocated * num_heaps;
16264 // In theory, the upper limit here is the physical memory of the machine, not
16265 // SIZE_T_MAX. This is not true today because total_physical_mem can be
16266 // larger than SIZE_T_MAX if running in wow64 on a machine with more than
16267 // 4GB of RAM. Once Local GC code divergence is resolved and code is flowing
16268 // more freely between branches, it would be good to clean this up to use
16269 // total_physical_mem instead of SIZE_T_MAX.
16270 assert(total_allowed_soh_allocation <= SIZE_T_MAX);
16271 uint64_t total_allowed_loh_allocation = SIZE_T_MAX;
16272 uint64_t total_allowed_soh_alloc_scaled = allocation_no_gc_soh > 0 ? static_cast<uint64_t>(total_allowed_soh_allocation / scale_factor) : 0;
16273 uint64_t total_allowed_loh_alloc_scaled = allocation_no_gc_loh > 0 ? static_cast<uint64_t>(total_allowed_loh_allocation / scale_factor) : 0;
16275 if (allocation_no_gc_soh > total_allowed_soh_alloc_scaled ||
16276 allocation_no_gc_loh > total_allowed_loh_alloc_scaled)
16278 status = start_no_gc_too_large;
16282 if (allocation_no_gc_soh > 0)
16284 allocation_no_gc_soh = static_cast<uint64_t>(allocation_no_gc_soh * scale_factor);
16285 allocation_no_gc_soh = min (allocation_no_gc_soh, total_allowed_soh_alloc_scaled);
16288 if (allocation_no_gc_loh > 0)
16290 allocation_no_gc_loh = static_cast<uint64_t>(allocation_no_gc_loh * scale_factor);
16291 allocation_no_gc_loh = min (allocation_no_gc_loh, total_allowed_loh_alloc_scaled);
16294 if (disallow_full_blocking)
16295 current_no_gc_region_info.minimal_gc_p = TRUE;
16297 if (allocation_no_gc_soh != 0)
16299 current_no_gc_region_info.soh_allocation_size = static_cast<size_t>(allocation_no_gc_soh);
16300 size_per_heap = current_no_gc_region_info.soh_allocation_size;
16301 #ifdef MULTIPLE_HEAPS
16302 size_per_heap /= n_heaps;
16303 for (int i = 0; i < n_heaps; i++)
16305 // due to heap balancing we need to allow some room before we even look to balance to another heap.
16306 g_heaps[i]->soh_allocation_no_gc = min (Align ((size_per_heap + min_balance_threshold), soh_align_const), max_soh_allocated);
16308 #else //MULTIPLE_HEAPS
16309 soh_allocation_no_gc = min (Align (size_per_heap, soh_align_const), max_soh_allocated);
16310 #endif //MULTIPLE_HEAPS
16313 if (allocation_no_gc_loh != 0)
16315 current_no_gc_region_info.loh_allocation_size = static_cast<size_t>(allocation_no_gc_loh);
16316 size_per_heap = current_no_gc_region_info.loh_allocation_size;
16317 #ifdef MULTIPLE_HEAPS
16318 size_per_heap /= n_heaps;
16319 for (int i = 0; i < n_heaps; i++)
16320 g_heaps[i]->loh_allocation_no_gc = Align (size_per_heap, get_alignment_constant (FALSE));
16321 #else //MULTIPLE_HEAPS
16322 loh_allocation_no_gc = Align (size_per_heap, get_alignment_constant (FALSE));
16323 #endif //MULTIPLE_HEAPS
16327 if (status != start_no_gc_success)
16328 restore_data_for_no_gc();
16332 void gc_heap::handle_failure_for_no_gc()
16334 gc_heap::restore_data_for_no_gc();
16335 // sets current_no_gc_region_info.started to FALSE here.
16336 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
16339 start_no_gc_region_status gc_heap::get_start_no_gc_region_status()
16341 return current_no_gc_region_info.start_status;
16344 void gc_heap::record_gcs_during_no_gc()
16346 if (current_no_gc_region_info.started)
16348 current_no_gc_region_info.num_gcs++;
16349 if (is_induced (settings.reason))
16350 current_no_gc_region_info.num_gcs_induced++;
16354 BOOL gc_heap::find_loh_free_for_no_gc()
16356 allocator* loh_allocator = generation_allocator (generation_of (max_generation + 1));
16357 size_t sz_list = loh_allocator->first_bucket_size();
16358 size_t size = loh_allocation_no_gc;
16359 for (unsigned int a_l_idx = 0; a_l_idx < loh_allocator->number_of_buckets(); a_l_idx++)
16361 if ((size < sz_list) || (a_l_idx == (loh_allocator->number_of_buckets()-1)))
16363 uint8_t* free_list = loh_allocator->alloc_list_head_of (a_l_idx);
16366 size_t free_list_size = unused_array_size(free_list);
16368 if (free_list_size > loh_allocation_no_gc)
16370 dprintf (3, ("free item %Ix(%Id) for no gc", (size_t)free_list, free_list_size));
16374 free_list = free_list_slot (free_list);
16377 sz_list = sz_list * 2;
16383 BOOL gc_heap::find_loh_space_for_no_gc()
16385 saved_loh_segment_no_gc = 0;
16387 if (find_loh_free_for_no_gc())
16390 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16394 size_t remaining = heap_segment_reserved (seg) - heap_segment_allocated (seg);
16395 if (remaining >= loh_allocation_no_gc)
16397 saved_loh_segment_no_gc = seg;
16400 seg = heap_segment_next (seg);
16403 if (!saved_loh_segment_no_gc && current_no_gc_region_info.minimal_gc_p)
16405 // If no full GC is allowed, we try to get a new seg right away.
16406 saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc)
16407 #ifdef MULTIPLE_HEAPS
16409 #endif //MULTIPLE_HEAPS
16413 return (saved_loh_segment_no_gc != 0);
16416 BOOL gc_heap::loh_allocated_for_no_gc()
16418 if (!saved_loh_segment_no_gc)
16421 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16424 if (seg == saved_loh_segment_no_gc)
16428 seg = heap_segment_next (seg);
16434 BOOL gc_heap::commit_loh_for_no_gc (heap_segment* seg)
16436 uint8_t* end_committed = heap_segment_allocated (seg) + loh_allocation_no_gc;
16437 assert (end_committed <= heap_segment_reserved (seg));
16438 return (grow_heap_segment (seg, end_committed));
16441 void gc_heap::thread_no_gc_loh_segments()
16443 #ifdef MULTIPLE_HEAPS
16444 for (int i = 0; i < n_heaps; i++)
16446 gc_heap* hp = g_heaps[i];
16447 if (hp->loh_allocated_for_no_gc())
16449 hp->thread_loh_segment (hp->saved_loh_segment_no_gc);
16450 hp->saved_loh_segment_no_gc = 0;
16453 #else //MULTIPLE_HEAPS
16454 if (loh_allocated_for_no_gc())
16456 thread_loh_segment (saved_loh_segment_no_gc);
16457 saved_loh_segment_no_gc = 0;
16459 #endif //MULTIPLE_HEAPS
16462 void gc_heap::set_loh_allocations_for_no_gc()
16464 if (current_no_gc_region_info.loh_allocation_size != 0)
16466 dynamic_data* dd = dynamic_data_of (max_generation + 1);
16467 dd_new_allocation (dd) = loh_allocation_no_gc;
16468 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
16472 void gc_heap::set_soh_allocations_for_no_gc()
16474 if (current_no_gc_region_info.soh_allocation_size != 0)
16476 dynamic_data* dd = dynamic_data_of (0);
16477 dd_new_allocation (dd) = soh_allocation_no_gc;
16478 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
16479 #ifdef MULTIPLE_HEAPS
16480 alloc_context_count = 0;
16481 #endif //MULTIPLE_HEAPS
16485 void gc_heap::set_allocations_for_no_gc()
16487 #ifdef MULTIPLE_HEAPS
16488 for (int i = 0; i < n_heaps; i++)
16490 gc_heap* hp = g_heaps[i];
16491 hp->set_loh_allocations_for_no_gc();
16492 hp->set_soh_allocations_for_no_gc();
16494 #else //MULTIPLE_HEAPS
16495 set_loh_allocations_for_no_gc();
16496 set_soh_allocations_for_no_gc();
16497 #endif //MULTIPLE_HEAPS
16500 BOOL gc_heap::should_proceed_for_no_gc()
16502 BOOL gc_requested = FALSE;
16503 BOOL loh_full_gc_requested = FALSE;
16504 BOOL soh_full_gc_requested = FALSE;
16505 BOOL no_gc_requested = FALSE;
16506 BOOL get_new_loh_segments = FALSE;
16508 if (current_no_gc_region_info.soh_allocation_size)
16510 #ifdef MULTIPLE_HEAPS
16511 for (int i = 0; i < n_heaps; i++)
16513 gc_heap* hp = g_heaps[i];
16514 if ((size_t)(heap_segment_reserved (hp->ephemeral_heap_segment) - hp->alloc_allocated) < hp->soh_allocation_no_gc)
16516 gc_requested = TRUE;
16520 #else //MULTIPLE_HEAPS
16521 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated) < soh_allocation_no_gc)
16522 gc_requested = TRUE;
16523 #endif //MULTIPLE_HEAPS
16527 #ifdef MULTIPLE_HEAPS
16528 for (int i = 0; i < n_heaps; i++)
16530 gc_heap* hp = g_heaps[i];
16531 if (!(hp->grow_heap_segment (hp->ephemeral_heap_segment, (hp->alloc_allocated + hp->soh_allocation_no_gc))))
16533 soh_full_gc_requested = TRUE;
16537 #else //MULTIPLE_HEAPS
16538 if (!grow_heap_segment (ephemeral_heap_segment, (alloc_allocated + soh_allocation_no_gc)))
16539 soh_full_gc_requested = TRUE;
16540 #endif //MULTIPLE_HEAPS
16544 if (!current_no_gc_region_info.minimal_gc_p && gc_requested)
16546 soh_full_gc_requested = TRUE;
16549 no_gc_requested = !(soh_full_gc_requested || gc_requested);
16551 if (soh_full_gc_requested && current_no_gc_region_info.minimal_gc_p)
16553 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16557 if (!soh_full_gc_requested && current_no_gc_region_info.loh_allocation_size)
16559 // Check to see if we have enough reserved space.
16560 #ifdef MULTIPLE_HEAPS
16561 for (int i = 0; i < n_heaps; i++)
16563 gc_heap* hp = g_heaps[i];
16564 if (!hp->find_loh_space_for_no_gc())
16566 loh_full_gc_requested = TRUE;
16570 #else //MULTIPLE_HEAPS
16571 if (!find_loh_space_for_no_gc())
16572 loh_full_gc_requested = TRUE;
16573 #endif //MULTIPLE_HEAPS
16575 // Check to see if we have committed space.
16576 if (!loh_full_gc_requested)
16578 #ifdef MULTIPLE_HEAPS
16579 for (int i = 0; i < n_heaps; i++)
16581 gc_heap* hp = g_heaps[i];
16582 if (hp->saved_loh_segment_no_gc &&!hp->commit_loh_for_no_gc (hp->saved_loh_segment_no_gc))
16584 loh_full_gc_requested = TRUE;
16588 #else //MULTIPLE_HEAPS
16589 if (saved_loh_segment_no_gc && !commit_loh_for_no_gc (saved_loh_segment_no_gc))
16590 loh_full_gc_requested = TRUE;
16591 #endif //MULTIPLE_HEAPS
16595 if (loh_full_gc_requested || soh_full_gc_requested)
16597 if (current_no_gc_region_info.minimal_gc_p)
16598 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16601 no_gc_requested = !(loh_full_gc_requested || soh_full_gc_requested || gc_requested);
16603 if (current_no_gc_region_info.start_status == start_no_gc_success)
16605 if (no_gc_requested)
16606 set_allocations_for_no_gc();
16611 if ((current_no_gc_region_info.start_status == start_no_gc_success) && !no_gc_requested)
16615 // We are done with starting the no_gc_region.
16616 current_no_gc_region_info.started = TRUE;
16621 end_no_gc_region_status gc_heap::end_no_gc_region()
16623 dprintf (1, ("end no gc called"));
16625 end_no_gc_region_status status = end_no_gc_success;
16627 if (!(current_no_gc_region_info.started))
16628 status = end_no_gc_not_in_progress;
16629 if (current_no_gc_region_info.num_gcs_induced)
16630 status = end_no_gc_induced;
16631 else if (current_no_gc_region_info.num_gcs)
16632 status = end_no_gc_alloc_exceeded;
16634 if (settings.pause_mode == pause_no_gc)
16635 restore_data_for_no_gc();
16637 // sets current_no_gc_region_info.started to FALSE here.
16638 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
16644 void gc_heap::update_collection_counts ()
16646 dynamic_data* dd0 = dynamic_data_of (0);
16647 dd_gc_clock (dd0) += 1;
16649 size_t now = GetHighPrecisionTimeStamp();
16651 for (int i = 0; i <= settings.condemned_generation;i++)
16653 dynamic_data* dd = dynamic_data_of (i);
16654 dd_collection_count (dd)++;
16655 //this is needed by the linear allocation model
16656 if (i == max_generation)
16657 dd_collection_count (dynamic_data_of (max_generation+1))++;
16658 dd_gc_clock (dd) = dd_gc_clock (dd0);
16659 dd_time_clock (dd) = now;
16663 BOOL gc_heap::expand_soh_with_minimal_gc()
16665 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment)) >= soh_allocation_no_gc)
16668 heap_segment* new_seg = soh_get_segment_to_expand();
16671 if (g_gc_card_table != card_table)
16672 copy_brick_card_table();
16674 settings.promotion = TRUE;
16675 settings.demotion = FALSE;
16676 ephemeral_promotion = TRUE;
16677 int condemned_gen_number = max_generation - 1;
16679 generation* gen = 0;
16680 int align_const = get_alignment_constant (TRUE);
16682 for (int i = 0; i <= condemned_gen_number; i++)
16684 gen = generation_of (i);
16685 saved_ephemeral_plan_start[i] = generation_allocation_start (gen);
16686 saved_ephemeral_plan_start_size[i] = Align (size (generation_allocation_start (gen)), align_const);
16689 // We do need to clear the bricks here as we are converting a bunch of ephemeral objects to gen2
16690 // and need to make sure that there are no left over bricks from the previous GCs for the space
16691 // we just used for gen0 allocation. We will need to go through the bricks for these objects for
16692 // ephemeral GCs later.
16693 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
16694 b < brick_of (align_on_brick (heap_segment_allocated (ephemeral_heap_segment)));
16700 size_t ephemeral_size = (heap_segment_allocated (ephemeral_heap_segment) -
16701 generation_allocation_start (generation_of (max_generation - 1)));
16702 heap_segment_next (ephemeral_heap_segment) = new_seg;
16703 ephemeral_heap_segment = new_seg;
16704 uint8_t* start = heap_segment_mem (ephemeral_heap_segment);
16706 for (int i = condemned_gen_number; i >= 0; i--)
16708 gen = generation_of (i);
16709 size_t gen_start_size = Align (min_obj_size);
16710 make_generation (generation_table[i], ephemeral_heap_segment, start, 0);
16711 generation_plan_allocation_start (gen) = start;
16712 generation_plan_allocation_start_size (gen) = gen_start_size;
16713 start += gen_start_size;
16715 heap_segment_used (ephemeral_heap_segment) = start - plug_skew;
16716 heap_segment_plan_allocated (ephemeral_heap_segment) = start;
16718 fix_generation_bounds (condemned_gen_number, generation_of (0));
16720 dd_gc_new_allocation (dynamic_data_of (max_generation)) -= ephemeral_size;
16721 dd_new_allocation (dynamic_data_of (max_generation)) = dd_gc_new_allocation (dynamic_data_of (max_generation));
16723 adjust_ephemeral_limits();
16730 // Only to be done on the thread that calls restart in a join for server GC
16731 // and reset the oom status per heap.
16732 void gc_heap::check_and_set_no_gc_oom()
16734 #ifdef MULTIPLE_HEAPS
16735 for (int i = 0; i < n_heaps; i++)
16737 gc_heap* hp = g_heaps[i];
16738 if (hp->no_gc_oom_p)
16740 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16741 hp->no_gc_oom_p = false;
16747 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16748 no_gc_oom_p = false;
16750 #endif //MULTIPLE_HEAPS
16753 void gc_heap::allocate_for_no_gc_after_gc()
16755 if (current_no_gc_region_info.minimal_gc_p)
16756 repair_allocation_contexts (TRUE);
16758 no_gc_oom_p = false;
16760 if (current_no_gc_region_info.start_status != start_no_gc_no_memory)
16762 if (current_no_gc_region_info.soh_allocation_size != 0)
16764 if (((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment)) < soh_allocation_no_gc) ||
16765 (!grow_heap_segment (ephemeral_heap_segment, (heap_segment_allocated (ephemeral_heap_segment) + soh_allocation_no_gc))))
16767 no_gc_oom_p = true;
16770 #ifdef MULTIPLE_HEAPS
16771 gc_t_join.join(this, gc_join_after_commit_soh_no_gc);
16772 if (gc_t_join.joined())
16774 #endif //MULTIPLE_HEAPS
16776 check_and_set_no_gc_oom();
16778 #ifdef MULTIPLE_HEAPS
16779 gc_t_join.restart();
16781 #endif //MULTIPLE_HEAPS
16784 if ((current_no_gc_region_info.start_status == start_no_gc_success) &&
16785 !(current_no_gc_region_info.minimal_gc_p) &&
16786 (current_no_gc_region_info.loh_allocation_size != 0))
16788 gc_policy = policy_compact;
16789 saved_loh_segment_no_gc = 0;
16791 if (!find_loh_free_for_no_gc())
16793 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16794 BOOL found_seg_p = FALSE;
16797 if ((size_t)(heap_segment_reserved (seg) - heap_segment_allocated (seg)) >= loh_allocation_no_gc)
16799 found_seg_p = TRUE;
16800 if (!commit_loh_for_no_gc (seg))
16802 no_gc_oom_p = true;
16806 seg = heap_segment_next (seg);
16810 gc_policy = policy_expand;
16813 #ifdef MULTIPLE_HEAPS
16814 gc_t_join.join(this, gc_join_expand_loh_no_gc);
16815 if (gc_t_join.joined())
16817 check_and_set_no_gc_oom();
16819 if (current_no_gc_region_info.start_status == start_no_gc_success)
16821 for (int i = 0; i < n_heaps; i++)
16823 gc_heap* hp = g_heaps[i];
16824 if (hp->gc_policy == policy_expand)
16826 hp->saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc), hp);
16827 if (!(hp->saved_loh_segment_no_gc))
16829 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16836 gc_t_join.restart();
16838 #else //MULTIPLE_HEAPS
16839 check_and_set_no_gc_oom();
16841 if ((current_no_gc_region_info.start_status == start_no_gc_success) && (gc_policy == policy_expand))
16843 saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc));
16844 if (!saved_loh_segment_no_gc)
16845 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16847 #endif //MULTIPLE_HEAPS
16849 if ((current_no_gc_region_info.start_status == start_no_gc_success) && saved_loh_segment_no_gc)
16851 if (!commit_loh_for_no_gc (saved_loh_segment_no_gc))
16853 no_gc_oom_p = true;
16859 #ifdef MULTIPLE_HEAPS
16860 gc_t_join.join(this, gc_join_final_no_gc);
16861 if (gc_t_join.joined())
16863 #endif //MULTIPLE_HEAPS
16865 check_and_set_no_gc_oom();
16867 if (current_no_gc_region_info.start_status == start_no_gc_success)
16869 set_allocations_for_no_gc();
16870 current_no_gc_region_info.started = TRUE;
16873 #ifdef MULTIPLE_HEAPS
16874 gc_t_join.restart();
16876 #endif //MULTIPLE_HEAPS
16879 void gc_heap::init_records()
16881 // An option is to move this to be after we figure out which gen to condemn so we don't
16882 // need to clear some generations' data 'cause we know they don't change, but that also means
16883 // we can't simply call memset here.
16884 memset (&gc_data_per_heap, 0, sizeof (gc_data_per_heap));
16885 gc_data_per_heap.heap_index = heap_number;
16886 if (heap_number == 0)
16887 memset (&gc_data_global, 0, sizeof (gc_data_global));
16889 #ifdef GC_CONFIG_DRIVEN
16890 memset (interesting_data_per_gc, 0, sizeof (interesting_data_per_gc));
16891 #endif //GC_CONFIG_DRIVEN
16892 memset (&fgm_result, 0, sizeof (fgm_result));
16894 for (int i = 0; i <= (max_generation + 1); i++)
16896 gc_data_per_heap.gen_data[i].size_before = generation_size (i);
16897 generation* gen = generation_of (i);
16898 gc_data_per_heap.gen_data[i].free_list_space_before = generation_free_list_space (gen);
16899 gc_data_per_heap.gen_data[i].free_obj_space_before = generation_free_obj_space (gen);
16902 sufficient_gen0_space_p = FALSE;
16904 #ifdef MULTIPLE_HEAPS
16905 gen0_allocated_after_gc_p = false;
16906 #endif //MULTIPLE_HEAPS
16908 #if defined (_DEBUG) && defined (VERIFY_HEAP)
16909 verify_pinned_queue_p = FALSE;
16910 #endif // _DEBUG && VERIFY_HEAP
16913 void gc_heap::pm_full_gc_init_or_clear()
16915 // This means the next GC will be a full blocking GC and we need to init.
16916 if (settings.condemned_generation == (max_generation - 1))
16918 if (pm_trigger_full_gc)
16920 #ifdef MULTIPLE_HEAPS
16922 #endif //MULTIPLE_HEAPS
16923 dprintf (GTC_LOG, ("init for PM triggered full GC"));
16924 uint32_t saved_entry_memory_load = settings.entry_memory_load;
16925 settings.init_mechanisms();
16926 settings.reason = reason_pm_full_gc;
16927 settings.condemned_generation = max_generation;
16928 settings.entry_memory_load = saved_entry_memory_load;
16929 // Can't assert this since we only check at the end of gen2 GCs,
16930 // during gen1 the memory load could have already dropped.
16931 // Although arguably we should just turn off PM then...
16932 //assert (settings.entry_memory_load >= high_memory_load_th);
16933 assert (settings.entry_memory_load > 0);
16934 settings.gc_index += 1;
16938 // This means we are in the progress of a full blocking GC triggered by
16940 else if (settings.reason == reason_pm_full_gc)
16942 assert (settings.condemned_generation == max_generation);
16943 assert (pm_trigger_full_gc);
16944 pm_trigger_full_gc = false;
16946 dprintf (GTC_LOG, ("PM triggered full GC done"));
16950 void gc_heap::garbage_collect_pm_full_gc()
16952 assert (settings.condemned_generation == max_generation);
16953 assert (settings.reason == reason_pm_full_gc);
16954 assert (!settings.concurrent);
16958 void gc_heap::garbage_collect (int n)
16960 //reset the number of alloc contexts
16961 alloc_contexts_used = 0;
16963 fix_allocation_contexts (TRUE);
16964 #ifdef MULTIPLE_HEAPS
16966 gc_t_join.start_ts(this);
16967 #endif //JOIN_STATS
16968 clear_gen0_bricks();
16969 #endif //MULTIPLE_HEAPS
16971 if ((settings.pause_mode == pause_no_gc) && current_no_gc_region_info.minimal_gc_p)
16973 #ifdef MULTIPLE_HEAPS
16974 gc_t_join.join(this, gc_join_minimal_gc);
16975 if (gc_t_join.joined())
16977 #endif //MULTIPLE_HEAPS
16979 #ifdef MULTIPLE_HEAPS
16980 // this is serialized because we need to get a segment
16981 for (int i = 0; i < n_heaps; i++)
16983 if (!(g_heaps[i]->expand_soh_with_minimal_gc()))
16984 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16987 if (!expand_soh_with_minimal_gc())
16988 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16989 #endif //MULTIPLE_HEAPS
16991 update_collection_counts_for_no_gc();
16993 #ifdef MULTIPLE_HEAPS
16994 gc_t_join.restart();
16996 #endif //MULTIPLE_HEAPS
17003 settings.reason = gc_trigger_reason;
17004 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
17005 num_pinned_objects = 0;
17006 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
17009 if (settings.reason == reason_gcstress)
17011 settings.reason = reason_induced;
17012 settings.stress_induced = TRUE;
17014 #endif // STRESS_HEAP
17016 #ifdef MULTIPLE_HEAPS
17017 //align all heaps on the max generation to condemn
17018 dprintf (3, ("Joining for max generation to condemn"));
17019 condemned_generation_num = generation_to_condemn (n,
17020 &blocking_collection,
17021 &elevation_requested,
17023 gc_t_join.join(this, gc_join_generation_determined);
17024 if (gc_t_join.joined())
17025 #endif //MULTIPLE_HEAPS
17027 #if !defined(SEG_MAPPING_TABLE) && !defined(FEATURE_BASICFREEZE)
17028 //delete old slots from the segment table
17029 seg_table->delete_old_slots();
17030 #endif //!SEG_MAPPING_TABLE && !FEATURE_BASICFREEZE
17032 #ifdef MULTIPLE_HEAPS
17033 for (int i = 0; i < n_heaps; i++)
17035 gc_heap* hp = g_heaps[i];
17036 // check for card table growth
17037 if (g_gc_card_table != hp->card_table)
17038 hp->copy_brick_card_table();
17040 hp->rearrange_large_heap_segments();
17041 #ifdef BACKGROUND_GC
17042 hp->background_delay_delete_loh_segments();
17043 if (!recursive_gc_sync::background_running_p())
17044 hp->rearrange_small_heap_segments();
17045 #endif //BACKGROUND_GC
17047 #else //MULTIPLE_HEAPS
17048 if (g_gc_card_table != card_table)
17049 copy_brick_card_table();
17051 rearrange_large_heap_segments();
17052 #ifdef BACKGROUND_GC
17053 background_delay_delete_loh_segments();
17054 if (!recursive_gc_sync::background_running_p())
17055 rearrange_small_heap_segments();
17056 #endif //BACKGROUND_GC
17057 #endif //MULTIPLE_HEAPS
17059 BOOL should_evaluate_elevation = TRUE;
17060 BOOL should_do_blocking_collection = FALSE;
17062 #ifdef MULTIPLE_HEAPS
17063 int gen_max = condemned_generation_num;
17064 for (int i = 0; i < n_heaps; i++)
17066 if (gen_max < g_heaps[i]->condemned_generation_num)
17067 gen_max = g_heaps[i]->condemned_generation_num;
17068 if (should_evaluate_elevation && !(g_heaps[i]->elevation_requested))
17069 should_evaluate_elevation = FALSE;
17070 if ((!should_do_blocking_collection) && (g_heaps[i]->blocking_collection))
17071 should_do_blocking_collection = TRUE;
17074 settings.condemned_generation = gen_max;
17075 #else //MULTIPLE_HEAPS
17076 settings.condemned_generation = generation_to_condemn (n,
17077 &blocking_collection,
17078 &elevation_requested,
17080 should_evaluate_elevation = elevation_requested;
17081 should_do_blocking_collection = blocking_collection;
17082 #endif //MULTIPLE_HEAPS
17084 settings.condemned_generation = joined_generation_to_condemn (
17085 should_evaluate_elevation,
17087 settings.condemned_generation,
17088 &should_do_blocking_collection
17092 STRESS_LOG1(LF_GCROOTS|LF_GC|LF_GCALLOC, LL_INFO10,
17093 "condemned generation num: %d\n", settings.condemned_generation);
17095 record_gcs_during_no_gc();
17097 if (settings.condemned_generation > 1)
17098 settings.promotion = TRUE;
17100 #ifdef HEAP_ANALYZE
17101 // At this point we've decided what generation is condemned
17102 // See if we've been requested to analyze survivors after the mark phase
17103 if (GCToEEInterface::AnalyzeSurvivorsRequested(settings.condemned_generation))
17105 heap_analyze_enabled = TRUE;
17107 #endif // HEAP_ANALYZE
17109 GCToEEInterface::DiagGCStart(settings.condemned_generation, settings.reason == reason_induced);
17111 #ifdef BACKGROUND_GC
17112 if ((settings.condemned_generation == max_generation) &&
17113 (recursive_gc_sync::background_running_p()))
17115 //TODO BACKGROUND_GC If we just wait for the end of gc, it won't work
17116 // because we have to collect 0 and 1 properly
17117 // in particular, the allocation contexts are gone.
17118 // For now, it is simpler to collect max_generation-1
17119 settings.condemned_generation = max_generation - 1;
17120 dprintf (GTC_LOG, ("bgc - 1 instead of 2"));
17123 if ((settings.condemned_generation == max_generation) &&
17124 (should_do_blocking_collection == FALSE) &&
17125 gc_can_use_concurrent &&
17126 !temp_disable_concurrent_p &&
17127 ((settings.pause_mode == pause_interactive) || (settings.pause_mode == pause_sustained_low_latency)))
17129 keep_bgc_threads_p = TRUE;
17130 c_write (settings.concurrent, TRUE);
17132 #endif //BACKGROUND_GC
17134 settings.gc_index = (uint32_t)dd_collection_count (dynamic_data_of (0)) + 1;
17136 // Call the EE for start of GC work
17137 // just one thread for MP GC
17138 GCToEEInterface::GcStartWork (settings.condemned_generation,
17141 // TODO: we could fire an ETW event to say this GC as a concurrent GC but later on due to not being able to
17142 // create threads or whatever, this could be a non concurrent GC. Maybe for concurrent GC we should fire
17143 // it in do_background_gc and if it failed to be a CGC we fire it in gc1... in other words, this should be
17147 #ifdef MULTIPLE_HEAPS
17148 gc_start_event.Reset();
17149 //start all threads on the roots.
17150 dprintf(3, ("Starting all gc threads for gc"));
17151 gc_t_join.restart();
17152 #endif //MULTIPLE_HEAPS
17155 descr_generations (TRUE);
17158 if ((GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC) &&
17159 !(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_POST_GC_ONLY))
17161 verify_heap (TRUE);
17163 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_BARRIERCHECK)
17164 checkGCWriteBarrier();
17166 #endif // VERIFY_HEAP
17168 #ifdef BACKGROUND_GC
17169 if (settings.concurrent)
17171 // We need to save the settings because we'll need to restore it after each FGC.
17172 assert (settings.condemned_generation == max_generation);
17173 settings.compaction = FALSE;
17174 saved_bgc_settings = settings;
17176 #ifdef MULTIPLE_HEAPS
17177 if (heap_number == 0)
17179 for (int i = 0; i < n_heaps; i++)
17181 prepare_bgc_thread (g_heaps[i]);
17183 dprintf (2, ("setting bgc_threads_sync_event"));
17184 bgc_threads_sync_event.Set();
17188 bgc_threads_sync_event.Wait(INFINITE, FALSE);
17189 dprintf (2, ("bgc_threads_sync_event is signalled"));
17192 prepare_bgc_thread(0);
17193 #endif //MULTIPLE_HEAPS
17195 #ifdef MULTIPLE_HEAPS
17196 gc_t_join.join(this, gc_join_start_bgc);
17197 if (gc_t_join.joined())
17198 #endif //MULTIPLE_HEAPS
17200 do_concurrent_p = TRUE;
17201 do_ephemeral_gc_p = FALSE;
17202 #ifdef MULTIPLE_HEAPS
17203 dprintf(2, ("Joined to perform a background GC"));
17205 for (int i = 0; i < n_heaps; i++)
17207 gc_heap* hp = g_heaps[i];
17208 if (!(hp->bgc_thread) || !hp->commit_mark_array_bgc_init (hp->mark_array))
17210 do_concurrent_p = FALSE;
17215 hp->background_saved_lowest_address = hp->lowest_address;
17216 hp->background_saved_highest_address = hp->highest_address;
17220 do_concurrent_p = (!!bgc_thread && commit_mark_array_bgc_init (mark_array));
17221 if (do_concurrent_p)
17223 background_saved_lowest_address = lowest_address;
17224 background_saved_highest_address = highest_address;
17226 #endif //MULTIPLE_HEAPS
17228 if (do_concurrent_p)
17230 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
17231 SoftwareWriteWatch::EnableForGCHeap();
17232 #endif //FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
17234 #ifdef MULTIPLE_HEAPS
17235 for (int i = 0; i < n_heaps; i++)
17236 g_heaps[i]->current_bgc_state = bgc_initialized;
17238 current_bgc_state = bgc_initialized;
17239 #endif //MULTIPLE_HEAPS
17241 int gen = check_for_ephemeral_alloc();
17242 // always do a gen1 GC before we start BGC.
17243 // This is temporary for testing purpose.
17244 //int gen = max_generation - 1;
17245 dont_restart_ee_p = TRUE;
17248 // If we decide to not do a GC before the BGC we need to
17249 // restore the gen0 alloc context.
17250 #ifdef MULTIPLE_HEAPS
17251 for (int i = 0; i < n_heaps; i++)
17253 generation_allocation_pointer (g_heaps[i]->generation_of (0)) = 0;
17254 generation_allocation_limit (g_heaps[i]->generation_of (0)) = 0;
17257 generation_allocation_pointer (youngest_generation) = 0;
17258 generation_allocation_limit (youngest_generation) = 0;
17259 #endif //MULTIPLE_HEAPS
17263 do_ephemeral_gc_p = TRUE;
17265 settings.init_mechanisms();
17266 settings.condemned_generation = gen;
17267 settings.gc_index = (size_t)dd_collection_count (dynamic_data_of (0)) + 2;
17270 // TODO BACKGROUND_GC need to add the profiling stuff here.
17271 dprintf (GTC_LOG, ("doing gen%d before doing a bgc", gen));
17274 //clear the cards so they don't bleed in gen 1 during collection
17275 // shouldn't this always be done at the beginning of any GC?
17276 //clear_card_for_addresses (
17277 // generation_allocation_start (generation_of (0)),
17278 // heap_segment_allocated (ephemeral_heap_segment));
17280 if (!do_ephemeral_gc_p)
17282 do_background_gc();
17287 settings.compaction = TRUE;
17288 c_write (settings.concurrent, FALSE);
17291 #ifdef MULTIPLE_HEAPS
17292 gc_t_join.restart();
17293 #endif //MULTIPLE_HEAPS
17296 if (do_concurrent_p)
17298 // At this point we are sure we'll be starting a BGC, so save its per heap data here.
17299 // global data is only calculated at the end of the GC so we don't need to worry about
17300 // FGCs overwriting it.
17301 memset (&bgc_data_per_heap, 0, sizeof (bgc_data_per_heap));
17302 memcpy (&bgc_data_per_heap, &gc_data_per_heap, sizeof(gc_data_per_heap));
17304 if (do_ephemeral_gc_p)
17306 dprintf (2, ("GC threads running, doing gen%d GC", settings.condemned_generation));
17308 gen_to_condemn_reasons.init();
17309 gen_to_condemn_reasons.set_condition (gen_before_bgc);
17310 gc_data_per_heap.gen_to_condemn_reasons.init (&gen_to_condemn_reasons);
17312 #ifdef MULTIPLE_HEAPS
17313 gc_t_join.join(this, gc_join_bgc_after_ephemeral);
17314 if (gc_t_join.joined())
17315 #endif //MULTIPLE_HEAPS
17317 #ifdef MULTIPLE_HEAPS
17319 #endif //MULTIPLE_HEAPS
17320 settings = saved_bgc_settings;
17321 assert (settings.concurrent);
17323 do_background_gc();
17325 #ifdef MULTIPLE_HEAPS
17326 gc_t_join.restart();
17327 #endif //MULTIPLE_HEAPS
17333 dprintf (2, ("couldn't create BGC threads, reverting to doing a blocking GC"));
17338 #endif //BACKGROUND_GC
17342 #ifndef MULTIPLE_HEAPS
17343 allocation_running_time = (size_t)GCToOSInterface::GetLowPrecisionTimeStamp();
17344 allocation_running_amount = dd_new_allocation (dynamic_data_of (0));
17345 fgn_last_alloc = dd_new_allocation (dynamic_data_of (0));
17346 #endif //MULTIPLE_HEAPS
17349 if (settings.pause_mode == pause_no_gc)
17350 allocate_for_no_gc_after_gc();
17354 #define mark_stack_empty_p() (mark_stack_base == mark_stack_tos)
17357 size_t& gc_heap::promoted_bytes(int thread)
17359 #ifdef MULTIPLE_HEAPS
17360 return g_promoted [thread*16];
17361 #else //MULTIPLE_HEAPS
17362 UNREFERENCED_PARAMETER(thread);
17364 #endif //MULTIPLE_HEAPS
17367 #ifdef INTERIOR_POINTERS
17368 heap_segment* gc_heap::find_segment (uint8_t* interior, BOOL small_segment_only_p)
17370 #ifdef SEG_MAPPING_TABLE
17371 heap_segment* seg = seg_mapping_table_segment_of (interior);
17374 if (small_segment_only_p && heap_segment_loh_p (seg))
17378 #else //SEG_MAPPING_TABLE
17379 #ifdef MULTIPLE_HEAPS
17380 for (int i = 0; i < gc_heap::n_heaps; i++)
17382 gc_heap* h = gc_heap::g_heaps [i];
17383 hs = h->find_segment_per_heap (o, small_segment_only_p);
17391 gc_heap* h = pGenGCHeap;
17392 hs = h->find_segment_per_heap (o, small_segment_only_p);
17394 #endif //MULTIPLE_HEAPS
17395 #endif //SEG_MAPPING_TABLE
17398 heap_segment* gc_heap::find_segment_per_heap (uint8_t* interior, BOOL small_segment_only_p)
17400 #ifdef SEG_MAPPING_TABLE
17401 return find_segment (interior, small_segment_only_p);
17402 #else //SEG_MAPPING_TABLE
17403 if (in_range_for_segment (interior, ephemeral_heap_segment))
17405 return ephemeral_heap_segment;
17409 heap_segment* found_seg = 0;
17412 heap_segment* seg = generation_start_segment (generation_of (max_generation));
17415 if (in_range_for_segment (interior, seg))
17418 goto end_find_segment;
17421 } while ((seg = heap_segment_next (seg)) != 0);
17423 if (!small_segment_only_p)
17425 #ifdef BACKGROUND_GC
17427 ptrdiff_t delta = 0;
17428 heap_segment* seg = segment_of (interior, delta);
17429 if (seg && in_range_for_segment (interior, seg))
17433 goto end_find_segment;
17435 #else //BACKGROUND_GC
17436 heap_segment* seg = generation_start_segment (generation_of (max_generation+1));
17439 if (in_range_for_segment(interior, seg))
17442 goto end_find_segment;
17445 } while ((seg = heap_segment_next (seg)) != 0);
17446 #endif //BACKGROUND_GC
17452 #endif //SEG_MAPPING_TABLE
17454 #endif //INTERIOR_POINTERS
17456 #if !defined(_DEBUG) && !defined(__GNUC__)
17457 inline // This causes link errors if global optimization is off
17458 #endif //!_DEBUG && !__GNUC__
17459 gc_heap* gc_heap::heap_of (uint8_t* o)
17461 #ifdef MULTIPLE_HEAPS
17463 return g_heaps [0];
17464 #ifdef SEG_MAPPING_TABLE
17465 gc_heap* hp = seg_mapping_table_heap_of (o);
17466 return (hp ? hp : g_heaps[0]);
17467 #else //SEG_MAPPING_TABLE
17468 ptrdiff_t delta = 0;
17469 heap_segment* seg = segment_of (o, delta);
17470 return (seg ? heap_segment_heap (seg) : g_heaps [0]);
17471 #endif //SEG_MAPPING_TABLE
17472 #else //MULTIPLE_HEAPS
17473 UNREFERENCED_PARAMETER(o);
17475 #endif //MULTIPLE_HEAPS
17479 gc_heap* gc_heap::heap_of_gc (uint8_t* o)
17481 #ifdef MULTIPLE_HEAPS
17483 return g_heaps [0];
17484 #ifdef SEG_MAPPING_TABLE
17485 gc_heap* hp = seg_mapping_table_heap_of_gc (o);
17486 return (hp ? hp : g_heaps[0]);
17487 #else //SEG_MAPPING_TABLE
17488 ptrdiff_t delta = 0;
17489 heap_segment* seg = segment_of (o, delta);
17490 return (seg ? heap_segment_heap (seg) : g_heaps [0]);
17491 #endif //SEG_MAPPING_TABLE
17492 #else //MULTIPLE_HEAPS
17493 UNREFERENCED_PARAMETER(o);
17495 #endif //MULTIPLE_HEAPS
17498 #ifdef INTERIOR_POINTERS
17499 // will find all heap objects (large and small)
17500 uint8_t* gc_heap::find_object (uint8_t* interior, uint8_t* low)
17502 if (!gen0_bricks_cleared)
17504 #ifdef MULTIPLE_HEAPS
17505 assert (!"Should have already been done in server GC");
17506 #endif //MULTIPLE_HEAPS
17507 gen0_bricks_cleared = TRUE;
17508 //initialize brick table for gen 0
17509 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
17510 b < brick_of (align_on_brick
17511 (heap_segment_allocated (ephemeral_heap_segment)));
17517 #ifdef FFIND_OBJECT
17518 //indicate that in the future this needs to be done during allocation
17519 #ifdef MULTIPLE_HEAPS
17520 gen0_must_clear_bricks = FFIND_DECAY*gc_heap::n_heaps;
17522 gen0_must_clear_bricks = FFIND_DECAY;
17523 #endif //MULTIPLE_HEAPS
17524 #endif //FFIND_OBJECT
17526 int brick_entry = get_brick_entry(brick_of (interior));
17527 if (brick_entry == 0)
17529 // this is a pointer to a large object
17530 heap_segment* seg = find_segment_per_heap (interior, FALSE);
17532 #ifdef FEATURE_CONSERVATIVE_GC
17533 && (GCConfig::GetConservativeGC() || interior <= heap_segment_allocated(seg))
17537 // If interior falls within the first free object at the beginning of a generation,
17538 // we don't have brick entry for it, and we may incorrectly treat it as on large object heap.
17539 int align_const = get_alignment_constant (heap_segment_read_only_p (seg)
17540 #ifdef FEATURE_CONSERVATIVE_GC
17541 || (GCConfig::GetConservativeGC() && !heap_segment_loh_p (seg))
17544 //int align_const = get_alignment_constant (heap_segment_read_only_p (seg));
17545 assert (interior < heap_segment_allocated (seg));
17547 uint8_t* o = heap_segment_mem (seg);
17548 while (o < heap_segment_allocated (seg))
17550 uint8_t* next_o = o + Align (size (o), align_const);
17551 assert (next_o > o);
17552 if ((o <= interior) && (interior < next_o))
17563 else if (interior >= low)
17565 heap_segment* seg = find_segment_per_heap (interior, TRUE);
17568 #ifdef FEATURE_CONSERVATIVE_GC
17569 if (interior >= heap_segment_allocated (seg))
17572 assert (interior < heap_segment_allocated (seg));
17574 uint8_t* o = find_first_object (interior, heap_segment_mem (seg));
17585 gc_heap::find_object_for_relocation (uint8_t* interior, uint8_t* low, uint8_t* high)
17587 uint8_t* old_address = interior;
17588 if (!((old_address >= low) && (old_address < high)))
17591 size_t brick = brick_of (old_address);
17592 int brick_entry = brick_table [ brick ];
17593 if (brick_entry != 0)
17597 while (brick_entry < 0)
17599 brick = (brick + brick_entry);
17600 brick_entry = brick_table [ brick ];
17602 uint8_t* old_loc = old_address;
17603 uint8_t* node = tree_search ((brick_address (brick) + brick_entry-1),
17605 if (node <= old_loc)
17610 brick_entry = brick_table [ brick ];
17616 //find the object by going along the plug
17618 while (o <= interior)
17620 uint8_t* next_o = o + Align (size (o));
17621 assert (next_o > o);
17622 if (next_o > interior)
17628 assert ((o <= interior) && ((o + Align (size (o))) > interior));
17633 // this is a pointer to a large object
17634 heap_segment* seg = find_segment_per_heap (interior, FALSE);
17637 assert (interior < heap_segment_allocated (seg));
17639 uint8_t* o = heap_segment_mem (seg);
17640 while (o < heap_segment_allocated (seg))
17642 uint8_t* next_o = o + Align (size (o));
17643 assert (next_o > o);
17644 if ((o < interior) && (interior < next_o))
17656 #else //INTERIOR_POINTERS
17658 uint8_t* gc_heap::find_object (uint8_t* o, uint8_t* low)
17662 #endif //INTERIOR_POINTERS
17664 #ifdef MULTIPLE_HEAPS
17667 #ifdef GC_CONFIG_DRIVEN
17668 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;} else {mark_list_index++;}}
17669 #else //GC_CONFIG_DRIVEN
17670 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;}}
17671 #endif //GC_CONFIG_DRIVEN
17673 #define m_boundary(o) {}
17676 #define m_boundary_fullgc(o) {}
17678 #else //MULTIPLE_HEAPS
17681 #ifdef GC_CONFIG_DRIVEN
17682 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;} else {mark_list_index++;} if (slow > o) slow = o; if (shigh < o) shigh = o;}
17684 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;}if (slow > o) slow = o; if (shigh < o) shigh = o;}
17685 #endif //GC_CONFIG_DRIVEN
17687 #define m_boundary(o) {if (slow > o) slow = o; if (shigh < o) shigh = o;}
17690 #define m_boundary_fullgc(o) {if (slow > o) slow = o; if (shigh < o) shigh = o;}
17692 #endif //MULTIPLE_HEAPS
17694 #define method_table(o) ((CObjectHeader*)(o))->GetMethodTable()
17697 BOOL gc_heap::gc_mark1 (uint8_t* o)
17699 BOOL marked = !marked (o);
17701 dprintf (3, ("*%Ix*, newly marked: %d", (size_t)o, marked));
17706 BOOL gc_heap::gc_mark (uint8_t* o, uint8_t* low, uint8_t* high)
17708 BOOL marked = FALSE;
17709 if ((o >= low) && (o < high))
17710 marked = gc_mark1 (o);
17711 #ifdef MULTIPLE_HEAPS
17715 gc_heap* hp = heap_of_gc (o);
17717 if ((o >= hp->gc_low) && (o < hp->gc_high))
17718 marked = gc_mark1 (o);
17721 snoop_stat.objects_checked_count++;
17725 snoop_stat.objects_marked_count++;
17729 snoop_stat.zero_ref_count++;
17732 #endif //SNOOP_STATS
17733 #endif //MULTIPLE_HEAPS
17737 #ifdef BACKGROUND_GC
17740 BOOL gc_heap::background_marked (uint8_t* o)
17742 return mark_array_marked (o);
17745 BOOL gc_heap::background_mark1 (uint8_t* o)
17747 BOOL to_mark = !mark_array_marked (o);
17749 dprintf (3, ("b*%Ix*b(%d)", (size_t)o, (to_mark ? 1 : 0)));
17752 mark_array_set_marked (o);
17753 dprintf (4, ("n*%Ix*n", (size_t)o));
17760 // TODO: we could consider filtering out NULL's here instead of going to
17761 // look for it on other heaps
17763 BOOL gc_heap::background_mark (uint8_t* o, uint8_t* low, uint8_t* high)
17765 BOOL marked = FALSE;
17766 if ((o >= low) && (o < high))
17767 marked = background_mark1 (o);
17768 #ifdef MULTIPLE_HEAPS
17772 gc_heap* hp = heap_of (o);
17774 if ((o >= hp->background_saved_lowest_address) && (o < hp->background_saved_highest_address))
17775 marked = background_mark1 (o);
17777 #endif //MULTIPLE_HEAPS
17781 #endif //BACKGROUND_GC
17784 uint8_t* gc_heap::next_end (heap_segment* seg, uint8_t* f)
17786 if (seg == ephemeral_heap_segment)
17789 return heap_segment_allocated (seg);
17792 #define new_start() {if (ppstop <= start) {break;} else {parm = start}}
17793 #define ignore_start 0
17794 #define use_start 1
17796 #define go_through_object(mt,o,size,parm,start,start_useful,limit,exp) \
17798 CGCDesc* map = CGCDesc::GetCGCDescFromMT((MethodTable*)(mt)); \
17799 CGCDescSeries* cur = map->GetHighestSeries(); \
17800 ptrdiff_t cnt = (ptrdiff_t) map->GetNumSeries(); \
17804 CGCDescSeries* last = map->GetLowestSeries(); \
17805 uint8_t** parm = 0; \
17808 assert (parm <= (uint8_t**)((o) + cur->GetSeriesOffset())); \
17809 parm = (uint8_t**)((o) + cur->GetSeriesOffset()); \
17810 uint8_t** ppstop = \
17811 (uint8_t**)((uint8_t*)parm + cur->GetSeriesSize() + (size));\
17812 if (!start_useful || (uint8_t*)ppstop > (start)) \
17814 if (start_useful && (uint8_t*)parm < (start)) parm = (uint8_t**)(start);\
17815 while (parm < ppstop) \
17823 } while (cur >= last); \
17827 /* Handle the repeating case - array of valuetypes */ \
17828 uint8_t** parm = (uint8_t**)((o) + cur->startoffset); \
17829 if (start_useful && start > (uint8_t*)parm) \
17831 ptrdiff_t cs = mt->RawGetComponentSize(); \
17832 parm = (uint8_t**)((uint8_t*)parm + (((start) - (uint8_t*)parm)/cs)*cs); \
17834 while ((uint8_t*)parm < ((o)+(size)-plug_skew)) \
17836 for (ptrdiff_t __i = 0; __i > cnt; __i--) \
17838 HALF_SIZE_T skip = cur->val_serie[__i].skip; \
17839 HALF_SIZE_T nptrs = cur->val_serie[__i].nptrs; \
17840 uint8_t** ppstop = parm + nptrs; \
17841 if (!start_useful || (uint8_t*)ppstop > (start)) \
17843 if (start_useful && (uint8_t*)parm < (start)) parm = (uint8_t**)(start); \
17848 } while (parm < ppstop); \
17850 parm = (uint8_t**)((uint8_t*)ppstop + skip); \
17856 #define go_through_object_nostart(mt,o,size,parm,exp) {go_through_object(mt,o,size,parm,o,ignore_start,(o + size),exp); }
17858 // 1 thing to note about this macro:
17859 // 1) you can use *parm safely but in general you don't want to use parm
17860 // because for the collectible types it's not an address on the managed heap.
17861 #ifndef COLLECTIBLE_CLASS
17862 #define go_through_object_cl(mt,o,size,parm,exp) \
17864 if (header(o)->ContainsPointers()) \
17866 go_through_object_nostart(mt,o,size,parm,exp); \
17869 #else //COLLECTIBLE_CLASS
17870 #define go_through_object_cl(mt,o,size,parm,exp) \
17872 if (header(o)->Collectible()) \
17874 uint8_t* class_obj = get_class_object (o); \
17875 uint8_t** parm = &class_obj; \
17876 do {exp} while (false); \
17878 if (header(o)->ContainsPointers()) \
17880 go_through_object_nostart(mt,o,size,parm,exp); \
17883 #endif //COLLECTIBLE_CLASS
17885 // This starts a plug. But mark_stack_tos isn't increased until set_pinned_info is called.
17886 void gc_heap::enque_pinned_plug (uint8_t* plug,
17887 BOOL save_pre_plug_info_p,
17888 uint8_t* last_object_in_last_plug)
17890 if (mark_stack_array_length <= mark_stack_tos)
17892 if (!grow_mark_stack (mark_stack_array, mark_stack_array_length, MARK_STACK_INITIAL_LENGTH))
17894 // we don't want to continue here due to security
17895 // risks. This happens very rarely and fixing it in the
17896 // way so that we can continue is a bit involved and will
17897 // not be done in Dev10.
17898 GCToEEInterface::HandleFatalError(CORINFO_EXCEPTION_GC);
17902 dprintf (3, ("enqueuing P #%Id(%Ix): %Ix. oldest: %Id, LO: %Ix, pre: %d",
17903 mark_stack_tos, &mark_stack_array[mark_stack_tos], plug, mark_stack_bos, last_object_in_last_plug, (save_pre_plug_info_p ? 1 : 0)));
17904 mark& m = mark_stack_array[mark_stack_tos];
17906 // Must be set now because if we have a short object we'll need the value of saved_pre_p.
17907 m.saved_pre_p = save_pre_plug_info_p;
17909 if (save_pre_plug_info_p)
17912 BOOL is_padded = is_plug_padded (last_object_in_last_plug);
17914 clear_plug_padded (last_object_in_last_plug);
17915 #endif //SHORT_PLUGS
17916 memcpy (&(m.saved_pre_plug), &(((plug_and_gap*)plug)[-1]), sizeof (gap_reloc_pair));
17919 set_plug_padded (last_object_in_last_plug);
17920 #endif //SHORT_PLUGS
17922 memcpy (&(m.saved_pre_plug_reloc), &(((plug_and_gap*)plug)[-1]), sizeof (gap_reloc_pair));
17924 // If the last object in the last plug is too short, it requires special handling.
17925 size_t last_obj_size = plug - last_object_in_last_plug;
17926 if (last_obj_size < min_pre_pin_obj_size)
17928 record_interesting_data_point (idp_pre_short);
17931 record_interesting_data_point (idp_pre_short_padded);
17932 #endif //SHORT_PLUGS
17933 dprintf (3, ("encountered a short object %Ix right before pinned plug %Ix!",
17934 last_object_in_last_plug, plug));
17935 // Need to set the short bit regardless of having refs or not because we need to
17936 // indicate that this object is not walkable.
17939 #ifdef COLLECTIBLE_CLASS
17940 if (is_collectible (last_object_in_last_plug))
17942 m.set_pre_short_collectible();
17944 #endif //COLLECTIBLE_CLASS
17946 if (contain_pointers (last_object_in_last_plug))
17948 dprintf (3, ("short object: %Ix(%Ix)", last_object_in_last_plug, last_obj_size));
17950 go_through_object_nostart (method_table(last_object_in_last_plug), last_object_in_last_plug, last_obj_size, pval,
17952 size_t gap_offset = (((size_t)pval - (size_t)(plug - sizeof (gap_reloc_pair) - plug_skew))) / sizeof (uint8_t*);
17953 dprintf (3, ("member: %Ix->%Ix, %Id ptrs from beginning of gap", (uint8_t*)pval, *pval, gap_offset));
17954 m.set_pre_short_bit (gap_offset);
17961 m.saved_post_p = FALSE;
17964 void gc_heap::save_post_plug_info (uint8_t* last_pinned_plug, uint8_t* last_object_in_last_plug, uint8_t* post_plug)
17966 UNREFERENCED_PARAMETER(last_pinned_plug);
17968 mark& m = mark_stack_array[mark_stack_tos - 1];
17969 assert (last_pinned_plug == m.first);
17970 m.saved_post_plug_info_start = (uint8_t*)&(((plug_and_gap*)post_plug)[-1]);
17973 BOOL is_padded = is_plug_padded (last_object_in_last_plug);
17975 clear_plug_padded (last_object_in_last_plug);
17976 #endif //SHORT_PLUGS
17977 memcpy (&(m.saved_post_plug), m.saved_post_plug_info_start, sizeof (gap_reloc_pair));
17980 set_plug_padded (last_object_in_last_plug);
17981 #endif //SHORT_PLUGS
17983 memcpy (&(m.saved_post_plug_reloc), m.saved_post_plug_info_start, sizeof (gap_reloc_pair));
17985 // This is important - we need to clear all bits here except the last one.
17986 m.saved_post_p = TRUE;
17989 m.saved_post_plug_debug.gap = 1;
17992 dprintf (3, ("PP %Ix has NP %Ix right after", last_pinned_plug, post_plug));
17994 size_t last_obj_size = post_plug - last_object_in_last_plug;
17995 if (last_obj_size < min_pre_pin_obj_size)
17997 dprintf (3, ("PP %Ix last obj %Ix is too short", last_pinned_plug, last_object_in_last_plug));
17998 record_interesting_data_point (idp_post_short);
18001 record_interesting_data_point (idp_post_short_padded);
18002 #endif //SHORT_PLUGS
18003 m.set_post_short();
18004 #if defined (_DEBUG) && defined (VERIFY_HEAP)
18005 verify_pinned_queue_p = TRUE;
18006 #endif // _DEBUG && VERIFY_HEAP
18008 #ifdef COLLECTIBLE_CLASS
18009 if (is_collectible (last_object_in_last_plug))
18011 m.set_post_short_collectible();
18013 #endif //COLLECTIBLE_CLASS
18015 if (contain_pointers (last_object_in_last_plug))
18017 dprintf (3, ("short object: %Ix(%Ix)", last_object_in_last_plug, last_obj_size));
18019 // TODO: since we won't be able to walk this object in relocation, we still need to
18020 // take care of collectible assemblies here.
18021 go_through_object_nostart (method_table(last_object_in_last_plug), last_object_in_last_plug, last_obj_size, pval,
18023 size_t gap_offset = (((size_t)pval - (size_t)(post_plug - sizeof (gap_reloc_pair) - plug_skew))) / sizeof (uint8_t*);
18024 dprintf (3, ("member: %Ix->%Ix, %Id ptrs from beginning of gap", (uint8_t*)pval, *pval, gap_offset));
18025 m.set_post_short_bit (gap_offset);
18034 __declspec(naked) void __fastcall Prefetch(void* addr)
18042 inline void Prefetch (void* addr)
18044 UNREFERENCED_PARAMETER(addr);
18049 VOLATILE(uint8_t*)& gc_heap::ref_mark_stack (gc_heap* hp, int index)
18051 return ((VOLATILE(uint8_t*)*)(hp->mark_stack_array))[index];
18054 #endif //MH_SC_MARK
18058 #define partial_object 3
18060 uint8_t* ref_from_slot (uint8_t* r)
18062 return (uint8_t*)((size_t)r & ~(stolen | partial));
18065 BOOL stolen_p (uint8_t* r)
18067 return (((size_t)r&2) && !((size_t)r&1));
18070 BOOL ready_p (uint8_t* r)
18072 return ((size_t)r != 1);
18075 BOOL partial_p (uint8_t* r)
18077 return (((size_t)r&1) && !((size_t)r&2));
18080 BOOL straight_ref_p (uint8_t* r)
18082 return (!stolen_p (r) && !partial_p (r));
18085 BOOL partial_object_p (uint8_t* r)
18087 return (((size_t)r & partial_object) == partial_object);
18090 BOOL ref_p (uint8_t* r)
18092 return (straight_ref_p (r) || partial_object_p (r));
18095 void gc_heap::mark_object_simple1 (uint8_t* oo, uint8_t* start THREAD_NUMBER_DCL)
18097 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_tos = (SERVER_SC_MARK_VOLATILE(uint8_t*)*)mark_stack_array;
18098 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_limit = (SERVER_SC_MARK_VOLATILE(uint8_t*)*)&mark_stack_array[mark_stack_array_length];
18099 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_base = mark_stack_tos;
18100 #ifdef SORT_MARK_STACK
18101 SERVER_SC_MARK_VOLATILE(uint8_t*)* sorted_tos = mark_stack_base;
18102 #endif //SORT_MARK_STACK
18104 // If we are doing a full GC we don't use mark list anyway so use m_boundary_fullgc that doesn't
18105 // update mark list.
18106 BOOL full_p = (settings.condemned_generation == max_generation);
18108 assert ((start >= oo) && (start < oo+size(oo)));
18111 *mark_stack_tos = oo;
18112 #endif //!MH_SC_MARK
18116 #ifdef MULTIPLE_HEAPS
18117 #else //MULTIPLE_HEAPS
18118 const int thread = 0;
18119 #endif //MULTIPLE_HEAPS
18121 if (oo && ((size_t)oo != 4))
18129 else if (!partial_p (oo) && ((s = size (oo)) < (partial_size_th*sizeof (uint8_t*))))
18131 BOOL overflow_p = FALSE;
18133 if (mark_stack_tos + (s) /sizeof (uint8_t*) >= (mark_stack_limit - 1))
18135 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
18136 if (mark_stack_tos + CGCDesc::GetNumPointers(method_table(oo), s, num_components) >= (mark_stack_limit - 1))
18142 if (overflow_p == FALSE)
18144 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18146 go_through_object_cl (method_table(oo), oo, s, ppslot,
18148 uint8_t* o = *ppslot;
18150 if (gc_mark (o, gc_low, gc_high))
18154 m_boundary_fullgc (o);
18160 size_t obj_size = size (o);
18161 promoted_bytes (thread) += obj_size;
18162 if (contain_pointers_or_collectible (o))
18164 *(mark_stack_tos++) = o;
18172 dprintf(3,("mark stack overflow for object %Ix ", (size_t)oo));
18173 min_overflow_address = min (min_overflow_address, oo);
18174 max_overflow_address = max (max_overflow_address, oo);
18179 if (partial_p (oo))
18181 start = ref_from_slot (oo);
18182 oo = ref_from_slot (*(--mark_stack_tos));
18183 dprintf (4, ("oo: %Ix, start: %Ix\n", (size_t)oo, (size_t)start));
18184 assert ((oo < start) && (start < (oo + size (oo))));
18186 #ifdef COLLECTIBLE_CLASS
18189 // If there's a class object, push it now. We are guaranteed to have the slot since
18190 // we just popped one object off.
18191 if (is_collectible (oo))
18193 uint8_t* class_obj = get_class_object (oo);
18194 if (gc_mark (class_obj, gc_low, gc_high))
18198 m_boundary_fullgc (class_obj);
18202 m_boundary (class_obj);
18205 size_t obj_size = size (class_obj);
18206 promoted_bytes (thread) += obj_size;
18207 *(mark_stack_tos++) = class_obj;
18208 // The code below expects that the oo is still stored in the stack slot that was
18209 // just popped and it "pushes" it back just by incrementing the mark_stack_tos.
18210 // But the class_obj has just overwritten that stack slot and so the oo needs to
18211 // be stored to the new slot that's pointed to by the mark_stack_tos.
18212 *mark_stack_tos = oo;
18216 if (!contain_pointers (oo))
18221 #endif //COLLECTIBLE_CLASS
18225 BOOL overflow_p = FALSE;
18227 if (mark_stack_tos + (num_partial_refs + 2) >= mark_stack_limit)
18231 if (overflow_p == FALSE)
18233 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18235 //push the object and its current
18236 SERVER_SC_MARK_VOLATILE(uint8_t*)* place = ++mark_stack_tos;
18240 *(place) = (uint8_t*)partial;
18241 #endif //MH_SC_MARK
18242 int i = num_partial_refs;
18243 uint8_t* ref_to_continue = 0;
18245 go_through_object (method_table(oo), oo, s, ppslot,
18246 start, use_start, (oo + s),
18248 uint8_t* o = *ppslot;
18250 if (gc_mark (o, gc_low, gc_high))
18254 m_boundary_fullgc (o);
18260 size_t obj_size = size (o);
18261 promoted_bytes (thread) += obj_size;
18262 if (contain_pointers_or_collectible (o))
18264 *(mark_stack_tos++) = o;
18267 ref_to_continue = (uint8_t*)((size_t)(ppslot+1) | partial);
18276 //we are finished with this object
18277 assert (ref_to_continue == 0);
18279 assert ((*(place-1)) == (uint8_t*)0);
18282 #endif //MH_SC_MARK
18284 // shouldn't we decrease tos by 2 here??
18287 if (ref_to_continue)
18291 assert ((*(place-1)) == (uint8_t*)0);
18292 *(place-1) = (uint8_t*)((size_t)oo | partial_object);
18293 assert (((*place) == (uint8_t*)1) || ((*place) == (uint8_t*)2));
18294 #endif //MH_SC_MARK
18295 *place = ref_to_continue;
18300 dprintf(3,("mark stack overflow for object %Ix ", (size_t)oo));
18301 min_overflow_address = min (min_overflow_address, oo);
18302 max_overflow_address = max (max_overflow_address, oo);
18305 #ifdef SORT_MARK_STACK
18306 if (mark_stack_tos > sorted_tos + mark_stack_array_length/8)
18308 rqsort1 (sorted_tos, mark_stack_tos-1);
18309 sorted_tos = mark_stack_tos-1;
18311 #endif //SORT_MARK_STACK
18314 if (!(mark_stack_empty_p()))
18316 oo = *(--mark_stack_tos);
18319 #ifdef SORT_MARK_STACK
18320 sorted_tos = min ((size_t)sorted_tos, (size_t)mark_stack_tos);
18321 #endif //SORT_MARK_STACK
18329 BOOL same_numa_node_p (int hn1, int hn2)
18331 return (heap_select::find_numa_node_from_heap_no (hn1) == heap_select::find_numa_node_from_heap_no (hn2));
18334 int find_next_buddy_heap (int this_heap_number, int current_buddy, int n_heaps)
18336 int hn = (current_buddy+1)%n_heaps;
18337 while (hn != current_buddy)
18339 if ((this_heap_number != hn) && (same_numa_node_p (this_heap_number, hn)))
18341 hn = (hn+1)%n_heaps;
18343 return current_buddy;
18347 gc_heap::mark_steal()
18349 mark_stack_busy() = 0;
18350 //clear the mark stack in the snooping range
18351 for (int i = 0; i < max_snoop_level; i++)
18353 ((VOLATILE(uint8_t*)*)(mark_stack_array))[i] = 0;
18356 //pick the next heap as our buddy
18357 int thpn = find_next_buddy_heap (heap_number, heap_number, n_heaps);
18360 dprintf (SNOOP_LOG, ("(GC%d)heap%d: start snooping %d", settings.gc_index, heap_number, (heap_number+1)%n_heaps));
18361 uint32_t begin_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
18362 #endif //SNOOP_STATS
18364 int idle_loop_count = 0;
18365 int first_not_ready_level = 0;
18369 gc_heap* hp = g_heaps [thpn];
18370 int level = first_not_ready_level;
18371 first_not_ready_level = 0;
18373 while (check_next_mark_stack (hp) && (level < (max_snoop_level-1)))
18375 idle_loop_count = 0;
18377 snoop_stat.busy_count++;
18378 dprintf (SNOOP_LOG, ("heap%d: looking at next heap level %d stack contents: %Ix",
18379 heap_number, level, (int)((uint8_t**)(hp->mark_stack_array))[level]));
18380 #endif //SNOOP_STATS
18382 uint8_t* o = ref_mark_stack (hp, level);
18384 uint8_t* start = o;
18387 mark_stack_busy() = 1;
18389 BOOL success = TRUE;
18390 uint8_t* next = (ref_mark_stack (hp, level+1));
18393 if (((size_t)o > 4) && !partial_object_p (o))
18395 //this is a normal object, not a partial mark tuple
18396 //success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level), 0, o)==o);
18397 success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level), (uint8_t*)4, o)==o);
18399 snoop_stat.interlocked_count++;
18401 snoop_stat.normal_count++;
18402 #endif //SNOOP_STATS
18406 //it is a stolen entry, or beginning/ending of a partial mark
18409 snoop_stat.stolen_or_pm_count++;
18410 #endif //SNOOP_STATS
18414 else if (stolen_p (next))
18416 //ignore the stolen guy and go to the next level
18420 snoop_stat.stolen_entry_count++;
18421 #endif //SNOOP_STATS
18425 assert (partial_p (next));
18426 start = ref_from_slot (next);
18427 //re-read the object
18428 o = ref_from_slot (ref_mark_stack (hp, level));
18432 success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level+1), (uint8_t*)stolen, next)==next);
18434 snoop_stat.interlocked_count++;
18437 snoop_stat.partial_mark_parent_count++;
18439 #endif //SNOOP_STATS
18443 // stack is not ready, or o is completely different from the last time we read from this stack level.
18444 // go up 2 levels to steal children or totally unrelated objects.
18446 if (first_not_ready_level == 0)
18448 first_not_ready_level = level;
18452 snoop_stat.pm_not_ready_count++;
18453 #endif //SNOOP_STATS
18460 dprintf (SNOOP_LOG, ("heap%d: marking %Ix from %d [%d] tl:%dms",
18461 heap_number, (size_t)o, (heap_number+1)%n_heaps, level,
18462 (GCToOSInterface::GetLowPrecisionTimeStamp()-begin_tick)));
18463 uint32_t start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
18464 #endif //SNOOP_STATS
18466 mark_object_simple1 (o, start, heap_number);
18469 dprintf (SNOOP_LOG, ("heap%d: done marking %Ix from %d [%d] %dms tl:%dms",
18470 heap_number, (size_t)o, (heap_number+1)%n_heaps, level,
18471 (GCToOSInterface::GetLowPrecisionTimeStamp()-start_tick),(GCToOSInterface::GetLowPrecisionTimeStamp()-begin_tick)));
18472 #endif //SNOOP_STATS
18474 mark_stack_busy() = 0;
18476 //clear the mark stack in snooping range
18477 for (int i = 0; i < max_snoop_level; i++)
18479 if (((uint8_t**)mark_stack_array)[i] != 0)
18481 ((VOLATILE(uint8_t*)*)(mark_stack_array))[i] = 0;
18483 snoop_stat.stack_bottom_clear_count++;
18484 #endif //SNOOP_STATS
18490 mark_stack_busy() = 0;
18494 //slot is either partial or stolen
18498 if ((first_not_ready_level != 0) && hp->mark_stack_busy())
18502 if (!hp->mark_stack_busy())
18504 first_not_ready_level = 0;
18507 if ((idle_loop_count % (6) )==1)
18510 snoop_stat.switch_to_thread_count++;
18511 #endif //SNOOP_STATS
18512 GCToOSInterface::Sleep(1);
18514 int free_count = 1;
18516 snoop_stat.stack_idle_count++;
18517 //dprintf (SNOOP_LOG, ("heap%d: counting idle threads", heap_number));
18518 #endif //SNOOP_STATS
18519 for (int hpn = (heap_number+1)%n_heaps; hpn != heap_number;)
18521 if (!((g_heaps [hpn])->mark_stack_busy()))
18525 dprintf (SNOOP_LOG, ("heap%d: %d idle", heap_number, free_count));
18526 #endif //SNOOP_STATS
18528 else if (same_numa_node_p (hpn, heap_number) || ((idle_loop_count%1000))==999)
18533 hpn = (hpn+1)%n_heaps;
18536 if (free_count == n_heaps)
18545 BOOL gc_heap::check_next_mark_stack (gc_heap* next_heap)
18548 snoop_stat.check_level_count++;
18549 #endif //SNOOP_STATS
18550 return (next_heap->mark_stack_busy()>=1);
18552 #endif //MH_SC_MARK
18555 void gc_heap::print_snoop_stat()
18557 dprintf (1234, ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s",
18558 "heap", "check", "zero", "mark", "stole", "pstack", "nstack", "nonsk"));
18559 dprintf (1234, ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d",
18560 snoop_stat.heap_index,
18561 snoop_stat.objects_checked_count,
18562 snoop_stat.zero_ref_count,
18563 snoop_stat.objects_marked_count,
18564 snoop_stat.stolen_stack_count,
18565 snoop_stat.partial_stack_count,
18566 snoop_stat.normal_stack_count,
18567 snoop_stat.non_stack_count));
18568 dprintf (1234, ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s",
18569 "heap", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "clear"));
18570 dprintf (1234, ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
18571 snoop_stat.heap_index,
18572 snoop_stat.check_level_count,
18573 snoop_stat.busy_count,
18574 snoop_stat.interlocked_count,
18575 snoop_stat.partial_mark_parent_count,
18576 snoop_stat.stolen_or_pm_count,
18577 snoop_stat.stolen_entry_count,
18578 snoop_stat.pm_not_ready_count,
18579 snoop_stat.normal_count,
18580 snoop_stat.stack_bottom_clear_count));
18582 printf ("\n%4s | %8s | %8s | %8s | %8s | %8s\n",
18583 "heap", "check", "zero", "mark", "idle", "switch");
18584 printf ("%4d | %8d | %8d | %8d | %8d | %8d\n",
18585 snoop_stat.heap_index,
18586 snoop_stat.objects_checked_count,
18587 snoop_stat.zero_ref_count,
18588 snoop_stat.objects_marked_count,
18589 snoop_stat.stack_idle_count,
18590 snoop_stat.switch_to_thread_count);
18591 printf ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s\n",
18592 "heap", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "normal", "clear");
18593 printf ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
18594 snoop_stat.heap_index,
18595 snoop_stat.check_level_count,
18596 snoop_stat.busy_count,
18597 snoop_stat.interlocked_count,
18598 snoop_stat.partial_mark_parent_count,
18599 snoop_stat.stolen_or_pm_count,
18600 snoop_stat.stolen_entry_count,
18601 snoop_stat.pm_not_ready_count,
18602 snoop_stat.normal_count,
18603 snoop_stat.stack_bottom_clear_count);
18605 #endif //SNOOP_STATS
18607 #ifdef HEAP_ANALYZE
18609 gc_heap::ha_mark_object_simple (uint8_t** po THREAD_NUMBER_DCL)
18611 if (!internal_root_array)
18613 internal_root_array = new (nothrow) uint8_t* [internal_root_array_length];
18614 if (!internal_root_array)
18616 heap_analyze_success = FALSE;
18620 if (heap_analyze_success && (internal_root_array_length <= internal_root_array_index))
18622 size_t new_size = 2*internal_root_array_length;
18624 uint64_t available_physical = 0;
18625 get_memory_info (NULL, &available_physical);
18626 if (new_size > (size_t)(available_physical / 10))
18628 heap_analyze_success = FALSE;
18632 uint8_t** tmp = new (nothrow) uint8_t* [new_size];
18635 memcpy (tmp, internal_root_array,
18636 internal_root_array_length*sizeof (uint8_t*));
18637 delete[] internal_root_array;
18638 internal_root_array = tmp;
18639 internal_root_array_length = new_size;
18643 heap_analyze_success = FALSE;
18648 if (heap_analyze_success)
18650 PREFIX_ASSUME(internal_root_array_index < internal_root_array_length);
18652 uint8_t* ref = (uint8_t*)po;
18653 if (!current_obj ||
18654 !((ref >= current_obj) && (ref < (current_obj + current_obj_size))))
18656 gc_heap* hp = gc_heap::heap_of (ref);
18657 current_obj = hp->find_object (ref, hp->lowest_address);
18658 current_obj_size = size (current_obj);
18660 internal_root_array[internal_root_array_index] = current_obj;
18661 internal_root_array_index++;
18665 mark_object_simple (po THREAD_NUMBER_ARG);
18667 #endif //HEAP_ANALYZE
18669 //this method assumes that *po is in the [low. high[ range
18671 gc_heap::mark_object_simple (uint8_t** po THREAD_NUMBER_DCL)
18674 #ifdef MULTIPLE_HEAPS
18675 #else //MULTIPLE_HEAPS
18676 const int thread = 0;
18677 #endif //MULTIPLE_HEAPS
18680 snoop_stat.objects_checked_count++;
18681 #endif //SNOOP_STATS
18686 size_t s = size (o);
18687 promoted_bytes (thread) += s;
18689 go_through_object_cl (method_table(o), o, s, poo,
18691 uint8_t* oo = *poo;
18692 if (gc_mark (oo, gc_low, gc_high))
18695 size_t obj_size = size (oo);
18696 promoted_bytes (thread) += obj_size;
18698 if (contain_pointers_or_collectible (oo))
18699 mark_object_simple1 (oo, oo THREAD_NUMBER_ARG);
18709 uint8_t* gc_heap::mark_object (uint8_t* o THREAD_NUMBER_DCL)
18711 if ((o >= gc_low) && (o < gc_high))
18712 mark_object_simple (&o THREAD_NUMBER_ARG);
18713 #ifdef MULTIPLE_HEAPS
18717 gc_heap* hp = heap_of (o);
18719 if ((o >= hp->gc_low) && (o < hp->gc_high))
18720 mark_object_simple (&o THREAD_NUMBER_ARG);
18722 #endif //MULTIPLE_HEAPS
18727 #ifdef BACKGROUND_GC
18729 void gc_heap::background_mark_simple1 (uint8_t* oo THREAD_NUMBER_DCL)
18731 uint8_t** mark_stack_limit = &background_mark_stack_array[background_mark_stack_array_length];
18733 #ifdef SORT_MARK_STACK
18734 uint8_t** sorted_tos = background_mark_stack_array;
18735 #endif //SORT_MARK_STACK
18737 background_mark_stack_tos = background_mark_stack_array;
18741 #ifdef MULTIPLE_HEAPS
18742 #else //MULTIPLE_HEAPS
18743 const int thread = 0;
18744 #endif //MULTIPLE_HEAPS
18748 if ((((size_t)oo & 1) == 0) && ((s = size (oo)) < (partial_size_th*sizeof (uint8_t*))))
18750 BOOL overflow_p = FALSE;
18752 if (background_mark_stack_tos + (s) /sizeof (uint8_t*) >= (mark_stack_limit - 1))
18754 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
18755 size_t num_pointers = CGCDesc::GetNumPointers(method_table(oo), s, num_components);
18756 if (background_mark_stack_tos + num_pointers >= (mark_stack_limit - 1))
18758 dprintf (2, ("h%d: %Id left, obj (mt: %Ix) %Id ptrs",
18760 (size_t)(mark_stack_limit - 1 - background_mark_stack_tos),
18764 bgc_overflow_count++;
18769 if (overflow_p == FALSE)
18771 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18773 go_through_object_cl (method_table(oo), oo, s, ppslot,
18775 uint8_t* o = *ppslot;
18777 if (background_mark (o,
18778 background_saved_lowest_address,
18779 background_saved_highest_address))
18782 size_t obj_size = size (o);
18783 bpromoted_bytes (thread) += obj_size;
18784 if (contain_pointers_or_collectible (o))
18786 *(background_mark_stack_tos++) = o;
18795 dprintf (3,("mark stack overflow for object %Ix ", (size_t)oo));
18796 background_min_overflow_address = min (background_min_overflow_address, oo);
18797 background_max_overflow_address = max (background_max_overflow_address, oo);
18802 uint8_t* start = oo;
18803 if ((size_t)oo & 1)
18805 oo = (uint8_t*)((size_t)oo & ~1);
18806 start = *(--background_mark_stack_tos);
18807 dprintf (4, ("oo: %Ix, start: %Ix\n", (size_t)oo, (size_t)start));
18809 #ifdef COLLECTIBLE_CLASS
18812 // If there's a class object, push it now. We are guaranteed to have the slot since
18813 // we just popped one object off.
18814 if (is_collectible (oo))
18816 uint8_t* class_obj = get_class_object (oo);
18817 if (background_mark (class_obj,
18818 background_saved_lowest_address,
18819 background_saved_highest_address))
18821 size_t obj_size = size (class_obj);
18822 bpromoted_bytes (thread) += obj_size;
18824 *(background_mark_stack_tos++) = class_obj;
18828 if (!contain_pointers (oo))
18833 #endif //COLLECTIBLE_CLASS
18837 BOOL overflow_p = FALSE;
18839 if (background_mark_stack_tos + (num_partial_refs + 2) >= mark_stack_limit)
18841 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
18842 size_t num_pointers = CGCDesc::GetNumPointers(method_table(oo), s, num_components);
18844 dprintf (2, ("h%d: PM: %Id left, obj %Ix (mt: %Ix) start: %Ix, total: %Id",
18846 (size_t)(mark_stack_limit - background_mark_stack_tos),
18852 bgc_overflow_count++;
18855 if (overflow_p == FALSE)
18857 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18859 //push the object and its current
18860 uint8_t** place = background_mark_stack_tos++;
18862 *(background_mark_stack_tos++) = (uint8_t*)((size_t)oo | 1);
18864 int i = num_partial_refs;
18866 go_through_object (method_table(oo), oo, s, ppslot,
18867 start, use_start, (oo + s),
18869 uint8_t* o = *ppslot;
18872 if (background_mark (o,
18873 background_saved_lowest_address,
18874 background_saved_highest_address))
18877 size_t obj_size = size (o);
18878 bpromoted_bytes (thread) += obj_size;
18879 if (contain_pointers_or_collectible (o))
18881 *(background_mark_stack_tos++) = o;
18885 *place = (uint8_t*)(ppslot+1);
18894 //we are finished with this object
18902 dprintf (3,("mark stack overflow for object %Ix ", (size_t)oo));
18903 background_min_overflow_address = min (background_min_overflow_address, oo);
18904 background_max_overflow_address = max (background_max_overflow_address, oo);
18908 #ifdef SORT_MARK_STACK
18909 if (background_mark_stack_tos > sorted_tos + mark_stack_array_length/8)
18911 rqsort1 (sorted_tos, background_mark_stack_tos-1);
18912 sorted_tos = background_mark_stack_tos-1;
18914 #endif //SORT_MARK_STACK
18916 #ifdef COLLECTIBLE_CLASS
18918 #endif // COLLECTIBLE_CLASS
18921 if (!(background_mark_stack_tos == background_mark_stack_array))
18923 oo = *(--background_mark_stack_tos);
18925 #ifdef SORT_MARK_STACK
18926 sorted_tos = (uint8_t**)min ((size_t)sorted_tos, (size_t)background_mark_stack_tos);
18927 #endif //SORT_MARK_STACK
18933 assert (background_mark_stack_tos == background_mark_stack_array);
18938 //this version is different than the foreground GC because
18939 //it can't keep pointers to the inside of an object
18940 //while calling background_mark_simple1. The object could be moved
18941 //by an intervening foreground gc.
18942 //this method assumes that *po is in the [low. high[ range
18944 gc_heap::background_mark_simple (uint8_t* o THREAD_NUMBER_DCL)
18946 #ifdef MULTIPLE_HEAPS
18947 #else //MULTIPLE_HEAPS
18948 const int thread = 0;
18949 #endif //MULTIPLE_HEAPS
18951 dprintf (3, ("bmarking %Ix", o));
18953 if (background_mark1 (o))
18956 size_t s = size (o);
18957 bpromoted_bytes (thread) += s;
18959 if (contain_pointers_or_collectible (o))
18961 background_mark_simple1 (o THREAD_NUMBER_ARG);
18968 uint8_t* gc_heap::background_mark_object (uint8_t* o THREAD_NUMBER_DCL)
18970 if ((o >= background_saved_lowest_address) && (o < background_saved_highest_address))
18972 background_mark_simple (o THREAD_NUMBER_ARG);
18978 dprintf (3, ("or-%Ix", o));
18984 void gc_heap::background_verify_mark (Object*& object, ScanContext* sc, uint32_t flags)
18986 UNREFERENCED_PARAMETER(sc);
18988 assert (settings.concurrent);
18989 uint8_t* o = (uint8_t*)object;
18991 gc_heap* hp = gc_heap::heap_of (o);
18992 #ifdef INTERIOR_POINTERS
18993 if (flags & GC_CALL_INTERIOR)
18995 o = hp->find_object (o, background_saved_lowest_address);
18997 #endif //INTERIOR_POINTERS
18999 if (!background_object_marked (o, FALSE))
19005 void gc_heap::background_promote (Object** ppObject, ScanContext* sc, uint32_t flags)
19007 UNREFERENCED_PARAMETER(sc);
19008 //in order to save space on the array, mark the object,
19009 //knowing that it will be visited later
19010 assert (settings.concurrent);
19012 THREAD_NUMBER_FROM_CONTEXT;
19013 #ifndef MULTIPLE_HEAPS
19014 const int thread = 0;
19015 #endif //!MULTIPLE_HEAPS
19017 uint8_t* o = (uint8_t*)*ppObject;
19022 #ifdef DEBUG_DestroyedHandleValue
19023 // we can race with destroy handle during concurrent scan
19024 if (o == (uint8_t*)DEBUG_DestroyedHandleValue)
19026 #endif //DEBUG_DestroyedHandleValue
19030 gc_heap* hp = gc_heap::heap_of (o);
19032 if ((o < hp->background_saved_lowest_address) || (o >= hp->background_saved_highest_address))
19037 #ifdef INTERIOR_POINTERS
19038 if (flags & GC_CALL_INTERIOR)
19040 o = hp->find_object (o, hp->background_saved_lowest_address);
19044 #endif //INTERIOR_POINTERS
19046 #ifdef FEATURE_CONSERVATIVE_GC
19047 // For conservative GC, a value on stack may point to middle of a free object.
19048 // In this case, we don't need to promote the pointer.
19049 if (GCConfig::GetConservativeGC() && ((CObjectHeader*)o)->IsFree())
19053 #endif //FEATURE_CONSERVATIVE_GC
19056 ((CObjectHeader*)o)->Validate();
19059 dprintf (BGC_LOG, ("Background Promote %Ix", (size_t)o));
19061 //needs to be called before the marking because it is possible for a foreground
19062 //gc to take place during the mark and move the object
19063 STRESS_LOG3(LF_GC|LF_GCROOTS, LL_INFO1000000, " GCHeap::Promote: Promote GC Root *%p = %p MT = %pT", ppObject, o, o ? ((Object*) o)->GetGCSafeMethodTable() : NULL);
19065 hpt->background_mark_simple (o THREAD_NUMBER_ARG);
19068 //used by the ephemeral collection to scan the local background structures
19069 //containing references.
19071 gc_heap::scan_background_roots (promote_func* fn, int hn, ScanContext *pSC)
19077 pSC->thread_number = hn;
19079 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
19080 pSC->pCurrentDomain = 0;
19083 BOOL relocate_p = (fn == &GCHeap::Relocate);
19085 dprintf (3, ("Scanning background mark list"));
19088 size_t mark_list_finger = 0;
19089 while (mark_list_finger < c_mark_list_index)
19091 uint8_t** o = &c_mark_list [mark_list_finger];
19094 // We may not be able to calculate the size during relocate as POPO
19095 // may have written over the object.
19096 size_t s = size (*o);
19097 assert (Align (s) >= Align (min_obj_size));
19098 dprintf(3,("background root %Ix", (size_t)*o));
19100 (*fn) ((Object**)o, pSC, 0);
19101 mark_list_finger++;
19104 //scan the mark stack
19105 dprintf (3, ("Scanning background mark stack"));
19107 uint8_t** finger = background_mark_stack_array;
19108 while (finger < background_mark_stack_tos)
19110 if ((finger + 1) < background_mark_stack_tos)
19112 // We need to check for the partial mark case here.
19113 uint8_t* parent_obj = *(finger + 1);
19114 if ((size_t)parent_obj & 1)
19116 uint8_t* place = *finger;
19117 size_t place_offset = 0;
19118 uint8_t* real_parent_obj = (uint8_t*)((size_t)parent_obj & ~1);
19122 *(finger + 1) = real_parent_obj;
19123 place_offset = place - real_parent_obj;
19124 dprintf(3,("relocating background root %Ix", (size_t)real_parent_obj));
19125 (*fn) ((Object**)(finger + 1), pSC, 0);
19126 real_parent_obj = *(finger + 1);
19127 *finger = real_parent_obj + place_offset;
19128 *(finger + 1) = (uint8_t*)((size_t)real_parent_obj | 1);
19129 dprintf(3,("roots changed to %Ix, %Ix", *finger, *(finger + 1)));
19133 uint8_t** temp = &real_parent_obj;
19134 dprintf(3,("marking background root %Ix", (size_t)real_parent_obj));
19135 (*fn) ((Object**)temp, pSC, 0);
19142 dprintf(3,("background root %Ix", (size_t)*finger));
19143 (*fn) ((Object**)finger, pSC, 0);
19149 void gc_heap::background_mark_through_object (uint8_t* oo THREAD_NUMBER_DCL)
19151 if (contain_pointers (oo))
19153 size_t total_refs = 0;
19154 size_t s = size (oo);
19155 go_through_object_nostart (method_table(oo), oo, s, po,
19159 background_mark_object (o THREAD_NUMBER_ARG);
19163 dprintf (3,("Background marking through %Ix went through %Id refs",
19169 uint8_t* gc_heap::background_seg_end (heap_segment* seg, BOOL concurrent_p)
19171 if (concurrent_p && (seg == saved_overflow_ephemeral_seg))
19173 // for now we stop at where gen1 started when we started processing
19174 return background_min_soh_overflow_address;
19178 return heap_segment_allocated (seg);
19182 uint8_t* gc_heap::background_first_overflow (uint8_t* min_add,
19185 BOOL small_object_p)
19189 if (small_object_p)
19191 if (in_range_for_segment (min_add, seg))
19193 // min_add was the beginning of gen1 when we did the concurrent
19194 // overflow. Now we could be in a situation where min_add is
19195 // actually the same as allocated for that segment (because
19196 // we expanded heap), in which case we can not call
19197 // find first on this address or we will AV.
19198 if (min_add >= heap_segment_allocated (seg))
19204 if (concurrent_p &&
19205 ((seg == saved_overflow_ephemeral_seg) && (min_add >= background_min_soh_overflow_address)))
19207 return background_min_soh_overflow_address;
19211 o = find_first_object (min_add, heap_segment_mem (seg));
19218 o = max (heap_segment_mem (seg), min_add);
19222 void gc_heap::background_process_mark_overflow_internal (int condemned_gen_number,
19223 uint8_t* min_add, uint8_t* max_add,
19228 current_bgc_state = bgc_overflow_soh;
19231 size_t total_marked_objects = 0;
19233 #ifdef MULTIPLE_HEAPS
19234 int thread = heap_number;
19235 #endif //MULTIPLE_HEAPS
19237 exclusive_sync* loh_alloc_lock = 0;
19239 dprintf (2,("Processing Mark overflow [%Ix %Ix]", (size_t)min_add, (size_t)max_add));
19240 #ifdef MULTIPLE_HEAPS
19241 // We don't have each heap scan all heaps concurrently because we are worried about
19242 // multiple threads calling things like find_first_object.
19243 int h_start = (concurrent_p ? heap_number : 0);
19244 int h_end = (concurrent_p ? (heap_number + 1) : n_heaps);
19245 for (int hi = h_start; hi < h_end; hi++)
19247 gc_heap* hp = (concurrent_p ? this : g_heaps [(heap_number + hi) % n_heaps]);
19253 #endif //MULTIPLE_HEAPS
19254 BOOL small_object_segments = TRUE;
19255 int align_const = get_alignment_constant (small_object_segments);
19256 generation* gen = hp->generation_of (condemned_gen_number);
19257 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
19258 PREFIX_ASSUME(seg != NULL);
19259 loh_alloc_lock = hp->bgc_alloc_lock;
19261 uint8_t* o = hp->background_first_overflow (min_add,
19264 small_object_segments);
19268 while ((o < hp->background_seg_end (seg, concurrent_p)) && (o <= max_add))
19270 dprintf (3, ("considering %Ix", (size_t)o));
19274 if (concurrent_p && !small_object_segments)
19276 loh_alloc_lock->bgc_mark_set (o);
19278 if (((CObjectHeader*)o)->IsFree())
19280 s = unused_array_size (o);
19292 if (background_object_marked (o, FALSE) && contain_pointers_or_collectible (o))
19294 total_marked_objects++;
19295 go_through_object_cl (method_table(o), o, s, poo,
19296 uint8_t* oo = *poo;
19297 background_mark_object (oo THREAD_NUMBER_ARG);
19301 if (concurrent_p && !small_object_segments)
19303 loh_alloc_lock->bgc_mark_done ();
19306 o = o + Align (s, align_const);
19314 dprintf (2, ("went through overflow objects in segment %Ix (%d) (so far %Id marked)",
19315 heap_segment_mem (seg), (small_object_segments ? 0 : 1), total_marked_objects));
19317 if ((concurrent_p && (seg == hp->saved_overflow_ephemeral_seg)) ||
19318 (seg = heap_segment_next_in_range (seg)) == 0)
19320 if (small_object_segments)
19324 current_bgc_state = bgc_overflow_loh;
19327 dprintf (2, ("h%d: SOH: ov-mo: %Id", heap_number, total_marked_objects));
19328 fire_overflow_event (min_add, max_add, total_marked_objects, !small_object_segments);
19329 concurrent_print_time_delta (concurrent_p ? "Cov SOH" : "Nov SOH");
19330 total_marked_objects = 0;
19331 small_object_segments = FALSE;
19332 align_const = get_alignment_constant (small_object_segments);
19333 seg = heap_segment_in_range (generation_start_segment (hp->generation_of (max_generation+1)));
19335 PREFIX_ASSUME(seg != NULL);
19337 o = max (heap_segment_mem (seg), min_add);
19342 dprintf (GTC_LOG, ("h%d: LOH: ov-mo: %Id", heap_number, total_marked_objects));
19343 fire_overflow_event (min_add, max_add, total_marked_objects, !small_object_segments);
19349 o = hp->background_first_overflow (min_add,
19352 small_object_segments);
19359 BOOL gc_heap::background_process_mark_overflow (BOOL concurrent_p)
19361 BOOL grow_mark_array_p = TRUE;
19365 assert (!processed_soh_overflow_p);
19367 if ((background_max_overflow_address != 0) &&
19368 (background_min_overflow_address != MAX_PTR))
19370 // We have overflow to process but we know we can't process the ephemeral generations
19371 // now (we actually could process till the current gen1 start but since we are going to
19372 // make overflow per segment, for now I'll just stop at the saved gen1 start.
19373 saved_overflow_ephemeral_seg = ephemeral_heap_segment;
19374 background_max_soh_overflow_address = heap_segment_reserved (saved_overflow_ephemeral_seg);
19375 background_min_soh_overflow_address = generation_allocation_start (generation_of (max_generation-1));
19380 assert ((saved_overflow_ephemeral_seg == 0) ||
19381 ((background_max_soh_overflow_address != 0) &&
19382 (background_min_soh_overflow_address != MAX_PTR)));
19384 if (!processed_soh_overflow_p)
19386 // if there was no more overflow we just need to process what we didn't process
19387 // on the saved ephemeral segment.
19388 if ((background_max_overflow_address == 0) && (background_min_overflow_address == MAX_PTR))
19390 dprintf (2, ("final processing mark overflow - no more overflow since last time"));
19391 grow_mark_array_p = FALSE;
19394 background_min_overflow_address = min (background_min_overflow_address,
19395 background_min_soh_overflow_address);
19396 background_max_overflow_address = max (background_max_overflow_address,
19397 background_max_soh_overflow_address);
19398 processed_soh_overflow_p = TRUE;
19402 BOOL overflow_p = FALSE;
19404 if ((! ((background_max_overflow_address == 0)) ||
19405 ! ((background_min_overflow_address == MAX_PTR))))
19409 if (grow_mark_array_p)
19411 // Try to grow the array.
19412 size_t new_size = max (MARK_STACK_INITIAL_LENGTH, 2*background_mark_stack_array_length);
19414 if ((new_size * sizeof(mark)) > 100*1024)
19416 size_t new_max_size = (get_total_heap_size() / 10) / sizeof(mark);
19418 new_size = min(new_max_size, new_size);
19421 if ((background_mark_stack_array_length < new_size) &&
19422 ((new_size - background_mark_stack_array_length) > (background_mark_stack_array_length / 2)))
19424 dprintf (2, ("h%d: ov grow to %Id", heap_number, new_size));
19426 uint8_t** tmp = new (nothrow) uint8_t* [new_size];
19429 delete background_mark_stack_array;
19430 background_mark_stack_array = tmp;
19431 background_mark_stack_array_length = new_size;
19432 background_mark_stack_tos = background_mark_stack_array;
19438 grow_mark_array_p = TRUE;
19441 uint8_t* min_add = background_min_overflow_address;
19442 uint8_t* max_add = background_max_overflow_address;
19444 background_max_overflow_address = 0;
19445 background_min_overflow_address = MAX_PTR;
19447 background_process_mark_overflow_internal (max_generation, min_add, max_add, concurrent_p);
19457 #endif //BACKGROUND_GC
19460 void gc_heap::mark_through_object (uint8_t* oo, BOOL mark_class_object_p THREAD_NUMBER_DCL)
19462 #ifndef COLLECTIBLE_CLASS
19463 UNREFERENCED_PARAMETER(mark_class_object_p);
19464 BOOL to_mark_class_object = FALSE;
19465 #else //COLLECTIBLE_CLASS
19466 BOOL to_mark_class_object = (mark_class_object_p && (is_collectible(oo)));
19467 #endif //COLLECTIBLE_CLASS
19468 if (contain_pointers (oo) || to_mark_class_object)
19470 dprintf(3,( "Marking through %Ix", (size_t)oo));
19471 size_t s = size (oo);
19473 #ifdef COLLECTIBLE_CLASS
19474 if (to_mark_class_object)
19476 uint8_t* class_obj = get_class_object (oo);
19477 mark_object (class_obj THREAD_NUMBER_ARG);
19479 #endif //COLLECTIBLE_CLASS
19481 if (contain_pointers (oo))
19483 go_through_object_nostart (method_table(oo), oo, s, po,
19485 mark_object (o THREAD_NUMBER_ARG);
19491 size_t gc_heap::get_total_heap_size()
19493 size_t total_heap_size = 0;
19495 #ifdef MULTIPLE_HEAPS
19498 for (hn = 0; hn < gc_heap::n_heaps; hn++)
19500 gc_heap* hp2 = gc_heap::g_heaps [hn];
19501 total_heap_size += hp2->generation_size (max_generation + 1) + hp2->generation_sizes (hp2->generation_of (max_generation));
19504 total_heap_size = generation_size (max_generation + 1) + generation_sizes (generation_of (max_generation));
19505 #endif //MULTIPLE_HEAPS
19507 return total_heap_size;
19510 size_t gc_heap::get_total_fragmentation()
19512 size_t total_fragmentation = 0;
19514 #ifdef MULTIPLE_HEAPS
19515 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
19517 gc_heap* hp = gc_heap::g_heaps[hn];
19518 #else //MULTIPLE_HEAPS
19520 gc_heap* hp = pGenGCHeap;
19521 #endif //MULTIPLE_HEAPS
19522 for (int i = 0; i <= (max_generation + 1); i++)
19524 generation* gen = hp->generation_of (i);
19525 total_fragmentation += (generation_free_list_space (gen) + generation_free_obj_space (gen));
19529 return total_fragmentation;
19532 size_t gc_heap::get_total_gen_fragmentation (int gen_number)
19534 size_t total_fragmentation = 0;
19536 #ifdef MULTIPLE_HEAPS
19537 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
19539 gc_heap* hp = gc_heap::g_heaps[hn];
19540 #else //MULTIPLE_HEAPS
19542 gc_heap* hp = pGenGCHeap;
19543 #endif //MULTIPLE_HEAPS
19544 generation* gen = hp->generation_of (gen_number);
19545 total_fragmentation += (generation_free_list_space (gen) + generation_free_obj_space (gen));
19548 return total_fragmentation;
19551 size_t gc_heap::get_total_gen_estimated_reclaim (int gen_number)
19553 size_t total_estimated_reclaim = 0;
19555 #ifdef MULTIPLE_HEAPS
19556 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
19558 gc_heap* hp = gc_heap::g_heaps[hn];
19559 #else //MULTIPLE_HEAPS
19561 gc_heap* hp = pGenGCHeap;
19562 #endif //MULTIPLE_HEAPS
19563 total_estimated_reclaim += hp->estimated_reclaim (gen_number);
19566 return total_estimated_reclaim;
19569 size_t gc_heap::committed_size()
19571 generation* gen = generation_of (max_generation);
19572 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
19573 size_t total_committed = 0;
19577 total_committed += heap_segment_committed (seg) - (uint8_t*)seg;
19579 seg = heap_segment_next (seg);
19582 if (gen != large_object_generation)
19584 gen = generation_of (max_generation + 1);
19585 seg = generation_start_segment (gen);
19592 return total_committed;
19595 size_t gc_heap::get_total_committed_size()
19597 size_t total_committed = 0;
19599 #ifdef MULTIPLE_HEAPS
19602 for (hn = 0; hn < gc_heap::n_heaps; hn++)
19604 gc_heap* hp = gc_heap::g_heaps [hn];
19605 total_committed += hp->committed_size();
19608 total_committed = committed_size();
19609 #endif //MULTIPLE_HEAPS
19611 return total_committed;
19614 size_t gc_heap::committed_size (bool loh_p, size_t* allocated)
19616 int gen_number = (loh_p ? (max_generation + 1) : max_generation);
19617 generation* gen = generation_of (gen_number);
19618 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
19619 size_t total_committed = 0;
19620 size_t total_allocated = 0;
19624 total_committed += heap_segment_committed (seg) - (uint8_t*)seg;
19625 total_allocated += heap_segment_allocated (seg) - (uint8_t*)seg;
19626 seg = heap_segment_next (seg);
19629 *allocated = total_allocated;
19630 return total_committed;
19633 void gc_heap::get_memory_info (uint32_t* memory_load,
19634 uint64_t* available_physical,
19635 uint64_t* available_page_file)
19637 GCToOSInterface::GetMemoryStatus(memory_load, available_physical, available_page_file);
19640 void fire_mark_event (int heap_num, int root_type, size_t bytes_marked)
19642 dprintf (DT_LOG_0, ("-----------[%d]mark %d: %Id", heap_num, root_type, bytes_marked));
19643 FIRE_EVENT(GCMarkWithType, heap_num, root_type, bytes_marked);
19646 //returns TRUE is an overflow happened.
19647 BOOL gc_heap::process_mark_overflow(int condemned_gen_number)
19649 size_t last_promoted_bytes = promoted_bytes (heap_number);
19650 BOOL overflow_p = FALSE;
19652 if ((! (max_overflow_address == 0) ||
19653 ! (min_overflow_address == MAX_PTR)))
19656 // Try to grow the array.
19658 max (MARK_STACK_INITIAL_LENGTH, 2*mark_stack_array_length);
19660 if ((new_size * sizeof(mark)) > 100*1024)
19662 size_t new_max_size = (get_total_heap_size() / 10) / sizeof(mark);
19664 new_size = min(new_max_size, new_size);
19667 if ((mark_stack_array_length < new_size) &&
19668 ((new_size - mark_stack_array_length) > (mark_stack_array_length / 2)))
19670 mark* tmp = new (nothrow) mark [new_size];
19673 delete mark_stack_array;
19674 mark_stack_array = tmp;
19675 mark_stack_array_length = new_size;
19679 uint8_t* min_add = min_overflow_address;
19680 uint8_t* max_add = max_overflow_address;
19681 max_overflow_address = 0;
19682 min_overflow_address = MAX_PTR;
19683 process_mark_overflow_internal (condemned_gen_number, min_add, max_add);
19687 size_t current_promoted_bytes = promoted_bytes (heap_number);
19689 if (current_promoted_bytes != last_promoted_bytes)
19690 fire_mark_event (heap_number, ETW::GC_ROOT_OVERFLOW, (current_promoted_bytes - last_promoted_bytes));
19694 void gc_heap::process_mark_overflow_internal (int condemned_gen_number,
19695 uint8_t* min_add, uint8_t* max_add)
19697 #ifdef MULTIPLE_HEAPS
19698 int thread = heap_number;
19699 #endif //MULTIPLE_HEAPS
19700 BOOL full_p = (condemned_gen_number == max_generation);
19702 dprintf(3,("Processing Mark overflow [%Ix %Ix]", (size_t)min_add, (size_t)max_add));
19703 #ifdef MULTIPLE_HEAPS
19704 for (int hi = 0; hi < n_heaps; hi++)
19706 gc_heap* hp = g_heaps [(heap_number + hi) % n_heaps];
19712 #endif //MULTIPLE_HEAPS
19713 BOOL small_object_segments = TRUE;
19714 int align_const = get_alignment_constant (small_object_segments);
19715 generation* gen = hp->generation_of (condemned_gen_number);
19716 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
19718 PREFIX_ASSUME(seg != NULL);
19719 uint8_t* o = max (heap_segment_mem (seg), min_add);
19722 uint8_t* end = heap_segment_allocated (seg);
19724 while ((o < end) && (o <= max_add))
19726 assert ((min_add <= o) && (max_add >= o));
19727 dprintf (3, ("considering %Ix", (size_t)o));
19730 mark_through_object (o, TRUE THREAD_NUMBER_ARG);
19733 o = o + Align (size (o), align_const);
19736 if (( seg = heap_segment_next_in_range (seg)) == 0)
19738 if (small_object_segments && full_p)
19740 small_object_segments = FALSE;
19741 align_const = get_alignment_constant (small_object_segments);
19742 seg = heap_segment_in_range (generation_start_segment (hp->generation_of (max_generation+1)));
19744 PREFIX_ASSUME(seg != NULL);
19746 o = max (heap_segment_mem (seg), min_add);
19756 o = max (heap_segment_mem (seg), min_add);
19763 // Scanning for promotion for dependent handles need special handling. Because the primary holds a strong
19764 // reference to the secondary (when the primary itself is reachable) and this can cause a cascading series of
19765 // promotions (the secondary of one handle is or promotes the primary of another) we might need to perform the
19766 // promotion scan multiple times.
19767 // This helper encapsulates the logic to complete all dependent handle promotions when running a server GC. It
19768 // also has the effect of processing any mark stack overflow.
19770 #ifdef MULTIPLE_HEAPS
19771 // When multiple heaps are enabled we have must utilize a more complex algorithm in order to keep all the GC
19772 // worker threads synchronized. The algorithms are sufficiently divergent that we have different
19773 // implementations based on whether MULTIPLE_HEAPS is defined or not.
19775 // Define some static variables used for synchronization in the method below. These should really be defined
19776 // locally but MSVC complains when the VOLATILE macro is expanded into an instantiation of the Volatile class.
19778 // A note about the synchronization used within this method. Communication between the worker threads is
19779 // achieved via two shared booleans (defined below). These both act as latches that are transitioned only from
19780 // false -> true by unsynchronized code. They are only read or reset to false by a single thread under the
19781 // protection of a join.
19782 static VOLATILE(BOOL) s_fUnpromotedHandles = FALSE;
19783 static VOLATILE(BOOL) s_fUnscannedPromotions = FALSE;
19784 static VOLATILE(BOOL) s_fScanRequired;
19785 void gc_heap::scan_dependent_handles (int condemned_gen_number, ScanContext *sc, BOOL initial_scan_p)
19787 // Whenever we call this method there may have been preceding object promotions. So set
19788 // s_fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
19789 // based on the how the scanning proceeded).
19790 s_fUnscannedPromotions = TRUE;
19792 // We don't know how many times we need to loop yet. In particular we can't base the loop condition on
19793 // the state of this thread's portion of the dependent handle table. That's because promotions on other
19794 // threads could cause handle promotions to become necessary here. Even if there are definitely no more
19795 // promotions possible in this thread's handles, we still have to stay in lock-step with those worker
19796 // threads that haven't finished yet (each GC worker thread has to join exactly the same number of times
19797 // as all the others or they'll get out of step).
19800 // The various worker threads are all currently racing in this code. We need to work out if at least
19801 // one of them think they have work to do this cycle. Each thread needs to rescan its portion of the
19802 // dependent handle table when both of the following conditions apply:
19803 // 1) At least one (arbitrary) object might have been promoted since the last scan (because if this
19804 // object happens to correspond to a primary in one of our handles we might potentially have to
19805 // promote the associated secondary).
19806 // 2) The table for this thread has at least one handle with a secondary that isn't promoted yet.
19808 // The first condition is represented by s_fUnscannedPromotions. This is always non-zero for the first
19809 // iteration of this loop (see comment above) and in subsequent cycles each thread updates this
19810 // whenever a mark stack overflow occurs or scanning their dependent handles results in a secondary
19811 // being promoted. This value is cleared back to zero in a synchronized fashion in the join that
19812 // follows below. Note that we can't read this outside of the join since on any iteration apart from
19813 // the first threads will be racing between reading this value and completing their previous
19814 // iteration's table scan.
19816 // The second condition is tracked by the dependent handle code itself on a per worker thread basis
19817 // (and updated by the GcDhReScan() method). We call GcDhUnpromotedHandlesExist() on each thread to
19818 // determine the local value and collect the results into the s_fUnpromotedHandles variable in what is
19819 // effectively an OR operation. As per s_fUnscannedPromotions we can't read the final result until
19820 // we're safely joined.
19821 if (GCScan::GcDhUnpromotedHandlesExist(sc))
19822 s_fUnpromotedHandles = TRUE;
19824 // Synchronize all the threads so we can read our state variables safely. The shared variable
19825 // s_fScanRequired, indicating whether we should scan the tables or terminate the loop, will be set by
19826 // a single thread inside the join.
19827 gc_t_join.join(this, gc_join_scan_dependent_handles);
19828 if (gc_t_join.joined())
19830 // We're synchronized so it's safe to read our shared state variables. We update another shared
19831 // variable to indicate to all threads whether we'll be scanning for another cycle or terminating
19832 // the loop. We scan if there has been at least one object promotion since last time and at least
19833 // one thread has a dependent handle table with a potential handle promotion possible.
19834 s_fScanRequired = s_fUnscannedPromotions && s_fUnpromotedHandles;
19836 // Reset our shared state variables (ready to be set again on this scan or with a good initial
19837 // value for the next call if we're terminating the loop).
19838 s_fUnscannedPromotions = FALSE;
19839 s_fUnpromotedHandles = FALSE;
19841 if (!s_fScanRequired)
19843 // We're terminating the loop. Perform any last operations that require single threaded access.
19844 if (!initial_scan_p)
19846 // On the second invocation we reconcile all mark overflow ranges across the heaps. This can help
19847 // load balance if some of the heaps have an abnormally large workload.
19848 uint8_t* all_heaps_max = 0;
19849 uint8_t* all_heaps_min = MAX_PTR;
19851 for (i = 0; i < n_heaps; i++)
19853 if (all_heaps_max < g_heaps[i]->max_overflow_address)
19854 all_heaps_max = g_heaps[i]->max_overflow_address;
19855 if (all_heaps_min > g_heaps[i]->min_overflow_address)
19856 all_heaps_min = g_heaps[i]->min_overflow_address;
19858 for (i = 0; i < n_heaps; i++)
19860 g_heaps[i]->max_overflow_address = all_heaps_max;
19861 g_heaps[i]->min_overflow_address = all_heaps_min;
19866 // Restart all the workers.
19867 dprintf(3, ("Starting all gc thread mark stack overflow processing"));
19868 gc_t_join.restart();
19871 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
19872 // being visible. If there really was an overflow (process_mark_overflow returns true) then set the
19873 // global flag indicating that at least one object promotion may have occurred (the usual comment
19874 // about races applies). (Note it's OK to set this flag even if we're about to terminate the loop and
19875 // exit the method since we unconditionally set this variable on method entry anyway).
19876 if (process_mark_overflow(condemned_gen_number))
19877 s_fUnscannedPromotions = TRUE;
19879 // If we decided that no scan was required we can terminate the loop now.
19880 if (!s_fScanRequired)
19883 // Otherwise we must join with the other workers to ensure that all mark stack overflows have been
19884 // processed before we start scanning dependent handle tables (if overflows remain while we scan we
19885 // could miss noting the promotion of some primary objects).
19886 gc_t_join.join(this, gc_join_rescan_dependent_handles);
19887 if (gc_t_join.joined())
19889 // Restart all the workers.
19890 dprintf(3, ("Starting all gc thread for dependent handle promotion"));
19891 gc_t_join.restart();
19894 // If the portion of the dependent handle table managed by this worker has handles that could still be
19895 // promoted perform a rescan. If the rescan resulted in at least one promotion note this fact since it
19896 // could require a rescan of handles on this or other workers.
19897 if (GCScan::GcDhUnpromotedHandlesExist(sc))
19898 if (GCScan::GcDhReScan(sc))
19899 s_fUnscannedPromotions = TRUE;
19902 #else //MULTIPLE_HEAPS
19903 // Non-multiple heap version of scan_dependent_handles: much simpler without the need to keep multiple worker
19904 // threads synchronized.
19905 void gc_heap::scan_dependent_handles (int condemned_gen_number, ScanContext *sc, BOOL initial_scan_p)
19907 UNREFERENCED_PARAMETER(initial_scan_p);
19909 // Whenever we call this method there may have been preceding object promotions. So set
19910 // fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
19911 // based on the how the scanning proceeded).
19912 bool fUnscannedPromotions = true;
19914 // Loop until there are either no more dependent handles that can have their secondary promoted or we've
19915 // managed to perform a scan without promoting anything new.
19916 while (GCScan::GcDhUnpromotedHandlesExist(sc) && fUnscannedPromotions)
19918 // On each iteration of the loop start with the assumption that no further objects have been promoted.
19919 fUnscannedPromotions = false;
19921 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
19922 // being visible. If there was an overflow (process_mark_overflow returned true) then additional
19923 // objects now appear to be promoted and we should set the flag.
19924 if (process_mark_overflow(condemned_gen_number))
19925 fUnscannedPromotions = true;
19927 // Perform the scan and set the flag if any promotions resulted.
19928 if (GCScan::GcDhReScan(sc))
19929 fUnscannedPromotions = true;
19932 // Process any mark stack overflow that may have resulted from scanning handles (or if we didn't need to
19933 // scan any handles at all this is the processing of overflows that may have occurred prior to this method
19935 process_mark_overflow(condemned_gen_number);
19937 #endif //MULTIPLE_HEAPS
19939 void gc_heap::mark_phase (int condemned_gen_number, BOOL mark_only_p)
19941 assert (settings.concurrent == FALSE);
19944 sc.thread_number = heap_number;
19945 sc.promotion = TRUE;
19946 sc.concurrent = FALSE;
19948 dprintf(2,("---- Mark Phase condemning %d ----", condemned_gen_number));
19949 BOOL full_p = (condemned_gen_number == max_generation);
19954 start = GetCycleCount32();
19957 int gen_to_init = condemned_gen_number;
19958 if (condemned_gen_number == max_generation)
19960 gen_to_init = max_generation + 1;
19962 for (int gen_idx = 0; gen_idx <= gen_to_init; gen_idx++)
19964 dynamic_data* dd = dynamic_data_of (gen_idx);
19965 dd_begin_data_size (dd) = generation_size (gen_idx) -
19966 dd_fragmentation (dd) -
19967 Align (size (generation_allocation_start (generation_of (gen_idx))));
19968 dprintf (2, ("begin data size for gen%d is %Id", gen_idx, dd_begin_data_size (dd)));
19969 dd_survived_size (dd) = 0;
19970 dd_pinned_survived_size (dd) = 0;
19971 dd_artificial_pinned_survived_size (dd) = 0;
19972 dd_added_pinned_size (dd) = 0;
19974 dd_padding_size (dd) = 0;
19975 #endif //SHORT_PLUGS
19976 #if defined (RESPECT_LARGE_ALIGNMENT) || defined (FEATURE_STRUCTALIGN)
19977 dd_num_npinned_plugs (dd) = 0;
19978 #endif //RESPECT_LARGE_ALIGNMENT || FEATURE_STRUCTALIGN
19981 #ifdef FFIND_OBJECT
19982 if (gen0_must_clear_bricks > 0)
19983 gen0_must_clear_bricks--;
19984 #endif //FFIND_OBJECT
19986 size_t last_promoted_bytes = 0;
19988 promoted_bytes (heap_number) = 0;
19989 reset_mark_stack();
19992 memset (&snoop_stat, 0, sizeof(snoop_stat));
19993 snoop_stat.heap_index = heap_number;
19994 #endif //SNOOP_STATS
19999 //initialize the mark stack
20000 for (int i = 0; i < max_snoop_level; i++)
20002 ((uint8_t**)(mark_stack_array))[i] = 0;
20005 mark_stack_busy() = 1;
20007 #endif //MH_SC_MARK
20009 static uint32_t num_sizedrefs = 0;
20012 static BOOL do_mark_steal_p = FALSE;
20013 #endif //MH_SC_MARK
20015 #ifdef MULTIPLE_HEAPS
20016 gc_t_join.join(this, gc_join_begin_mark_phase);
20017 if (gc_t_join.joined())
20019 #endif //MULTIPLE_HEAPS
20021 maxgen_size_inc_p = false;
20023 num_sizedrefs = GCToEEInterface::GetTotalNumSizedRefHandles();
20025 #ifdef MULTIPLE_HEAPS
20030 size_t total_heap_size = get_total_heap_size();
20032 if (total_heap_size > (100 * 1024 * 1024))
20034 do_mark_steal_p = TRUE;
20038 do_mark_steal_p = FALSE;
20043 do_mark_steal_p = FALSE;
20045 #endif //MH_SC_MARK
20047 gc_t_join.restart();
20049 #endif //MULTIPLE_HEAPS
20054 //set up the mark lists from g_mark_list
20055 assert (g_mark_list);
20056 #ifdef MULTIPLE_HEAPS
20057 mark_list = &g_mark_list [heap_number*mark_list_size];
20059 mark_list = g_mark_list;
20060 #endif //MULTIPLE_HEAPS
20061 //dont use the mark list for full gc
20062 //because multiple segments are more complex to handle and the list
20063 //is likely to overflow
20064 if (condemned_gen_number != max_generation)
20065 mark_list_end = &mark_list [mark_list_size-1];
20067 mark_list_end = &mark_list [0];
20068 mark_list_index = &mark_list [0];
20071 #ifndef MULTIPLE_HEAPS
20072 shigh = (uint8_t*) 0;
20074 #endif //MULTIPLE_HEAPS
20076 //%type% category = quote (mark);
20078 if ((condemned_gen_number == max_generation) && (num_sizedrefs > 0))
20080 GCScan::GcScanSizedRefs(GCHeap::Promote, condemned_gen_number, max_generation, &sc);
20081 fire_mark_event (heap_number, ETW::GC_ROOT_SIZEDREF, (promoted_bytes (heap_number) - last_promoted_bytes));
20082 last_promoted_bytes = promoted_bytes (heap_number);
20084 #ifdef MULTIPLE_HEAPS
20085 gc_t_join.join(this, gc_join_scan_sizedref_done);
20086 if (gc_t_join.joined())
20088 dprintf(3, ("Done with marking all sized refs. Starting all gc thread for marking other strong roots"));
20089 gc_t_join.restart();
20091 #endif //MULTIPLE_HEAPS
20094 dprintf(3,("Marking Roots"));
20096 GCScan::GcScanRoots(GCHeap::Promote,
20097 condemned_gen_number, max_generation,
20100 fire_mark_event (heap_number, ETW::GC_ROOT_STACK, (promoted_bytes (heap_number) - last_promoted_bytes));
20101 last_promoted_bytes = promoted_bytes (heap_number);
20103 #ifdef BACKGROUND_GC
20104 if (recursive_gc_sync::background_running_p())
20106 scan_background_roots (GCHeap::Promote, heap_number, &sc);
20108 #endif //BACKGROUND_GC
20110 #ifdef FEATURE_PREMORTEM_FINALIZATION
20111 dprintf(3, ("Marking finalization data"));
20112 finalize_queue->GcScanRoots(GCHeap::Promote, heap_number, 0);
20113 #endif // FEATURE_PREMORTEM_FINALIZATION
20115 fire_mark_event (heap_number, ETW::GC_ROOT_FQ, (promoted_bytes (heap_number) - last_promoted_bytes));
20116 last_promoted_bytes = promoted_bytes (heap_number);
20121 dprintf(3,("Marking handle table"));
20122 GCScan::GcScanHandles(GCHeap::Promote,
20123 condemned_gen_number, max_generation,
20125 fire_mark_event (heap_number, ETW::GC_ROOT_HANDLES, (promoted_bytes (heap_number) - last_promoted_bytes));
20126 last_promoted_bytes = promoted_bytes (heap_number);
20130 size_t promoted_before_cards = promoted_bytes (heap_number);
20133 dprintf (3, ("before cards: %Id", promoted_before_cards));
20137 #ifdef MULTIPLE_HEAPS
20138 if (gc_t_join.r_join(this, gc_r_join_update_card_bundle))
20140 #endif //MULTIPLE_HEAPS
20142 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
20143 // If we are manually managing card bundles, every write to the card table should already be
20144 // accounted for in the card bundle table so there's nothing to update here.
20145 update_card_table_bundle();
20147 if (card_bundles_enabled())
20149 verify_card_bundles();
20152 #ifdef MULTIPLE_HEAPS
20153 gc_t_join.r_restart();
20155 #endif //MULTIPLE_HEAPS
20156 #endif //CARD_BUNDLE
20158 card_fn mark_object_fn = &gc_heap::mark_object_simple;
20159 #ifdef HEAP_ANALYZE
20160 heap_analyze_success = TRUE;
20161 if (heap_analyze_enabled)
20163 internal_root_array_index = 0;
20165 current_obj_size = 0;
20166 mark_object_fn = &gc_heap::ha_mark_object_simple;
20168 #endif //HEAP_ANALYZE
20170 dprintf(3,("Marking cross generation pointers"));
20171 mark_through_cards_for_segments (mark_object_fn, FALSE);
20173 dprintf(3,("Marking cross generation pointers for large objects"));
20174 mark_through_cards_for_large_objects (mark_object_fn, FALSE);
20176 dprintf (3, ("marked by cards: %Id",
20177 (promoted_bytes (heap_number) - promoted_before_cards)));
20178 fire_mark_event (heap_number, ETW::GC_ROOT_OLDER, (promoted_bytes (heap_number) - last_promoted_bytes));
20179 last_promoted_bytes = promoted_bytes (heap_number);
20184 if (do_mark_steal_p)
20188 #endif //MH_SC_MARK
20190 // Dependent handles need to be scanned with a special algorithm (see the header comment on
20191 // scan_dependent_handles for more detail). We perform an initial scan without synchronizing with other
20192 // worker threads or processing any mark stack overflow. This is not guaranteed to complete the operation
20193 // but in a common case (where there are no dependent handles that are due to be collected) it allows us
20194 // to optimize away further scans. The call to scan_dependent_handles is what will cycle through more
20195 // iterations if required and will also perform processing of any mark stack overflow once the dependent
20196 // handle table has been fully promoted.
20197 GCScan::GcDhInitialScan(GCHeap::Promote, condemned_gen_number, max_generation, &sc);
20198 scan_dependent_handles(condemned_gen_number, &sc, true);
20200 #ifdef MULTIPLE_HEAPS
20201 dprintf(3, ("Joining for short weak handle scan"));
20202 gc_t_join.join(this, gc_join_null_dead_short_weak);
20203 if (gc_t_join.joined())
20204 #endif //MULTIPLE_HEAPS
20206 #ifdef HEAP_ANALYZE
20207 heap_analyze_enabled = FALSE;
20208 GCToEEInterface::AnalyzeSurvivorsFinished(condemned_gen_number);
20209 #endif // HEAP_ANALYZE
20210 GCToEEInterface::AfterGcScanRoots (condemned_gen_number, max_generation, &sc);
20212 #ifdef MULTIPLE_HEAPS
20215 // we used r_join and need to reinitialize states for it here.
20216 gc_t_join.r_init();
20219 //start all threads on the roots.
20220 dprintf(3, ("Starting all gc thread for short weak handle scan"));
20221 gc_t_join.restart();
20222 #endif //MULTIPLE_HEAPS
20226 // null out the target of short weakref that were not promoted.
20227 GCScan::GcShortWeakPtrScan(GCHeap::Promote, condemned_gen_number, max_generation,&sc);
20229 // MTHTS: keep by single thread
20230 #ifdef MULTIPLE_HEAPS
20231 dprintf(3, ("Joining for finalization"));
20232 gc_t_join.join(this, gc_join_scan_finalization);
20233 if (gc_t_join.joined())
20234 #endif //MULTIPLE_HEAPS
20237 #ifdef MULTIPLE_HEAPS
20238 //start all threads on the roots.
20239 dprintf(3, ("Starting all gc thread for Finalization"));
20240 gc_t_join.restart();
20241 #endif //MULTIPLE_HEAPS
20244 //Handle finalization.
20245 size_t promoted_bytes_live = promoted_bytes (heap_number);
20247 #ifdef FEATURE_PREMORTEM_FINALIZATION
20248 dprintf (3, ("Finalize marking"));
20249 finalize_queue->ScanForFinalization (GCHeap::Promote, condemned_gen_number, mark_only_p, __this);
20251 GCToEEInterface::DiagWalkFReachableObjects(__this);
20252 #endif // FEATURE_PREMORTEM_FINALIZATION
20254 // Scan dependent handles again to promote any secondaries associated with primaries that were promoted
20255 // for finalization. As before scan_dependent_handles will also process any mark stack overflow.
20256 scan_dependent_handles(condemned_gen_number, &sc, false);
20258 #ifdef MULTIPLE_HEAPS
20259 dprintf(3, ("Joining for weak pointer deletion"));
20260 gc_t_join.join(this, gc_join_null_dead_long_weak);
20261 if (gc_t_join.joined())
20263 //start all threads on the roots.
20264 dprintf(3, ("Starting all gc thread for weak pointer deletion"));
20265 gc_t_join.restart();
20267 #endif //MULTIPLE_HEAPS
20269 // null out the target of long weakref that were not promoted.
20270 GCScan::GcWeakPtrScan (GCHeap::Promote, condemned_gen_number, max_generation, &sc);
20272 // MTHTS: keep by single thread
20273 #ifdef MULTIPLE_HEAPS
20275 #ifdef PARALLEL_MARK_LIST_SORT
20276 // unsigned long start = GetCycleCount32();
20278 // printf("sort_mark_list took %u cycles\n", GetCycleCount32() - start);
20279 #endif //PARALLEL_MARK_LIST_SORT
20282 dprintf (3, ("Joining for sync block cache entry scanning"));
20283 gc_t_join.join(this, gc_join_null_dead_syncblk);
20284 if (gc_t_join.joined())
20285 #endif //MULTIPLE_HEAPS
20287 // scan for deleted entries in the syncblk cache
20288 GCScan::GcWeakPtrScanBySingleThread (condemned_gen_number, max_generation, &sc);
20290 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
20291 if (g_fEnableAppDomainMonitoring)
20293 size_t promoted_all_heaps = 0;
20294 #ifdef MULTIPLE_HEAPS
20295 for (int i = 0; i < n_heaps; i++)
20297 promoted_all_heaps += promoted_bytes (i);
20300 promoted_all_heaps = promoted_bytes (heap_number);
20301 #endif //MULTIPLE_HEAPS
20302 GCToEEInterface::RecordTotalSurvivedBytes(promoted_all_heaps);
20304 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
20306 #ifdef MULTIPLE_HEAPS
20309 #ifndef PARALLEL_MARK_LIST_SORT
20310 //compact g_mark_list and sort it.
20311 combine_mark_lists();
20312 #endif //PARALLEL_MARK_LIST_SORT
20315 //decide on promotion
20316 if (!settings.promotion)
20319 for (int n = 0; n <= condemned_gen_number;n++)
20321 m += (size_t)(dd_min_size (dynamic_data_of (n))*(n+1)*0.1);
20324 for (int i = 0; i < n_heaps; i++)
20326 dynamic_data* dd = g_heaps[i]->dynamic_data_of (min (condemned_gen_number +1,
20328 size_t older_gen_size = (dd_current_size (dd) +
20329 (dd_desired_allocation (dd) -
20330 dd_new_allocation (dd)));
20332 if ((m > (older_gen_size)) ||
20333 (promoted_bytes (i) > m))
20335 settings.promotion = TRUE;
20341 if (do_mark_steal_p)
20343 size_t objects_checked_count = 0;
20344 size_t zero_ref_count = 0;
20345 size_t objects_marked_count = 0;
20346 size_t check_level_count = 0;
20347 size_t busy_count = 0;
20348 size_t interlocked_count = 0;
20349 size_t partial_mark_parent_count = 0;
20350 size_t stolen_or_pm_count = 0;
20351 size_t stolen_entry_count = 0;
20352 size_t pm_not_ready_count = 0;
20353 size_t normal_count = 0;
20354 size_t stack_bottom_clear_count = 0;
20356 for (int i = 0; i < n_heaps; i++)
20358 gc_heap* hp = g_heaps[i];
20359 hp->print_snoop_stat();
20360 objects_checked_count += hp->snoop_stat.objects_checked_count;
20361 zero_ref_count += hp->snoop_stat.zero_ref_count;
20362 objects_marked_count += hp->snoop_stat.objects_marked_count;
20363 check_level_count += hp->snoop_stat.check_level_count;
20364 busy_count += hp->snoop_stat.busy_count;
20365 interlocked_count += hp->snoop_stat.interlocked_count;
20366 partial_mark_parent_count += hp->snoop_stat.partial_mark_parent_count;
20367 stolen_or_pm_count += hp->snoop_stat.stolen_or_pm_count;
20368 stolen_entry_count += hp->snoop_stat.stolen_entry_count;
20369 pm_not_ready_count += hp->snoop_stat.pm_not_ready_count;
20370 normal_count += hp->snoop_stat.normal_count;
20371 stack_bottom_clear_count += hp->snoop_stat.stack_bottom_clear_count;
20376 printf ("-------total stats-------\n");
20377 printf ("%8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s\n",
20378 "checked", "zero", "marked", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "normal", "clear");
20379 printf ("%8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
20380 objects_checked_count,
20382 objects_marked_count,
20386 partial_mark_parent_count,
20387 stolen_or_pm_count,
20388 stolen_entry_count,
20389 pm_not_ready_count,
20391 stack_bottom_clear_count);
20393 #endif //SNOOP_STATS
20395 //start all threads.
20396 dprintf(3, ("Starting all threads for end of mark phase"));
20397 gc_t_join.restart();
20398 #else //MULTIPLE_HEAPS
20400 //decide on promotion
20401 if (!settings.promotion)
20404 for (int n = 0; n <= condemned_gen_number;n++)
20406 m += (size_t)(dd_min_size (dynamic_data_of (n))*(n+1)*0.06);
20408 dynamic_data* dd = dynamic_data_of (min (condemned_gen_number +1,
20410 size_t older_gen_size = (dd_current_size (dd) +
20411 (dd_desired_allocation (dd) -
20412 dd_new_allocation (dd)));
20414 dprintf (2, ("promotion threshold: %Id, promoted bytes: %Id size n+1: %Id",
20415 m, promoted_bytes (heap_number), older_gen_size));
20417 if ((m > older_gen_size) ||
20418 (promoted_bytes (heap_number) > m))
20420 settings.promotion = TRUE;
20424 #endif //MULTIPLE_HEAPS
20427 #ifdef MULTIPLE_HEAPS
20429 #ifdef PARALLEL_MARK_LIST_SORT
20430 // start = GetCycleCount32();
20431 merge_mark_lists();
20432 // printf("merge_mark_lists took %u cycles\n", GetCycleCount32() - start);
20433 #endif //PARALLEL_MARK_LIST_SORT
20435 #endif //MULTIPLE_HEAPS
20437 #ifdef BACKGROUND_GC
20438 total_promoted_bytes = promoted_bytes (heap_number);
20439 #endif //BACKGROUND_GC
20441 promoted_bytes (heap_number) -= promoted_bytes_live;
20444 finish = GetCycleCount32();
20445 mark_time = finish - start;
20448 dprintf(2,("---- End of mark phase ----"));
20452 void gc_heap::pin_object (uint8_t* o, uint8_t** ppObject, uint8_t* low, uint8_t* high)
20454 dprintf (3, ("Pinning %Ix", (size_t)o));
20455 if ((o >= low) && (o < high))
20457 dprintf(3,("^%Ix^", (size_t)o));
20460 #ifdef FEATURE_EVENT_TRACE
20461 if(EVENT_ENABLED(PinObjectAtGCTime))
20463 fire_etw_pin_object_event(o, ppObject);
20465 #endif // FEATURE_EVENT_TRACE
20467 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
20468 num_pinned_objects++;
20469 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
20473 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
20474 size_t gc_heap::get_total_pinned_objects()
20476 #ifdef MULTIPLE_HEAPS
20477 size_t total_num_pinned_objects = 0;
20478 for (int i = 0; i < gc_heap::n_heaps; i++)
20480 gc_heap* hp = gc_heap::g_heaps[i];
20481 total_num_pinned_objects += hp->num_pinned_objects;
20483 return total_num_pinned_objects;
20484 #else //MULTIPLE_HEAPS
20485 return num_pinned_objects;
20486 #endif //MULTIPLE_HEAPS
20488 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
20490 void gc_heap::reset_mark_stack ()
20492 reset_pinned_queue();
20493 max_overflow_address = 0;
20494 min_overflow_address = MAX_PTR;
20497 #ifdef FEATURE_STRUCTALIGN
20499 // The word with left child, right child, and align info is laid out as follows:
20501 // | upper short word | lower short word |
20502 // |<------------> <----->|<------------> <----->|
20503 // | left child info hi| right child info lo|
20504 // x86: | 10 bits 6 bits| 10 bits 6 bits|
20506 // where left/right child are signed values and concat(info hi, info lo) is unsigned.
20508 // The "align info" encodes two numbers: the required alignment (a power of two)
20509 // and the misalignment (the number of machine words the destination address needs
20510 // to be adjusted by to provide alignment - so this number is always smaller than
20511 // the required alignment). Thus, the two can be represented as the "logical or"
20512 // of the two numbers. Note that the actual pad is computed from the misalignment
20513 // by adding the alignment iff the misalignment is non-zero and less than min_obj_size.
20516 // The number of bits in a brick.
20517 #if defined (_TARGET_AMD64_)
20518 #define brick_bits (12)
20520 #define brick_bits (11)
20521 #endif //_TARGET_AMD64_
20522 C_ASSERT(brick_size == (1 << brick_bits));
20524 // The number of bits needed to represent the offset to a child node.
20525 // "brick_bits + 1" allows us to represent a signed offset within a brick.
20526 #define child_bits (brick_bits + 1 - LOG2_PTRSIZE)
20528 // The number of bits in each of the pad hi, pad lo fields.
20529 #define pad_bits (sizeof(short) * 8 - child_bits)
20531 #define child_from_short(w) (((signed short)(w) / (1 << (pad_bits - LOG2_PTRSIZE))) & ~((1 << LOG2_PTRSIZE) - 1))
20532 #define pad_mask ((1 << pad_bits) - 1)
20533 #define pad_from_short(w) ((size_t)(w) & pad_mask)
20534 #else // FEATURE_STRUCTALIGN
20535 #define child_from_short(w) (w)
20536 #endif // FEATURE_STRUCTALIGN
20539 short node_left_child(uint8_t* node)
20541 return child_from_short(((plug_and_pair*)node)[-1].m_pair.left);
20545 void set_node_left_child(uint8_t* node, ptrdiff_t val)
20547 assert (val > -(ptrdiff_t)brick_size);
20548 assert (val < (ptrdiff_t)brick_size);
20549 assert (Aligned (val));
20550 #ifdef FEATURE_STRUCTALIGN
20551 size_t pad = pad_from_short(((plug_and_pair*)node)[-1].m_pair.left);
20552 ((plug_and_pair*)node)[-1].m_pair.left = ((short)val << (pad_bits - LOG2_PTRSIZE)) | (short)pad;
20553 #else // FEATURE_STRUCTALIGN
20554 ((plug_and_pair*)node)[-1].m_pair.left = (short)val;
20555 #endif // FEATURE_STRUCTALIGN
20556 assert (node_left_child (node) == val);
20560 short node_right_child(uint8_t* node)
20562 return child_from_short(((plug_and_pair*)node)[-1].m_pair.right);
20566 void set_node_right_child(uint8_t* node, ptrdiff_t val)
20568 assert (val > -(ptrdiff_t)brick_size);
20569 assert (val < (ptrdiff_t)brick_size);
20570 assert (Aligned (val));
20571 #ifdef FEATURE_STRUCTALIGN
20572 size_t pad = pad_from_short(((plug_and_pair*)node)[-1].m_pair.right);
20573 ((plug_and_pair*)node)[-1].m_pair.right = ((short)val << (pad_bits - LOG2_PTRSIZE)) | (short)pad;
20574 #else // FEATURE_STRUCTALIGN
20575 ((plug_and_pair*)node)[-1].m_pair.right = (short)val;
20576 #endif // FEATURE_STRUCTALIGN
20577 assert (node_right_child (node) == val);
20580 #ifdef FEATURE_STRUCTALIGN
20581 void node_aligninfo (uint8_t* node, int& requiredAlignment, ptrdiff_t& pad)
20583 // Extract the single-number aligninfo from the fields.
20584 short left = ((plug_and_pair*)node)[-1].m_pair.left;
20585 short right = ((plug_and_pair*)node)[-1].m_pair.right;
20586 ptrdiff_t pad_shifted = (pad_from_short(left) << pad_bits) | pad_from_short(right);
20587 ptrdiff_t aligninfo = pad_shifted * DATA_ALIGNMENT;
20589 // Replicate the topmost bit into all lower bits.
20590 ptrdiff_t x = aligninfo;
20596 // Clear all bits but the highest.
20597 requiredAlignment = (int)(x ^ (x >> 1));
20598 pad = aligninfo - requiredAlignment;
20599 pad += AdjustmentForMinPadSize(pad, requiredAlignment);
20603 ptrdiff_t node_alignpad (uint8_t* node)
20605 int requiredAlignment;
20606 ptrdiff_t alignpad;
20607 node_aligninfo (node, requiredAlignment, alignpad);
20611 void clear_node_aligninfo (uint8_t* node)
20613 ((plug_and_pair*)node)[-1].m_pair.left &= ~0 << pad_bits;
20614 ((plug_and_pair*)node)[-1].m_pair.right &= ~0 << pad_bits;
20617 void set_node_aligninfo (uint8_t* node, int requiredAlignment, ptrdiff_t pad)
20619 // Encode the alignment requirement and alignment offset as a single number
20620 // as described above.
20621 ptrdiff_t aligninfo = (size_t)requiredAlignment + (pad & (requiredAlignment-1));
20622 assert (Aligned (aligninfo));
20623 ptrdiff_t aligninfo_shifted = aligninfo / DATA_ALIGNMENT;
20624 assert (aligninfo_shifted < (1 << (pad_bits + pad_bits)));
20626 ptrdiff_t hi = aligninfo_shifted >> pad_bits;
20627 assert (pad_from_short(((plug_and_gap*)node)[-1].m_pair.left) == 0);
20628 ((plug_and_pair*)node)[-1].m_pair.left |= hi;
20630 ptrdiff_t lo = aligninfo_shifted & pad_mask;
20631 assert (pad_from_short(((plug_and_gap*)node)[-1].m_pair.right) == 0);
20632 ((plug_and_pair*)node)[-1].m_pair.right |= lo;
20635 int requiredAlignment2;
20637 node_aligninfo (node, requiredAlignment2, pad2);
20638 assert (requiredAlignment == requiredAlignment2);
20639 assert (pad == pad2);
20642 #endif // FEATURE_STRUCTALIGN
20645 void loh_set_node_relocation_distance(uint8_t* node, ptrdiff_t val)
20647 ptrdiff_t* place = &(((loh_obj_and_pad*)node)[-1].reloc);
20652 ptrdiff_t loh_node_relocation_distance(uint8_t* node)
20654 return (((loh_obj_and_pad*)node)[-1].reloc);
20658 ptrdiff_t node_relocation_distance (uint8_t* node)
20660 return (((plug_and_reloc*)(node))[-1].reloc & ~3);
20664 void set_node_relocation_distance(uint8_t* node, ptrdiff_t val)
20666 assert (val == (val & ~3));
20667 ptrdiff_t* place = &(((plug_and_reloc*)node)[-1].reloc);
20668 //clear the left bit and the relocation field
20674 #define node_left_p(node) (((plug_and_reloc*)(node))[-1].reloc & 2)
20676 #define set_node_left(node) ((plug_and_reloc*)(node))[-1].reloc |= 2;
20678 #ifndef FEATURE_STRUCTALIGN
20679 void set_node_realigned(uint8_t* node)
20681 ((plug_and_reloc*)(node))[-1].reloc |= 1;
20684 void clear_node_realigned(uint8_t* node)
20686 #ifdef RESPECT_LARGE_ALIGNMENT
20687 ((plug_and_reloc*)(node))[-1].reloc &= ~1;
20689 UNREFERENCED_PARAMETER(node);
20690 #endif //RESPECT_LARGE_ALIGNMENT
20692 #endif // FEATURE_STRUCTALIGN
20695 size_t node_gap_size (uint8_t* node)
20697 return ((plug_and_gap *)node)[-1].gap;
20700 void set_gap_size (uint8_t* node, size_t size)
20702 assert (Aligned (size));
20704 // clear the 2 uint32_t used by the node.
20705 ((plug_and_gap *)node)[-1].reloc = 0;
20706 ((plug_and_gap *)node)[-1].lr =0;
20707 ((plug_and_gap *)node)[-1].gap = size;
20709 assert ((size == 0 )||(size >= sizeof(plug_and_reloc)));
20713 uint8_t* gc_heap::insert_node (uint8_t* new_node, size_t sequence_number,
20714 uint8_t* tree, uint8_t* last_node)
20716 dprintf (3, ("IN: %Ix(%Ix), T: %Ix(%Ix), L: %Ix(%Ix) [%Ix]",
20717 (size_t)new_node, brick_of(new_node),
20718 (size_t)tree, brick_of(tree),
20719 (size_t)last_node, brick_of(last_node),
20721 if (power_of_two_p (sequence_number))
20723 set_node_left_child (new_node, (tree - new_node));
20724 dprintf (3, ("NT: %Ix, LC->%Ix", (size_t)new_node, (tree - new_node)));
20729 if (oddp (sequence_number))
20731 set_node_right_child (last_node, (new_node - last_node));
20732 dprintf (3, ("%Ix RC->%Ix", last_node, (new_node - last_node)));
20736 uint8_t* earlier_node = tree;
20737 size_t imax = logcount(sequence_number) - 2;
20738 for (size_t i = 0; i != imax; i++)
20740 earlier_node = earlier_node + node_right_child (earlier_node);
20742 int tmp_offset = node_right_child (earlier_node);
20743 assert (tmp_offset); // should never be empty
20744 set_node_left_child (new_node, ((earlier_node + tmp_offset ) - new_node));
20745 set_node_right_child (earlier_node, (new_node - earlier_node));
20747 dprintf (3, ("%Ix LC->%Ix, %Ix RC->%Ix",
20748 new_node, ((earlier_node + tmp_offset ) - new_node),
20749 earlier_node, (new_node - earlier_node)));
20755 size_t gc_heap::update_brick_table (uint8_t* tree, size_t current_brick,
20756 uint8_t* x, uint8_t* plug_end)
20758 dprintf (3, ("tree: %Ix, current b: %Ix, x: %Ix, plug_end: %Ix",
20759 tree, current_brick, x, plug_end));
20763 dprintf (3, ("b- %Ix->%Ix pointing to tree %Ix",
20764 current_brick, (size_t)(tree - brick_address (current_brick)), tree));
20765 set_brick (current_brick, (tree - brick_address (current_brick)));
20769 dprintf (3, ("b- %Ix->-1", current_brick));
20770 set_brick (current_brick, -1);
20772 size_t b = 1 + current_brick;
20773 ptrdiff_t offset = 0;
20774 size_t last_br = brick_of (plug_end-1);
20775 current_brick = brick_of (x-1);
20776 dprintf (3, ("ubt: %Ix->%Ix]->%Ix]", b, last_br, current_brick));
20777 while (b <= current_brick)
20781 set_brick (b, --offset);
20789 return brick_of (x);
20792 void gc_heap::plan_generation_start (generation* gen, generation* consing_gen, uint8_t* next_plug_to_allocate)
20795 // We should never demote big plugs to gen0.
20796 if (gen == youngest_generation)
20798 heap_segment* seg = ephemeral_heap_segment;
20799 size_t mark_stack_large_bos = mark_stack_bos;
20800 size_t large_plug_pos = 0;
20801 while (mark_stack_large_bos < mark_stack_tos)
20803 if (mark_stack_array[mark_stack_large_bos].len > demotion_plug_len_th)
20805 while (mark_stack_bos <= mark_stack_large_bos)
20807 size_t entry = deque_pinned_plug();
20808 size_t len = pinned_len (pinned_plug_of (entry));
20809 uint8_t* plug = pinned_plug (pinned_plug_of(entry));
20810 if (len > demotion_plug_len_th)
20812 dprintf (2, ("ps(%d): S %Ix (%Id)(%Ix)", gen->gen_num, plug, len, (plug+len)));
20814 pinned_len (pinned_plug_of (entry)) = plug - generation_allocation_pointer (consing_gen);
20815 assert(mark_stack_array[entry].len == 0 ||
20816 mark_stack_array[entry].len >= Align(min_obj_size));
20817 generation_allocation_pointer (consing_gen) = plug + len;
20818 generation_allocation_limit (consing_gen) = heap_segment_plan_allocated (seg);
20819 set_allocator_next_pin (consing_gen);
20823 mark_stack_large_bos++;
20828 generation_plan_allocation_start (gen) =
20829 allocate_in_condemned_generations (consing_gen, Align (min_obj_size), -1);
20830 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
20831 size_t allocation_left = (size_t)(generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
20832 if (next_plug_to_allocate)
20834 size_t dist_to_next_plug = (size_t)(next_plug_to_allocate - generation_allocation_pointer (consing_gen));
20835 if (allocation_left > dist_to_next_plug)
20837 allocation_left = dist_to_next_plug;
20840 if (allocation_left < Align (min_obj_size))
20842 generation_plan_allocation_start_size (gen) += allocation_left;
20843 generation_allocation_pointer (consing_gen) += allocation_left;
20846 dprintf (2, ("plan alloc gen%d(%Ix) start at %Ix (ptr: %Ix, limit: %Ix, next: %Ix)", gen->gen_num,
20847 generation_plan_allocation_start (gen),
20848 generation_plan_allocation_start_size (gen),
20849 generation_allocation_pointer (consing_gen), generation_allocation_limit (consing_gen),
20850 next_plug_to_allocate));
20853 void gc_heap::realloc_plan_generation_start (generation* gen, generation* consing_gen)
20855 BOOL adjacentp = FALSE;
20857 generation_plan_allocation_start (gen) =
20858 allocate_in_expanded_heap (consing_gen, Align(min_obj_size), adjacentp, 0,
20861 #endif //SHORT_PLUGS
20862 FALSE, -1 REQD_ALIGN_AND_OFFSET_ARG);
20864 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
20865 size_t allocation_left = (size_t)(generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
20866 if ((allocation_left < Align (min_obj_size)) &&
20867 (generation_allocation_limit (consing_gen)!=heap_segment_plan_allocated (generation_allocation_segment (consing_gen))))
20869 generation_plan_allocation_start_size (gen) += allocation_left;
20870 generation_allocation_pointer (consing_gen) += allocation_left;
20873 dprintf (1, ("plan re-alloc gen%d start at %Ix (ptr: %Ix, limit: %Ix)", gen->gen_num,
20874 generation_plan_allocation_start (consing_gen),
20875 generation_allocation_pointer (consing_gen),
20876 generation_allocation_limit (consing_gen)));
20879 void gc_heap::plan_generation_starts (generation*& consing_gen)
20881 //make sure that every generation has a planned allocation start
20882 int gen_number = settings.condemned_generation;
20883 while (gen_number >= 0)
20885 if (gen_number < max_generation)
20887 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
20889 generation* gen = generation_of (gen_number);
20890 if (0 == generation_plan_allocation_start (gen))
20892 plan_generation_start (gen, consing_gen, 0);
20893 assert (generation_plan_allocation_start (gen));
20897 // now we know the planned allocation size
20898 heap_segment_plan_allocated (ephemeral_heap_segment) =
20899 generation_allocation_pointer (consing_gen);
20902 void gc_heap::advance_pins_for_demotion (generation* gen)
20904 uint8_t* original_youngest_start = generation_allocation_start (youngest_generation);
20905 heap_segment* seg = ephemeral_heap_segment;
20907 if ((!(pinned_plug_que_empty_p())))
20909 size_t gen1_pinned_promoted = generation_pinned_allocation_compact_size (generation_of (max_generation));
20910 size_t gen1_pins_left = dd_pinned_survived_size (dynamic_data_of (max_generation - 1)) - gen1_pinned_promoted;
20911 size_t total_space_to_skip = last_gen1_pin_end - generation_allocation_pointer (gen);
20912 float pin_frag_ratio = (float)gen1_pins_left / (float)total_space_to_skip;
20913 float pin_surv_ratio = (float)gen1_pins_left / (float)(dd_survived_size (dynamic_data_of (max_generation - 1)));
20914 if ((pin_frag_ratio > 0.15) && (pin_surv_ratio > 0.30))
20916 while (!pinned_plug_que_empty_p() &&
20917 (pinned_plug (oldest_pin()) < original_youngest_start))
20919 size_t entry = deque_pinned_plug();
20920 size_t len = pinned_len (pinned_plug_of (entry));
20921 uint8_t* plug = pinned_plug (pinned_plug_of(entry));
20922 pinned_len (pinned_plug_of (entry)) = plug - generation_allocation_pointer (gen);
20923 assert(mark_stack_array[entry].len == 0 ||
20924 mark_stack_array[entry].len >= Align(min_obj_size));
20925 generation_allocation_pointer (gen) = plug + len;
20926 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20927 set_allocator_next_pin (gen);
20929 //Add the size of the pinned plug to the right pinned allocations
20930 //find out which gen this pinned plug came from
20931 int frgn = object_gennum (plug);
20932 if ((frgn != (int)max_generation) && settings.promotion)
20934 int togn = object_gennum_plan (plug);
20935 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
20938 generation_pinned_allocation_compact_size (generation_of (togn)) += len;
20942 dprintf (2, ("skipping gap %d, pin %Ix (%Id)",
20943 pinned_len (pinned_plug_of (entry)), plug, len));
20946 dprintf (2, ("ad_p_d: PL: %Id, SL: %Id, pfr: %d, psr: %d",
20947 gen1_pins_left, total_space_to_skip, (int)(pin_frag_ratio*100), (int)(pin_surv_ratio*100)));
20951 void gc_heap::process_ephemeral_boundaries (uint8_t* x,
20952 int& active_new_gen_number,
20953 int& active_old_gen_number,
20954 generation*& consing_gen,
20955 BOOL& allocate_in_condemned)
20958 if ((active_old_gen_number > 0) &&
20959 (x >= generation_allocation_start (generation_of (active_old_gen_number - 1))))
20961 dprintf (2, ("crossing gen%d, x is %Ix", active_old_gen_number - 1, x));
20963 if (!pinned_plug_que_empty_p())
20965 dprintf (2, ("oldest pin: %Ix(%Id)",
20966 pinned_plug (oldest_pin()),
20967 (x - pinned_plug (oldest_pin()))));
20970 if (active_old_gen_number <= (settings.promotion ? (max_generation - 1) : max_generation))
20972 active_new_gen_number--;
20975 active_old_gen_number--;
20976 assert ((!settings.promotion) || (active_new_gen_number>0));
20978 if (active_new_gen_number == (max_generation - 1))
20980 #ifdef FREE_USAGE_STATS
20981 if (settings.condemned_generation == max_generation)
20983 // We need to do this before we skip the rest of the pinned plugs.
20984 generation* gen_2 = generation_of (max_generation);
20985 generation* gen_1 = generation_of (max_generation - 1);
20987 size_t total_num_pinned_free_spaces_left = 0;
20989 // We are about to allocate gen1, check to see how efficient fitting in gen2 pinned free spaces is.
20990 for (int j = 0; j < NUM_GEN_POWER2; j++)
20992 dprintf (1, ("[h%d][#%Id]2^%d: current: %Id, S: 2: %Id, 1: %Id(%Id)",
20996 gen_2->gen_current_pinned_free_spaces[j],
20997 gen_2->gen_plugs[j], gen_1->gen_plugs[j],
20998 (gen_2->gen_plugs[j] + gen_1->gen_plugs[j])));
21000 total_num_pinned_free_spaces_left += gen_2->gen_current_pinned_free_spaces[j];
21003 float pinned_free_list_efficiency = 0;
21004 size_t total_pinned_free_space = generation_allocated_in_pinned_free (gen_2) + generation_pinned_free_obj_space (gen_2);
21005 if (total_pinned_free_space != 0)
21007 pinned_free_list_efficiency = (float)(generation_allocated_in_pinned_free (gen_2)) / (float)total_pinned_free_space;
21010 dprintf (1, ("[h%d] gen2 allocated %Id bytes with %Id bytes pinned free spaces (effi: %d%%), %Id (%Id) left",
21012 generation_allocated_in_pinned_free (gen_2),
21013 total_pinned_free_space,
21014 (int)(pinned_free_list_efficiency * 100),
21015 generation_pinned_free_obj_space (gen_2),
21016 total_num_pinned_free_spaces_left));
21018 #endif //FREE_USAGE_STATS
21020 //Go past all of the pinned plugs for this generation.
21021 while (!pinned_plug_que_empty_p() &&
21022 (!in_range_for_segment ((pinned_plug (oldest_pin())), ephemeral_heap_segment)))
21024 size_t entry = deque_pinned_plug();
21025 mark* m = pinned_plug_of (entry);
21026 uint8_t* plug = pinned_plug (m);
21027 size_t len = pinned_len (m);
21028 // detect pinned block in different segment (later) than
21029 // allocation segment, skip those until the oldest pin is in the ephemeral seg.
21030 // adjust the allocation segment along the way (at the end it will
21031 // be the ephemeral segment.
21032 heap_segment* nseg = heap_segment_in_range (generation_allocation_segment (consing_gen));
21034 PREFIX_ASSUME(nseg != NULL);
21036 while (!((plug >= generation_allocation_pointer (consing_gen))&&
21037 (plug < heap_segment_allocated (nseg))))
21039 //adjust the end of the segment to be the end of the plug
21040 assert (generation_allocation_pointer (consing_gen)>=
21041 heap_segment_mem (nseg));
21042 assert (generation_allocation_pointer (consing_gen)<=
21043 heap_segment_committed (nseg));
21045 heap_segment_plan_allocated (nseg) =
21046 generation_allocation_pointer (consing_gen);
21047 //switch allocation segment
21048 nseg = heap_segment_next_rw (nseg);
21049 generation_allocation_segment (consing_gen) = nseg;
21050 //reset the allocation pointer and limits
21051 generation_allocation_pointer (consing_gen) =
21052 heap_segment_mem (nseg);
21054 set_new_pin_info (m, generation_allocation_pointer (consing_gen));
21055 assert(pinned_len(m) == 0 || pinned_len(m) >= Align(min_obj_size));
21056 generation_allocation_pointer (consing_gen) = plug + len;
21057 generation_allocation_limit (consing_gen) =
21058 generation_allocation_pointer (consing_gen);
21060 allocate_in_condemned = TRUE;
21061 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
21064 if (active_new_gen_number != max_generation)
21066 if (active_new_gen_number == (max_generation - 1))
21068 maxgen_pinned_compact_before_advance = generation_pinned_allocation_compact_size (generation_of (max_generation));
21069 if (!demote_gen1_p)
21070 advance_pins_for_demotion (consing_gen);
21073 plan_generation_start (generation_of (active_new_gen_number), consing_gen, x);
21075 dprintf (2, ("process eph: allocated gen%d start at %Ix",
21076 active_new_gen_number,
21077 generation_plan_allocation_start (generation_of (active_new_gen_number))));
21079 if ((demotion_low == MAX_PTR) && !pinned_plug_que_empty_p())
21081 uint8_t* pplug = pinned_plug (oldest_pin());
21082 if (object_gennum (pplug) > 0)
21084 demotion_low = pplug;
21085 dprintf (3, ("process eph: dlow->%Ix", demotion_low));
21089 assert (generation_plan_allocation_start (generation_of (active_new_gen_number)));
21097 void gc_heap::seg_clear_mark_bits (heap_segment* seg)
21099 uint8_t* o = heap_segment_mem (seg);
21100 while (o < heap_segment_allocated (seg))
21106 o = o + Align (size (o));
21110 #ifdef FEATURE_BASICFREEZE
21111 void gc_heap::sweep_ro_segments (heap_segment* start_seg)
21113 //go through all of the segment in range and reset the mark bit
21114 //TODO works only on small object segments
21116 heap_segment* seg = start_seg;
21120 if (heap_segment_read_only_p (seg) &&
21121 heap_segment_in_range_p (seg))
21123 #ifdef BACKGROUND_GC
21124 if (settings.concurrent)
21126 seg_clear_mark_array_bits_soh (seg);
21130 seg_clear_mark_bits (seg);
21132 #else //BACKGROUND_GC
21135 if(gc_can_use_concurrent)
21137 clear_mark_array (max (heap_segment_mem (seg), lowest_address),
21138 min (heap_segment_allocated (seg), highest_address),
21139 FALSE); // read_only segments need the mark clear
21142 seg_clear_mark_bits (seg);
21143 #endif //MARK_ARRAY
21145 #endif //BACKGROUND_GC
21147 seg = heap_segment_next (seg);
21150 #endif // FEATURE_BASICFREEZE
21152 #ifdef FEATURE_LOH_COMPACTION
21154 BOOL gc_heap::loh_pinned_plug_que_empty_p()
21156 return (loh_pinned_queue_bos == loh_pinned_queue_tos);
21159 void gc_heap::loh_set_allocator_next_pin()
21161 if (!(loh_pinned_plug_que_empty_p()))
21163 mark* oldest_entry = loh_oldest_pin();
21164 uint8_t* plug = pinned_plug (oldest_entry);
21165 generation* gen = large_object_generation;
21166 if ((plug >= generation_allocation_pointer (gen)) &&
21167 (plug < generation_allocation_limit (gen)))
21169 generation_allocation_limit (gen) = pinned_plug (oldest_entry);
21172 assert (!((plug < generation_allocation_pointer (gen)) &&
21173 (plug >= heap_segment_mem (generation_allocation_segment (gen)))));
21177 size_t gc_heap::loh_deque_pinned_plug ()
21179 size_t m = loh_pinned_queue_bos;
21180 loh_pinned_queue_bos++;
21185 mark* gc_heap::loh_pinned_plug_of (size_t bos)
21187 return &loh_pinned_queue[bos];
21191 mark* gc_heap::loh_oldest_pin()
21193 return loh_pinned_plug_of (loh_pinned_queue_bos);
21196 // If we can't grow the queue, then don't compact.
21197 BOOL gc_heap::loh_enque_pinned_plug (uint8_t* plug, size_t len)
21199 assert(len >= Align(min_obj_size, get_alignment_constant (FALSE)));
21201 if (loh_pinned_queue_length <= loh_pinned_queue_tos)
21203 if (!grow_mark_stack (loh_pinned_queue, loh_pinned_queue_length, LOH_PIN_QUEUE_LENGTH))
21208 dprintf (3, (" P: %Ix(%Id)", plug, len));
21209 mark& m = loh_pinned_queue[loh_pinned_queue_tos];
21212 loh_pinned_queue_tos++;
21213 loh_set_allocator_next_pin();
21218 BOOL gc_heap::loh_size_fit_p (size_t size, uint8_t* alloc_pointer, uint8_t* alloc_limit)
21220 dprintf (1235, ("trying to fit %Id(%Id) between %Ix and %Ix (%Id)",
21222 (2* AlignQword (loh_padding_obj_size) + size),
21225 (alloc_limit - alloc_pointer)));
21227 return ((alloc_pointer + 2* AlignQword (loh_padding_obj_size) + size) <= alloc_limit);
21230 uint8_t* gc_heap::loh_allocate_in_condemned (uint8_t* old_loc, size_t size)
21232 UNREFERENCED_PARAMETER(old_loc);
21234 generation* gen = large_object_generation;
21235 dprintf (1235, ("E: p:%Ix, l:%Ix, s: %Id",
21236 generation_allocation_pointer (gen),
21237 generation_allocation_limit (gen),
21242 heap_segment* seg = generation_allocation_segment (gen);
21243 if (!(loh_size_fit_p (size, generation_allocation_pointer (gen), generation_allocation_limit (gen))))
21245 if ((!(loh_pinned_plug_que_empty_p()) &&
21246 (generation_allocation_limit (gen) ==
21247 pinned_plug (loh_oldest_pin()))))
21249 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
21250 size_t len = pinned_len (m);
21251 uint8_t* plug = pinned_plug (m);
21252 dprintf (1235, ("AIC: %Ix->%Ix(%Id)", generation_allocation_pointer (gen), plug, plug - generation_allocation_pointer (gen)));
21253 pinned_len (m) = plug - generation_allocation_pointer (gen);
21254 generation_allocation_pointer (gen) = plug + len;
21256 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
21257 loh_set_allocator_next_pin();
21258 dprintf (1235, ("s: p: %Ix, l: %Ix (%Id)",
21259 generation_allocation_pointer (gen),
21260 generation_allocation_limit (gen),
21261 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21266 if (generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))
21268 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
21269 dprintf (1235, ("l->pa(%Ix)", generation_allocation_limit (gen)));
21273 if (heap_segment_plan_allocated (seg) != heap_segment_committed (seg))
21275 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
21276 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
21277 dprintf (1235, ("l->c(%Ix)", generation_allocation_limit (gen)));
21281 if (loh_size_fit_p (size, generation_allocation_pointer (gen), heap_segment_reserved (seg)) &&
21282 (grow_heap_segment (seg, (generation_allocation_pointer (gen) + size + 2* AlignQword (loh_padding_obj_size)))))
21284 dprintf (1235, ("growing seg from %Ix to %Ix\n", heap_segment_committed (seg),
21285 (generation_allocation_pointer (gen) + size)));
21287 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
21288 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
21290 dprintf (1235, ("g: p: %Ix, l: %Ix (%Id)",
21291 generation_allocation_pointer (gen),
21292 generation_allocation_limit (gen),
21293 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21297 heap_segment* next_seg = heap_segment_next (seg);
21298 assert (generation_allocation_pointer (gen)>=
21299 heap_segment_mem (seg));
21300 // Verify that all pinned plugs for this segment are consumed
21301 if (!loh_pinned_plug_que_empty_p() &&
21302 ((pinned_plug (loh_oldest_pin()) <
21303 heap_segment_allocated (seg)) &&
21304 (pinned_plug (loh_oldest_pin()) >=
21305 generation_allocation_pointer (gen))))
21307 LOG((LF_GC, LL_INFO10, "remaining pinned plug %Ix while leaving segment on allocation",
21308 pinned_plug (loh_oldest_pin())));
21309 dprintf (1236, ("queue empty: %d", loh_pinned_plug_que_empty_p()));
21312 assert (generation_allocation_pointer (gen)>=
21313 heap_segment_mem (seg));
21314 assert (generation_allocation_pointer (gen)<=
21315 heap_segment_committed (seg));
21316 heap_segment_plan_allocated (seg) = generation_allocation_pointer (gen);
21320 // for LOH do we want to try starting from the first LOH every time though?
21321 generation_allocation_segment (gen) = next_seg;
21322 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
21323 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
21325 dprintf (1235, ("n: p: %Ix, l: %Ix (%Id)",
21326 generation_allocation_pointer (gen),
21327 generation_allocation_limit (gen),
21328 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21332 dprintf (1, ("We ran out of space compacting, shouldn't happen"));
21338 loh_set_allocator_next_pin();
21340 dprintf (1235, ("r: p: %Ix, l: %Ix (%Id)",
21341 generation_allocation_pointer (gen),
21342 generation_allocation_limit (gen),
21343 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21350 assert (generation_allocation_pointer (gen)>=
21351 heap_segment_mem (generation_allocation_segment (gen)));
21352 uint8_t* result = generation_allocation_pointer (gen);
21353 size_t loh_pad = AlignQword (loh_padding_obj_size);
21355 generation_allocation_pointer (gen) += size + loh_pad;
21356 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
21358 dprintf (1235, ("p: %Ix, l: %Ix (%Id)",
21359 generation_allocation_pointer (gen),
21360 generation_allocation_limit (gen),
21361 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
21363 assert (result + loh_pad);
21364 return result + loh_pad;
21368 BOOL gc_heap::should_compact_loh()
21370 // If hard limit is specified GC will automatically decide if LOH needs to be compacted.
21371 return (heap_hard_limit || loh_compaction_always_p || (loh_compaction_mode != loh_compaction_default));
21375 void gc_heap::check_loh_compact_mode (BOOL all_heaps_compacted_p)
21377 if (settings.loh_compaction && (loh_compaction_mode == loh_compaction_once))
21379 if (all_heaps_compacted_p)
21381 // If the compaction mode says to compact once and we are going to compact LOH,
21382 // we need to revert it back to no compaction.
21383 loh_compaction_mode = loh_compaction_default;
21388 BOOL gc_heap::plan_loh()
21390 if (!loh_pinned_queue)
21392 loh_pinned_queue = new (nothrow) (mark [LOH_PIN_QUEUE_LENGTH]);
21393 if (!loh_pinned_queue)
21395 dprintf (1, ("Cannot allocate the LOH pinned queue (%Id bytes), no compaction",
21396 LOH_PIN_QUEUE_LENGTH * sizeof (mark)));
21400 loh_pinned_queue_length = LOH_PIN_QUEUE_LENGTH;
21403 if (heap_number == 0)
21404 loh_pinned_queue_decay = LOH_PIN_DECAY;
21406 loh_pinned_queue_tos = 0;
21407 loh_pinned_queue_bos = 0;
21409 generation* gen = large_object_generation;
21410 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
21411 PREFIX_ASSUME(start_seg != NULL);
21412 heap_segment* seg = start_seg;
21413 uint8_t* o = generation_allocation_start (gen);
21415 dprintf (1235, ("before GC LOH size: %Id, free list: %Id, free obj: %Id\n",
21416 generation_size (max_generation + 1),
21417 generation_free_list_space (gen),
21418 generation_free_obj_space (gen)));
21422 heap_segment_plan_allocated (seg) = heap_segment_mem (seg);
21423 seg = heap_segment_next (seg);
21428 //Skip the generation gap object
21429 o = o + AlignQword (size (o));
21430 // We don't need to ever realloc gen3 start so don't touch it.
21431 heap_segment_plan_allocated (seg) = o;
21432 generation_allocation_pointer (gen) = o;
21433 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
21434 generation_allocation_segment (gen) = start_seg;
21436 uint8_t* free_space_start = o;
21437 uint8_t* free_space_end = o;
21438 uint8_t* new_address = 0;
21442 if (o >= heap_segment_allocated (seg))
21444 seg = heap_segment_next (seg);
21450 o = heap_segment_mem (seg);
21455 free_space_end = o;
21456 size_t size = AlignQword (size (o));
21457 dprintf (1235, ("%Ix(%Id) M", o, size));
21461 // We don't clear the pinned bit yet so we can check in
21462 // compact phase how big a free object we should allocate
21463 // in front of the pinned object. We use the reloc address
21464 // field to store this.
21465 if (!loh_enque_pinned_plug (o, size))
21473 new_address = loh_allocate_in_condemned (o, size);
21476 loh_set_node_relocation_distance (o, (new_address - o));
21477 dprintf (1235, ("lobj %Ix-%Ix -> %Ix-%Ix (%Id)", o, (o + size), new_address, (new_address + size), (new_address - o)));
21480 free_space_start = o;
21481 if (o < heap_segment_allocated (seg))
21483 assert (!marked (o));
21488 while (o < heap_segment_allocated (seg) && !marked (o))
21490 dprintf (1235, ("%Ix(%Id) F (%d)", o, AlignQword (size (o)), ((method_table (o) == g_gc_pFreeObjectMethodTable) ? 1 : 0)));
21491 o = o + AlignQword (size (o));
21496 while (!loh_pinned_plug_que_empty_p())
21498 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
21499 size_t len = pinned_len (m);
21500 uint8_t* plug = pinned_plug (m);
21502 // detect pinned block in different segment (later) than
21503 // allocation segment
21504 heap_segment* nseg = heap_segment_rw (generation_allocation_segment (gen));
21506 while ((plug < generation_allocation_pointer (gen)) ||
21507 (plug >= heap_segment_allocated (nseg)))
21509 assert ((plug < heap_segment_mem (nseg)) ||
21510 (plug > heap_segment_reserved (nseg)));
21511 //adjust the end of the segment to be the end of the plug
21512 assert (generation_allocation_pointer (gen)>=
21513 heap_segment_mem (nseg));
21514 assert (generation_allocation_pointer (gen)<=
21515 heap_segment_committed (nseg));
21517 heap_segment_plan_allocated (nseg) =
21518 generation_allocation_pointer (gen);
21519 //switch allocation segment
21520 nseg = heap_segment_next_rw (nseg);
21521 generation_allocation_segment (gen) = nseg;
21522 //reset the allocation pointer and limits
21523 generation_allocation_pointer (gen) =
21524 heap_segment_mem (nseg);
21527 dprintf (1235, ("SP: %Ix->%Ix(%Id)", generation_allocation_pointer (gen), plug, plug - generation_allocation_pointer (gen)));
21528 pinned_len (m) = plug - generation_allocation_pointer (gen);
21529 generation_allocation_pointer (gen) = plug + len;
21532 heap_segment_plan_allocated (generation_allocation_segment (gen)) = generation_allocation_pointer (gen);
21533 generation_allocation_pointer (gen) = 0;
21534 generation_allocation_limit (gen) = 0;
21539 void gc_heap::compact_loh()
21541 assert (should_compact_loh());
21543 generation* gen = large_object_generation;
21544 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
21545 PREFIX_ASSUME(start_seg != NULL);
21546 heap_segment* seg = start_seg;
21547 heap_segment* prev_seg = 0;
21548 uint8_t* o = generation_allocation_start (gen);
21550 //Skip the generation gap object
21551 o = o + AlignQword (size (o));
21552 // We don't need to ever realloc gen3 start so don't touch it.
21553 uint8_t* free_space_start = o;
21554 uint8_t* free_space_end = o;
21555 generation_allocator (gen)->clear();
21556 generation_free_list_space (gen) = 0;
21557 generation_free_obj_space (gen) = 0;
21559 loh_pinned_queue_bos = 0;
21563 if (o >= heap_segment_allocated (seg))
21565 heap_segment* next_seg = heap_segment_next (seg);
21567 if ((heap_segment_plan_allocated (seg) == heap_segment_mem (seg)) &&
21568 (seg != start_seg) && !heap_segment_read_only_p (seg))
21570 dprintf (3, ("Preparing empty large segment %Ix", (size_t)seg));
21572 heap_segment_next (prev_seg) = next_seg;
21573 heap_segment_next (seg) = freeable_large_heap_segment;
21574 freeable_large_heap_segment = seg;
21578 if (!heap_segment_read_only_p (seg))
21580 // We grew the segment to accommodate allocations.
21581 if (heap_segment_plan_allocated (seg) > heap_segment_allocated (seg))
21583 if ((heap_segment_plan_allocated (seg) - plug_skew) > heap_segment_used (seg))
21585 heap_segment_used (seg) = heap_segment_plan_allocated (seg) - plug_skew;
21589 heap_segment_allocated (seg) = heap_segment_plan_allocated (seg);
21590 dprintf (3, ("Trimming seg to %Ix[", heap_segment_allocated (seg)));
21591 decommit_heap_segment_pages (seg, 0);
21592 dprintf (1236, ("CLOH: seg: %Ix, alloc: %Ix, used: %Ix, committed: %Ix",
21594 heap_segment_allocated (seg),
21595 heap_segment_used (seg),
21596 heap_segment_committed (seg)));
21597 //heap_segment_used (seg) = heap_segment_allocated (seg) - plug_skew;
21598 dprintf (1236, ("CLOH: used is set to %Ix", heap_segment_used (seg)));
21608 o = heap_segment_mem (seg);
21614 free_space_end = o;
21615 size_t size = AlignQword (size (o));
21618 uint8_t* reloc = o;
21623 // We are relying on the fact the pinned objects are always looked at in the same order
21624 // in plan phase and in compact phase.
21625 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
21626 uint8_t* plug = pinned_plug (m);
21627 assert (plug == o);
21629 loh_pad = pinned_len (m);
21634 loh_pad = AlignQword (loh_padding_obj_size);
21636 reloc += loh_node_relocation_distance (o);
21637 gcmemcopy (reloc, o, size, TRUE);
21640 thread_gap ((reloc - loh_pad), loh_pad, gen);
21643 free_space_start = o;
21644 if (o < heap_segment_allocated (seg))
21646 assert (!marked (o));
21651 while (o < heap_segment_allocated (seg) && !marked (o))
21653 o = o + AlignQword (size (o));
21658 assert (loh_pinned_plug_que_empty_p());
21660 dprintf (1235, ("after GC LOH size: %Id, free list: %Id, free obj: %Id\n\n",
21661 generation_size (max_generation + 1),
21662 generation_free_list_space (gen),
21663 generation_free_obj_space (gen)));
21666 void gc_heap::relocate_in_loh_compact()
21668 generation* gen = large_object_generation;
21669 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
21670 uint8_t* o = generation_allocation_start (gen);
21672 //Skip the generation gap object
21673 o = o + AlignQword (size (o));
21675 relocate_args args;
21677 args.high = gc_high;
21678 args.last_plug = 0;
21682 if (o >= heap_segment_allocated (seg))
21684 seg = heap_segment_next (seg);
21690 o = heap_segment_mem (seg);
21695 size_t size = AlignQword (size (o));
21697 check_class_object_demotion (o);
21698 if (contain_pointers (o))
21700 go_through_object_nostart (method_table (o), o, size(o), pval,
21702 reloc_survivor_helper (pval);
21707 if (o < heap_segment_allocated (seg))
21709 assert (!marked (o));
21714 while (o < heap_segment_allocated (seg) && !marked (o))
21716 o = o + AlignQword (size (o));
21721 dprintf (1235, ("after GC LOH size: %Id, free list: %Id, free obj: %Id\n\n",
21722 generation_size (max_generation + 1),
21723 generation_free_list_space (gen),
21724 generation_free_obj_space (gen)));
21727 void gc_heap::walk_relocation_for_loh (void* profiling_context, record_surv_fn fn)
21729 generation* gen = large_object_generation;
21730 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
21731 uint8_t* o = generation_allocation_start (gen);
21733 //Skip the generation gap object
21734 o = o + AlignQword (size (o));
21738 if (o >= heap_segment_allocated (seg))
21740 seg = heap_segment_next (seg);
21746 o = heap_segment_mem (seg);
21751 size_t size = AlignQword (size (o));
21753 ptrdiff_t reloc = loh_node_relocation_distance (o);
21755 STRESS_LOG_PLUG_MOVE(o, (o + size), -reloc);
21757 fn (o, (o + size), reloc, profiling_context, !!settings.compaction, false);
21760 if (o < heap_segment_allocated (seg))
21762 assert (!marked (o));
21767 while (o < heap_segment_allocated (seg) && !marked (o))
21769 o = o + AlignQword (size (o));
21775 BOOL gc_heap::loh_object_p (uint8_t* o)
21777 #ifdef MULTIPLE_HEAPS
21778 gc_heap* hp = gc_heap::g_heaps [0];
21779 int brick_entry = hp->brick_table[hp->brick_of (o)];
21780 #else //MULTIPLE_HEAPS
21781 int brick_entry = brick_table[brick_of (o)];
21782 #endif //MULTIPLE_HEAPS
21784 return (brick_entry == 0);
21786 #endif //FEATURE_LOH_COMPACTION
21788 void gc_heap::convert_to_pinned_plug (BOOL& last_npinned_plug_p,
21789 BOOL& last_pinned_plug_p,
21790 BOOL& pinned_plug_p,
21792 size_t& artificial_pinned_size)
21794 last_npinned_plug_p = FALSE;
21795 last_pinned_plug_p = TRUE;
21796 pinned_plug_p = TRUE;
21797 artificial_pinned_size = ps;
21800 // Because we have the artificial pinning, we can't guarantee that pinned and npinned
21801 // plugs are always interleaved.
21802 void gc_heap::store_plug_gap_info (uint8_t* plug_start,
21804 BOOL& last_npinned_plug_p,
21805 BOOL& last_pinned_plug_p,
21806 uint8_t*& last_pinned_plug,
21807 BOOL& pinned_plug_p,
21808 uint8_t* last_object_in_last_plug,
21809 BOOL& merge_with_last_pin_p,
21810 // this is only for verification purpose
21811 size_t last_plug_len)
21813 UNREFERENCED_PARAMETER(last_plug_len);
21815 if (!last_npinned_plug_p && !last_pinned_plug_p)
21817 //dprintf (3, ("last full plug end: %Ix, full plug start: %Ix", plug_end, plug_start));
21818 dprintf (3, ("Free: %Ix", (plug_start - plug_end)));
21819 assert ((plug_start == plug_end) || ((size_t)(plug_start - plug_end) >= Align (min_obj_size)));
21820 set_gap_size (plug_start, plug_start - plug_end);
21823 if (pinned (plug_start))
21825 BOOL save_pre_plug_info_p = FALSE;
21827 if (last_npinned_plug_p || last_pinned_plug_p)
21829 //if (last_plug_len == Align (min_obj_size))
21831 // dprintf (3, ("debugging only - last npinned plug is min, check to see if it's correct"));
21832 // GCToOSInterface::DebugBreak();
21834 save_pre_plug_info_p = TRUE;
21837 pinned_plug_p = TRUE;
21838 last_npinned_plug_p = FALSE;
21840 if (last_pinned_plug_p)
21842 dprintf (3, ("last plug %Ix was also pinned, should merge", last_pinned_plug));
21843 merge_with_last_pin_p = TRUE;
21847 last_pinned_plug_p = TRUE;
21848 last_pinned_plug = plug_start;
21850 enque_pinned_plug (last_pinned_plug, save_pre_plug_info_p, last_object_in_last_plug);
21852 if (save_pre_plug_info_p)
21854 set_gap_size (plug_start, sizeof (gap_reloc_pair));
21860 if (last_pinned_plug_p)
21862 //if (Align (last_plug_len) < min_pre_pin_obj_size)
21864 // dprintf (3, ("debugging only - last pinned plug is min, check to see if it's correct"));
21865 // GCToOSInterface::DebugBreak();
21868 save_post_plug_info (last_pinned_plug, last_object_in_last_plug, plug_start);
21869 set_gap_size (plug_start, sizeof (gap_reloc_pair));
21871 verify_pins_with_post_plug_info("after saving post plug info");
21873 last_npinned_plug_p = TRUE;
21874 last_pinned_plug_p = FALSE;
21878 void gc_heap::record_interesting_data_point (interesting_data_point idp)
21880 #ifdef GC_CONFIG_DRIVEN
21881 (interesting_data_per_gc[idp])++;
21883 UNREFERENCED_PARAMETER(idp);
21884 #endif //GC_CONFIG_DRIVEN
21888 #pragma warning(push)
21889 #pragma warning(disable:21000) // Suppress PREFast warning about overly large function
21891 void gc_heap::plan_phase (int condemned_gen_number)
21893 size_t old_gen2_allocated = 0;
21894 size_t old_gen2_size = 0;
21896 if (condemned_gen_number == (max_generation - 1))
21898 old_gen2_allocated = generation_free_list_allocated (generation_of (max_generation));
21899 old_gen2_size = generation_size (max_generation);
21902 assert (settings.concurrent == FALSE);
21904 // %type% category = quote (plan);
21908 start = GetCycleCount32();
21911 dprintf (2,("---- Plan Phase ---- Condemned generation %d, promotion: %d",
21912 condemned_gen_number, settings.promotion ? 1 : 0));
21914 generation* condemned_gen1 = generation_of (condemned_gen_number);
21917 BOOL use_mark_list = FALSE;
21918 uint8_t** mark_list_next = &mark_list[0];
21919 #ifdef GC_CONFIG_DRIVEN
21920 dprintf (3, ("total number of marked objects: %Id (%Id)",
21921 (mark_list_index - &mark_list[0]), ((mark_list_end - &mark_list[0]))));
21923 if (mark_list_index >= (mark_list_end + 1))
21924 mark_list_index = mark_list_end + 1;
21926 dprintf (3, ("mark_list length: %Id",
21927 (mark_list_index - &mark_list[0])));
21928 #endif //GC_CONFIG_DRIVEN
21930 if ((condemned_gen_number < max_generation) &&
21931 (mark_list_index <= mark_list_end)
21932 #ifdef BACKGROUND_GC
21933 && (!recursive_gc_sync::background_running_p())
21934 #endif //BACKGROUND_GC
21937 #ifndef MULTIPLE_HEAPS
21938 _sort (&mark_list[0], mark_list_index-1, 0);
21939 //printf ("using mark list at GC #%d", dd_collection_count (dynamic_data_of (0)));
21940 //verify_qsort_array (&mark_list[0], mark_list_index-1);
21941 #endif //!MULTIPLE_HEAPS
21942 use_mark_list = TRUE;
21943 get_gc_data_per_heap()->set_mechanism_bit (gc_mark_list_bit);
21947 dprintf (3, ("mark_list not used"));
21952 #ifdef FEATURE_BASICFREEZE
21953 if ((generation_start_segment (condemned_gen1) != ephemeral_heap_segment) &&
21954 ro_segments_in_range)
21956 sweep_ro_segments (generation_start_segment (condemned_gen1));
21958 #endif // FEATURE_BASICFREEZE
21960 #ifndef MULTIPLE_HEAPS
21961 if (shigh != (uint8_t*)0)
21963 heap_segment* seg = heap_segment_rw (generation_start_segment (condemned_gen1));
21965 PREFIX_ASSUME(seg != NULL);
21967 heap_segment* fseg = seg;
21970 if (slow > heap_segment_mem (seg) &&
21971 slow < heap_segment_reserved (seg))
21975 uint8_t* o = generation_allocation_start (condemned_gen1) +
21976 Align (size (generation_allocation_start (condemned_gen1)));
21979 assert ((slow - o) >= (int)Align (min_obj_size));
21980 #ifdef BACKGROUND_GC
21981 if (current_c_gc_state == c_gc_state_marking)
21983 bgc_clear_batch_mark_array_bits (o, slow);
21985 #endif //BACKGROUND_GC
21986 make_unused_array (o, slow - o);
21991 assert (condemned_gen_number == max_generation);
21992 make_unused_array (heap_segment_mem (seg),
21993 slow - heap_segment_mem (seg));
21996 if (in_range_for_segment (shigh, seg))
21998 #ifdef BACKGROUND_GC
21999 if (current_c_gc_state == c_gc_state_marking)
22001 bgc_clear_batch_mark_array_bits ((shigh + Align (size (shigh))), heap_segment_allocated (seg));
22003 #endif //BACKGROUND_GC
22004 heap_segment_allocated (seg) = shigh + Align (size (shigh));
22006 // test if the segment is in the range of [slow, shigh]
22007 if (!((heap_segment_reserved (seg) >= slow) &&
22008 (heap_segment_mem (seg) <= shigh)))
22010 // shorten it to minimum
22011 heap_segment_allocated (seg) = heap_segment_mem (seg);
22013 seg = heap_segment_next_rw (seg);
22018 heap_segment* seg = heap_segment_rw (generation_start_segment (condemned_gen1));
22020 PREFIX_ASSUME(seg != NULL);
22022 heap_segment* sseg = seg;
22025 // shorten it to minimum
22028 // no survivors make all generations look empty
22029 uint8_t* o = generation_allocation_start (condemned_gen1) +
22030 Align (size (generation_allocation_start (condemned_gen1)));
22031 #ifdef BACKGROUND_GC
22032 if (current_c_gc_state == c_gc_state_marking)
22034 bgc_clear_batch_mark_array_bits (o, heap_segment_allocated (seg));
22036 #endif //BACKGROUND_GC
22037 heap_segment_allocated (seg) = o;
22041 assert (condemned_gen_number == max_generation);
22042 #ifdef BACKGROUND_GC
22043 if (current_c_gc_state == c_gc_state_marking)
22045 bgc_clear_batch_mark_array_bits (heap_segment_mem (seg), heap_segment_allocated (seg));
22047 #endif //BACKGROUND_GC
22048 heap_segment_allocated (seg) = heap_segment_mem (seg);
22050 seg = heap_segment_next_rw (seg);
22054 #endif //MULTIPLE_HEAPS
22056 heap_segment* seg1 = heap_segment_rw (generation_start_segment (condemned_gen1));
22058 PREFIX_ASSUME(seg1 != NULL);
22060 uint8_t* end = heap_segment_allocated (seg1);
22061 uint8_t* first_condemned_address = generation_allocation_start (condemned_gen1);
22062 uint8_t* x = first_condemned_address;
22064 assert (!marked (x));
22065 uint8_t* plug_end = x;
22067 size_t sequence_number = 0;
22068 uint8_t* last_node = 0;
22069 size_t current_brick = brick_of (x);
22070 BOOL allocate_in_condemned = ((condemned_gen_number == max_generation)||
22071 (settings.promotion == FALSE));
22072 int active_old_gen_number = condemned_gen_number;
22073 int active_new_gen_number = (allocate_in_condemned ? condemned_gen_number:
22074 (1 + condemned_gen_number));
22075 generation* older_gen = 0;
22076 generation* consing_gen = condemned_gen1;
22077 alloc_list r_free_list [MAX_BUCKET_COUNT];
22079 size_t r_free_list_space = 0;
22080 size_t r_free_obj_space = 0;
22081 size_t r_older_gen_free_list_allocated = 0;
22082 size_t r_older_gen_condemned_allocated = 0;
22083 size_t r_older_gen_end_seg_allocated = 0;
22084 uint8_t* r_allocation_pointer = 0;
22085 uint8_t* r_allocation_limit = 0;
22086 uint8_t* r_allocation_start_region = 0;
22087 heap_segment* r_allocation_segment = 0;
22088 #ifdef FREE_USAGE_STATS
22089 size_t r_older_gen_free_space[NUM_GEN_POWER2];
22090 #endif //FREE_USAGE_STATS
22092 if ((condemned_gen_number < max_generation))
22094 older_gen = generation_of (min (max_generation, 1 + condemned_gen_number));
22095 generation_allocator (older_gen)->copy_to_alloc_list (r_free_list);
22097 r_free_list_space = generation_free_list_space (older_gen);
22098 r_free_obj_space = generation_free_obj_space (older_gen);
22099 #ifdef FREE_USAGE_STATS
22100 memcpy (r_older_gen_free_space, older_gen->gen_free_spaces, sizeof (r_older_gen_free_space));
22101 #endif //FREE_USAGE_STATS
22102 generation_allocate_end_seg_p (older_gen) = FALSE;
22103 r_older_gen_free_list_allocated = generation_free_list_allocated (older_gen);
22104 r_older_gen_condemned_allocated = generation_condemned_allocated (older_gen);
22105 r_older_gen_end_seg_allocated = generation_end_seg_allocated (older_gen);
22106 r_allocation_limit = generation_allocation_limit (older_gen);
22107 r_allocation_pointer = generation_allocation_pointer (older_gen);
22108 r_allocation_start_region = generation_allocation_context_start_region (older_gen);
22109 r_allocation_segment = generation_allocation_segment (older_gen);
22110 heap_segment* start_seg = heap_segment_rw (generation_start_segment (older_gen));
22112 PREFIX_ASSUME(start_seg != NULL);
22114 if (start_seg != ephemeral_heap_segment)
22116 assert (condemned_gen_number == (max_generation - 1));
22117 while (start_seg && (start_seg != ephemeral_heap_segment))
22119 assert (heap_segment_allocated (start_seg) >=
22120 heap_segment_mem (start_seg));
22121 assert (heap_segment_allocated (start_seg) <=
22122 heap_segment_reserved (start_seg));
22123 heap_segment_plan_allocated (start_seg) =
22124 heap_segment_allocated (start_seg);
22125 start_seg = heap_segment_next_rw (start_seg);
22130 //reset all of the segment allocated sizes
22132 heap_segment* seg2 = heap_segment_rw (generation_start_segment (condemned_gen1));
22134 PREFIX_ASSUME(seg2 != NULL);
22138 heap_segment_plan_allocated (seg2) =
22139 heap_segment_mem (seg2);
22140 seg2 = heap_segment_next_rw (seg2);
22143 int condemned_gn = condemned_gen_number;
22145 int bottom_gen = 0;
22146 init_free_and_plug();
22148 while (condemned_gn >= bottom_gen)
22150 generation* condemned_gen2 = generation_of (condemned_gn);
22151 generation_allocator (condemned_gen2)->clear();
22152 generation_free_list_space (condemned_gen2) = 0;
22153 generation_free_obj_space (condemned_gen2) = 0;
22154 generation_allocation_size (condemned_gen2) = 0;
22155 generation_condemned_allocated (condemned_gen2) = 0;
22156 generation_pinned_allocated (condemned_gen2) = 0;
22157 generation_free_list_allocated(condemned_gen2) = 0;
22158 generation_end_seg_allocated (condemned_gen2) = 0;
22159 generation_pinned_allocation_sweep_size (condemned_gen2) = 0;
22160 generation_pinned_allocation_compact_size (condemned_gen2) = 0;
22161 #ifdef FREE_USAGE_STATS
22162 generation_pinned_free_obj_space (condemned_gen2) = 0;
22163 generation_allocated_in_pinned_free (condemned_gen2) = 0;
22164 generation_allocated_since_last_pin (condemned_gen2) = 0;
22165 #endif //FREE_USAGE_STATS
22166 generation_plan_allocation_start (condemned_gen2) = 0;
22167 generation_allocation_segment (condemned_gen2) =
22168 heap_segment_rw (generation_start_segment (condemned_gen2));
22170 PREFIX_ASSUME(generation_allocation_segment(condemned_gen2) != NULL);
22172 if (generation_start_segment (condemned_gen2) != ephemeral_heap_segment)
22174 generation_allocation_pointer (condemned_gen2) =
22175 heap_segment_mem (generation_allocation_segment (condemned_gen2));
22179 generation_allocation_pointer (condemned_gen2) = generation_allocation_start (condemned_gen2);
22182 generation_allocation_limit (condemned_gen2) = generation_allocation_pointer (condemned_gen2);
22183 generation_allocation_context_start_region (condemned_gen2) = generation_allocation_pointer (condemned_gen2);
22188 BOOL allocate_first_generation_start = FALSE;
22190 if (allocate_in_condemned)
22192 allocate_first_generation_start = TRUE;
22195 dprintf(3,( " From %Ix to %Ix", (size_t)x, (size_t)end));
22197 demotion_low = MAX_PTR;
22198 demotion_high = heap_segment_allocated (ephemeral_heap_segment);
22200 // If we are doing a gen1 only because of cards, it means we should not demote any pinned plugs
22201 // from gen1. They should get promoted to gen2.
22202 demote_gen1_p = !(settings.promotion &&
22203 (settings.condemned_generation == (max_generation - 1)) &&
22204 gen_to_condemn_reasons.is_only_condition (gen_low_card_p));
22206 total_ephemeral_size = 0;
22208 print_free_and_plug ("BP");
22210 for (int gen_idx = 0; gen_idx <= max_generation; gen_idx++)
22212 generation* temp_gen = generation_of (gen_idx);
22214 dprintf (2, ("gen%d start %Ix, plan start %Ix",
22216 generation_allocation_start (temp_gen),
22217 generation_plan_allocation_start (temp_gen)));
22220 BOOL fire_pinned_plug_events_p = EVENT_ENABLED(PinPlugAtGCTime);
22221 size_t last_plug_len = 0;
22228 assert (heap_segment_allocated (seg1) == end);
22229 heap_segment_allocated (seg1) = plug_end;
22231 current_brick = update_brick_table (tree, current_brick, x, plug_end);
22232 dprintf (3, ("end of seg: new tree, sequence# 0"));
22233 sequence_number = 0;
22236 if (heap_segment_next_rw (seg1))
22238 seg1 = heap_segment_next_rw (seg1);
22239 end = heap_segment_allocated (seg1);
22240 plug_end = x = heap_segment_mem (seg1);
22241 current_brick = brick_of (x);
22242 dprintf(3,( " From %Ix to %Ix", (size_t)x, (size_t)end));
22251 BOOL last_npinned_plug_p = FALSE;
22252 BOOL last_pinned_plug_p = FALSE;
22254 // last_pinned_plug is the beginning of the last pinned plug. If we merge a plug into a pinned
22255 // plug we do not change the value of last_pinned_plug. This happens with artificially pinned plugs -
22256 // it can be merged with a previous pinned plug and a pinned plug after it can be merged with it.
22257 uint8_t* last_pinned_plug = 0;
22258 size_t num_pinned_plugs_in_plug = 0;
22260 uint8_t* last_object_in_plug = 0;
22262 while ((x < end) && marked (x))
22264 uint8_t* plug_start = x;
22265 uint8_t* saved_plug_end = plug_end;
22266 BOOL pinned_plug_p = FALSE;
22267 BOOL npin_before_pin_p = FALSE;
22268 BOOL saved_last_npinned_plug_p = last_npinned_plug_p;
22269 uint8_t* saved_last_object_in_plug = last_object_in_plug;
22270 BOOL merge_with_last_pin_p = FALSE;
22272 size_t added_pinning_size = 0;
22273 size_t artificial_pinned_size = 0;
22275 store_plug_gap_info (plug_start, plug_end, last_npinned_plug_p, last_pinned_plug_p,
22276 last_pinned_plug, pinned_plug_p, last_object_in_plug,
22277 merge_with_last_pin_p, last_plug_len);
22279 #ifdef FEATURE_STRUCTALIGN
22280 int requiredAlignment = ((CObjectHeader*)plug_start)->GetRequiredAlignment();
22281 size_t alignmentOffset = OBJECT_ALIGNMENT_OFFSET;
22282 #endif // FEATURE_STRUCTALIGN
22286 while ((xl < end) && marked (xl) && (pinned (xl) == pinned_plug_p))
22293 #ifdef FEATURE_STRUCTALIGN
22296 int obj_requiredAlignment = ((CObjectHeader*)xl)->GetRequiredAlignment();
22297 if (obj_requiredAlignment > requiredAlignment)
22299 requiredAlignment = obj_requiredAlignment;
22300 alignmentOffset = xl - plug_start + OBJECT_ALIGNMENT_OFFSET;
22303 #endif // FEATURE_STRUCTALIGN
22307 dprintf(4, ("+%Ix+", (size_t)xl));
22308 assert ((size (xl) > 0));
22309 assert ((size (xl) <= loh_size_threshold));
22311 last_object_in_plug = xl;
22313 xl = xl + Align (size (xl));
22317 BOOL next_object_marked_p = ((xl < end) && marked (xl));
22321 // If it is pinned we need to extend to the next marked object as we can't use part of
22322 // a pinned object to make the artificial gap (unless the last 3 ptr sized words are all
22323 // references but for now I am just using the next non pinned object for that).
22324 if (next_object_marked_p)
22327 last_object_in_plug = xl;
22328 size_t extra_size = Align (size (xl));
22329 xl = xl + extra_size;
22330 added_pinning_size = extra_size;
22335 if (next_object_marked_p)
22336 npin_before_pin_p = TRUE;
22339 assert (xl <= end);
22342 dprintf (3, ( "%Ix[", (size_t)x));
22344 size_t ps = plug_end - plug_start;
22345 last_plug_len = ps;
22346 dprintf (3, ( "%Ix[(%Ix)", (size_t)x, ps));
22347 uint8_t* new_address = 0;
22349 if (!pinned_plug_p)
22351 if (allocate_in_condemned &&
22352 (settings.condemned_generation == max_generation) &&
22353 (ps > OS_PAGE_SIZE))
22355 ptrdiff_t reloc = plug_start - generation_allocation_pointer (consing_gen);
22356 //reloc should >=0 except when we relocate
22357 //across segments and the dest seg is higher then the src
22359 if ((ps > (8*OS_PAGE_SIZE)) &&
22361 ((size_t)reloc < (ps/16)))
22363 dprintf (3, ("Pinning %Ix; reloc would have been: %Ix",
22364 (size_t)plug_start, reloc));
22365 // The last plug couldn't have been a npinned plug or it would have
22366 // included this plug.
22367 assert (!saved_last_npinned_plug_p);
22369 if (last_pinned_plug)
22371 dprintf (3, ("artificially pinned plug merged with last pinned plug"));
22372 merge_with_last_pin_p = TRUE;
22376 enque_pinned_plug (plug_start, FALSE, 0);
22377 last_pinned_plug = plug_start;
22380 convert_to_pinned_plug (last_npinned_plug_p, last_pinned_plug_p, pinned_plug_p,
22381 ps, artificial_pinned_size);
22386 if (allocate_first_generation_start)
22388 allocate_first_generation_start = FALSE;
22389 plan_generation_start (condemned_gen1, consing_gen, plug_start);
22390 assert (generation_plan_allocation_start (condemned_gen1));
22393 if (seg1 == ephemeral_heap_segment)
22395 process_ephemeral_boundaries (plug_start, active_new_gen_number,
22396 active_old_gen_number,
22398 allocate_in_condemned);
22401 dprintf (3, ("adding %Id to gen%d surv", ps, active_old_gen_number));
22403 dynamic_data* dd_active_old = dynamic_data_of (active_old_gen_number);
22404 dd_survived_size (dd_active_old) += ps;
22406 BOOL convert_to_pinned_p = FALSE;
22408 if (!pinned_plug_p)
22410 #if defined (RESPECT_LARGE_ALIGNMENT) || defined (FEATURE_STRUCTALIGN)
22411 dd_num_npinned_plugs (dd_active_old)++;
22412 #endif //RESPECT_LARGE_ALIGNMENT || FEATURE_STRUCTALIGN
22414 add_gen_plug (active_old_gen_number, ps);
22416 if (allocate_in_condemned)
22418 verify_pins_with_post_plug_info("before aic");
22421 allocate_in_condemned_generations (consing_gen,
22423 active_old_gen_number,
22425 &convert_to_pinned_p,
22426 (npin_before_pin_p ? plug_end : 0),
22428 #endif //SHORT_PLUGS
22429 plug_start REQD_ALIGN_AND_OFFSET_ARG);
22430 verify_pins_with_post_plug_info("after aic");
22434 new_address = allocate_in_older_generation (older_gen, ps, active_old_gen_number, plug_start REQD_ALIGN_AND_OFFSET_ARG);
22436 if (new_address != 0)
22438 if (settings.condemned_generation == (max_generation - 1))
22440 dprintf (3, (" NA: %Ix-%Ix -> %Ix, %Ix (%Ix)",
22441 plug_start, plug_end,
22442 (size_t)new_address, (size_t)new_address + (plug_end - plug_start),
22443 (size_t)(plug_end - plug_start)));
22448 if (generation_allocator(older_gen)->discard_if_no_fit_p())
22450 allocate_in_condemned = TRUE;
22453 new_address = allocate_in_condemned_generations (consing_gen, ps, active_old_gen_number,
22455 &convert_to_pinned_p,
22456 (npin_before_pin_p ? plug_end : 0),
22458 #endif //SHORT_PLUGS
22459 plug_start REQD_ALIGN_AND_OFFSET_ARG);
22463 if (convert_to_pinned_p)
22465 assert (last_npinned_plug_p != FALSE);
22466 assert (last_pinned_plug_p == FALSE);
22467 convert_to_pinned_plug (last_npinned_plug_p, last_pinned_plug_p, pinned_plug_p,
22468 ps, artificial_pinned_size);
22469 enque_pinned_plug (plug_start, FALSE, 0);
22470 last_pinned_plug = plug_start;
22476 //verify that we are at then end of the ephemeral segment
22477 assert (generation_allocation_segment (consing_gen) ==
22478 ephemeral_heap_segment);
22479 //verify that we are near the end
22480 assert ((generation_allocation_pointer (consing_gen) + Align (ps)) <
22481 heap_segment_allocated (ephemeral_heap_segment));
22482 assert ((generation_allocation_pointer (consing_gen) + Align (ps)) >
22483 (heap_segment_allocated (ephemeral_heap_segment) + Align (min_obj_size)));
22487 #ifdef SIMPLE_DPRINTF
22488 dprintf (3, ("(%Ix)[%Ix->%Ix, NA: [%Ix(%Id), %Ix[: %Ix(%d)",
22489 (size_t)(node_gap_size (plug_start)),
22490 plug_start, plug_end, (size_t)new_address, (size_t)(plug_start - new_address),
22491 (size_t)new_address + ps, ps,
22492 (is_plug_padded (plug_start) ? 1 : 0)));
22493 #endif //SIMPLE_DPRINTF
22496 if (is_plug_padded (plug_start))
22498 dprintf (3, ("%Ix was padded", plug_start));
22499 dd_padding_size (dd_active_old) += Align (min_obj_size);
22501 #endif //SHORT_PLUGS
22508 if (fire_pinned_plug_events_p)
22510 FIRE_EVENT(PinPlugAtGCTime, plug_start, plug_end,
22511 (merge_with_last_pin_p ? 0 : (uint8_t*)node_gap_size (plug_start)));
22514 if (merge_with_last_pin_p)
22516 merge_with_last_pinned_plug (last_pinned_plug, ps);
22520 assert (last_pinned_plug == plug_start);
22521 set_pinned_info (plug_start, ps, consing_gen);
22524 new_address = plug_start;
22526 dprintf (3, ( "(%Ix)PP: [%Ix, %Ix[%Ix](m:%d)",
22527 (size_t)(node_gap_size (plug_start)), (size_t)plug_start,
22528 (size_t)plug_end, ps,
22529 (merge_with_last_pin_p ? 1 : 0)));
22531 dprintf (3, ("adding %Id to gen%d pinned surv", plug_end - plug_start, active_old_gen_number));
22532 dd_pinned_survived_size (dd_active_old) += plug_end - plug_start;
22533 dd_added_pinned_size (dd_active_old) += added_pinning_size;
22534 dd_artificial_pinned_survived_size (dd_active_old) += artificial_pinned_size;
22536 if (!demote_gen1_p && (active_old_gen_number == (max_generation - 1)))
22538 last_gen1_pin_end = plug_end;
22543 // detect forward allocation in the same segment
22544 assert (!((new_address > plug_start) &&
22545 (new_address < heap_segment_reserved (seg1))));
22548 if (!merge_with_last_pin_p)
22550 if (current_brick != brick_of (plug_start))
22552 current_brick = update_brick_table (tree, current_brick, plug_start, saved_plug_end);
22553 sequence_number = 0;
22557 set_node_relocation_distance (plug_start, (new_address - plug_start));
22558 if (last_node && (node_relocation_distance (last_node) ==
22559 (node_relocation_distance (plug_start) +
22560 (ptrdiff_t)node_gap_size (plug_start))))
22562 //dprintf(3,( " Lb"));
22563 dprintf (3, ("%Ix Lb", plug_start));
22564 set_node_left (plug_start);
22566 if (0 == sequence_number)
22568 dprintf (2, ("sn: 0, tree is set to %Ix", plug_start));
22572 verify_pins_with_post_plug_info("before insert node");
22574 tree = insert_node (plug_start, ++sequence_number, tree, last_node);
22575 dprintf (3, ("tree is %Ix (b: %Ix) after insert_node", tree, brick_of (tree)));
22576 last_node = plug_start;
22579 // If we detect if the last plug is pinned plug right before us, we should save this gap info
22580 if (!pinned_plug_p)
22582 if (mark_stack_tos > 0)
22584 mark& m = mark_stack_array[mark_stack_tos - 1];
22585 if (m.has_post_plug_info())
22587 uint8_t* post_plug_info_start = m.saved_post_plug_info_start;
22588 size_t* current_plug_gap_start = (size_t*)(plug_start - sizeof (plug_and_gap));
22589 if ((uint8_t*)current_plug_gap_start == post_plug_info_start)
22591 dprintf (3, ("Ginfo: %Ix, %Ix, %Ix",
22592 *current_plug_gap_start, *(current_plug_gap_start + 1),
22593 *(current_plug_gap_start + 2)));
22594 memcpy (&(m.saved_post_plug_debug), current_plug_gap_start, sizeof (gap_reloc_pair));
22601 verify_pins_with_post_plug_info("after insert node");
22605 if (num_pinned_plugs_in_plug > 1)
22607 dprintf (3, ("more than %Id pinned plugs in this plug", num_pinned_plugs_in_plug));
22614 while ((mark_list_next < mark_list_index) &&
22615 (*mark_list_next <= x))
22619 if ((mark_list_next < mark_list_index)
22620 #ifdef MULTIPLE_HEAPS
22621 && (*mark_list_next < end) //for multiple segments
22622 #endif //MULTIPLE_HEAPS
22624 x = *mark_list_next;
22632 #ifdef BACKGROUND_GC
22633 if (current_c_gc_state == c_gc_state_marking)
22635 assert (recursive_gc_sync::background_running_p());
22636 while ((xl < end) && !marked (xl))
22638 dprintf (4, ("-%Ix-", (size_t)xl));
22639 assert ((size (xl) > 0));
22640 background_object_marked (xl, TRUE);
22641 xl = xl + Align (size (xl));
22646 #endif //BACKGROUND_GC
22648 while ((xl < end) && !marked (xl))
22650 dprintf (4, ("-%Ix-", (size_t)xl));
22651 assert ((size (xl) > 0));
22652 xl = xl + Align (size (xl));
22656 assert (xl <= end);
22662 while (!pinned_plug_que_empty_p())
22664 if (settings.promotion)
22666 uint8_t* pplug = pinned_plug (oldest_pin());
22667 if (in_range_for_segment (pplug, ephemeral_heap_segment))
22669 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
22670 //allocate all of the generation gaps
22671 while (active_new_gen_number > 0)
22673 active_new_gen_number--;
22675 if (active_new_gen_number == (max_generation - 1))
22677 maxgen_pinned_compact_before_advance = generation_pinned_allocation_compact_size (generation_of (max_generation));
22678 if (!demote_gen1_p)
22679 advance_pins_for_demotion (consing_gen);
22682 generation* gen = generation_of (active_new_gen_number);
22683 plan_generation_start (gen, consing_gen, 0);
22685 if (demotion_low == MAX_PTR)
22687 demotion_low = pplug;
22688 dprintf (3, ("end plan: dlow->%Ix", demotion_low));
22691 dprintf (2, ("(%d)gen%d plan start: %Ix",
22692 heap_number, active_new_gen_number, (size_t)generation_plan_allocation_start (gen)));
22693 assert (generation_plan_allocation_start (gen));
22698 if (pinned_plug_que_empty_p())
22701 size_t entry = deque_pinned_plug();
22702 mark* m = pinned_plug_of (entry);
22703 uint8_t* plug = pinned_plug (m);
22704 size_t len = pinned_len (m);
22706 // detect pinned block in different segment (later) than
22707 // allocation segment
22708 heap_segment* nseg = heap_segment_rw (generation_allocation_segment (consing_gen));
22710 while ((plug < generation_allocation_pointer (consing_gen)) ||
22711 (plug >= heap_segment_allocated (nseg)))
22713 assert ((plug < heap_segment_mem (nseg)) ||
22714 (plug > heap_segment_reserved (nseg)));
22715 //adjust the end of the segment to be the end of the plug
22716 assert (generation_allocation_pointer (consing_gen)>=
22717 heap_segment_mem (nseg));
22718 assert (generation_allocation_pointer (consing_gen)<=
22719 heap_segment_committed (nseg));
22721 heap_segment_plan_allocated (nseg) =
22722 generation_allocation_pointer (consing_gen);
22723 //switch allocation segment
22724 nseg = heap_segment_next_rw (nseg);
22725 generation_allocation_segment (consing_gen) = nseg;
22726 //reset the allocation pointer and limits
22727 generation_allocation_pointer (consing_gen) =
22728 heap_segment_mem (nseg);
22731 set_new_pin_info (m, generation_allocation_pointer (consing_gen));
22732 dprintf (2, ("pin %Ix b: %Ix->%Ix", plug, brick_of (plug),
22733 (size_t)(brick_table[brick_of (plug)])));
22735 generation_allocation_pointer (consing_gen) = plug + len;
22736 generation_allocation_limit (consing_gen) =
22737 generation_allocation_pointer (consing_gen);
22738 //Add the size of the pinned plug to the right pinned allocations
22739 //find out which gen this pinned plug came from
22740 int frgn = object_gennum (plug);
22741 if ((frgn != (int)max_generation) && settings.promotion)
22743 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
22748 plan_generation_starts (consing_gen);
22749 print_free_and_plug ("AP");
22752 #ifdef SIMPLE_DPRINTF
22753 for (int gen_idx = 0; gen_idx <= max_generation; gen_idx++)
22755 generation* temp_gen = generation_of (gen_idx);
22756 dynamic_data* temp_dd = dynamic_data_of (gen_idx);
22758 int added_pinning_ratio = 0;
22759 int artificial_pinned_ratio = 0;
22761 if (dd_pinned_survived_size (temp_dd) != 0)
22763 added_pinning_ratio = (int)((float)dd_added_pinned_size (temp_dd) * 100 / (float)dd_pinned_survived_size (temp_dd));
22764 artificial_pinned_ratio = (int)((float)dd_artificial_pinned_survived_size (temp_dd) * 100 / (float)dd_pinned_survived_size (temp_dd));
22767 size_t padding_size =
22769 dd_padding_size (temp_dd);
22772 #endif //SHORT_PLUGS
22773 dprintf (1, ("gen%d: %Ix, %Ix(%Id), NON PIN alloc: %Id, pin com: %Id, sweep: %Id, surv: %Id, pinsurv: %Id(%d%% added, %d%% art), np surv: %Id, pad: %Id",
22775 generation_allocation_start (temp_gen),
22776 generation_plan_allocation_start (temp_gen),
22777 (size_t)(generation_plan_allocation_start (temp_gen) - generation_allocation_start (temp_gen)),
22778 generation_allocation_size (temp_gen),
22779 generation_pinned_allocation_compact_size (temp_gen),
22780 generation_pinned_allocation_sweep_size (temp_gen),
22781 dd_survived_size (temp_dd),
22782 dd_pinned_survived_size (temp_dd),
22783 added_pinning_ratio,
22784 artificial_pinned_ratio,
22785 (dd_survived_size (temp_dd) - dd_pinned_survived_size (temp_dd)),
22788 #endif //SIMPLE_DPRINTF
22791 if (settings.condemned_generation == (max_generation - 1 ))
22793 size_t plan_gen2_size = generation_plan_size (max_generation);
22794 size_t growth = plan_gen2_size - old_gen2_size;
22796 generation* older_gen = generation_of (settings.condemned_generation + 1);
22797 size_t rejected_free_space = generation_free_obj_space (older_gen) - r_free_obj_space;
22798 size_t free_list_allocated = generation_free_list_allocated (older_gen) - r_older_gen_free_list_allocated;
22799 size_t end_seg_allocated = generation_end_seg_allocated (older_gen) - r_older_gen_end_seg_allocated;
22800 size_t condemned_allocated = generation_condemned_allocated (older_gen) - r_older_gen_condemned_allocated;
22804 dprintf (1, ("gen2 grew %Id (end seg alloc: %Id, condemned alloc: %Id",
22805 growth, end_seg_allocated, condemned_allocated));
22807 maxgen_size_inc_p = true;
22811 dprintf (2, ("gen2 shrank %Id (end seg alloc: %Id, , condemned alloc: %Id, gen1 c alloc: %Id",
22812 (old_gen2_size - plan_gen2_size), end_seg_allocated, condemned_allocated,
22813 generation_condemned_allocated (generation_of (max_generation - 1))));
22816 dprintf (1, ("older gen's free alloc: %Id->%Id, seg alloc: %Id->%Id, condemned alloc: %Id->%Id",
22817 r_older_gen_free_list_allocated, generation_free_list_allocated (older_gen),
22818 r_older_gen_end_seg_allocated, generation_end_seg_allocated (older_gen),
22819 r_older_gen_condemned_allocated, generation_condemned_allocated (older_gen)));
22821 dprintf (1, ("this GC did %Id free list alloc(%Id bytes free space rejected)",
22822 free_list_allocated, rejected_free_space));
22824 maxgen_size_increase* maxgen_size_info = &(get_gc_data_per_heap()->maxgen_size_info);
22825 maxgen_size_info->free_list_allocated = free_list_allocated;
22826 maxgen_size_info->free_list_rejected = rejected_free_space;
22827 maxgen_size_info->end_seg_allocated = end_seg_allocated;
22828 maxgen_size_info->condemned_allocated = condemned_allocated;
22829 maxgen_size_info->pinned_allocated = maxgen_pinned_compact_before_advance;
22830 maxgen_size_info->pinned_allocated_advance = generation_pinned_allocation_compact_size (generation_of (max_generation)) - maxgen_pinned_compact_before_advance;
22832 #ifdef FREE_USAGE_STATS
22833 int free_list_efficiency = 0;
22834 if ((free_list_allocated + rejected_free_space) != 0)
22835 free_list_efficiency = (int)(((float) (free_list_allocated) / (float)(free_list_allocated + rejected_free_space)) * (float)100);
22837 int running_free_list_efficiency = (int)(generation_allocator_efficiency(older_gen)*100);
22839 dprintf (1, ("gen%d free list alloc effi: %d%%, current effi: %d%%",
22840 older_gen->gen_num,
22841 free_list_efficiency, running_free_list_efficiency));
22843 dprintf (1, ("gen2 free list change"));
22844 for (int j = 0; j < NUM_GEN_POWER2; j++)
22846 dprintf (1, ("[h%d][#%Id]: 2^%d: F: %Id->%Id(%Id), P: %Id",
22849 (j + 10), r_older_gen_free_space[j], older_gen->gen_free_spaces[j],
22850 (ptrdiff_t)(r_older_gen_free_space[j] - older_gen->gen_free_spaces[j]),
22851 (generation_of(max_generation - 1))->gen_plugs[j]));
22853 #endif //FREE_USAGE_STATS
22856 size_t fragmentation =
22857 generation_fragmentation (generation_of (condemned_gen_number),
22859 heap_segment_allocated (ephemeral_heap_segment));
22861 dprintf (2,("Fragmentation: %Id", fragmentation));
22862 dprintf (2,("---- End of Plan phase ----"));
22865 finish = GetCycleCount32();
22866 plan_time = finish - start;
22869 // We may update write barrier code. We assume here EE has been suspended if we are on a GC thread.
22870 assert(IsGCInProgress());
22872 BOOL should_expand = FALSE;
22873 BOOL should_compact= FALSE;
22874 ephemeral_promotion = FALSE;
22877 if ((!settings.concurrent) &&
22878 !provisional_mode_triggered &&
22879 ((condemned_gen_number < max_generation) &&
22880 ((settings.gen0_reduction_count > 0) || (settings.entry_memory_load >= 95))))
22882 dprintf (GTC_LOG, ("gen0 reduction count is %d, condemning %d, mem load %d",
22883 settings.gen0_reduction_count,
22884 condemned_gen_number,
22885 settings.entry_memory_load));
22886 should_compact = TRUE;
22888 get_gc_data_per_heap()->set_mechanism (gc_heap_compact,
22889 ((settings.gen0_reduction_count > 0) ? compact_fragmented_gen0 : compact_high_mem_load));
22891 if ((condemned_gen_number >= (max_generation - 1)) &&
22892 dt_low_ephemeral_space_p (tuning_deciding_expansion))
22894 dprintf (GTC_LOG, ("Not enough space for all ephemeral generations with compaction"));
22895 should_expand = TRUE;
22901 should_compact = decide_on_compacting (condemned_gen_number, fragmentation, should_expand);
22906 #ifdef FEATURE_LOH_COMPACTION
22907 loh_compacted_p = FALSE;
22908 #endif //FEATURE_LOH_COMPACTION
22910 if (condemned_gen_number == max_generation)
22912 #ifdef FEATURE_LOH_COMPACTION
22913 if (settings.loh_compaction)
22917 should_compact = TRUE;
22918 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_loh_forced);
22919 loh_compacted_p = TRUE;
22924 if ((heap_number == 0) && (loh_pinned_queue))
22926 loh_pinned_queue_decay--;
22928 if (!loh_pinned_queue_decay)
22930 delete loh_pinned_queue;
22931 loh_pinned_queue = 0;
22936 if (!loh_compacted_p)
22937 #endif //FEATURE_LOH_COMPACTION
22939 GCToEEInterface::DiagWalkLOHSurvivors(__this);
22940 sweep_large_objects();
22945 settings.loh_compaction = FALSE;
22948 #ifdef MULTIPLE_HEAPS
22950 new_heap_segment = NULL;
22952 if (should_compact && should_expand)
22953 gc_policy = policy_expand;
22954 else if (should_compact)
22955 gc_policy = policy_compact;
22957 gc_policy = policy_sweep;
22959 //vote for result of should_compact
22960 dprintf (3, ("Joining for compaction decision"));
22961 gc_t_join.join(this, gc_join_decide_on_compaction);
22962 if (gc_t_join.joined())
22964 //safe place to delete large heap segments
22965 if (condemned_gen_number == max_generation)
22967 for (int i = 0; i < n_heaps; i++)
22969 g_heaps [i]->rearrange_large_heap_segments ();
22973 if (maxgen_size_inc_p && provisional_mode_triggered)
22975 pm_trigger_full_gc = true;
22976 dprintf (GTC_LOG, ("in PM: maxgen size inc, doing a sweeping gen1 and trigger NGC2"));
22980 settings.demotion = FALSE;
22981 int pol_max = policy_sweep;
22982 #ifdef GC_CONFIG_DRIVEN
22983 BOOL is_compaction_mandatory = FALSE;
22984 #endif //GC_CONFIG_DRIVEN
22987 for (i = 0; i < n_heaps; i++)
22989 if (pol_max < g_heaps[i]->gc_policy)
22990 pol_max = policy_compact;
22991 // set the demotion flag is any of the heap has demotion
22992 if (g_heaps[i]->demotion_high >= g_heaps[i]->demotion_low)
22994 (g_heaps[i]->get_gc_data_per_heap())->set_mechanism_bit (gc_demotion_bit);
22995 settings.demotion = TRUE;
22998 #ifdef GC_CONFIG_DRIVEN
22999 if (!is_compaction_mandatory)
23001 int compact_reason = (g_heaps[i]->get_gc_data_per_heap())->get_mechanism (gc_heap_compact);
23002 if (compact_reason >= 0)
23004 if (gc_heap_compact_reason_mandatory_p[compact_reason])
23005 is_compaction_mandatory = TRUE;
23008 #endif //GC_CONFIG_DRIVEN
23011 #ifdef GC_CONFIG_DRIVEN
23012 if (!is_compaction_mandatory)
23014 // If compaction is not mandatory we can feel free to change it to a sweeping GC.
23015 // Note that we may want to change this to only checking every so often instead of every single GC.
23016 if (should_do_sweeping_gc (pol_max >= policy_compact))
23018 pol_max = policy_sweep;
23022 if (pol_max == policy_sweep)
23023 pol_max = policy_compact;
23026 #endif //GC_CONFIG_DRIVEN
23028 for (i = 0; i < n_heaps; i++)
23030 if (pol_max > g_heaps[i]->gc_policy)
23031 g_heaps[i]->gc_policy = pol_max;
23032 //get the segment while we are serialized
23033 if (g_heaps[i]->gc_policy == policy_expand)
23035 g_heaps[i]->new_heap_segment =
23036 g_heaps[i]->soh_get_segment_to_expand();
23037 if (!g_heaps[i]->new_heap_segment)
23039 set_expand_in_full_gc (condemned_gen_number);
23040 //we are out of memory, cancel the expansion
23041 g_heaps[i]->gc_policy = policy_compact;
23046 BOOL is_full_compacting_gc = FALSE;
23048 if ((gc_policy >= policy_compact) && (condemned_gen_number == max_generation))
23050 full_gc_counts[gc_type_compacting]++;
23051 is_full_compacting_gc = TRUE;
23054 for (i = 0; i < n_heaps; i++)
23056 //copy the card and brick tables
23057 if (g_gc_card_table!= g_heaps[i]->card_table)
23059 g_heaps[i]->copy_brick_card_table();
23062 if (is_full_compacting_gc)
23064 g_heaps[i]->loh_alloc_since_cg = 0;
23069 //start all threads on the roots.
23070 dprintf(3, ("Starting all gc threads after compaction decision"));
23071 gc_t_join.restart();
23074 //reset the local variable accordingly
23075 should_compact = (gc_policy >= policy_compact);
23076 should_expand = (gc_policy >= policy_expand);
23078 #else //MULTIPLE_HEAPS
23080 //safe place to delete large heap segments
23081 if (condemned_gen_number == max_generation)
23083 rearrange_large_heap_segments ();
23086 if (maxgen_size_inc_p && provisional_mode_triggered)
23088 pm_trigger_full_gc = true;
23089 dprintf (GTC_LOG, ("in PM: maxgen size inc, doing a sweeping gen1 and trigger NGC2"));
23093 settings.demotion = ((demotion_high >= demotion_low) ? TRUE : FALSE);
23094 if (settings.demotion)
23095 get_gc_data_per_heap()->set_mechanism_bit (gc_demotion_bit);
23097 #ifdef GC_CONFIG_DRIVEN
23098 BOOL is_compaction_mandatory = FALSE;
23099 int compact_reason = get_gc_data_per_heap()->get_mechanism (gc_heap_compact);
23100 if (compact_reason >= 0)
23101 is_compaction_mandatory = gc_heap_compact_reason_mandatory_p[compact_reason];
23103 if (!is_compaction_mandatory)
23105 if (should_do_sweeping_gc (should_compact))
23106 should_compact = FALSE;
23108 should_compact = TRUE;
23110 #endif //GC_CONFIG_DRIVEN
23112 if (should_compact && (condemned_gen_number == max_generation))
23114 full_gc_counts[gc_type_compacting]++;
23115 loh_alloc_since_cg = 0;
23118 #endif //MULTIPLE_HEAPS
23120 if (!pm_trigger_full_gc && pm_stress_on && provisional_mode_triggered)
23122 if ((settings.condemned_generation == (max_generation - 1)) &&
23123 ((settings.gc_index % 5) == 0))
23125 pm_trigger_full_gc = true;
23129 if (settings.condemned_generation == (max_generation - 1))
23131 if (provisional_mode_triggered)
23135 should_expand = FALSE;
23136 dprintf (GTC_LOG, ("h%d in PM cannot expand", heap_number));
23140 if (pm_trigger_full_gc)
23142 should_compact = FALSE;
23143 dprintf (GTC_LOG, ("h%d PM doing sweeping", heap_number));
23147 if (should_compact)
23149 dprintf (2,( "**** Doing Compacting GC ****"));
23153 #ifndef MULTIPLE_HEAPS
23154 heap_segment* new_heap_segment = soh_get_segment_to_expand();
23155 #endif //!MULTIPLE_HEAPS
23156 if (new_heap_segment)
23158 consing_gen = expand_heap(condemned_gen_number,
23163 // If we couldn't get a new segment, or we were able to
23164 // reserve one but no space to commit, we couldn't
23166 if (ephemeral_heap_segment != new_heap_segment)
23168 set_expand_in_full_gc (condemned_gen_number);
23169 should_expand = FALSE;
23172 generation_allocation_limit (condemned_gen1) =
23173 generation_allocation_pointer (condemned_gen1);
23174 if ((condemned_gen_number < max_generation))
23176 generation_allocator (older_gen)->commit_alloc_list_changes();
23178 // Fix the allocation area of the older generation
23179 fix_older_allocation_area (older_gen);
23181 assert (generation_allocation_segment (consing_gen) ==
23182 ephemeral_heap_segment);
23184 GCToEEInterface::DiagWalkSurvivors(__this);
23186 relocate_phase (condemned_gen_number, first_condemned_address);
23187 compact_phase (condemned_gen_number, first_condemned_address,
23188 (!settings.demotion && settings.promotion));
23189 fix_generation_bounds (condemned_gen_number, consing_gen);
23190 assert (generation_allocation_limit (youngest_generation) ==
23191 generation_allocation_pointer (youngest_generation));
23192 if (condemned_gen_number >= (max_generation -1))
23194 #ifdef MULTIPLE_HEAPS
23195 // this needs be serialized just because we have one
23196 // segment_standby_list/seg_table for all heaps. We should make it at least
23197 // so that when hoarding is not on we don't need this join because
23198 // decommitting memory can take a long time.
23199 //must serialize on deleting segments
23200 gc_t_join.join(this, gc_join_rearrange_segs_compaction);
23201 if (gc_t_join.joined())
23203 for (int i = 0; i < n_heaps; i++)
23205 g_heaps[i]->rearrange_heap_segments(TRUE);
23207 gc_t_join.restart();
23210 rearrange_heap_segments(TRUE);
23211 #endif //MULTIPLE_HEAPS
23215 //fix the start_segment for the ephemeral generations
23216 for (int i = 0; i < max_generation; i++)
23218 generation* gen = generation_of (i);
23219 generation_start_segment (gen) = ephemeral_heap_segment;
23220 generation_allocation_segment (gen) = ephemeral_heap_segment;
23226 #ifdef FEATURE_PREMORTEM_FINALIZATION
23227 finalize_queue->UpdatePromotedGenerations (condemned_gen_number,
23228 (!settings.demotion && settings.promotion));
23229 #endif // FEATURE_PREMORTEM_FINALIZATION
23231 #ifdef MULTIPLE_HEAPS
23232 dprintf(3, ("Joining after end of compaction"));
23233 gc_t_join.join(this, gc_join_adjust_handle_age_compact);
23234 if (gc_t_join.joined())
23235 #endif //MULTIPLE_HEAPS
23237 #ifdef MULTIPLE_HEAPS
23238 //join all threads to make sure they are synchronized
23239 dprintf(3, ("Restarting after Promotion granted"));
23240 gc_t_join.restart();
23241 #endif //MULTIPLE_HEAPS
23245 sc.thread_number = heap_number;
23246 sc.promotion = FALSE;
23247 sc.concurrent = FALSE;
23248 // new generations bounds are set can call this guy
23249 if (settings.promotion && !settings.demotion)
23251 dprintf (2, ("Promoting EE roots for gen %d",
23252 condemned_gen_number));
23253 GCScan::GcPromotionsGranted(condemned_gen_number,
23254 max_generation, &sc);
23256 else if (settings.demotion)
23258 dprintf (2, ("Demoting EE roots for gen %d",
23259 condemned_gen_number));
23260 GCScan::GcDemote (condemned_gen_number, max_generation, &sc);
23265 gen0_big_free_spaces = 0;
23267 reset_pinned_queue_bos();
23268 unsigned int gen_number = min (max_generation, 1 + condemned_gen_number);
23269 generation* gen = generation_of (gen_number);
23270 uint8_t* low = generation_allocation_start (generation_of (gen_number-1));
23271 uint8_t* high = heap_segment_allocated (ephemeral_heap_segment);
23273 while (!pinned_plug_que_empty_p())
23275 mark* m = pinned_plug_of (deque_pinned_plug());
23276 size_t len = pinned_len (m);
23277 uint8_t* arr = (pinned_plug (m) - len);
23278 dprintf(3,("free [%Ix %Ix[ pin",
23279 (size_t)arr, (size_t)arr + len));
23282 assert (len >= Align (min_obj_size));
23283 make_unused_array (arr, len);
23284 // fix fully contained bricks + first one
23285 // if the array goes beyond the first brick
23286 size_t start_brick = brick_of (arr);
23287 size_t end_brick = brick_of (arr + len);
23288 if (end_brick != start_brick)
23291 ("Fixing bricks [%Ix, %Ix[ to point to unused array %Ix",
23292 start_brick, end_brick, (size_t)arr));
23293 set_brick (start_brick,
23294 arr - brick_address (start_brick));
23295 size_t brick = start_brick+1;
23296 while (brick < end_brick)
23298 set_brick (brick, start_brick - brick);
23303 //when we take an old segment to make the new
23304 //ephemeral segment. we can have a bunch of
23305 //pinned plugs out of order going to the new ephemeral seg
23306 //and then the next plugs go back to max_generation
23307 if ((heap_segment_mem (ephemeral_heap_segment) <= arr) &&
23308 (heap_segment_reserved (ephemeral_heap_segment) > arr))
23311 while ((low <= arr) && (high > arr))
23314 assert ((gen_number >= 1) || (demotion_low != MAX_PTR) ||
23315 settings.demotion || !settings.promotion);
23316 dprintf (3, ("new free list generation %d", gen_number));
23318 gen = generation_of (gen_number);
23319 if (gen_number >= 1)
23320 low = generation_allocation_start (generation_of (gen_number-1));
23327 dprintf (3, ("new free list generation %d", max_generation));
23328 gen_number = max_generation;
23329 gen = generation_of (gen_number);
23332 dprintf(3,("threading it into generation %d", gen_number));
23333 thread_gap (arr, len, gen);
23334 add_gen_free (gen_number, len);
23340 for (int x = 0; x <= max_generation; x++)
23342 assert (generation_allocation_start (generation_of (x)));
23346 if (!settings.demotion && settings.promotion)
23348 //clear card for generation 1. generation 0 is empty
23349 clear_card_for_addresses (
23350 generation_allocation_start (generation_of (1)),
23351 generation_allocation_start (generation_of (0)));
23353 if (settings.promotion && !settings.demotion)
23355 uint8_t* start = generation_allocation_start (youngest_generation);
23356 MAYBE_UNUSED_VAR(start);
23357 assert (heap_segment_allocated (ephemeral_heap_segment) ==
23358 (start + Align (size (start))));
23363 //force promotion for sweep
23364 settings.promotion = TRUE;
23365 settings.compaction = FALSE;
23368 sc.thread_number = heap_number;
23369 sc.promotion = FALSE;
23370 sc.concurrent = FALSE;
23372 dprintf (2, ("**** Doing Mark and Sweep GC****"));
23374 if ((condemned_gen_number < max_generation))
23376 generation_allocator (older_gen)->copy_from_alloc_list (r_free_list);
23377 generation_free_list_space (older_gen) = r_free_list_space;
23378 generation_free_obj_space (older_gen) = r_free_obj_space;
23379 generation_free_list_allocated (older_gen) = r_older_gen_free_list_allocated;
23380 generation_end_seg_allocated (older_gen) = r_older_gen_end_seg_allocated;
23381 generation_condemned_allocated (older_gen) = r_older_gen_condemned_allocated;
23382 generation_allocation_limit (older_gen) = r_allocation_limit;
23383 generation_allocation_pointer (older_gen) = r_allocation_pointer;
23384 generation_allocation_context_start_region (older_gen) = r_allocation_start_region;
23385 generation_allocation_segment (older_gen) = r_allocation_segment;
23388 if ((condemned_gen_number < max_generation))
23390 // Fix the allocation area of the older generation
23391 fix_older_allocation_area (older_gen);
23394 GCToEEInterface::DiagWalkSurvivors(__this);
23396 gen0_big_free_spaces = 0;
23397 make_free_lists (condemned_gen_number);
23398 recover_saved_pinned_info();
23400 #ifdef FEATURE_PREMORTEM_FINALIZATION
23401 finalize_queue->UpdatePromotedGenerations (condemned_gen_number, TRUE);
23402 #endif // FEATURE_PREMORTEM_FINALIZATION
23403 // MTHTS: leave single thread for HT processing on plan_phase
23404 #ifdef MULTIPLE_HEAPS
23405 dprintf(3, ("Joining after end of sweep"));
23406 gc_t_join.join(this, gc_join_adjust_handle_age_sweep);
23407 if (gc_t_join.joined())
23408 #endif //MULTIPLE_HEAPS
23410 GCScan::GcPromotionsGranted(condemned_gen_number,
23411 max_generation, &sc);
23412 if (condemned_gen_number >= (max_generation -1))
23414 #ifdef MULTIPLE_HEAPS
23415 for (int i = 0; i < n_heaps; i++)
23417 g_heaps[i]->rearrange_heap_segments(FALSE);
23420 rearrange_heap_segments(FALSE);
23421 #endif //MULTIPLE_HEAPS
23424 #ifdef MULTIPLE_HEAPS
23425 //join all threads to make sure they are synchronized
23426 dprintf(3, ("Restarting after Promotion granted"));
23427 gc_t_join.restart();
23428 #endif //MULTIPLE_HEAPS
23432 for (int x = 0; x <= max_generation; x++)
23434 assert (generation_allocation_start (generation_of (x)));
23438 //clear card for generation 1. generation 0 is empty
23439 clear_card_for_addresses (
23440 generation_allocation_start (generation_of (1)),
23441 generation_allocation_start (generation_of (0)));
23442 assert ((heap_segment_allocated (ephemeral_heap_segment) ==
23443 (generation_allocation_start (youngest_generation) +
23444 Align (min_obj_size))));
23447 //verify_partial();
23450 #pragma warning(pop)
23454 /*****************************
23455 Called after compact phase to fix all generation gaps
23456 ********************************/
23457 void gc_heap::fix_generation_bounds (int condemned_gen_number,
23458 generation* consing_gen)
23460 UNREFERENCED_PARAMETER(consing_gen);
23462 assert (generation_allocation_segment (consing_gen) ==
23463 ephemeral_heap_segment);
23465 //assign the planned allocation start to the generation
23466 int gen_number = condemned_gen_number;
23467 int bottom_gen = 0;
23469 while (gen_number >= bottom_gen)
23471 generation* gen = generation_of (gen_number);
23472 dprintf(3,("Fixing generation pointers for %Ix", gen_number));
23473 if ((gen_number < max_generation) && ephemeral_promotion)
23475 make_unused_array (saved_ephemeral_plan_start[gen_number],
23476 saved_ephemeral_plan_start_size[gen_number]);
23478 reset_allocation_pointers (gen, generation_plan_allocation_start (gen));
23479 make_unused_array (generation_allocation_start (gen), generation_plan_allocation_start_size (gen));
23480 dprintf(3,(" start %Ix", (size_t)generation_allocation_start (gen)));
23483 #ifdef MULTIPLE_HEAPS
23484 if (ephemeral_promotion)
23486 //we are creating a generation fault. set the cards.
23487 // and we are only doing this for multiple heaps because in the single heap scenario the
23488 // new ephemeral generations will be empty and there'll be no need to set cards for the
23489 // old ephemeral generations that got promoted into max_generation.
23490 ptrdiff_t delta = 0;
23491 #ifdef SEG_MAPPING_TABLE
23492 heap_segment* old_ephemeral_seg = seg_mapping_table_segment_of (saved_ephemeral_plan_start[max_generation-1]);
23493 #else //SEG_MAPPING_TABLE
23494 heap_segment* old_ephemeral_seg = segment_of (saved_ephemeral_plan_start[max_generation-1], delta);
23495 #endif //SEG_MAPPING_TABLE
23497 assert (in_range_for_segment (saved_ephemeral_plan_start[max_generation-1], old_ephemeral_seg));
23498 size_t end_card = card_of (align_on_card (heap_segment_plan_allocated (old_ephemeral_seg)));
23499 size_t card = card_of (saved_ephemeral_plan_start[max_generation-1]);
23500 while (card != end_card)
23506 #endif //MULTIPLE_HEAPS
23508 alloc_allocated = heap_segment_plan_allocated(ephemeral_heap_segment);
23509 //reset the allocated size
23510 uint8_t* start = generation_allocation_start (youngest_generation);
23511 MAYBE_UNUSED_VAR(start);
23512 if (settings.promotion && !settings.demotion)
23514 assert ((start + Align (size (start))) ==
23515 heap_segment_plan_allocated(ephemeral_heap_segment));
23518 heap_segment_allocated(ephemeral_heap_segment)=
23519 heap_segment_plan_allocated(ephemeral_heap_segment);
23523 uint8_t* gc_heap::generation_limit (int gen_number)
23525 if (settings.promotion)
23527 if (gen_number <= 1)
23528 return heap_segment_reserved (ephemeral_heap_segment);
23530 return generation_allocation_start (generation_of ((gen_number - 2)));
23534 if (gen_number <= 0)
23535 return heap_segment_reserved (ephemeral_heap_segment);
23537 return generation_allocation_start (generation_of ((gen_number - 1)));
23541 BOOL gc_heap::ensure_gap_allocation (int condemned_gen_number)
23543 uint8_t* start = heap_segment_allocated (ephemeral_heap_segment);
23544 size_t size = Align (min_obj_size)*(condemned_gen_number+1);
23545 assert ((start + size) <=
23546 heap_segment_reserved (ephemeral_heap_segment));
23547 if ((start + size) >
23548 heap_segment_committed (ephemeral_heap_segment))
23550 if (!grow_heap_segment (ephemeral_heap_segment, start + size))
23558 uint8_t* gc_heap::allocate_at_end (size_t size)
23560 uint8_t* start = heap_segment_allocated (ephemeral_heap_segment);
23561 size = Align (size);
23562 uint8_t* result = start;
23563 // only called to allocate a min obj so can't overflow here.
23564 assert ((start + size) <=
23565 heap_segment_reserved (ephemeral_heap_segment));
23566 //ensure_gap_allocation took care of it
23567 assert ((start + size) <=
23568 heap_segment_committed (ephemeral_heap_segment));
23569 heap_segment_allocated (ephemeral_heap_segment) += size;
23574 void gc_heap::make_free_lists (int condemned_gen_number)
23579 start = GetCycleCount32();
23582 //Promotion has to happen in sweep case.
23583 assert (settings.promotion);
23585 generation* condemned_gen = generation_of (condemned_gen_number);
23586 uint8_t* start_address = generation_allocation_start (condemned_gen);
23588 size_t current_brick = brick_of (start_address);
23589 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
23591 PREFIX_ASSUME(current_heap_segment != NULL);
23593 uint8_t* end_address = heap_segment_allocated (current_heap_segment);
23594 size_t end_brick = brick_of (end_address-1);
23595 make_free_args args;
23596 args.free_list_gen_number = min (max_generation, 1 + condemned_gen_number);
23597 args.current_gen_limit = (((condemned_gen_number == max_generation)) ?
23599 (generation_limit (args.free_list_gen_number)));
23600 args.free_list_gen = generation_of (args.free_list_gen_number);
23601 args.highest_plug = 0;
23603 if ((start_address < end_address) ||
23604 (condemned_gen_number == max_generation))
23608 if ((current_brick > end_brick))
23610 if (args.current_gen_limit == MAX_PTR)
23612 //We had an empty segment
23613 //need to allocate the generation start
23615 generation* gen = generation_of (max_generation);
23617 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
23619 PREFIX_ASSUME(start_seg != NULL);
23621 uint8_t* gap = heap_segment_mem (start_seg);
23623 generation_allocation_start (gen) = gap;
23624 heap_segment_allocated (start_seg) = gap + Align (min_obj_size);
23625 make_unused_array (gap, Align (min_obj_size));
23626 reset_allocation_pointers (gen, gap);
23627 dprintf (3, ("Start segment empty, fixing generation start of %d to: %Ix",
23628 max_generation, (size_t)gap));
23629 args.current_gen_limit = generation_limit (args.free_list_gen_number);
23631 if (heap_segment_next_rw (current_heap_segment))
23633 current_heap_segment = heap_segment_next_rw (current_heap_segment);
23634 current_brick = brick_of (heap_segment_mem (current_heap_segment));
23635 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
23645 int brick_entry = brick_table [ current_brick ];
23646 if ((brick_entry >= 0))
23648 make_free_list_in_brick (brick_address (current_brick) + brick_entry-1, &args);
23649 dprintf(3,("Fixing brick entry %Ix to %Ix",
23650 current_brick, (size_t)args.highest_plug));
23651 set_brick (current_brick,
23652 (args.highest_plug - brick_address (current_brick)));
23656 if ((brick_entry > -32768))
23660 ptrdiff_t offset = brick_of (args.highest_plug) - current_brick;
23661 if ((brick_entry != -32767) && (! ((offset == brick_entry))))
23663 assert ((brick_entry == -1));
23666 //init to -1 for faster find_first_object
23667 set_brick (current_brick, -1);
23675 int bottom_gen = 0;
23676 args.free_list_gen_number--;
23677 while (args.free_list_gen_number >= bottom_gen)
23680 generation* gen2 = generation_of (args.free_list_gen_number);
23681 gap = allocate_at_end (Align(min_obj_size));
23682 generation_allocation_start (gen2) = gap;
23683 reset_allocation_pointers (gen2, gap);
23684 dprintf(3,("Fixing generation start of %d to: %Ix",
23685 args.free_list_gen_number, (size_t)gap));
23686 PREFIX_ASSUME(gap != NULL);
23687 make_unused_array (gap, Align (min_obj_size));
23689 args.free_list_gen_number--;
23692 //reset the allocated size
23693 uint8_t* start2 = generation_allocation_start (youngest_generation);
23694 alloc_allocated = start2 + Align (size (start2));
23698 finish = GetCycleCount32();
23699 sweep_time = finish - start;
23703 void gc_heap::make_free_list_in_brick (uint8_t* tree, make_free_args* args)
23705 assert ((tree != NULL));
23707 int right_node = node_right_child (tree);
23708 int left_node = node_left_child (tree);
23709 args->highest_plug = 0;
23712 if (! (0 == left_node))
23714 make_free_list_in_brick (tree + left_node, args);
23718 uint8_t* plug = tree;
23719 size_t gap_size = node_gap_size (tree);
23720 uint8_t* gap = (plug - gap_size);
23721 dprintf (3,("Making free list %Ix len %d in %d",
23722 //dprintf (3,("F: %Ix len %Ix in %d",
23723 (size_t)gap, gap_size, args->free_list_gen_number));
23724 args->highest_plug = tree;
23726 if (is_plug_padded (plug))
23728 dprintf (3, ("%Ix padded", plug));
23729 clear_plug_padded (plug);
23731 #endif //SHORT_PLUGS
23734 if ((args->current_gen_limit == MAX_PTR) ||
23735 ((plug >= args->current_gen_limit) &&
23736 ephemeral_pointer_p (plug)))
23738 dprintf(3,(" Crossing Generation boundary at %Ix",
23739 (size_t)args->current_gen_limit));
23740 if (!(args->current_gen_limit == MAX_PTR))
23742 args->free_list_gen_number--;
23743 args->free_list_gen = generation_of (args->free_list_gen_number);
23745 dprintf(3,( " Fixing generation start of %d to: %Ix",
23746 args->free_list_gen_number, (size_t)gap));
23748 reset_allocation_pointers (args->free_list_gen, gap);
23749 args->current_gen_limit = generation_limit (args->free_list_gen_number);
23751 if ((gap_size >= (2*Align (min_obj_size))))
23753 dprintf(3,(" Splitting the gap in two %Id left",
23755 make_unused_array (gap, Align(min_obj_size));
23756 gap_size = (gap_size - Align(min_obj_size));
23757 gap = (gap + Align(min_obj_size));
23761 make_unused_array (gap, gap_size);
23768 thread_gap (gap, gap_size, args->free_list_gen);
23769 add_gen_free (args->free_list_gen->gen_num, gap_size);
23771 if (! (0 == right_node))
23773 make_free_list_in_brick (tree + right_node, args);
23779 void gc_heap::thread_gap (uint8_t* gap_start, size_t size, generation* gen)
23781 assert (generation_allocation_start (gen));
23784 if ((gen->gen_num == 0) && (size > CLR_SIZE))
23786 gen0_big_free_spaces += size;
23789 assert ((heap_segment_rw (generation_start_segment (gen))!=
23790 ephemeral_heap_segment) ||
23791 (gap_start > generation_allocation_start (gen)));
23792 // The beginning of a segment gap is not aligned
23793 assert (size >= Align (min_obj_size));
23794 make_unused_array (gap_start, size,
23795 (!settings.concurrent && (gen != youngest_generation)),
23796 (gen->gen_num == max_generation));
23797 dprintf (3, ("fr: [%Ix, %Ix[", (size_t)gap_start, (size_t)gap_start+size));
23799 if ((size >= min_free_list))
23801 generation_free_list_space (gen) += size;
23802 generation_allocator (gen)->thread_item (gap_start, size);
23806 generation_free_obj_space (gen) += size;
23811 void gc_heap::loh_thread_gap_front (uint8_t* gap_start, size_t size, generation* gen)
23813 assert (generation_allocation_start (gen));
23814 if (size >= min_free_list)
23816 generation_free_list_space (gen) += size;
23817 generation_allocator (gen)->thread_item_front (gap_start, size);
23821 void gc_heap::make_unused_array (uint8_t* x, size_t size, BOOL clearp, BOOL resetp)
23823 dprintf (3, ("Making unused array [%Ix, %Ix[",
23824 (size_t)x, (size_t)(x+size)));
23825 assert (size >= Align (min_obj_size));
23827 //#if defined (VERIFY_HEAP) && defined (BACKGROUND_GC)
23828 // check_batch_mark_array_bits (x, x+size);
23829 //#endif //VERIFY_HEAP && BACKGROUND_GC
23832 reset_memory (x, size);
23834 ((CObjectHeader*)x)->SetFree(size);
23839 #error "This won't work on big endian platforms"
23842 size_t size_as_object = (uint32_t)(size - free_object_base_size) + free_object_base_size;
23844 if (size_as_object < size)
23847 // If the size is more than 4GB, we need to create multiple objects because of
23848 // the Array::m_NumComponents is uint32_t and the high 32 bits of unused array
23849 // size is ignored in regular object size computation.
23851 uint8_t * tmp = x + size_as_object;
23852 size_t remaining_size = size - size_as_object;
23854 while (remaining_size > UINT32_MAX)
23856 // Make sure that there will be at least Align(min_obj_size) left
23857 size_t current_size = UINT32_MAX - get_alignment_constant (FALSE)
23858 - Align (min_obj_size, get_alignment_constant (FALSE));
23860 ((CObjectHeader*)tmp)->SetFree(current_size);
23862 remaining_size -= current_size;
23863 tmp += current_size;
23866 ((CObjectHeader*)tmp)->SetFree(remaining_size);
23871 clear_card_for_addresses (x, x + Align(size));
23874 // Clear memory set by make_unused_array.
23875 void gc_heap::clear_unused_array (uint8_t* x, size_t size)
23877 // Also clear the sync block
23878 *(((PTR_PTR)x)-1) = 0;
23880 ((CObjectHeader*)x)->UnsetFree();
23885 #error "This won't work on big endian platforms"
23888 // The memory could have been cleared in the meantime. We have to mirror the algorithm
23889 // from make_unused_array since we cannot depend on the object sizes in memory.
23890 size_t size_as_object = (uint32_t)(size - free_object_base_size) + free_object_base_size;
23892 if (size_as_object < size)
23894 uint8_t * tmp = x + size_as_object;
23895 size_t remaining_size = size - size_as_object;
23897 while (remaining_size > UINT32_MAX)
23899 size_t current_size = UINT32_MAX - get_alignment_constant (FALSE)
23900 - Align (min_obj_size, get_alignment_constant (FALSE));
23902 ((CObjectHeader*)tmp)->UnsetFree();
23904 remaining_size -= current_size;
23905 tmp += current_size;
23908 ((CObjectHeader*)tmp)->UnsetFree();
23911 UNREFERENCED_PARAMETER(size);
23916 uint8_t* tree_search (uint8_t* tree, uint8_t* old_address)
23918 uint8_t* candidate = 0;
23922 if (tree < old_address)
23924 if ((cn = node_right_child (tree)) != 0)
23926 assert (candidate < tree);
23929 Prefetch (tree - 8);
23935 else if (tree > old_address)
23937 if ((cn = node_left_child (tree)) != 0)
23940 Prefetch (tree - 8);
23948 if (tree <= old_address)
23950 else if (candidate)
23956 #ifdef FEATURE_BASICFREEZE
23957 bool gc_heap::frozen_object_p (Object* obj)
23959 #ifdef MULTIPLE_HEAPS
23960 #ifdef SEG_MAPPING_TABLE
23961 heap_segment* pSegment = seg_mapping_table_segment_of((uint8_t*)obj);
23963 ptrdiff_t delta = 0;
23964 heap_segment* pSegment = segment_of ((uint8_t*)obj, delta);
23966 #else //MULTIPLE_HEAPS
23967 heap_segment* pSegment = gc_heap::find_segment ((uint8_t*)obj, FALSE);
23968 _ASSERTE(pSegment);
23969 #endif //MULTIPLE_HEAPS
23971 return heap_segment_read_only_p(pSegment);
23973 #endif // FEATURE_BASICFREEZE
23975 #ifdef FEATURE_REDHAWK
23976 // TODO: this was added on RH, we have not done perf runs to see if this is the right
23977 // thing to do for other versions of the CLR.
23979 #endif // FEATURE_REDHAWK
23980 void gc_heap::relocate_address (uint8_t** pold_address THREAD_NUMBER_DCL)
23982 uint8_t* old_address = *pold_address;
23983 if (!((old_address >= gc_low) && (old_address < gc_high)))
23984 #ifdef MULTIPLE_HEAPS
23986 UNREFERENCED_PARAMETER(thread);
23987 if (old_address == 0)
23989 gc_heap* hp = heap_of (old_address);
23990 if ((hp == this) ||
23991 !((old_address >= hp->gc_low) && (old_address < hp->gc_high)))
23994 #else //MULTIPLE_HEAPS
23996 #endif //MULTIPLE_HEAPS
23997 // delta translates old_address into address_gc (old_address);
23998 size_t brick = brick_of (old_address);
23999 int brick_entry = brick_table [ brick ];
24000 uint8_t* new_address = old_address;
24001 if (! ((brick_entry == 0)))
24005 while (brick_entry < 0)
24007 brick = (brick + brick_entry);
24008 brick_entry = brick_table [ brick ];
24010 uint8_t* old_loc = old_address;
24012 uint8_t* node = tree_search ((brick_address (brick) + brick_entry-1),
24014 if ((node <= old_loc))
24015 new_address = (old_address + node_relocation_distance (node));
24018 if (node_left_p (node))
24020 dprintf(3,(" L: %Ix", (size_t)node));
24021 new_address = (old_address +
24022 (node_relocation_distance (node) +
24023 node_gap_size (node)));
24028 brick_entry = brick_table [ brick ];
24034 *pold_address = new_address;
24038 #ifdef FEATURE_LOH_COMPACTION
24039 if (loh_compacted_p
24040 #ifdef FEATURE_BASICFREEZE
24041 && !frozen_object_p((Object*)old_address)
24042 #endif // FEATURE_BASICFREEZE
24045 *pold_address = old_address + loh_node_relocation_distance (old_address);
24048 #endif //FEATURE_LOH_COMPACTION
24050 *pold_address = new_address;
24055 gc_heap::check_class_object_demotion (uint8_t* obj)
24057 #ifdef COLLECTIBLE_CLASS
24058 if (is_collectible(obj))
24060 check_class_object_demotion_internal (obj);
24063 UNREFERENCED_PARAMETER(obj);
24064 #endif //COLLECTIBLE_CLASS
24067 #ifdef COLLECTIBLE_CLASS
24069 gc_heap::check_class_object_demotion_internal (uint8_t* obj)
24071 if (settings.demotion)
24073 #ifdef MULTIPLE_HEAPS
24074 // We set the card without checking the demotion range 'cause at this point
24075 // the handle that points to the loader allocator object may or may not have
24076 // been relocated by other GC threads.
24077 set_card (card_of (obj));
24080 uint8_t* class_obj = get_class_object (obj);
24081 dprintf (3, ("%Ix: got classobj %Ix", obj, class_obj));
24082 uint8_t* temp_class_obj = class_obj;
24083 uint8_t** temp = &temp_class_obj;
24084 relocate_address (temp THREAD_NUMBER_ARG);
24086 check_demotion_helper (temp, obj);
24087 #endif //MULTIPLE_HEAPS
24091 #endif //COLLECTIBLE_CLASS
24094 gc_heap::check_demotion_helper (uint8_t** pval, uint8_t* parent_obj)
24096 // detect if we are demoting an object
24097 if ((*pval < demotion_high) &&
24098 (*pval >= demotion_low))
24100 dprintf(3, ("setting card %Ix:%Ix",
24101 card_of((uint8_t*)pval),
24104 set_card (card_of (parent_obj));
24106 #ifdef MULTIPLE_HEAPS
24107 else if (settings.demotion)
24109 dprintf (4, ("Demotion active, computing heap_of object"));
24110 gc_heap* hp = heap_of (*pval);
24111 if ((*pval < hp->demotion_high) &&
24112 (*pval >= hp->demotion_low))
24114 dprintf(3, ("setting card %Ix:%Ix",
24115 card_of((uint8_t*)pval),
24118 set_card (card_of (parent_obj));
24121 #endif //MULTIPLE_HEAPS
24125 gc_heap::reloc_survivor_helper (uint8_t** pval)
24128 relocate_address (pval THREAD_NUMBER_ARG);
24130 check_demotion_helper (pval, (uint8_t*)pval);
24134 gc_heap::relocate_obj_helper (uint8_t* x, size_t s)
24137 if (contain_pointers (x))
24139 dprintf (3, ("$%Ix$", (size_t)x));
24141 go_through_object_nostart (method_table(x), x, s, pval,
24143 uint8_t* child = *pval;
24144 reloc_survivor_helper (pval);
24147 dprintf (3, ("%Ix->%Ix->%Ix", (uint8_t*)pval, child, *pval));
24152 check_class_object_demotion (x);
24156 void gc_heap::reloc_ref_in_shortened_obj (uint8_t** address_to_set_card, uint8_t** address_to_reloc)
24160 uint8_t* old_val = (address_to_reloc ? *address_to_reloc : 0);
24161 relocate_address (address_to_reloc THREAD_NUMBER_ARG);
24162 if (address_to_reloc)
24164 dprintf (3, ("SR %Ix: %Ix->%Ix", (uint8_t*)address_to_reloc, old_val, *address_to_reloc));
24167 //check_demotion_helper (current_saved_info_to_relocate, (uint8_t*)pval);
24168 uint8_t* relocated_addr = *address_to_reloc;
24169 if ((relocated_addr < demotion_high) &&
24170 (relocated_addr >= demotion_low))
24172 dprintf (3, ("set card for location %Ix(%Ix)",
24173 (size_t)address_to_set_card, card_of((uint8_t*)address_to_set_card)));
24175 set_card (card_of ((uint8_t*)address_to_set_card));
24177 #ifdef MULTIPLE_HEAPS
24178 else if (settings.demotion)
24180 gc_heap* hp = heap_of (relocated_addr);
24181 if ((relocated_addr < hp->demotion_high) &&
24182 (relocated_addr >= hp->demotion_low))
24184 dprintf (3, ("%Ix on h%d, set card for location %Ix(%Ix)",
24185 relocated_addr, hp->heap_number, (size_t)address_to_set_card, card_of((uint8_t*)address_to_set_card)));
24187 set_card (card_of ((uint8_t*)address_to_set_card));
24190 #endif //MULTIPLE_HEAPS
24193 void gc_heap::relocate_pre_plug_info (mark* pinned_plug_entry)
24196 uint8_t* plug = pinned_plug (pinned_plug_entry);
24197 uint8_t* pre_plug_start = plug - sizeof (plug_and_gap);
24198 // Note that we need to add one ptr size here otherwise we may not be able to find the relocated
24199 // address. Consider this scenario:
24200 // gen1 start | 3-ptr sized NP | PP
24202 // If we are asking for the reloc address of 0x10 we will AV in relocate_address because
24203 // the first plug we saw in the brick is 0x18 which means 0x10 will cause us to go back a brick
24204 // which is 0, and then we'll AV in tree_search when we try to do node_right_child (tree).
24205 pre_plug_start += sizeof (uint8_t*);
24206 uint8_t** old_address = &pre_plug_start;
24208 uint8_t* old_val = (old_address ? *old_address : 0);
24209 relocate_address (old_address THREAD_NUMBER_ARG);
24212 dprintf (3, ("PreR %Ix: %Ix->%Ix, set reloc: %Ix",
24213 (uint8_t*)old_address, old_val, *old_address, (pre_plug_start - sizeof (uint8_t*))));
24216 pinned_plug_entry->set_pre_plug_info_reloc_start (pre_plug_start - sizeof (uint8_t*));
24220 void gc_heap::relocate_shortened_obj_helper (uint8_t* x, size_t s, uint8_t* end, mark* pinned_plug_entry, BOOL is_pinned)
24223 uint8_t* plug = pinned_plug (pinned_plug_entry);
24227 //// Temporary - we just wanna make sure we are doing things right when padding is needed.
24228 //if ((x + s) < plug)
24230 // dprintf (3, ("obj %Ix needed padding: end %Ix is %d bytes from pinned obj %Ix",
24231 // x, (x + s), (plug- (x + s)), plug));
24232 // GCToOSInterface::DebugBreak();
24235 relocate_pre_plug_info (pinned_plug_entry);
24238 verify_pins_with_post_plug_info("after relocate_pre_plug_info");
24240 uint8_t* saved_plug_info_start = 0;
24241 uint8_t** saved_info_to_relocate = 0;
24245 saved_plug_info_start = (uint8_t*)(pinned_plug_entry->get_post_plug_info_start());
24246 saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_post_plug_reloc_info());
24250 saved_plug_info_start = (plug - sizeof (plug_and_gap));
24251 saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_pre_plug_reloc_info());
24254 uint8_t** current_saved_info_to_relocate = 0;
24255 uint8_t* child = 0;
24257 dprintf (3, ("x: %Ix, pp: %Ix, end: %Ix", x, plug, end));
24259 if (contain_pointers (x))
24261 dprintf (3,("$%Ix$", (size_t)x));
24263 go_through_object_nostart (method_table(x), x, s, pval,
24265 dprintf (3, ("obj %Ix, member: %Ix->%Ix", x, (uint8_t*)pval, *pval));
24267 if ((uint8_t*)pval >= end)
24269 current_saved_info_to_relocate = saved_info_to_relocate + ((uint8_t*)pval - saved_plug_info_start) / sizeof (uint8_t**);
24270 child = *current_saved_info_to_relocate;
24271 reloc_ref_in_shortened_obj (pval, current_saved_info_to_relocate);
24272 dprintf (3, ("last part: R-%Ix(saved: %Ix)->%Ix ->%Ix",
24273 (uint8_t*)pval, current_saved_info_to_relocate, child, *current_saved_info_to_relocate));
24277 reloc_survivor_helper (pval);
24282 check_class_object_demotion (x);
24285 void gc_heap::relocate_survivor_helper (uint8_t* plug, uint8_t* plug_end)
24288 while (x < plug_end)
24290 size_t s = size (x);
24291 uint8_t* next_obj = x + Align (s);
24292 Prefetch (next_obj);
24293 relocate_obj_helper (x, s);
24299 // if we expanded, right now we are not handling it as We are not saving the new reloc info.
24300 void gc_heap::verify_pins_with_post_plug_info (const char* msg)
24302 #if defined (_DEBUG) && defined (VERIFY_HEAP)
24303 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
24305 if (!verify_pinned_queue_p)
24308 if (settings.heap_expansion)
24311 for (size_t i = 0; i < mark_stack_tos; i++)
24313 mark& m = mark_stack_array[i];
24315 mark* pinned_plug_entry = pinned_plug_of(i);
24317 if (pinned_plug_entry->has_post_plug_info() &&
24318 pinned_plug_entry->post_short_p() &&
24319 (pinned_plug_entry->saved_post_plug_debug.gap != 1))
24321 uint8_t* next_obj = pinned_plug_entry->get_post_plug_info_start() + sizeof (plug_and_gap);
24322 // object after pin
24323 dprintf (3, ("OFP: %Ix, G: %Ix, R: %Ix, LC: %d, RC: %d",
24324 next_obj, node_gap_size (next_obj), node_relocation_distance (next_obj),
24325 (int)node_left_child (next_obj), (int)node_right_child (next_obj)));
24327 size_t* post_plug_debug = (size_t*)(&m.saved_post_plug_debug);
24329 if (node_gap_size (next_obj) != *post_plug_debug)
24331 dprintf (3, ("obj: %Ix gap should be %Ix but it is %Ix",
24332 next_obj, *post_plug_debug, (size_t)(node_gap_size (next_obj))));
24336 // can't do node_relocation_distance here as it clears the left bit.
24337 //if (node_relocation_distance (next_obj) != *post_plug_debug)
24338 if (*((size_t*)(next_obj - 3 * sizeof (size_t))) != *post_plug_debug)
24340 dprintf (3, ("obj: %Ix reloc should be %Ix but it is %Ix",
24341 next_obj, *post_plug_debug, (size_t)(node_relocation_distance (next_obj))));
24344 if (node_left_child (next_obj) > 0)
24346 dprintf (3, ("obj: %Ix, vLC: %d\n", next_obj, (int)(node_left_child (next_obj))));
24352 dprintf (3, ("%s verified", msg));
24354 #else // _DEBUG && VERIFY_HEAP
24355 UNREFERENCED_PARAMETER(msg);
24356 #endif // _DEBUG && VERIFY_HEAP
24359 #ifdef COLLECTIBLE_CLASS
24360 // We don't want to burn another ptr size space for pinned plugs to record this so just
24361 // set the card unconditionally for collectible objects if we are demoting.
24363 gc_heap::unconditional_set_card_collectible (uint8_t* obj)
24365 if (settings.demotion)
24367 set_card (card_of (obj));
24370 #endif //COLLECTIBLE_CLASS
24372 void gc_heap::relocate_shortened_survivor_helper (uint8_t* plug, uint8_t* plug_end, mark* pinned_plug_entry)
24375 uint8_t* p_plug = pinned_plug (pinned_plug_entry);
24376 BOOL is_pinned = (plug == p_plug);
24377 BOOL check_short_obj_p = (is_pinned ? pinned_plug_entry->post_short_p() : pinned_plug_entry->pre_short_p());
24379 plug_end += sizeof (gap_reloc_pair);
24381 //dprintf (3, ("%s %Ix is shortened, and last object %s overwritten", (is_pinned ? "PP" : "NP"), plug, (check_short_obj_p ? "is" : "is not")));
24382 dprintf (3, ("%s %Ix-%Ix short, LO: %s OW", (is_pinned ? "PP" : "NP"), plug, plug_end, (check_short_obj_p ? "is" : "is not")));
24384 verify_pins_with_post_plug_info("begin reloc short surv");
24386 while (x < plug_end)
24388 if (check_short_obj_p && ((plug_end - x) < min_pre_pin_obj_size))
24390 dprintf (3, ("last obj %Ix is short", x));
24394 #ifdef COLLECTIBLE_CLASS
24395 if (pinned_plug_entry->post_short_collectible_p())
24396 unconditional_set_card_collectible (x);
24397 #endif //COLLECTIBLE_CLASS
24399 // Relocate the saved references based on bits set.
24400 uint8_t** saved_plug_info_start = (uint8_t**)(pinned_plug_entry->get_post_plug_info_start());
24401 uint8_t** saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_post_plug_reloc_info());
24402 for (size_t i = 0; i < pinned_plug_entry->get_max_short_bits(); i++)
24404 if (pinned_plug_entry->post_short_bit_p (i))
24406 reloc_ref_in_shortened_obj ((saved_plug_info_start + i), (saved_info_to_relocate + i));
24412 #ifdef COLLECTIBLE_CLASS
24413 if (pinned_plug_entry->pre_short_collectible_p())
24414 unconditional_set_card_collectible (x);
24415 #endif //COLLECTIBLE_CLASS
24417 relocate_pre_plug_info (pinned_plug_entry);
24419 // Relocate the saved references based on bits set.
24420 uint8_t** saved_plug_info_start = (uint8_t**)(p_plug - sizeof (plug_and_gap));
24421 uint8_t** saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_pre_plug_reloc_info());
24422 for (size_t i = 0; i < pinned_plug_entry->get_max_short_bits(); i++)
24424 if (pinned_plug_entry->pre_short_bit_p (i))
24426 reloc_ref_in_shortened_obj ((saved_plug_info_start + i), (saved_info_to_relocate + i));
24434 size_t s = size (x);
24435 uint8_t* next_obj = x + Align (s);
24436 Prefetch (next_obj);
24438 if (next_obj >= plug_end)
24440 dprintf (3, ("object %Ix is at the end of the plug %Ix->%Ix",
24441 next_obj, plug, plug_end));
24443 verify_pins_with_post_plug_info("before reloc short obj");
24445 relocate_shortened_obj_helper (x, s, (x + Align (s) - sizeof (plug_and_gap)), pinned_plug_entry, is_pinned);
24449 relocate_obj_helper (x, s);
24456 verify_pins_with_post_plug_info("end reloc short surv");
24459 void gc_heap::relocate_survivors_in_plug (uint8_t* plug, uint8_t* plug_end,
24460 BOOL check_last_object_p,
24461 mark* pinned_plug_entry)
24463 //dprintf(3,("Relocating pointers in Plug [%Ix,%Ix[", (size_t)plug, (size_t)plug_end));
24464 dprintf (3,("RP: [%Ix,%Ix[", (size_t)plug, (size_t)plug_end));
24466 if (check_last_object_p)
24468 relocate_shortened_survivor_helper (plug, plug_end, pinned_plug_entry);
24472 relocate_survivor_helper (plug, plug_end);
24476 void gc_heap::relocate_survivors_in_brick (uint8_t* tree, relocate_args* args)
24478 assert ((tree != NULL));
24480 dprintf (3, ("tree: %Ix, args->last_plug: %Ix, left: %Ix, right: %Ix, gap(t): %Ix",
24481 tree, args->last_plug,
24482 (tree + node_left_child (tree)),
24483 (tree + node_right_child (tree)),
24484 node_gap_size (tree)));
24486 if (node_left_child (tree))
24488 relocate_survivors_in_brick (tree + node_left_child (tree), args);
24491 uint8_t* plug = tree;
24492 BOOL has_post_plug_info_p = FALSE;
24493 BOOL has_pre_plug_info_p = FALSE;
24495 if (tree == oldest_pinned_plug)
24497 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
24498 &has_post_plug_info_p);
24499 assert (tree == pinned_plug (args->pinned_plug_entry));
24501 dprintf (3, ("tree is the oldest pin: %Ix", tree));
24503 if (args->last_plug)
24505 size_t gap_size = node_gap_size (tree);
24506 uint8_t* gap = (plug - gap_size);
24507 dprintf (3, ("tree: %Ix, gap: %Ix (%Ix)", tree, gap, gap_size));
24508 assert (gap_size >= Align (min_obj_size));
24509 uint8_t* last_plug_end = gap;
24511 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
24514 relocate_survivors_in_plug (args->last_plug, last_plug_end, check_last_object_p, args->pinned_plug_entry);
24519 assert (!has_pre_plug_info_p);
24522 args->last_plug = plug;
24523 args->is_shortened = has_post_plug_info_p;
24524 if (has_post_plug_info_p)
24526 dprintf (3, ("setting %Ix as shortened", plug));
24528 dprintf (3, ("last_plug: %Ix(shortened: %d)", plug, (args->is_shortened ? 1 : 0)));
24530 if (node_right_child (tree))
24532 relocate_survivors_in_brick (tree + node_right_child (tree), args);
24537 void gc_heap::update_oldest_pinned_plug()
24539 oldest_pinned_plug = (pinned_plug_que_empty_p() ? 0 : pinned_plug (oldest_pin()));
24542 void gc_heap::relocate_survivors (int condemned_gen_number,
24543 uint8_t* first_condemned_address)
24545 generation* condemned_gen = generation_of (condemned_gen_number);
24546 uint8_t* start_address = first_condemned_address;
24547 size_t current_brick = brick_of (start_address);
24548 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
24550 PREFIX_ASSUME(current_heap_segment != NULL);
24552 uint8_t* end_address = 0;
24554 reset_pinned_queue_bos();
24555 update_oldest_pinned_plug();
24557 end_address = heap_segment_allocated (current_heap_segment);
24559 size_t end_brick = brick_of (end_address - 1);
24560 relocate_args args;
24562 args.high = gc_high;
24563 args.is_shortened = FALSE;
24564 args.pinned_plug_entry = 0;
24565 args.last_plug = 0;
24568 if (current_brick > end_brick)
24570 if (args.last_plug)
24573 assert (!(args.is_shortened));
24574 relocate_survivors_in_plug (args.last_plug,
24575 heap_segment_allocated (current_heap_segment),
24577 args.pinned_plug_entry);
24580 args.last_plug = 0;
24583 if (heap_segment_next_rw (current_heap_segment))
24585 current_heap_segment = heap_segment_next_rw (current_heap_segment);
24586 current_brick = brick_of (heap_segment_mem (current_heap_segment));
24587 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24596 int brick_entry = brick_table [ current_brick ];
24598 if (brick_entry >= 0)
24600 relocate_survivors_in_brick (brick_address (current_brick) +
24609 void gc_heap::walk_plug (uint8_t* plug, size_t size, BOOL check_last_object_p, walk_relocate_args* args)
24611 if (check_last_object_p)
24613 size += sizeof (gap_reloc_pair);
24614 mark* entry = args->pinned_plug_entry;
24616 if (args->is_shortened)
24618 assert (entry->has_post_plug_info());
24619 entry->swap_post_plug_and_saved_for_profiler();
24623 assert (entry->has_pre_plug_info());
24624 entry->swap_pre_plug_and_saved_for_profiler();
24628 ptrdiff_t last_plug_relocation = node_relocation_distance (plug);
24629 STRESS_LOG_PLUG_MOVE(plug, (plug + size), -last_plug_relocation);
24630 ptrdiff_t reloc = settings.compaction ? last_plug_relocation : 0;
24632 (args->fn) (plug, (plug + size), reloc, args->profiling_context, !!settings.compaction, false);
24634 if (check_last_object_p)
24636 mark* entry = args->pinned_plug_entry;
24638 if (args->is_shortened)
24640 entry->swap_post_plug_and_saved_for_profiler();
24644 entry->swap_pre_plug_and_saved_for_profiler();
24649 void gc_heap::walk_relocation_in_brick (uint8_t* tree, walk_relocate_args* args)
24651 assert ((tree != NULL));
24652 if (node_left_child (tree))
24654 walk_relocation_in_brick (tree + node_left_child (tree), args);
24657 uint8_t* plug = tree;
24658 BOOL has_pre_plug_info_p = FALSE;
24659 BOOL has_post_plug_info_p = FALSE;
24661 if (tree == oldest_pinned_plug)
24663 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
24664 &has_post_plug_info_p);
24665 assert (tree == pinned_plug (args->pinned_plug_entry));
24668 if (args->last_plug != 0)
24670 size_t gap_size = node_gap_size (tree);
24671 uint8_t* gap = (plug - gap_size);
24672 uint8_t* last_plug_end = gap;
24673 size_t last_plug_size = (last_plug_end - args->last_plug);
24674 dprintf (3, ("tree: %Ix, last_plug: %Ix, gap: %Ix(%Ix), last_plug_end: %Ix, size: %Ix",
24675 tree, args->last_plug, gap, gap_size, last_plug_end, last_plug_size));
24677 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
24678 if (!check_last_object_p)
24680 assert (last_plug_size >= Align (min_obj_size));
24683 walk_plug (args->last_plug, last_plug_size, check_last_object_p, args);
24687 assert (!has_pre_plug_info_p);
24690 dprintf (3, ("set args last plug to plug: %Ix", plug));
24691 args->last_plug = plug;
24692 args->is_shortened = has_post_plug_info_p;
24694 if (node_right_child (tree))
24696 walk_relocation_in_brick (tree + node_right_child (tree), args);
24700 void gc_heap::walk_relocation (void* profiling_context, record_surv_fn fn)
24702 generation* condemned_gen = generation_of (settings.condemned_generation);
24703 uint8_t* start_address = generation_allocation_start (condemned_gen);
24704 size_t current_brick = brick_of (start_address);
24705 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
24707 PREFIX_ASSUME(current_heap_segment != NULL);
24709 reset_pinned_queue_bos();
24710 update_oldest_pinned_plug();
24711 size_t end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24712 walk_relocate_args args;
24713 args.is_shortened = FALSE;
24714 args.pinned_plug_entry = 0;
24715 args.last_plug = 0;
24716 args.profiling_context = profiling_context;
24721 if (current_brick > end_brick)
24723 if (args.last_plug)
24725 walk_plug (args.last_plug,
24726 (heap_segment_allocated (current_heap_segment) - args.last_plug),
24729 args.last_plug = 0;
24731 if (heap_segment_next_rw (current_heap_segment))
24733 current_heap_segment = heap_segment_next_rw (current_heap_segment);
24734 current_brick = brick_of (heap_segment_mem (current_heap_segment));
24735 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24744 int brick_entry = brick_table [ current_brick ];
24745 if (brick_entry >= 0)
24747 walk_relocation_in_brick (brick_address (current_brick) +
24756 void gc_heap::walk_survivors (record_surv_fn fn, void* context, walk_surv_type type)
24758 if (type == walk_for_gc)
24759 walk_survivors_relocation (context, fn);
24760 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24761 else if (type == walk_for_bgc)
24762 walk_survivors_for_bgc (context, fn);
24763 #endif //BACKGROUND_GC && FEATURE_EVENT_TRACE
24764 else if (type == walk_for_loh)
24765 walk_survivors_for_loh (context, fn);
24767 assert (!"unknown type!");
24770 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24771 void gc_heap::walk_survivors_for_bgc (void* profiling_context, record_surv_fn fn)
24773 // This should only be called for BGCs
24774 assert(settings.concurrent);
24776 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
24778 BOOL small_object_segments = TRUE;
24779 int align_const = get_alignment_constant (small_object_segments);
24785 if (small_object_segments)
24787 //switch to large segment
24788 small_object_segments = FALSE;
24790 align_const = get_alignment_constant (small_object_segments);
24791 seg = heap_segment_rw (generation_start_segment (large_object_generation));
24793 PREFIX_ASSUME(seg != NULL);
24801 uint8_t* o = heap_segment_mem (seg);
24802 uint8_t* end = heap_segment_allocated (seg);
24806 if (method_table(o) == g_gc_pFreeObjectMethodTable)
24808 o += Align (size (o), align_const);
24812 // It's survived. Make a fake plug, starting at o,
24813 // and send the event
24815 uint8_t* plug_start = o;
24817 while (method_table(o) != g_gc_pFreeObjectMethodTable)
24819 o += Align (size (o), align_const);
24826 uint8_t* plug_end = o;
24830 0, // Reloc distance == 0 as this is non-compacting
24832 false, // Non-compacting
24836 seg = heap_segment_next (seg);
24839 #endif // defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24841 void gc_heap::relocate_phase (int condemned_gen_number,
24842 uint8_t* first_condemned_address)
24845 sc.thread_number = heap_number;
24846 sc.promotion = FALSE;
24847 sc.concurrent = FALSE;
24853 start = GetCycleCount32();
24856 // %type% category = quote (relocate);
24857 dprintf (2,("---- Relocate phase -----"));
24859 #ifdef MULTIPLE_HEAPS
24860 //join all threads to make sure they are synchronized
24861 dprintf(3, ("Joining after end of plan"));
24862 gc_t_join.join(this, gc_join_begin_relocate_phase);
24863 if (gc_t_join.joined())
24864 #endif //MULTIPLE_HEAPS
24867 #ifdef MULTIPLE_HEAPS
24869 //join all threads to make sure they are synchronized
24870 dprintf(3, ("Restarting for relocation"));
24871 gc_t_join.restart();
24872 #endif //MULTIPLE_HEAPS
24875 dprintf(3,("Relocating roots"));
24876 GCScan::GcScanRoots(GCHeap::Relocate,
24877 condemned_gen_number, max_generation, &sc);
24879 verify_pins_with_post_plug_info("after reloc stack");
24881 #ifdef BACKGROUND_GC
24882 if (recursive_gc_sync::background_running_p())
24884 scan_background_roots (GCHeap::Relocate, heap_number, &sc);
24886 #endif //BACKGROUND_GC
24888 if (condemned_gen_number != max_generation)
24890 dprintf(3,("Relocating cross generation pointers"));
24891 mark_through_cards_for_segments (&gc_heap::relocate_address, TRUE);
24892 verify_pins_with_post_plug_info("after reloc cards");
24894 if (condemned_gen_number != max_generation)
24896 dprintf(3,("Relocating cross generation pointers for large objects"));
24897 mark_through_cards_for_large_objects (&gc_heap::relocate_address, TRUE);
24901 #ifdef FEATURE_LOH_COMPACTION
24902 if (loh_compacted_p)
24904 assert (settings.condemned_generation == max_generation);
24905 relocate_in_loh_compact();
24908 #endif //FEATURE_LOH_COMPACTION
24910 relocate_in_large_objects ();
24914 dprintf(3,("Relocating survivors"));
24915 relocate_survivors (condemned_gen_number,
24916 first_condemned_address);
24919 #ifdef FEATURE_PREMORTEM_FINALIZATION
24920 dprintf(3,("Relocating finalization data"));
24921 finalize_queue->RelocateFinalizationData (condemned_gen_number,
24923 #endif // FEATURE_PREMORTEM_FINALIZATION
24928 dprintf(3,("Relocating handle table"));
24929 GCScan::GcScanHandles(GCHeap::Relocate,
24930 condemned_gen_number, max_generation, &sc);
24933 #ifdef MULTIPLE_HEAPS
24934 //join all threads to make sure they are synchronized
24935 dprintf(3, ("Joining after end of relocation"));
24936 gc_t_join.join(this, gc_join_relocate_phase_done);
24938 #endif //MULTIPLE_HEAPS
24941 finish = GetCycleCount32();
24942 reloc_time = finish - start;
24945 dprintf(2,( "---- End of Relocate phase ----"));
24948 // This compares to see if tree is the current pinned plug and returns info
24949 // for this pinned plug. Also advances the pinned queue if that's the case.
24951 // We don't change the values of the plug info if tree is not the same as
24952 // the current pinned plug - the caller is responsible for setting the right
24953 // values to begin with.
24955 // POPO TODO: We are keeping this temporarily as this is also used by realloc
24956 // where it passes FALSE to deque_p, change it to use the same optimization
24957 // as relocate. Not as essential since realloc is already a slow path.
24958 mark* gc_heap::get_next_pinned_entry (uint8_t* tree,
24959 BOOL* has_pre_plug_info_p,
24960 BOOL* has_post_plug_info_p,
24963 if (!pinned_plug_que_empty_p())
24965 mark* oldest_entry = oldest_pin();
24966 uint8_t* oldest_plug = pinned_plug (oldest_entry);
24967 if (tree == oldest_plug)
24969 *has_pre_plug_info_p = oldest_entry->has_pre_plug_info();
24970 *has_post_plug_info_p = oldest_entry->has_post_plug_info();
24974 deque_pinned_plug();
24977 dprintf (3, ("found a pinned plug %Ix, pre: %d, post: %d",
24979 (*has_pre_plug_info_p ? 1 : 0),
24980 (*has_post_plug_info_p ? 1 : 0)));
24982 return oldest_entry;
24989 // This also deques the oldest entry and update the oldest plug
24990 mark* gc_heap::get_oldest_pinned_entry (BOOL* has_pre_plug_info_p,
24991 BOOL* has_post_plug_info_p)
24993 mark* oldest_entry = oldest_pin();
24994 *has_pre_plug_info_p = oldest_entry->has_pre_plug_info();
24995 *has_post_plug_info_p = oldest_entry->has_post_plug_info();
24997 deque_pinned_plug();
24998 update_oldest_pinned_plug();
24999 return oldest_entry;
25003 void gc_heap::copy_cards_range (uint8_t* dest, uint8_t* src, size_t len, BOOL copy_cards_p)
25006 copy_cards_for_addresses (dest, src, len);
25008 clear_card_for_addresses (dest, dest + len);
25011 // POPO TODO: We should actually just recover the artifically made gaps here..because when we copy
25012 // we always copy the earlier plugs first which means we won't need the gap sizes anymore. This way
25013 // we won't need to individually recover each overwritten part of plugs.
25015 void gc_heap::gcmemcopy (uint8_t* dest, uint8_t* src, size_t len, BOOL copy_cards_p)
25019 #ifdef BACKGROUND_GC
25020 if (current_c_gc_state == c_gc_state_marking)
25022 //TODO: should look to see whether we should consider changing this
25023 // to copy a consecutive region of the mark array instead.
25024 copy_mark_bits_for_addresses (dest, src, len);
25026 #endif //BACKGROUND_GC
25027 //dprintf(3,(" Memcopy [%Ix->%Ix, %Ix->%Ix[", (size_t)src, (size_t)dest, (size_t)src+len, (size_t)dest+len));
25028 dprintf(3,(" mc: [%Ix->%Ix, %Ix->%Ix[", (size_t)src, (size_t)dest, (size_t)src+len, (size_t)dest+len));
25029 memcopy (dest - plug_skew, src - plug_skew, len);
25030 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25031 if (SoftwareWriteWatch::IsEnabledForGCHeap())
25033 // The ranges [src - plug_kew .. src[ and [src + len - plug_skew .. src + len[ are ObjHeaders, which don't have GC
25034 // references, and are not relevant for write watch. The latter range actually corresponds to the ObjHeader for the
25035 // object at (src + len), so it can be ignored anyway.
25036 SoftwareWriteWatch::SetDirtyRegion(dest, len - plug_skew);
25038 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25039 copy_cards_range (dest, src, len, copy_cards_p);
25043 void gc_heap::compact_plug (uint8_t* plug, size_t size, BOOL check_last_object_p, compact_args* args)
25046 uint8_t* reloc_plug = plug + args->last_plug_relocation;
25048 if (check_last_object_p)
25050 size += sizeof (gap_reloc_pair);
25051 mark* entry = args->pinned_plug_entry;
25053 if (args->is_shortened)
25055 assert (entry->has_post_plug_info());
25056 entry->swap_post_plug_and_saved();
25060 assert (entry->has_pre_plug_info());
25061 entry->swap_pre_plug_and_saved();
25065 int old_brick_entry = brick_table [brick_of (plug)];
25067 assert (node_relocation_distance (plug) == args->last_plug_relocation);
25069 #ifdef FEATURE_STRUCTALIGN
25070 ptrdiff_t alignpad = node_alignpad(plug);
25073 make_unused_array (reloc_plug - alignpad, alignpad);
25074 if (brick_of (reloc_plug - alignpad) != brick_of (reloc_plug))
25076 // The alignment padding is straddling one or more bricks;
25077 // it has to be the last "object" of its first brick.
25078 fix_brick_to_highest (reloc_plug - alignpad, reloc_plug);
25081 #else // FEATURE_STRUCTALIGN
25082 size_t unused_arr_size = 0;
25083 BOOL already_padded_p = FALSE;
25085 if (is_plug_padded (plug))
25087 already_padded_p = TRUE;
25088 clear_plug_padded (plug);
25089 unused_arr_size = Align (min_obj_size);
25091 #endif //SHORT_PLUGS
25092 if (node_realigned (plug))
25094 unused_arr_size += switch_alignment_size (already_padded_p);
25097 if (unused_arr_size != 0)
25099 make_unused_array (reloc_plug - unused_arr_size, unused_arr_size);
25101 if (brick_of (reloc_plug - unused_arr_size) != brick_of (reloc_plug))
25103 dprintf (3, ("fix B for padding: %Id: %Ix->%Ix",
25104 unused_arr_size, (reloc_plug - unused_arr_size), reloc_plug));
25105 // The alignment padding is straddling one or more bricks;
25106 // it has to be the last "object" of its first brick.
25107 fix_brick_to_highest (reloc_plug - unused_arr_size, reloc_plug);
25110 #endif // FEATURE_STRUCTALIGN
25113 if (is_plug_padded (plug))
25115 make_unused_array (reloc_plug - Align (min_obj_size), Align (min_obj_size));
25117 if (brick_of (reloc_plug - Align (min_obj_size)) != brick_of (reloc_plug))
25119 // The alignment padding is straddling one or more bricks;
25120 // it has to be the last "object" of its first brick.
25121 fix_brick_to_highest (reloc_plug - Align (min_obj_size), reloc_plug);
25124 #endif //SHORT_PLUGS
25126 gcmemcopy (reloc_plug, plug, size, args->copy_cards_p);
25128 if (args->check_gennum_p)
25130 int src_gennum = args->src_gennum;
25131 if (src_gennum == -1)
25133 src_gennum = object_gennum (plug);
25136 int dest_gennum = object_gennum_plan (reloc_plug);
25138 if (src_gennum < dest_gennum)
25140 generation_allocation_size (generation_of (dest_gennum)) += size;
25144 size_t current_reloc_brick = args->current_compacted_brick;
25146 if (brick_of (reloc_plug) != current_reloc_brick)
25148 dprintf (3, ("last reloc B: %Ix, current reloc B: %Ix",
25149 current_reloc_brick, brick_of (reloc_plug)));
25151 if (args->before_last_plug)
25153 dprintf (3,(" fixing last brick %Ix to point to last plug %Ix(%Ix)",
25154 current_reloc_brick,
25155 args->before_last_plug,
25156 (args->before_last_plug - brick_address (current_reloc_brick))));
25159 set_brick (current_reloc_brick,
25160 args->before_last_plug - brick_address (current_reloc_brick));
25163 current_reloc_brick = brick_of (reloc_plug);
25165 size_t end_brick = brick_of (reloc_plug + size-1);
25166 if (end_brick != current_reloc_brick)
25168 // The plug is straddling one or more bricks
25169 // It has to be the last plug of its first brick
25170 dprintf (3,("plug spanning multiple bricks, fixing first brick %Ix to %Ix(%Ix)",
25171 current_reloc_brick, (size_t)reloc_plug,
25172 (reloc_plug - brick_address (current_reloc_brick))));
25175 set_brick (current_reloc_brick,
25176 reloc_plug - brick_address (current_reloc_brick));
25178 // update all intervening brick
25179 size_t brick = current_reloc_brick + 1;
25180 dprintf (3,("setting intervening bricks %Ix->%Ix to -1",
25181 brick, (end_brick - 1)));
25182 while (brick < end_brick)
25184 set_brick (brick, -1);
25187 // code last brick offset as a plug address
25188 args->before_last_plug = brick_address (end_brick) -1;
25189 current_reloc_brick = end_brick;
25190 dprintf (3, ("setting before last to %Ix, last brick to %Ix",
25191 args->before_last_plug, current_reloc_brick));
25195 dprintf (3, ("still in the same brick: %Ix", end_brick));
25196 args->before_last_plug = reloc_plug;
25198 args->current_compacted_brick = current_reloc_brick;
25200 if (check_last_object_p)
25202 mark* entry = args->pinned_plug_entry;
25204 if (args->is_shortened)
25206 entry->swap_post_plug_and_saved();
25210 entry->swap_pre_plug_and_saved();
25215 void gc_heap::compact_in_brick (uint8_t* tree, compact_args* args)
25217 assert (tree != NULL);
25218 int left_node = node_left_child (tree);
25219 int right_node = node_right_child (tree);
25220 ptrdiff_t relocation = node_relocation_distance (tree);
25226 dprintf (3, ("B: L: %d->%Ix", left_node, (tree + left_node)));
25227 compact_in_brick ((tree + left_node), args);
25230 uint8_t* plug = tree;
25231 BOOL has_pre_plug_info_p = FALSE;
25232 BOOL has_post_plug_info_p = FALSE;
25234 if (tree == oldest_pinned_plug)
25236 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
25237 &has_post_plug_info_p);
25238 assert (tree == pinned_plug (args->pinned_plug_entry));
25241 if (args->last_plug != 0)
25243 size_t gap_size = node_gap_size (tree);
25244 uint8_t* gap = (plug - gap_size);
25245 uint8_t* last_plug_end = gap;
25246 size_t last_plug_size = (last_plug_end - args->last_plug);
25247 dprintf (3, ("tree: %Ix, last_plug: %Ix, gap: %Ix(%Ix), last_plug_end: %Ix, size: %Ix",
25248 tree, args->last_plug, gap, gap_size, last_plug_end, last_plug_size));
25250 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
25251 if (!check_last_object_p)
25253 assert (last_plug_size >= Align (min_obj_size));
25256 compact_plug (args->last_plug, last_plug_size, check_last_object_p, args);
25260 assert (!has_pre_plug_info_p);
25263 dprintf (3, ("set args last plug to plug: %Ix, reloc: %Ix", plug, relocation));
25264 args->last_plug = plug;
25265 args->last_plug_relocation = relocation;
25266 args->is_shortened = has_post_plug_info_p;
25270 dprintf (3, ("B: R: %d->%Ix", right_node, (tree + right_node)));
25271 compact_in_brick ((tree + right_node), args);
25275 void gc_heap::recover_saved_pinned_info()
25277 reset_pinned_queue_bos();
25279 while (!(pinned_plug_que_empty_p()))
25281 mark* oldest_entry = oldest_pin();
25282 oldest_entry->recover_plug_info();
25283 #ifdef GC_CONFIG_DRIVEN
25284 if (oldest_entry->has_pre_plug_info() && oldest_entry->has_post_plug_info())
25285 record_interesting_data_point (idp_pre_and_post_pin);
25286 else if (oldest_entry->has_pre_plug_info())
25287 record_interesting_data_point (idp_pre_pin);
25288 else if (oldest_entry->has_post_plug_info())
25289 record_interesting_data_point (idp_post_pin);
25290 #endif //GC_CONFIG_DRIVEN
25292 deque_pinned_plug();
25296 void gc_heap::compact_phase (int condemned_gen_number,
25297 uint8_t* first_condemned_address,
25300 // %type% category = quote (compact);
25304 start = GetCycleCount32();
25306 generation* condemned_gen = generation_of (condemned_gen_number);
25307 uint8_t* start_address = first_condemned_address;
25308 size_t current_brick = brick_of (start_address);
25309 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
25311 PREFIX_ASSUME(current_heap_segment != NULL);
25313 reset_pinned_queue_bos();
25314 update_oldest_pinned_plug();
25316 BOOL reused_seg = expand_reused_seg_p();
25319 for (int i = 1; i <= max_generation; i++)
25321 generation_allocation_size (generation_of (i)) = 0;
25325 uint8_t* end_address = heap_segment_allocated (current_heap_segment);
25327 size_t end_brick = brick_of (end_address-1);
25329 args.last_plug = 0;
25330 args.before_last_plug = 0;
25331 args.current_compacted_brick = ~((size_t)1);
25332 args.is_shortened = FALSE;
25333 args.pinned_plug_entry = 0;
25334 args.copy_cards_p = (condemned_gen_number >= 1) || !clear_cards;
25335 args.check_gennum_p = reused_seg;
25336 if (args.check_gennum_p)
25338 args.src_gennum = ((current_heap_segment == ephemeral_heap_segment) ? -1 : 2);
25341 dprintf (2,("---- Compact Phase: %Ix(%Ix)----",
25342 first_condemned_address, brick_of (first_condemned_address)));
25344 #ifdef MULTIPLE_HEAPS
25346 if (gc_t_join.joined())
25348 #endif //MULTIPLE_HEAPS
25350 #ifdef MULTIPLE_HEAPS
25351 dprintf(3, ("Restarting for compaction"));
25352 gc_t_join.restart();
25354 #endif //MULTIPLE_HEAPS
25356 reset_pinned_queue_bos();
25358 #ifdef FEATURE_LOH_COMPACTION
25359 if (loh_compacted_p)
25363 #endif //FEATURE_LOH_COMPACTION
25365 if ((start_address < end_address) ||
25366 (condemned_gen_number == max_generation))
25370 if (current_brick > end_brick)
25372 if (args.last_plug != 0)
25374 dprintf (3, ("compacting last plug: %Ix", args.last_plug))
25375 compact_plug (args.last_plug,
25376 (heap_segment_allocated (current_heap_segment) - args.last_plug),
25381 if (heap_segment_next_rw (current_heap_segment))
25383 current_heap_segment = heap_segment_next_rw (current_heap_segment);
25384 current_brick = brick_of (heap_segment_mem (current_heap_segment));
25385 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
25386 args.last_plug = 0;
25387 if (args.check_gennum_p)
25389 args.src_gennum = ((current_heap_segment == ephemeral_heap_segment) ? -1 : 2);
25395 if (args.before_last_plug !=0)
25397 dprintf (3, ("Fixing last brick %Ix to point to plug %Ix",
25398 args.current_compacted_brick, (size_t)args.before_last_plug));
25399 assert (args.current_compacted_brick != ~1u);
25400 set_brick (args.current_compacted_brick,
25401 args.before_last_plug - brick_address (args.current_compacted_brick));
25407 int brick_entry = brick_table [ current_brick ];
25408 dprintf (3, ("B: %Ix(%Ix)->%Ix",
25409 current_brick, (size_t)brick_entry, (brick_address (current_brick) + brick_entry - 1)));
25411 if (brick_entry >= 0)
25413 compact_in_brick ((brick_address (current_brick) + brick_entry -1),
25422 recover_saved_pinned_info();
25425 finish = GetCycleCount32();
25426 compact_time = finish - start;
25429 concurrent_print_time_delta ("compact end");
25431 dprintf(2,("---- End of Compact phase ----"));
25434 #ifdef MULTIPLE_HEAPS
25437 #pragma warning(push)
25438 #pragma warning(disable:4702) // C4702: unreachable code: gc_thread_function may not return
25440 void gc_heap::gc_thread_stub (void* arg)
25442 gc_heap* heap = (gc_heap*)arg;
25443 if (!gc_thread_no_affinitize_p)
25445 GCThreadAffinity affinity;
25446 affinity.Group = GCThreadAffinity::None;
25447 affinity.Processor = GCThreadAffinity::None;
25449 // We are about to set affinity for GC threads. It is a good place to set up NUMA and
25450 // CPU groups because the process mask, processor number, and group number are all
25451 // readily available.
25452 if (GCToOSInterface::CanEnableGCCPUGroups())
25453 set_thread_group_affinity_for_heap(heap->heap_number, &affinity);
25455 set_thread_affinity_mask_for_heap(heap->heap_number, &affinity);
25457 if (!GCToOSInterface::SetThreadAffinity(&affinity))
25459 dprintf(1, ("Failed to set thread affinity for server GC thread"));
25463 // server GC threads run at a higher priority than normal.
25464 GCToOSInterface::BoostThreadPriority();
25465 _alloca (256*heap->heap_number);
25466 heap->gc_thread_function();
25469 #pragma warning(pop)
25472 #endif //MULTIPLE_HEAPS
25474 #ifdef BACKGROUND_GC
25477 #pragma warning(push)
25478 #pragma warning(disable:4702) // C4702: unreachable code: gc_thread_function may not return
25480 void gc_heap::bgc_thread_stub (void* arg)
25482 gc_heap* heap = (gc_heap*)arg;
25483 heap->bgc_thread = GCToEEInterface::GetThread();
25484 assert(heap->bgc_thread != nullptr);
25485 heap->bgc_thread_function();
25488 #pragma warning(pop)
25491 #endif //BACKGROUND_GC
25493 /*------------------ Background GC ----------------------------*/
25495 #ifdef BACKGROUND_GC
25497 void gc_heap::background_drain_mark_list (int thread)
25499 UNREFERENCED_PARAMETER(thread);
25501 size_t saved_c_mark_list_index = c_mark_list_index;
25503 if (saved_c_mark_list_index)
25505 concurrent_print_time_delta ("SML");
25507 while (c_mark_list_index != 0)
25509 size_t current_index = c_mark_list_index - 1;
25510 uint8_t* o = c_mark_list [current_index];
25511 background_mark_object (o THREAD_NUMBER_ARG);
25512 c_mark_list_index--;
25514 if (saved_c_mark_list_index)
25517 concurrent_print_time_delta ("EML");
25520 fire_drain_mark_list_event (saved_c_mark_list_index);
25524 // The background GC version of scan_dependent_handles (see that method for a more in-depth comment).
25525 #ifdef MULTIPLE_HEAPS
25526 // Since we only scan dependent handles while we are stopped we'll never interfere with FGCs scanning
25527 // them. So we can use the same static variables.
25528 void gc_heap::background_scan_dependent_handles (ScanContext *sc)
25530 // Whenever we call this method there may have been preceding object promotions. So set
25531 // s_fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
25532 // based on the how the scanning proceeded).
25533 s_fUnscannedPromotions = TRUE;
25535 // We don't know how many times we need to loop yet. In particular we can't base the loop condition on
25536 // the state of this thread's portion of the dependent handle table. That's because promotions on other
25537 // threads could cause handle promotions to become necessary here. Even if there are definitely no more
25538 // promotions possible in this thread's handles, we still have to stay in lock-step with those worker
25539 // threads that haven't finished yet (each GC worker thread has to join exactly the same number of times
25540 // as all the others or they'll get out of step).
25543 // The various worker threads are all currently racing in this code. We need to work out if at least
25544 // one of them think they have work to do this cycle. Each thread needs to rescan its portion of the
25545 // dependent handle table when both of the following conditions apply:
25546 // 1) At least one (arbitrary) object might have been promoted since the last scan (because if this
25547 // object happens to correspond to a primary in one of our handles we might potentially have to
25548 // promote the associated secondary).
25549 // 2) The table for this thread has at least one handle with a secondary that isn't promoted yet.
25551 // The first condition is represented by s_fUnscannedPromotions. This is always non-zero for the first
25552 // iteration of this loop (see comment above) and in subsequent cycles each thread updates this
25553 // whenever a mark stack overflow occurs or scanning their dependent handles results in a secondary
25554 // being promoted. This value is cleared back to zero in a synchronized fashion in the join that
25555 // follows below. Note that we can't read this outside of the join since on any iteration apart from
25556 // the first threads will be racing between reading this value and completing their previous
25557 // iteration's table scan.
25559 // The second condition is tracked by the dependent handle code itself on a per worker thread basis
25560 // (and updated by the GcDhReScan() method). We call GcDhUnpromotedHandlesExist() on each thread to
25561 // determine the local value and collect the results into the s_fUnpromotedHandles variable in what is
25562 // effectively an OR operation. As per s_fUnscannedPromotions we can't read the final result until
25563 // we're safely joined.
25564 if (GCScan::GcDhUnpromotedHandlesExist(sc))
25565 s_fUnpromotedHandles = TRUE;
25567 // Synchronize all the threads so we can read our state variables safely. The following shared
25568 // variable (indicating whether we should scan the tables or terminate the loop) will be set by a
25569 // single thread inside the join.
25570 bgc_t_join.join(this, gc_join_scan_dependent_handles);
25571 if (bgc_t_join.joined())
25573 // We're synchronized so it's safe to read our shared state variables. We update another shared
25574 // variable to indicate to all threads whether we'll be scanning for another cycle or terminating
25575 // the loop. We scan if there has been at least one object promotion since last time and at least
25576 // one thread has a dependent handle table with a potential handle promotion possible.
25577 s_fScanRequired = s_fUnscannedPromotions && s_fUnpromotedHandles;
25579 // Reset our shared state variables (ready to be set again on this scan or with a good initial
25580 // value for the next call if we're terminating the loop).
25581 s_fUnscannedPromotions = FALSE;
25582 s_fUnpromotedHandles = FALSE;
25584 if (!s_fScanRequired)
25586 uint8_t* all_heaps_max = 0;
25587 uint8_t* all_heaps_min = MAX_PTR;
25589 for (i = 0; i < n_heaps; i++)
25591 if (all_heaps_max < g_heaps[i]->background_max_overflow_address)
25592 all_heaps_max = g_heaps[i]->background_max_overflow_address;
25593 if (all_heaps_min > g_heaps[i]->background_min_overflow_address)
25594 all_heaps_min = g_heaps[i]->background_min_overflow_address;
25596 for (i = 0; i < n_heaps; i++)
25598 g_heaps[i]->background_max_overflow_address = all_heaps_max;
25599 g_heaps[i]->background_min_overflow_address = all_heaps_min;
25603 // Restart all the workers.
25604 dprintf(2, ("Starting all gc thread mark stack overflow processing"));
25605 bgc_t_join.restart();
25608 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
25609 // being visible. If there really was an overflow (process_mark_overflow returns true) then set the
25610 // global flag indicating that at least one object promotion may have occurred (the usual comment
25611 // about races applies). (Note it's OK to set this flag even if we're about to terminate the loop and
25612 // exit the method since we unconditionally set this variable on method entry anyway).
25613 if (background_process_mark_overflow (sc->concurrent))
25614 s_fUnscannedPromotions = TRUE;
25616 // If we decided that no scan was required we can terminate the loop now.
25617 if (!s_fScanRequired)
25620 // Otherwise we must join with the other workers to ensure that all mark stack overflows have been
25621 // processed before we start scanning dependent handle tables (if overflows remain while we scan we
25622 // could miss noting the promotion of some primary objects).
25623 bgc_t_join.join(this, gc_join_rescan_dependent_handles);
25624 if (bgc_t_join.joined())
25626 // Restart all the workers.
25627 dprintf(3, ("Starting all gc thread for dependent handle promotion"));
25628 bgc_t_join.restart();
25631 // If the portion of the dependent handle table managed by this worker has handles that could still be
25632 // promoted perform a rescan. If the rescan resulted in at least one promotion note this fact since it
25633 // could require a rescan of handles on this or other workers.
25634 if (GCScan::GcDhUnpromotedHandlesExist(sc))
25635 if (GCScan::GcDhReScan(sc))
25636 s_fUnscannedPromotions = TRUE;
25640 void gc_heap::background_scan_dependent_handles (ScanContext *sc)
25642 // Whenever we call this method there may have been preceding object promotions. So set
25643 // fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
25644 // based on the how the scanning proceeded).
25645 bool fUnscannedPromotions = true;
25647 // Scan dependent handles repeatedly until there are no further promotions that can be made or we made a
25648 // scan without performing any new promotions.
25649 while (GCScan::GcDhUnpromotedHandlesExist(sc) && fUnscannedPromotions)
25651 // On each iteration of the loop start with the assumption that no further objects have been promoted.
25652 fUnscannedPromotions = false;
25654 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
25655 // being visible. If there was an overflow (background_process_mark_overflow returned true) then
25656 // additional objects now appear to be promoted and we should set the flag.
25657 if (background_process_mark_overflow (sc->concurrent))
25658 fUnscannedPromotions = true;
25660 // Perform the scan and set the flag if any promotions resulted.
25661 if (GCScan::GcDhReScan (sc))
25662 fUnscannedPromotions = true;
25665 // Perform a last processing of any overflowed mark stack.
25666 background_process_mark_overflow (sc->concurrent);
25668 #endif //MULTIPLE_HEAPS
25670 void gc_heap::recover_bgc_settings()
25672 if ((settings.condemned_generation < max_generation) && recursive_gc_sync::background_running_p())
25674 dprintf (2, ("restoring bgc settings"));
25675 settings = saved_bgc_settings;
25676 GCHeap::GcCondemnedGeneration = gc_heap::settings.condemned_generation;
25680 void gc_heap::allow_fgc()
25682 assert (bgc_thread == GCToEEInterface::GetThread());
25683 bool bToggleGC = false;
25685 if (g_fSuspensionPending > 0)
25687 bToggleGC = GCToEEInterface::EnablePreemptiveGC();
25690 GCToEEInterface::DisablePreemptiveGC();
25695 BOOL gc_heap::should_commit_mark_array()
25697 return (recursive_gc_sync::background_running_p() || (current_bgc_state == bgc_initialized));
25700 void gc_heap::clear_commit_flag()
25702 generation* gen = generation_of (max_generation);
25703 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
25708 if (gen != large_object_generation)
25710 gen = large_object_generation;
25711 seg = heap_segment_in_range (generation_start_segment (gen));
25719 if (seg->flags & heap_segment_flags_ma_committed)
25721 seg->flags &= ~heap_segment_flags_ma_committed;
25724 if (seg->flags & heap_segment_flags_ma_pcommitted)
25726 seg->flags &= ~heap_segment_flags_ma_pcommitted;
25729 seg = heap_segment_next (seg);
25733 void gc_heap::clear_commit_flag_global()
25735 #ifdef MULTIPLE_HEAPS
25736 for (int i = 0; i < n_heaps; i++)
25738 g_heaps[i]->clear_commit_flag();
25741 clear_commit_flag();
25742 #endif //MULTIPLE_HEAPS
25745 void gc_heap::verify_mark_array_cleared (uint8_t* begin, uint8_t* end, uint32_t* mark_array_addr)
25748 size_t markw = mark_word_of (begin);
25749 size_t markw_end = mark_word_of (end);
25751 while (markw < markw_end)
25753 if (mark_array_addr[markw])
25755 dprintf (1, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
25756 markw, mark_array_addr[markw], mark_word_address (markw)));
25762 UNREFERENCED_PARAMETER(begin);
25763 UNREFERENCED_PARAMETER(end);
25764 UNREFERENCED_PARAMETER(mark_array_addr);
25768 void gc_heap::verify_mark_array_cleared (heap_segment* seg, uint32_t* mark_array_addr)
25770 verify_mark_array_cleared (heap_segment_mem (seg), heap_segment_reserved (seg), mark_array_addr);
25773 BOOL gc_heap::commit_mark_array_new_seg (gc_heap* hp,
25775 uint32_t* new_card_table,
25776 uint8_t* new_lowest_address)
25778 UNREFERENCED_PARAMETER(hp); // compiler bug? -- this *is*, indeed, referenced
25780 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
25781 uint8_t* end = heap_segment_reserved (seg);
25783 uint8_t* lowest = hp->background_saved_lowest_address;
25784 uint8_t* highest = hp->background_saved_highest_address;
25786 uint8_t* commit_start = NULL;
25787 uint8_t* commit_end = NULL;
25788 size_t commit_flag = 0;
25790 if ((highest >= start) &&
25793 if ((start >= lowest) && (end <= highest))
25795 dprintf (GC_TABLE_LOG, ("completely in bgc range: seg %Ix-%Ix, bgc: %Ix-%Ix",
25796 start, end, lowest, highest));
25797 commit_flag = heap_segment_flags_ma_committed;
25801 dprintf (GC_TABLE_LOG, ("partially in bgc range: seg %Ix-%Ix, bgc: %Ix-%Ix",
25802 start, end, lowest, highest));
25803 commit_flag = heap_segment_flags_ma_pcommitted;
25806 commit_start = max (lowest, start);
25807 commit_end = min (highest, end);
25809 if (!commit_mark_array_by_range (commit_start, commit_end, hp->mark_array))
25814 if (new_card_table == 0)
25816 new_card_table = g_gc_card_table;
25819 if (hp->card_table != new_card_table)
25821 if (new_lowest_address == 0)
25823 new_lowest_address = g_gc_lowest_address;
25826 uint32_t* ct = &new_card_table[card_word (gcard_of (new_lowest_address))];
25827 uint32_t* ma = (uint32_t*)((uint8_t*)card_table_mark_array (ct) - size_mark_array_of (0, new_lowest_address));
25829 dprintf (GC_TABLE_LOG, ("table realloc-ed: %Ix->%Ix, MA: %Ix->%Ix",
25830 hp->card_table, new_card_table,
25831 hp->mark_array, ma));
25833 if (!commit_mark_array_by_range (commit_start, commit_end, ma))
25839 seg->flags |= commit_flag;
25845 BOOL gc_heap::commit_mark_array_by_range (uint8_t* begin, uint8_t* end, uint32_t* mark_array_addr)
25847 size_t beg_word = mark_word_of (begin);
25848 size_t end_word = mark_word_of (align_on_mark_word (end));
25849 uint8_t* commit_start = align_lower_page ((uint8_t*)&mark_array_addr[beg_word]);
25850 uint8_t* commit_end = align_on_page ((uint8_t*)&mark_array_addr[end_word]);
25851 size_t size = (size_t)(commit_end - commit_start);
25853 #ifdef SIMPLE_DPRINTF
25854 dprintf (GC_TABLE_LOG, ("range: %Ix->%Ix mark word: %Ix->%Ix(%Id), mark array: %Ix->%Ix(%Id), commit %Ix->%Ix(%Id)",
25856 beg_word, end_word,
25857 (end_word - beg_word) * sizeof (uint32_t),
25858 &mark_array_addr[beg_word],
25859 &mark_array_addr[end_word],
25860 (size_t)(&mark_array_addr[end_word] - &mark_array_addr[beg_word]),
25861 commit_start, commit_end,
25863 #endif //SIMPLE_DPRINTF
25865 if (virtual_commit (commit_start, size))
25867 // We can only verify the mark array is cleared from begin to end, the first and the last
25868 // page aren't necessarily all cleared 'cause they could be used by other segments or
25870 verify_mark_array_cleared (begin, end, mark_array_addr);
25875 dprintf (GC_TABLE_LOG, ("failed to commit %Id bytes", (end_word - beg_word) * sizeof (uint32_t)));
25880 BOOL gc_heap::commit_mark_array_with_check (heap_segment* seg, uint32_t* new_mark_array_addr)
25882 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
25883 uint8_t* end = heap_segment_reserved (seg);
25885 #ifdef MULTIPLE_HEAPS
25886 uint8_t* lowest = heap_segment_heap (seg)->background_saved_lowest_address;
25887 uint8_t* highest = heap_segment_heap (seg)->background_saved_highest_address;
25889 uint8_t* lowest = background_saved_lowest_address;
25890 uint8_t* highest = background_saved_highest_address;
25891 #endif //MULTIPLE_HEAPS
25893 if ((highest >= start) &&
25896 start = max (lowest, start);
25897 end = min (highest, end);
25898 if (!commit_mark_array_by_range (start, end, new_mark_array_addr))
25907 BOOL gc_heap::commit_mark_array_by_seg (heap_segment* seg, uint32_t* mark_array_addr)
25909 dprintf (GC_TABLE_LOG, ("seg: %Ix->%Ix; MA: %Ix",
25911 heap_segment_reserved (seg),
25913 uint8_t* start = (heap_segment_read_only_p (seg) ? heap_segment_mem (seg) : (uint8_t*)seg);
25915 return commit_mark_array_by_range (start, heap_segment_reserved (seg), mark_array_addr);
25918 BOOL gc_heap::commit_mark_array_bgc_init (uint32_t* mark_array_addr)
25920 UNREFERENCED_PARAMETER(mark_array_addr);
25922 dprintf (GC_TABLE_LOG, ("BGC init commit: lowest: %Ix, highest: %Ix, mark_array: %Ix",
25923 lowest_address, highest_address, mark_array));
25925 generation* gen = generation_of (max_generation);
25926 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
25931 if (gen != large_object_generation)
25933 gen = large_object_generation;
25934 seg = heap_segment_in_range (generation_start_segment (gen));
25942 dprintf (GC_TABLE_LOG, ("seg: %Ix, flags: %Id", seg, seg->flags));
25944 if (!(seg->flags & heap_segment_flags_ma_committed))
25946 // For ro segments they could always be only partially in range so we'd
25947 // be calling this at the beginning of every BGC. We are not making this
25948 // more efficient right now - ro segments are currently only used by redhawk.
25949 if (heap_segment_read_only_p (seg))
25951 if ((heap_segment_mem (seg) >= lowest_address) &&
25952 (heap_segment_reserved (seg) <= highest_address))
25954 if (commit_mark_array_by_seg (seg, mark_array))
25956 seg->flags |= heap_segment_flags_ma_committed;
25965 uint8_t* start = max (lowest_address, heap_segment_mem (seg));
25966 uint8_t* end = min (highest_address, heap_segment_reserved (seg));
25967 if (commit_mark_array_by_range (start, end, mark_array))
25969 seg->flags |= heap_segment_flags_ma_pcommitted;
25979 // For normal segments they are by design completely in range so just
25980 // commit the whole mark array for each seg.
25981 if (commit_mark_array_by_seg (seg, mark_array))
25983 if (seg->flags & heap_segment_flags_ma_pcommitted)
25985 seg->flags &= ~heap_segment_flags_ma_pcommitted;
25987 seg->flags |= heap_segment_flags_ma_committed;
25996 seg = heap_segment_next (seg);
26002 // This function doesn't check the commit flag since it's for a new array -
26003 // the mark_array flag for these segments will remain the same.
26004 BOOL gc_heap::commit_new_mark_array (uint32_t* new_mark_array_addr)
26006 dprintf (GC_TABLE_LOG, ("commiting existing segs on MA %Ix", new_mark_array_addr));
26007 generation* gen = generation_of (max_generation);
26008 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
26013 if (gen != large_object_generation)
26015 gen = large_object_generation;
26016 seg = heap_segment_in_range (generation_start_segment (gen));
26024 if (!commit_mark_array_with_check (seg, new_mark_array_addr))
26029 seg = heap_segment_next (seg);
26032 #ifdef MULTIPLE_HEAPS
26033 if (new_heap_segment)
26035 if (!commit_mark_array_with_check (new_heap_segment, new_mark_array_addr))
26040 #endif //MULTIPLE_HEAPS
26045 BOOL gc_heap::commit_new_mark_array_global (uint32_t* new_mark_array)
26047 #ifdef MULTIPLE_HEAPS
26048 for (int i = 0; i < n_heaps; i++)
26050 if (!g_heaps[i]->commit_new_mark_array (new_mark_array))
26056 if (!commit_new_mark_array (new_mark_array))
26060 #endif //MULTIPLE_HEAPS
26065 void gc_heap::decommit_mark_array_by_seg (heap_segment* seg)
26067 // if BGC is disabled (the finalize watchdog does this at shutdown), the mark array could have
26068 // been set to NULL.
26069 if (mark_array == NULL)
26074 dprintf (GC_TABLE_LOG, ("decommitting seg %Ix(%Ix), MA: %Ix", seg, seg->flags, mark_array));
26076 size_t flags = seg->flags;
26078 if ((flags & heap_segment_flags_ma_committed) ||
26079 (flags & heap_segment_flags_ma_pcommitted))
26081 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
26082 uint8_t* end = heap_segment_reserved (seg);
26084 if (flags & heap_segment_flags_ma_pcommitted)
26086 start = max (lowest_address, start);
26087 end = min (highest_address, end);
26090 size_t beg_word = mark_word_of (start);
26091 size_t end_word = mark_word_of (align_on_mark_word (end));
26092 uint8_t* decommit_start = align_on_page ((uint8_t*)&mark_array[beg_word]);
26093 uint8_t* decommit_end = align_lower_page ((uint8_t*)&mark_array[end_word]);
26094 size_t size = (size_t)(decommit_end - decommit_start);
26096 #ifdef SIMPLE_DPRINTF
26097 dprintf (GC_TABLE_LOG, ("seg: %Ix mark word: %Ix->%Ix(%Id), mark array: %Ix->%Ix(%Id), decommit %Ix->%Ix(%Id)",
26099 beg_word, end_word,
26100 (end_word - beg_word) * sizeof (uint32_t),
26101 &mark_array[beg_word],
26102 &mark_array[end_word],
26103 (size_t)(&mark_array[end_word] - &mark_array[beg_word]),
26104 decommit_start, decommit_end,
26106 #endif //SIMPLE_DPRINTF
26108 if (decommit_start < decommit_end)
26110 if (!virtual_decommit (decommit_start, size))
26112 dprintf (GC_TABLE_LOG, ("decommit on %Ix for %Id bytes failed",
26113 decommit_start, size));
26114 assert (!"decommit failed");
26118 dprintf (GC_TABLE_LOG, ("decommited [%Ix for address [%Ix", beg_word, seg));
26122 void gc_heap::background_mark_phase ()
26124 verify_mark_array_cleared();
26127 sc.thread_number = heap_number;
26128 sc.promotion = TRUE;
26129 sc.concurrent = FALSE;
26132 BOOL cooperative_mode = TRUE;
26133 #ifndef MULTIPLE_HEAPS
26134 const int thread = heap_number;
26135 #endif //!MULTIPLE_HEAPS
26137 dprintf(2,("-(GC%d)BMark-", VolatileLoad(&settings.gc_index)));
26139 assert (settings.concurrent);
26144 start = GetCycleCount32();
26147 #ifdef FFIND_OBJECT
26148 if (gen0_must_clear_bricks > 0)
26149 gen0_must_clear_bricks--;
26150 #endif //FFIND_OBJECT
26152 background_soh_alloc_count = 0;
26153 background_loh_alloc_count = 0;
26154 bgc_overflow_count = 0;
26156 bpromoted_bytes (heap_number) = 0;
26157 static uint32_t num_sizedrefs = 0;
26159 background_min_overflow_address = MAX_PTR;
26160 background_max_overflow_address = 0;
26161 background_min_soh_overflow_address = MAX_PTR;
26162 background_max_soh_overflow_address = 0;
26163 processed_soh_overflow_p = FALSE;
26166 //set up the mark lists from g_mark_list
26167 assert (g_mark_list);
26168 mark_list = g_mark_list;
26169 //dont use the mark list for full gc
26170 //because multiple segments are more complex to handle and the list
26171 //is likely to overflow
26172 mark_list_end = &mark_list [0];
26173 mark_list_index = &mark_list [0];
26175 c_mark_list_index = 0;
26177 #ifndef MULTIPLE_HEAPS
26178 shigh = (uint8_t*) 0;
26180 #endif //MULTIPLE_HEAPS
26182 generation* gen = generation_of (max_generation);
26184 dprintf(3,("BGC: stack marking"));
26185 sc.concurrent = TRUE;
26187 GCScan::GcScanRoots(background_promote_callback,
26188 max_generation, max_generation,
26193 dprintf(3,("BGC: finalization marking"));
26194 finalize_queue->GcScanRoots(background_promote_callback, heap_number, 0);
26197 size_t total_loh_size = generation_size (max_generation + 1);
26198 bgc_begin_loh_size = total_loh_size;
26199 bgc_alloc_spin_loh = 0;
26200 bgc_loh_size_increased = 0;
26201 bgc_loh_allocated_in_free = 0;
26202 size_t total_soh_size = generation_sizes (generation_of (max_generation));
26204 dprintf (GTC_LOG, ("BM: h%d: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
26207 //concurrent_print_time_delta ("copying stack roots");
26208 concurrent_print_time_delta ("CS");
26210 FIRE_EVENT(BGC1stNonConEnd);
26212 expanded_in_fgc = FALSE;
26213 saved_overflow_ephemeral_seg = 0;
26214 current_bgc_state = bgc_reset_ww;
26216 // we don't need a join here - just whichever thread that gets here
26217 // first can change the states and call restart_vm.
26218 // this is not true - we can't let the EE run when we are scanning stack.
26219 // since we now allow reset ww to run concurrently and have a join for it,
26220 // we can do restart ee on the 1st thread that got here. Make sure we handle the
26221 // sizedref handles correctly.
26222 #ifdef MULTIPLE_HEAPS
26223 bgc_t_join.join(this, gc_join_restart_ee);
26224 if (bgc_t_join.joined())
26225 #endif //MULTIPLE_HEAPS
26227 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26228 // Resetting write watch for software write watch is pretty fast, much faster than for hardware write watch. Reset
26229 // can be done while the runtime is suspended or after the runtime is restarted, the preference was to reset while
26230 // the runtime is suspended. The reset for hardware write watch is done after the runtime is restarted below.
26232 concurrent_print_time_delta ("CRWW begin");
26234 #ifdef MULTIPLE_HEAPS
26235 for (int i = 0; i < n_heaps; i++)
26237 g_heaps[i]->reset_write_watch (FALSE);
26240 reset_write_watch (FALSE);
26241 #endif //MULTIPLE_HEAPS
26243 concurrent_print_time_delta ("CRWW");
26244 #endif //WRITE_WATCH
26245 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26247 num_sizedrefs = GCToEEInterface::GetTotalNumSizedRefHandles();
26249 // this c_write is not really necessary because restart_vm
26250 // has an instruction that will flush the cpu cache (interlocked
26251 // or whatever) but we don't want to rely on that.
26252 dprintf (BGC_LOG, ("setting cm_in_progress"));
26253 c_write (cm_in_progress, TRUE);
26255 //restart all thread, doing the marking from the array
26256 assert (dont_restart_ee_p);
26257 dont_restart_ee_p = FALSE;
26260 GCToOSInterface::YieldThread (0);
26261 #ifdef MULTIPLE_HEAPS
26262 dprintf(3, ("Starting all gc threads for gc"));
26263 bgc_t_join.restart();
26264 #endif //MULTIPLE_HEAPS
26267 #ifdef MULTIPLE_HEAPS
26268 bgc_t_join.join(this, gc_join_after_reset);
26269 if (bgc_t_join.joined())
26270 #endif //MULTIPLE_HEAPS
26272 disable_preemptive (true);
26274 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26275 // When software write watch is enabled, resetting write watch is done while the runtime is suspended above. The
26276 // post-reset call to revisit_written_pages is only necessary for concurrent reset_write_watch, to discard dirtied
26277 // pages during the concurrent reset.
26280 concurrent_print_time_delta ("CRWW begin");
26282 #ifdef MULTIPLE_HEAPS
26283 for (int i = 0; i < n_heaps; i++)
26285 g_heaps[i]->reset_write_watch (TRUE);
26288 reset_write_watch (TRUE);
26289 #endif //MULTIPLE_HEAPS
26291 concurrent_print_time_delta ("CRWW");
26292 #endif //WRITE_WATCH
26294 #ifdef MULTIPLE_HEAPS
26295 for (int i = 0; i < n_heaps; i++)
26297 g_heaps[i]->revisit_written_pages (TRUE, TRUE);
26300 revisit_written_pages (TRUE, TRUE);
26301 #endif //MULTIPLE_HEAPS
26303 concurrent_print_time_delta ("CRW");
26304 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26306 #ifdef MULTIPLE_HEAPS
26307 for (int i = 0; i < n_heaps; i++)
26309 g_heaps[i]->current_bgc_state = bgc_mark_handles;
26312 current_bgc_state = bgc_mark_handles;
26313 #endif //MULTIPLE_HEAPS
26315 current_c_gc_state = c_gc_state_marking;
26317 enable_preemptive ();
26319 #ifdef MULTIPLE_HEAPS
26320 dprintf(3, ("Joining BGC threads after resetting writewatch"));
26321 bgc_t_join.restart();
26322 #endif //MULTIPLE_HEAPS
26325 disable_preemptive (true);
26327 if (num_sizedrefs > 0)
26329 GCScan::GcScanSizedRefs(background_promote, max_generation, max_generation, &sc);
26331 enable_preemptive ();
26333 #ifdef MULTIPLE_HEAPS
26334 bgc_t_join.join(this, gc_join_scan_sizedref_done);
26335 if (bgc_t_join.joined())
26337 dprintf(3, ("Done with marking all sized refs. Starting all bgc thread for marking other strong roots"));
26338 bgc_t_join.restart();
26340 #endif //MULTIPLE_HEAPS
26342 disable_preemptive (true);
26345 dprintf (3,("BGC: handle table marking"));
26346 GCScan::GcScanHandles(background_promote,
26347 max_generation, max_generation,
26349 //concurrent_print_time_delta ("concurrent marking handle table");
26350 concurrent_print_time_delta ("CRH");
26352 current_bgc_state = bgc_mark_stack;
26353 dprintf (2,("concurrent draining mark list"));
26354 background_drain_mark_list (thread);
26355 //concurrent_print_time_delta ("concurrent marking stack roots");
26356 concurrent_print_time_delta ("CRS");
26358 dprintf (2,("concurrent revisiting dirtied pages"));
26359 revisit_written_pages (TRUE);
26360 revisit_written_pages (TRUE);
26361 //concurrent_print_time_delta ("concurrent marking dirtied pages on LOH");
26362 concurrent_print_time_delta ("CRre");
26364 enable_preemptive ();
26366 #ifdef MULTIPLE_HEAPS
26367 bgc_t_join.join(this, gc_join_concurrent_overflow);
26368 if (bgc_t_join.joined())
26370 uint8_t* all_heaps_max = 0;
26371 uint8_t* all_heaps_min = MAX_PTR;
26373 for (i = 0; i < n_heaps; i++)
26375 dprintf (3, ("heap %d overflow max is %Ix, min is %Ix",
26377 g_heaps[i]->background_max_overflow_address,
26378 g_heaps[i]->background_min_overflow_address));
26379 if (all_heaps_max < g_heaps[i]->background_max_overflow_address)
26380 all_heaps_max = g_heaps[i]->background_max_overflow_address;
26381 if (all_heaps_min > g_heaps[i]->background_min_overflow_address)
26382 all_heaps_min = g_heaps[i]->background_min_overflow_address;
26384 for (i = 0; i < n_heaps; i++)
26386 g_heaps[i]->background_max_overflow_address = all_heaps_max;
26387 g_heaps[i]->background_min_overflow_address = all_heaps_min;
26389 dprintf(3, ("Starting all bgc threads after updating the overflow info"));
26390 bgc_t_join.restart();
26392 #endif //MULTIPLE_HEAPS
26394 disable_preemptive (true);
26396 dprintf (2, ("before CRov count: %d", bgc_overflow_count));
26397 bgc_overflow_count = 0;
26398 background_process_mark_overflow (TRUE);
26399 dprintf (2, ("after CRov count: %d", bgc_overflow_count));
26400 bgc_overflow_count = 0;
26401 //concurrent_print_time_delta ("concurrent processing mark overflow");
26402 concurrent_print_time_delta ("CRov");
26404 // Stop all threads, crawl all stacks and revisit changed pages.
26405 FIRE_EVENT(BGC1stConEnd);
26407 dprintf (2, ("Stopping the EE"));
26409 enable_preemptive ();
26411 #ifdef MULTIPLE_HEAPS
26412 bgc_t_join.join(this, gc_join_suspend_ee);
26413 if (bgc_t_join.joined())
26415 bgc_threads_sync_event.Reset();
26417 dprintf(3, ("Joining BGC threads for non concurrent final marking"));
26418 bgc_t_join.restart();
26420 #endif //MULTIPLE_HEAPS
26422 if (heap_number == 0)
26424 enter_spin_lock (&gc_lock);
26428 bgc_threads_sync_event.Set();
26432 bgc_threads_sync_event.Wait(INFINITE, FALSE);
26433 dprintf (2, ("bgc_threads_sync_event is signalled"));
26436 assert (settings.concurrent);
26437 assert (settings.condemned_generation == max_generation);
26439 dprintf (2, ("clearing cm_in_progress"));
26440 c_write (cm_in_progress, FALSE);
26442 bgc_alloc_lock->check();
26444 current_bgc_state = bgc_final_marking;
26446 //concurrent_print_time_delta ("concurrent marking ended");
26447 concurrent_print_time_delta ("CR");
26449 FIRE_EVENT(BGC2ndNonConBegin);
26451 mark_absorb_new_alloc();
26453 // We need a join here 'cause find_object would complain if the gen0
26454 // bricks of another heap haven't been fixed up. So we need to make sure
26455 // that every heap's gen0 bricks are fixed up before we proceed.
26456 #ifdef MULTIPLE_HEAPS
26457 bgc_t_join.join(this, gc_join_after_absorb);
26458 if (bgc_t_join.joined())
26460 dprintf(3, ("Joining BGC threads after absorb"));
26461 bgc_t_join.restart();
26463 #endif //MULTIPLE_HEAPS
26465 // give VM a chance to do work
26466 GCToEEInterface::GcBeforeBGCSweepWork();
26468 //reset the flag, indicating that the EE no longer expect concurrent
26470 sc.concurrent = FALSE;
26472 total_loh_size = generation_size (max_generation + 1);
26473 total_soh_size = generation_sizes (generation_of (max_generation));
26475 dprintf (GTC_LOG, ("FM: h%d: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
26477 dprintf (2, ("nonconcurrent marking stack roots"));
26478 GCScan::GcScanRoots(background_promote,
26479 max_generation, max_generation,
26481 //concurrent_print_time_delta ("nonconcurrent marking stack roots");
26482 concurrent_print_time_delta ("NRS");
26484 // finalize_queue->EnterFinalizeLock();
26485 finalize_queue->GcScanRoots(background_promote, heap_number, 0);
26486 // finalize_queue->LeaveFinalizeLock();
26488 dprintf (2, ("nonconcurrent marking handle table"));
26489 GCScan::GcScanHandles(background_promote,
26490 max_generation, max_generation,
26492 //concurrent_print_time_delta ("nonconcurrent marking handle table");
26493 concurrent_print_time_delta ("NRH");
26495 dprintf (2,("---- (GC%d)final going through written pages ----", VolatileLoad(&settings.gc_index)));
26496 revisit_written_pages (FALSE);
26497 //concurrent_print_time_delta ("nonconcurrent revisit dirtied pages on LOH");
26498 concurrent_print_time_delta ("NRre LOH");
26500 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26501 #ifdef MULTIPLE_HEAPS
26502 bgc_t_join.join(this, gc_join_disable_software_write_watch);
26503 if (bgc_t_join.joined())
26504 #endif // MULTIPLE_HEAPS
26506 // The runtime is suspended, and we will be doing a final query of dirty pages, so pause tracking written pages to
26507 // avoid further perf penalty after the runtime is restarted
26508 SoftwareWriteWatch::DisableForGCHeap();
26510 #ifdef MULTIPLE_HEAPS
26511 dprintf(3, ("Restarting BGC threads after disabling software write watch"));
26512 bgc_t_join.restart();
26513 #endif // MULTIPLE_HEAPS
26515 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26517 dprintf (2, ("before NR 1st Hov count: %d", bgc_overflow_count));
26518 bgc_overflow_count = 0;
26520 // Dependent handles need to be scanned with a special algorithm (see the header comment on
26521 // scan_dependent_handles for more detail). We perform an initial scan without processing any mark
26522 // stack overflow. This is not guaranteed to complete the operation but in a common case (where there
26523 // are no dependent handles that are due to be collected) it allows us to optimize away further scans.
26524 // The call to background_scan_dependent_handles is what will cycle through more iterations if
26525 // required and will also perform processing of any mark stack overflow once the dependent handle
26526 // table has been fully promoted.
26527 dprintf (2, ("1st dependent handle scan and process mark overflow"));
26528 GCScan::GcDhInitialScan(background_promote, max_generation, max_generation, &sc);
26529 background_scan_dependent_handles (&sc);
26530 //concurrent_print_time_delta ("1st nonconcurrent dependent handle scan and process mark overflow");
26531 concurrent_print_time_delta ("NR 1st Hov");
26533 dprintf (2, ("after NR 1st Hov count: %d", bgc_overflow_count));
26534 bgc_overflow_count = 0;
26536 #ifdef MULTIPLE_HEAPS
26537 bgc_t_join.join(this, gc_join_null_dead_short_weak);
26538 if (bgc_t_join.joined())
26539 #endif //MULTIPLE_HEAPS
26541 GCToEEInterface::AfterGcScanRoots (max_generation, max_generation, &sc);
26543 #ifdef MULTIPLE_HEAPS
26544 dprintf(3, ("Joining BGC threads for short weak handle scan"));
26545 bgc_t_join.restart();
26546 #endif //MULTIPLE_HEAPS
26549 // null out the target of short weakref that were not promoted.
26550 GCScan::GcShortWeakPtrScan(background_promote, max_generation, max_generation,&sc);
26552 //concurrent_print_time_delta ("bgc GcShortWeakPtrScan");
26553 concurrent_print_time_delta ("NR GcShortWeakPtrScan");
26557 #ifdef MULTIPLE_HEAPS
26558 bgc_t_join.join(this, gc_join_scan_finalization);
26559 if (bgc_t_join.joined())
26561 dprintf(3, ("Joining BGC threads for finalization"));
26562 bgc_t_join.restart();
26564 #endif //MULTIPLE_HEAPS
26566 //Handle finalization.
26567 dprintf(3,("Marking finalization data"));
26568 //concurrent_print_time_delta ("bgc joined to mark finalization");
26569 concurrent_print_time_delta ("NRj");
26571 // finalize_queue->EnterFinalizeLock();
26572 finalize_queue->ScanForFinalization (background_promote, max_generation, FALSE, __this);
26573 // finalize_queue->LeaveFinalizeLock();
26575 concurrent_print_time_delta ("NRF");
26578 dprintf (2, ("before NR 2nd Hov count: %d", bgc_overflow_count));
26579 bgc_overflow_count = 0;
26581 // Scan dependent handles again to promote any secondaries associated with primaries that were promoted
26582 // for finalization. As before background_scan_dependent_handles will also process any mark stack
26584 dprintf (2, ("2nd dependent handle scan and process mark overflow"));
26585 background_scan_dependent_handles (&sc);
26586 //concurrent_print_time_delta ("2nd nonconcurrent dependent handle scan and process mark overflow");
26587 concurrent_print_time_delta ("NR 2nd Hov");
26589 #ifdef MULTIPLE_HEAPS
26590 bgc_t_join.join(this, gc_join_null_dead_long_weak);
26591 if (bgc_t_join.joined())
26593 dprintf(2, ("Joining BGC threads for weak pointer deletion"));
26594 bgc_t_join.restart();
26596 #endif //MULTIPLE_HEAPS
26598 // null out the target of long weakref that were not promoted.
26599 GCScan::GcWeakPtrScan (background_promote, max_generation, max_generation, &sc);
26600 concurrent_print_time_delta ("NR GcWeakPtrScan");
26602 #ifdef MULTIPLE_HEAPS
26603 bgc_t_join.join(this, gc_join_null_dead_syncblk);
26604 if (bgc_t_join.joined())
26605 #endif //MULTIPLE_HEAPS
26607 dprintf (2, ("calling GcWeakPtrScanBySingleThread"));
26608 // scan for deleted entries in the syncblk cache
26609 GCScan::GcWeakPtrScanBySingleThread (max_generation, max_generation, &sc);
26610 concurrent_print_time_delta ("NR GcWeakPtrScanBySingleThread");
26611 #ifdef MULTIPLE_HEAPS
26612 dprintf(2, ("Starting BGC threads for end of background mark phase"));
26613 bgc_t_join.restart();
26614 #endif //MULTIPLE_HEAPS
26617 gen0_bricks_cleared = FALSE;
26619 dprintf (2, ("end of bgc mark: loh: %d, soh: %d",
26620 generation_size (max_generation + 1),
26621 generation_sizes (generation_of (max_generation))));
26623 for (int gen_idx = max_generation; gen_idx <= (max_generation + 1); gen_idx++)
26625 generation* gen = generation_of (gen_idx);
26626 dynamic_data* dd = dynamic_data_of (gen_idx);
26627 dd_begin_data_size (dd) = generation_size (gen_idx) -
26628 (generation_free_list_space (gen) + generation_free_obj_space (gen)) -
26629 Align (size (generation_allocation_start (gen)));
26630 dd_survived_size (dd) = 0;
26631 dd_pinned_survived_size (dd) = 0;
26632 dd_artificial_pinned_survived_size (dd) = 0;
26633 dd_added_pinned_size (dd) = 0;
26636 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
26637 PREFIX_ASSUME(seg != NULL);
26641 seg->flags &= ~heap_segment_flags_swept;
26643 if (heap_segment_allocated (seg) == heap_segment_mem (seg))
26645 // This can't happen...
26649 if (seg == ephemeral_heap_segment)
26651 heap_segment_background_allocated (seg) = generation_allocation_start (generation_of (max_generation - 1));
26655 heap_segment_background_allocated (seg) = heap_segment_allocated (seg);
26658 dprintf (2, ("seg %Ix background allocated is %Ix",
26659 heap_segment_mem (seg),
26660 heap_segment_background_allocated (seg)));
26661 seg = heap_segment_next_rw (seg);
26664 // We need to void alloc contexts here 'cause while background_ephemeral_sweep is running
26665 // we can't let the user code consume the left over parts in these alloc contexts.
26666 repair_allocation_contexts (FALSE);
26669 finish = GetCycleCount32();
26670 mark_time = finish - start;
26673 dprintf (2, ("end of bgc mark: gen2 free list space: %d, free obj space: %d",
26674 generation_free_list_space (generation_of (max_generation)),
26675 generation_free_obj_space (generation_of (max_generation))));
26677 dprintf(2,("---- (GC%d)End of background mark phase ----", VolatileLoad(&settings.gc_index)));
26681 gc_heap::suspend_EE ()
26683 dprintf (2, ("suspend_EE"));
26684 #ifdef MULTIPLE_HEAPS
26685 gc_heap* hp = gc_heap::g_heaps[0];
26686 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26688 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26689 #endif //MULTIPLE_HEAPS
26692 #ifdef MULTIPLE_HEAPS
26694 gc_heap::bgc_suspend_EE ()
26696 for (int i = 0; i < n_heaps; i++)
26698 gc_heap::g_heaps[i]->reset_gc_done();
26701 dprintf (2, ("bgc_suspend_EE"));
26702 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26704 gc_started = FALSE;
26705 for (int i = 0; i < n_heaps; i++)
26707 gc_heap::g_heaps[i]->set_gc_done();
26712 gc_heap::bgc_suspend_EE ()
26716 dprintf (2, ("bgc_suspend_EE"));
26717 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26718 gc_started = FALSE;
26721 #endif //MULTIPLE_HEAPS
26724 gc_heap::restart_EE ()
26726 dprintf (2, ("restart_EE"));
26727 #ifdef MULTIPLE_HEAPS
26728 GCToEEInterface::RestartEE(FALSE);
26730 GCToEEInterface::RestartEE(FALSE);
26731 #endif //MULTIPLE_HEAPS
26734 inline uint8_t* gc_heap::high_page ( heap_segment* seg, BOOL concurrent_p)
26738 uint8_t* end = ((seg == ephemeral_heap_segment) ?
26739 generation_allocation_start (generation_of (max_generation-1)) :
26740 heap_segment_allocated (seg));
26741 return align_lower_page (end);
26745 return heap_segment_allocated (seg);
26749 void gc_heap::revisit_written_page (uint8_t* page,
26753 uint8_t*& last_page,
26754 uint8_t*& last_object,
26755 BOOL large_objects_p,
26756 size_t& num_marked_objects)
26758 UNREFERENCED_PARAMETER(seg);
26760 uint8_t* start_address = page;
26762 int align_const = get_alignment_constant (!large_objects_p);
26763 uint8_t* high_address = end;
26764 uint8_t* current_lowest_address = background_saved_lowest_address;
26765 uint8_t* current_highest_address = background_saved_highest_address;
26766 BOOL no_more_loop_p = FALSE;
26769 #ifndef MULTIPLE_HEAPS
26770 const int thread = heap_number;
26771 #endif //!MULTIPLE_HEAPS
26773 if (large_objects_p)
26779 if (((last_page + WRITE_WATCH_UNIT_SIZE) == page)
26780 || (start_address <= last_object))
26786 o = find_first_object (start_address, last_object);
26787 // We can visit the same object again, but on a different page.
26788 assert (o >= last_object);
26792 dprintf (3,("page %Ix start: %Ix, %Ix[ ",
26793 (size_t)page, (size_t)o,
26794 (size_t)(min (high_address, page + WRITE_WATCH_UNIT_SIZE))));
26796 while (o < (min (high_address, page + WRITE_WATCH_UNIT_SIZE)))
26800 if (concurrent_p && large_objects_p)
26802 bgc_alloc_lock->bgc_mark_set (o);
26804 if (((CObjectHeader*)o)->IsFree())
26806 s = unused_array_size (o);
26818 dprintf (3,("Considering object %Ix(%s)", (size_t)o, (background_object_marked (o, FALSE) ? "bm" : "nbm")));
26820 assert (Align (s) >= Align (min_obj_size));
26822 uint8_t* next_o = o + Align (s, align_const);
26824 if (next_o >= start_address)
26826 #ifdef MULTIPLE_HEAPS
26829 // We set last_object here for SVR BGC here because SVR BGC has more than
26830 // one GC thread. When we have more than one GC thread we would run into this
26831 // situation if we skipped unmarked objects:
26832 // bgc thread 1 calls GWW, and detect object X not marked so it would skip it
26834 // bgc thread 2 marks X and all its current children.
26835 // user thread comes along and dirties more (and later) pages in X.
26836 // bgc thread 1 calls GWW again and gets those later pages but it will not mark anything
26837 // on them because it had already skipped X. We need to detect that this object is now
26838 // marked and mark the children on the dirtied pages.
26839 // In the future if we have less BGC threads than we have heaps we should add
26840 // the check to the number of BGC threads.
26843 #endif //MULTIPLE_HEAPS
26845 if (contain_pointers (o) &&
26846 (!((o >= current_lowest_address) && (o < current_highest_address)) ||
26847 background_marked (o)))
26849 dprintf (3, ("going through %Ix", (size_t)o));
26850 go_through_object (method_table(o), o, s, poo, start_address, use_start, (o + s),
26851 if ((uint8_t*)poo >= min (high_address, page + WRITE_WATCH_UNIT_SIZE))
26853 no_more_loop_p = TRUE;
26856 uint8_t* oo = *poo;
26858 num_marked_objects++;
26859 background_mark_object (oo THREAD_NUMBER_ARG);
26864 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP // see comment below
26866 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26867 ((CObjectHeader*)o)->IsFree() &&
26868 (next_o > min (high_address, page + WRITE_WATCH_UNIT_SIZE)))
26870 // We need to not skip the object here because of this corner scenario:
26871 // A large object was being allocated during BGC mark so we first made it
26872 // into a free object, then cleared its memory. In this loop we would detect
26873 // that it's a free object which normally we would skip. But by the next time
26874 // we call GetWriteWatch we could still be on this object and the object had
26875 // been made into a valid object and some of its memory was changed. We need
26876 // to be sure to process those written pages so we can't skip the object just
26879 // Similarly, when using software write watch, don't advance last_object when
26880 // the current object is a free object that spans beyond the current page or
26881 // high_address. Software write watch acquires gc_lock before the concurrent
26882 // GetWriteWatch() call during revisit_written_pages(). A foreground GC may
26883 // happen at that point and allocate from this free region, so when
26884 // revisit_written_pages() continues, it cannot skip now-valid objects in this
26886 no_more_loop_p = TRUE;
26891 if (concurrent_p && large_objects_p)
26893 bgc_alloc_lock->bgc_mark_done ();
26895 if (no_more_loop_p)
26902 #ifdef MULTIPLE_HEAPS
26905 assert (last_object < (min (high_address, page + WRITE_WATCH_UNIT_SIZE)));
26908 #endif //MULTIPLE_HEAPS
26913 dprintf (3,("Last object: %Ix", (size_t)last_object));
26914 last_page = align_write_watch_lower_page (o);
26917 // When reset_only_p is TRUE, we should only reset pages that are in range
26918 // because we need to consider the segments or part of segments that were
26919 // allocated out of range all live.
26920 void gc_heap::revisit_written_pages (BOOL concurrent_p, BOOL reset_only_p)
26923 if (concurrent_p && !reset_only_p)
26925 current_bgc_state = bgc_revisit_soh;
26928 size_t total_dirtied_pages = 0;
26929 size_t total_marked_objects = 0;
26931 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
26933 PREFIX_ASSUME(seg != NULL);
26935 bool reset_watch_state = !!concurrent_p;
26936 bool is_runtime_suspended = !concurrent_p;
26937 BOOL small_object_segments = TRUE;
26938 int align_const = get_alignment_constant (small_object_segments);
26944 if (small_object_segments)
26946 //switch to large segment
26947 if (concurrent_p && !reset_only_p)
26949 current_bgc_state = bgc_revisit_loh;
26954 dprintf (GTC_LOG, ("h%d: SOH: dp:%Id; mo: %Id", heap_number, total_dirtied_pages, total_marked_objects));
26955 fire_revisit_event (total_dirtied_pages, total_marked_objects, !small_object_segments);
26956 concurrent_print_time_delta (concurrent_p ? "CR SOH" : "NR SOH");
26957 total_dirtied_pages = 0;
26958 total_marked_objects = 0;
26961 small_object_segments = FALSE;
26962 //concurrent_print_time_delta (concurrent_p ? "concurrent marking dirtied pages on SOH" : "nonconcurrent marking dirtied pages on SOH");
26964 dprintf (3, ("now revisiting large object segments"));
26965 align_const = get_alignment_constant (small_object_segments);
26966 seg = heap_segment_rw (generation_start_segment (large_object_generation));
26968 PREFIX_ASSUME(seg != NULL);
26976 dprintf (GTC_LOG, ("h%d: tdp: %Id", heap_number, total_dirtied_pages));
26980 dprintf (GTC_LOG, ("h%d: LOH: dp:%Id; mo: %Id", heap_number, total_dirtied_pages, total_marked_objects));
26981 fire_revisit_event (total_dirtied_pages, total_marked_objects, !small_object_segments);
26986 uint8_t* base_address = (uint8_t*)heap_segment_mem (seg);
26987 //we need to truncate to the base of the page because
26988 //some newly allocated could exist beyond heap_segment_allocated
26989 //and if we reset the last page write watch status,
26990 // they wouldn't be guaranteed to be visited -> gc hole.
26991 uintptr_t bcount = array_size;
26992 uint8_t* last_page = 0;
26993 uint8_t* last_object = heap_segment_mem (seg);
26994 uint8_t* high_address = 0;
26996 BOOL skip_seg_p = FALSE;
27000 if ((heap_segment_mem (seg) >= background_saved_lowest_address) ||
27001 (heap_segment_reserved (seg) <= background_saved_highest_address))
27003 dprintf (3, ("h%d: sseg: %Ix(-%Ix)", heap_number,
27004 heap_segment_mem (seg), heap_segment_reserved (seg)));
27011 dprintf (3, ("looking at seg %Ix", (size_t)last_object));
27015 base_address = max (base_address, background_saved_lowest_address);
27016 dprintf (3, ("h%d: reset only starting %Ix", heap_number, base_address));
27019 dprintf (3, ("h%d: starting: %Ix, seg %Ix-%Ix", heap_number, base_address,
27020 heap_segment_mem (seg), heap_segment_reserved (seg)));
27027 high_address = ((seg == ephemeral_heap_segment) ? alloc_allocated : heap_segment_allocated (seg));
27028 high_address = min (high_address, background_saved_highest_address);
27032 high_address = high_page (seg, concurrent_p);
27035 if ((base_address < high_address) &&
27036 (bcount >= array_size))
27038 ptrdiff_t region_size = high_address - base_address;
27039 dprintf (3, ("h%d: gw: [%Ix(%Id)", heap_number, (size_t)base_address, (size_t)region_size));
27041 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
27042 // When the runtime is not suspended, it's possible for the table to be resized concurrently with the scan
27043 // for dirty pages below. Prevent that by synchronizing with grow_brick_card_tables(). When the runtime is
27044 // suspended, it's ok to scan for dirty pages concurrently from multiple background GC threads for disjoint
27046 if (!is_runtime_suspended)
27048 enter_spin_lock(&gc_lock);
27050 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
27052 get_write_watch_for_gc_heap (reset_watch_state, base_address, region_size,
27053 (void**)background_written_addresses,
27054 &bcount, is_runtime_suspended);
27056 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
27057 if (!is_runtime_suspended)
27059 leave_spin_lock(&gc_lock);
27061 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
27065 total_dirtied_pages += bcount;
27067 dprintf (3, ("Found %d pages [%Ix, %Ix[",
27068 bcount, (size_t)base_address, (size_t)high_address));
27073 for (unsigned i = 0; i < bcount; i++)
27075 uint8_t* page = (uint8_t*)background_written_addresses[i];
27076 dprintf (3, ("looking at page %d at %Ix(h: %Ix)", i,
27077 (size_t)page, (size_t)high_address));
27078 if (page < high_address)
27080 //search for marked objects in the page
27081 revisit_written_page (page, high_address, concurrent_p,
27082 seg, last_page, last_object,
27083 !small_object_segments,
27084 total_marked_objects);
27088 dprintf (3, ("page %d at %Ix is >= %Ix!", i, (size_t)page, (size_t)high_address));
27089 assert (!"page shouldn't have exceeded limit");
27094 if (bcount >= array_size){
27095 base_address = background_written_addresses [array_size-1] + WRITE_WATCH_UNIT_SIZE;
27096 bcount = array_size;
27106 seg = heap_segment_next_rw (seg);
27109 #endif //WRITE_WATCH
27112 void gc_heap::background_grow_c_mark_list()
27114 assert (c_mark_list_index >= c_mark_list_length);
27115 BOOL should_drain_p = FALSE;
27117 #ifndef MULTIPLE_HEAPS
27118 const int thread = heap_number;
27119 #endif //!MULTIPLE_HEAPS
27121 dprintf (2, ("stack copy buffer overflow"));
27122 uint8_t** new_c_mark_list = 0;
27125 if (c_mark_list_length >= (SIZE_T_MAX / (2 * sizeof (uint8_t*))))
27127 should_drain_p = TRUE;
27131 new_c_mark_list = new (nothrow) uint8_t*[c_mark_list_length*2];
27132 if (new_c_mark_list == 0)
27134 should_drain_p = TRUE;
27138 if (should_drain_p)
27141 dprintf (2, ("No more memory for the stacks copy, draining.."));
27142 //drain the list by marking its elements
27143 background_drain_mark_list (thread);
27147 assert (new_c_mark_list);
27148 memcpy (new_c_mark_list, c_mark_list, c_mark_list_length*sizeof(uint8_t*));
27149 c_mark_list_length = c_mark_list_length*2;
27150 delete c_mark_list;
27151 c_mark_list = new_c_mark_list;
27155 void gc_heap::background_promote_callback (Object** ppObject, ScanContext* sc,
27158 UNREFERENCED_PARAMETER(sc);
27159 //in order to save space on the array, mark the object,
27160 //knowing that it will be visited later
27161 assert (settings.concurrent);
27163 THREAD_NUMBER_FROM_CONTEXT;
27164 #ifndef MULTIPLE_HEAPS
27165 const int thread = 0;
27166 #endif //!MULTIPLE_HEAPS
27168 uint8_t* o = (uint8_t*)*ppObject;
27175 gc_heap* hp = gc_heap::heap_of (o);
27177 if ((o < hp->background_saved_lowest_address) || (o >= hp->background_saved_highest_address))
27182 #ifdef INTERIOR_POINTERS
27183 if (flags & GC_CALL_INTERIOR)
27185 o = hp->find_object (o, hp->background_saved_lowest_address);
27189 #endif //INTERIOR_POINTERS
27191 #ifdef FEATURE_CONSERVATIVE_GC
27192 // For conservative GC, a value on stack may point to middle of a free object.
27193 // In this case, we don't need to promote the pointer.
27194 if (GCConfig::GetConservativeGC() && ((CObjectHeader*)o)->IsFree())
27198 #endif //FEATURE_CONSERVATIVE_GC
27201 ((CObjectHeader*)o)->Validate();
27204 dprintf (3, ("Concurrent Background Promote %Ix", (size_t)o));
27205 if (o && (size (o) > loh_size_threshold))
27207 dprintf (3, ("Brc %Ix", (size_t)o));
27210 if (hpt->c_mark_list_index >= hpt->c_mark_list_length)
27212 hpt->background_grow_c_mark_list();
27214 dprintf (3, ("pushing %08x into mark_list", (size_t)o));
27215 hpt->c_mark_list [hpt->c_mark_list_index++] = o;
27217 STRESS_LOG3(LF_GC|LF_GCROOTS, LL_INFO1000000, " GCHeap::Background Promote: Promote GC Root *%p = %p MT = %pT", ppObject, o, o ? ((Object*) o)->GetGCSafeMethodTable() : NULL);
27220 void gc_heap::mark_absorb_new_alloc()
27222 fix_allocation_contexts (FALSE);
27224 gen0_bricks_cleared = FALSE;
27226 clear_gen0_bricks();
27229 BOOL gc_heap::prepare_bgc_thread(gc_heap* gh)
27231 BOOL success = FALSE;
27232 BOOL thread_created = FALSE;
27233 dprintf (2, ("Preparing gc thread"));
27234 gh->bgc_threads_timeout_cs.Enter();
27235 if (!(gh->bgc_thread_running))
27237 dprintf (2, ("GC thread not runnning"));
27238 if ((gh->bgc_thread == 0) && create_bgc_thread(gh))
27241 thread_created = TRUE;
27246 dprintf (3, ("GC thread already running"));
27249 gh->bgc_threads_timeout_cs.Leave();
27252 FIRE_EVENT(GCCreateConcurrentThread_V1);
27257 BOOL gc_heap::create_bgc_thread(gc_heap* gh)
27259 assert (background_gc_done_event.IsValid());
27261 //dprintf (2, ("Creating BGC thread"));
27263 gh->bgc_thread_running = GCToEEInterface::CreateThread(gh->bgc_thread_stub, gh, true, ".NET Background GC");
27264 return gh->bgc_thread_running;
27267 BOOL gc_heap::create_bgc_threads_support (int number_of_heaps)
27270 dprintf (3, ("Creating concurrent GC thread for the first time"));
27271 if (!background_gc_done_event.CreateManualEventNoThrow(TRUE))
27275 if (!bgc_threads_sync_event.CreateManualEventNoThrow(FALSE))
27279 if (!ee_proceed_event.CreateAutoEventNoThrow(FALSE))
27283 if (!bgc_start_event.CreateManualEventNoThrow(FALSE))
27288 #ifdef MULTIPLE_HEAPS
27289 bgc_t_join.init (number_of_heaps, join_flavor_bgc);
27291 UNREFERENCED_PARAMETER(number_of_heaps);
27292 #endif //MULTIPLE_HEAPS
27300 if (background_gc_done_event.IsValid())
27302 background_gc_done_event.CloseEvent();
27304 if (bgc_threads_sync_event.IsValid())
27306 bgc_threads_sync_event.CloseEvent();
27308 if (ee_proceed_event.IsValid())
27310 ee_proceed_event.CloseEvent();
27312 if (bgc_start_event.IsValid())
27314 bgc_start_event.CloseEvent();
27321 BOOL gc_heap::create_bgc_thread_support()
27326 if (!gc_lh_block_event.CreateManualEventNoThrow(FALSE))
27331 //needs to have room for enough smallest objects fitting on a page
27332 parr = new (nothrow) uint8_t*[1 + OS_PAGE_SIZE / MIN_OBJECT_SIZE];
27338 make_c_mark_list (parr);
27346 if (gc_lh_block_event.IsValid())
27348 gc_lh_block_event.CloseEvent();
27355 int gc_heap::check_for_ephemeral_alloc()
27357 int gen = ((settings.reason == reason_oos_soh) ? (max_generation - 1) : -1);
27361 #ifdef MULTIPLE_HEAPS
27362 for (int heap_index = 0; heap_index < n_heaps; heap_index++)
27363 #endif //MULTIPLE_HEAPS
27365 for (int i = 0; i <= (max_generation - 1); i++)
27367 #ifdef MULTIPLE_HEAPS
27368 if (g_heaps[heap_index]->get_new_allocation (i) <= 0)
27370 if (get_new_allocation (i) <= 0)
27371 #endif //MULTIPLE_HEAPS
27373 gen = max (gen, i);
27384 // Wait for gc to finish sequential part
27385 void gc_heap::wait_to_proceed()
27387 assert (background_gc_done_event.IsValid());
27388 assert (bgc_start_event.IsValid());
27390 user_thread_wait(&ee_proceed_event, FALSE);
27393 // Start a new concurrent gc
27394 void gc_heap::start_c_gc()
27396 assert (background_gc_done_event.IsValid());
27397 assert (bgc_start_event.IsValid());
27399 //Need to make sure that the gc thread is in the right place.
27400 background_gc_done_event.Wait(INFINITE, FALSE);
27401 background_gc_done_event.Reset();
27402 bgc_start_event.Set();
27405 void gc_heap::do_background_gc()
27407 dprintf (2, ("starting a BGC"));
27408 #ifdef MULTIPLE_HEAPS
27409 for (int i = 0; i < n_heaps; i++)
27411 g_heaps[i]->init_background_gc();
27414 init_background_gc();
27415 #endif //MULTIPLE_HEAPS
27416 //start the background gc
27419 //wait until we get restarted by the BGC.
27423 void gc_heap::kill_gc_thread()
27425 //assert (settings.concurrent == FALSE);
27427 // We are doing a two-stage shutdown now.
27428 // In the first stage, we do minimum work, and call ExitProcess at the end.
27429 // In the secodn stage, we have the Loader lock and only one thread is
27430 // alive. Hence we do not need to kill gc thread.
27431 background_gc_done_event.CloseEvent();
27432 gc_lh_block_event.CloseEvent();
27433 bgc_start_event.CloseEvent();
27434 bgc_threads_timeout_cs.Destroy();
27436 recursive_gc_sync::shutdown();
27439 void gc_heap::bgc_thread_function()
27441 assert (background_gc_done_event.IsValid());
27442 assert (bgc_start_event.IsValid());
27444 dprintf (3, ("gc_thread thread starting..."));
27446 BOOL do_exit = FALSE;
27448 bool cooperative_mode = true;
27449 bgc_thread_id.SetToCurrentThread();
27450 dprintf (1, ("bgc_thread_id is set to %x", (uint32_t)GCToOSInterface::GetCurrentThreadIdForLogging()));
27453 // Wait for work to do...
27454 dprintf (3, ("bgc thread: waiting..."));
27456 cooperative_mode = enable_preemptive ();
27457 //current_thread->m_fPreemptiveGCDisabled = 0;
27459 uint32_t result = bgc_start_event.Wait(
27461 #ifdef MULTIPLE_HEAPS
27465 #endif //MULTIPLE_HEAPS
27467 #ifdef MULTIPLE_HEAPS
27471 #endif //MULTIPLE_HEAPS
27474 dprintf (2, ("gc thread: finished waiting"));
27476 // not calling disable_preemptive here 'cause we
27477 // can't wait for GC complete here - RestartEE will be called
27478 // when we've done the init work.
27480 if (result == WAIT_TIMEOUT)
27482 // Should join the bgc threads and terminate all of them
27484 dprintf (1, ("GC thread timeout"));
27485 bgc_threads_timeout_cs.Enter();
27486 if (!keep_bgc_threads_p)
27488 dprintf (2, ("GC thread exiting"));
27489 bgc_thread_running = FALSE;
27491 bgc_thread_id.Clear();
27494 bgc_threads_timeout_cs.Leave();
27499 dprintf (3, ("GC thread needed, not exiting"));
27503 // if we signal the thread with no concurrent work to do -> exit
27504 if (!settings.concurrent)
27506 dprintf (3, ("no concurrent GC needed, exiting"));
27512 recursive_gc_sync::begin_background();
27513 dprintf (2, ("beginning of bgc: gen2 FL: %d, FO: %d, frag: %d",
27514 generation_free_list_space (generation_of (max_generation)),
27515 generation_free_obj_space (generation_of (max_generation)),
27516 dd_fragmentation (dynamic_data_of (max_generation))));
27520 current_bgc_state = bgc_not_in_process;
27523 //trace_gc = FALSE;
27526 enable_preemptive ();
27527 #ifdef MULTIPLE_HEAPS
27528 bgc_t_join.join(this, gc_join_done);
27529 if (bgc_t_join.joined())
27530 #endif //MULTIPLE_HEAPS
27532 enter_spin_lock (&gc_lock);
27533 dprintf (SPINLOCK_LOG, ("bgc Egc"));
27535 bgc_start_event.Reset();
27537 #ifdef MULTIPLE_HEAPS
27538 for (int gen = max_generation; gen <= (max_generation + 1); gen++)
27540 size_t desired_per_heap = 0;
27541 size_t total_desired = 0;
27544 for (int i = 0; i < n_heaps; i++)
27547 dd = hp->dynamic_data_of (gen);
27548 size_t temp_total_desired = total_desired + dd_desired_allocation (dd);
27549 if (temp_total_desired < total_desired)
27552 total_desired = (size_t)MAX_PTR;
27555 total_desired = temp_total_desired;
27558 desired_per_heap = Align ((total_desired/n_heaps), get_alignment_constant (FALSE));
27560 for (int i = 0; i < n_heaps; i++)
27562 hp = gc_heap::g_heaps[i];
27563 dd = hp->dynamic_data_of (gen);
27564 dd_desired_allocation (dd) = desired_per_heap;
27565 dd_gc_new_allocation (dd) = desired_per_heap;
27566 dd_new_allocation (dd) = desired_per_heap;
27569 #endif //MULTIPLE_HEAPS
27570 #ifdef MULTIPLE_HEAPS
27572 #endif //MULTIPLE_HEAPS
27574 c_write (settings.concurrent, FALSE);
27575 recursive_gc_sync::end_background();
27576 keep_bgc_threads_p = FALSE;
27577 background_gc_done_event.Set();
27579 dprintf (SPINLOCK_LOG, ("bgc Lgc"));
27580 leave_spin_lock (&gc_lock);
27581 #ifdef MULTIPLE_HEAPS
27582 dprintf(1, ("End of BGC - starting all BGC threads"));
27583 bgc_t_join.restart();
27584 #endif //MULTIPLE_HEAPS
27586 // We can't disable preempt here because there might've been a GC already
27587 // started and decided to do a BGC and waiting for a BGC thread to restart
27588 // vm. That GC will be waiting in wait_to_proceed and we are waiting for it
27589 // to restart the VM so we deadlock.
27590 //gc_heap::disable_preemptive (true);
27593 FIRE_EVENT(GCTerminateConcurrentThread_V1);
27595 dprintf (3, ("bgc_thread thread exiting"));
27599 #endif //BACKGROUND_GC
27601 //Clear the cards [start_card, end_card[
27602 void gc_heap::clear_cards (size_t start_card, size_t end_card)
27604 if (start_card < end_card)
27606 size_t start_word = card_word (start_card);
27607 size_t end_word = card_word (end_card);
27608 if (start_word < end_word)
27610 // Figure out the bit positions of the cards within their words
27611 unsigned bits = card_bit (start_card);
27612 card_table [start_word] &= lowbits (~0, bits);
27613 for (size_t i = start_word+1; i < end_word; i++)
27614 card_table [i] = 0;
27615 bits = card_bit (end_card);
27616 // Don't write beyond end_card (and possibly uncommitted card table space).
27619 card_table [end_word] &= highbits (~0, bits);
27624 // If the start and end cards are in the same word, just clear the appropriate card
27625 // bits in that word.
27626 card_table [start_word] &= (lowbits (~0, card_bit (start_card)) |
27627 highbits (~0, card_bit (end_card)));
27629 #ifdef VERYSLOWDEBUG
27630 size_t card = start_card;
27631 while (card < end_card)
27633 assert (! (card_set_p (card)));
27636 #endif //VERYSLOWDEBUG
27637 dprintf (3,("Cleared cards [%Ix:%Ix, %Ix:%Ix[",
27638 start_card, (size_t)card_address (start_card),
27639 end_card, (size_t)card_address (end_card)));
27643 void gc_heap::clear_card_for_addresses (uint8_t* start_address, uint8_t* end_address)
27645 size_t start_card = card_of (align_on_card (start_address));
27646 size_t end_card = card_of (align_lower_card (end_address));
27647 clear_cards (start_card, end_card);
27650 // copy [srccard, ...[ to [dst_card, end_card[
27651 // This will set the same bit twice. Can be optimized.
27653 void gc_heap::copy_cards (size_t dst_card,
27658 // If the range is empty, this function is a no-op - with the subtlety that
27659 // either of the accesses card_table[srcwrd] or card_table[dstwrd] could be
27660 // outside the committed region. To avoid the access, leave early.
27661 if (!(dst_card < end_card))
27664 unsigned int srcbit = card_bit (src_card);
27665 unsigned int dstbit = card_bit (dst_card);
27666 size_t srcwrd = card_word (src_card);
27667 size_t dstwrd = card_word (dst_card);
27668 unsigned int srctmp = card_table[srcwrd];
27669 unsigned int dsttmp = card_table[dstwrd];
27671 for (size_t card = dst_card; card < end_card; card++)
27673 if (srctmp & (1 << srcbit))
27674 dsttmp |= 1 << dstbit;
27676 dsttmp &= ~(1 << dstbit);
27677 if (!(++srcbit % 32))
27679 srctmp = card_table[++srcwrd];
27685 if (srctmp & (1 << srcbit))
27686 dsttmp |= 1 << dstbit;
27689 if (!(++dstbit % 32))
27691 card_table[dstwrd] = dsttmp;
27693 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27696 card_bundle_set(cardw_card_bundle(dstwrd));
27701 dsttmp = card_table[dstwrd];
27706 card_table[dstwrd] = dsttmp;
27708 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27711 card_bundle_set(cardw_card_bundle(dstwrd));
27716 void gc_heap::copy_cards_for_addresses (uint8_t* dest, uint8_t* src, size_t len)
27718 ptrdiff_t relocation_distance = src - dest;
27719 size_t start_dest_card = card_of (align_on_card (dest));
27720 size_t end_dest_card = card_of (dest + len - 1);
27721 size_t dest_card = start_dest_card;
27722 size_t src_card = card_of (card_address (dest_card)+relocation_distance);
27723 dprintf (3,("Copying cards [%Ix:%Ix->%Ix:%Ix, ",
27724 src_card, (size_t)src, dest_card, (size_t)dest));
27725 dprintf (3,(" %Ix->%Ix:%Ix[",
27726 (size_t)src+len, end_dest_card, (size_t)dest+len));
27728 dprintf (3, ("dest: %Ix, src: %Ix, len: %Ix, reloc: %Ix, align_on_card(dest) is %Ix",
27729 dest, src, len, relocation_distance, (align_on_card (dest))));
27731 dprintf (3, ("start_dest_card: %Ix (address: %Ix), end_dest_card: %Ix(addr: %Ix), card_of (dest): %Ix",
27732 start_dest_card, card_address (start_dest_card), end_dest_card, card_address (end_dest_card), card_of (dest)));
27734 //First card has two boundaries
27735 if (start_dest_card != card_of (dest))
27737 if ((card_of (card_address (start_dest_card) + relocation_distance) <= card_of (src + len - 1))&&
27738 card_set_p (card_of (card_address (start_dest_card) + relocation_distance)))
27740 dprintf (3, ("card_address (start_dest_card) + reloc is %Ix, card: %Ix(set), src+len-1: %Ix, card: %Ix",
27741 (card_address (start_dest_card) + relocation_distance),
27742 card_of (card_address (start_dest_card) + relocation_distance),
27744 card_of (src + len - 1)));
27746 dprintf (3, ("setting card: %Ix", card_of (dest)));
27747 set_card (card_of (dest));
27751 if (card_set_p (card_of (src)))
27752 set_card (card_of (dest));
27755 copy_cards (dest_card, src_card, end_dest_card,
27756 ((dest - align_lower_card (dest)) != (src - align_lower_card (src))));
27758 //Last card has two boundaries.
27759 if ((card_of (card_address (end_dest_card) + relocation_distance) >= card_of (src)) &&
27760 card_set_p (card_of (card_address (end_dest_card) + relocation_distance)))
27762 dprintf (3, ("card_address (end_dest_card) + reloc is %Ix, card: %Ix(set), src: %Ix, card: %Ix",
27763 (card_address (end_dest_card) + relocation_distance),
27764 card_of (card_address (end_dest_card) + relocation_distance),
27768 dprintf (3, ("setting card: %Ix", end_dest_card));
27769 set_card (end_dest_card);
27772 if (card_set_p (card_of (src + len - 1)))
27773 set_card (end_dest_card);
27775 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27776 card_bundles_set(cardw_card_bundle(card_word(card_of(dest))), cardw_card_bundle(align_cardw_on_bundle(card_word(end_dest_card))));
27780 #ifdef BACKGROUND_GC
27781 // this does not need the Interlocked version of mark_array_set_marked.
27782 void gc_heap::copy_mark_bits_for_addresses (uint8_t* dest, uint8_t* src, size_t len)
27784 dprintf (3, ("Copying mark_bits for addresses [%Ix->%Ix, %Ix->%Ix[",
27785 (size_t)src, (size_t)dest,
27786 (size_t)src+len, (size_t)dest+len));
27788 uint8_t* src_o = src;
27790 uint8_t* src_end = src + len;
27791 int align_const = get_alignment_constant (TRUE);
27792 ptrdiff_t reloc = dest - src;
27794 while (src_o < src_end)
27796 uint8_t* next_o = src_o + Align (size (src_o), align_const);
27798 if (background_object_marked (src_o, TRUE))
27800 dest_o = src_o + reloc;
27802 //if (background_object_marked (dest_o, FALSE))
27804 // dprintf (3, ("*%Ix shouldn't have already been marked!", (size_t)(dest_o)));
27805 // FATAL_GC_ERROR();
27808 background_mark (dest_o,
27809 background_saved_lowest_address,
27810 background_saved_highest_address);
27811 dprintf (3, ("bc*%Ix*bc, b*%Ix*b", (size_t)src_o, (size_t)(dest_o)));
27817 #endif //BACKGROUND_GC
27819 void gc_heap::fix_brick_to_highest (uint8_t* o, uint8_t* next_o)
27821 size_t new_current_brick = brick_of (o);
27822 set_brick (new_current_brick,
27823 (o - brick_address (new_current_brick)));
27824 size_t b = 1 + new_current_brick;
27825 size_t limit = brick_of (next_o);
27826 //dprintf(3,(" fixing brick %Ix to point to object %Ix, till %Ix(%Ix)",
27827 dprintf(3,("b:%Ix->%Ix-%Ix",
27828 new_current_brick, (size_t)o, (size_t)next_o));
27831 set_brick (b,(new_current_brick - b));
27836 // start can not be >= heap_segment_allocated for the segment.
27837 uint8_t* gc_heap::find_first_object (uint8_t* start, uint8_t* first_object)
27839 size_t brick = brick_of (start);
27841 //last_object == null -> no search shortcut needed
27842 if ((brick == brick_of (first_object) || (start <= first_object)))
27848 ptrdiff_t min_brick = (ptrdiff_t)brick_of (first_object);
27849 ptrdiff_t prev_brick = (ptrdiff_t)brick - 1;
27850 int brick_entry = 0;
27853 if (prev_brick < min_brick)
27857 if ((brick_entry = get_brick_entry(prev_brick)) >= 0)
27861 assert (! ((brick_entry == 0)));
27862 prev_brick = (brick_entry + prev_brick);
27865 o = ((prev_brick < min_brick) ? first_object :
27866 brick_address (prev_brick) + brick_entry - 1);
27867 assert (o <= start);
27870 assert (Align (size (o)) >= Align (min_obj_size));
27871 uint8_t* next_o = o + Align (size (o));
27872 size_t curr_cl = (size_t)next_o / brick_size;
27873 size_t min_cl = (size_t)first_object / brick_size;
27875 //dprintf (3,( "Looking for intersection with %Ix from %Ix", (size_t)start, (size_t)o));
27877 unsigned int n_o = 1;
27880 uint8_t* next_b = min (align_lower_brick (next_o) + brick_size, start+1);
27882 while (next_o <= start)
27890 assert (Align (size (o)) >= Align (min_obj_size));
27891 next_o = o + Align (size (o));
27893 }while (next_o < next_b);
27895 if (((size_t)next_o / brick_size) != curr_cl)
27897 if (curr_cl >= min_cl)
27899 fix_brick_to_highest (o, next_o);
27901 curr_cl = (size_t) next_o / brick_size;
27903 next_b = min (align_lower_brick (next_o) + brick_size, start+1);
27906 size_t bo = brick_of (o);
27907 //dprintf (3, ("Looked at %Id objects, fixing brick [%Ix-[%Ix",
27908 dprintf (3, ("%Id o, [%Ix-[%Ix",
27912 set_brick (bo, (o - brick_address(bo)));
27927 // Find the first non-zero card word between cardw and cardw_end.
27928 // The index of the word we find is returned in cardw.
27929 BOOL gc_heap::find_card_dword (size_t& cardw, size_t cardw_end)
27931 dprintf (3, ("gc: %d, find_card_dword cardw: %Ix, cardw_end: %Ix",
27932 dd_collection_count (dynamic_data_of (0)), cardw, cardw_end));
27934 if (card_bundles_enabled())
27936 size_t cardb = cardw_card_bundle (cardw);
27937 size_t end_cardb = cardw_card_bundle (align_cardw_on_bundle (cardw_end));
27940 // Find a non-zero bundle
27941 while ((cardb < end_cardb) && (card_bundle_set_p (cardb) == 0))
27945 if (cardb == end_cardb)
27948 uint32_t* card_word = &card_table[max(card_bundle_cardw (cardb),cardw)];
27949 uint32_t* card_word_end = &card_table[min(card_bundle_cardw (cardb+1),cardw_end)];
27950 while ((card_word < card_word_end) && !(*card_word))
27955 if (card_word != card_word_end)
27957 cardw = (card_word - &card_table[0]);
27960 else if ((cardw <= card_bundle_cardw (cardb)) &&
27961 (card_word == &card_table [card_bundle_cardw (cardb+1)]))
27963 // a whole bundle was explored and is empty
27964 dprintf (3, ("gc: %d, find_card_dword clear bundle: %Ix cardw:[%Ix,%Ix[",
27965 dd_collection_count (dynamic_data_of (0)),
27966 cardb, card_bundle_cardw (cardb),
27967 card_bundle_cardw (cardb+1)));
27968 card_bundle_clear (cardb);
27976 uint32_t* card_word = &card_table[cardw];
27977 uint32_t* card_word_end = &card_table [cardw_end];
27979 while (card_word < card_word_end)
27981 if ((*card_word) != 0)
27983 cardw = (card_word - &card_table [0]);
27995 #endif //CARD_BUNDLE
27997 // Find cards that are set between two points in a card table.
27999 // card_table : The card table.
28000 // card : [in/out] As input, the card to start searching from.
28001 // As output, the first card that's set.
28002 // card_word_end : The card word at which to stop looking.
28003 // end_card : [out] The last card which is set.
28004 BOOL gc_heap::find_card(uint32_t* card_table,
28006 size_t card_word_end,
28009 uint32_t* last_card_word;
28010 uint32_t card_word_value;
28011 uint32_t bit_position;
28013 // Find the first card which is set
28014 last_card_word = &card_table [card_word (card)];
28015 bit_position = card_bit (card);
28016 card_word_value = (*last_card_word) >> bit_position;
28017 if (!card_word_value)
28021 // Using the card bundle, go through the remaining card words between here and
28022 // card_word_end until we find one that is non-zero.
28023 size_t lcw = card_word(card) + 1;
28024 if (gc_heap::find_card_dword (lcw, card_word_end) == FALSE)
28030 last_card_word = &card_table [lcw];
28031 card_word_value = *last_card_word;
28034 #else //CARD_BUNDLE
28035 // Go through the remaining card words between here and card_word_end until we find
28036 // one that is non-zero.
28042 while ((last_card_word < &card_table [card_word_end]) && !(*last_card_word));
28043 if (last_card_word < &card_table [card_word_end])
28045 card_word_value = *last_card_word;
28049 // We failed to find any non-zero card words before we got to card_word_end
28052 #endif //CARD_BUNDLE
28056 // Look for the lowest bit set
28057 if (card_word_value)
28059 while (!(card_word_value & 1))
28062 card_word_value = card_word_value / 2;
28066 // card is the card word index * card size + the bit index within the card
28067 card = (last_card_word - &card_table[0]) * card_word_width + bit_position;
28071 // Keep going until we get to an un-set card.
28073 card_word_value = card_word_value / 2;
28075 // If we reach the end of the card word and haven't hit a 0 yet, start going
28076 // card word by card word until we get to one that's not fully set (0xFFFF...)
28077 // or we reach card_word_end.
28078 if ((bit_position == card_word_width) && (last_card_word < &card_table [card_word_end]))
28082 card_word_value = *(++last_card_word);
28083 } while ((last_card_word < &card_table [card_word_end]) &&
28086 (card_word_value == (1 << card_word_width)-1)
28088 // if left shift count >= width of type,
28089 // gcc reports error.
28090 (card_word_value == ~0u)
28095 } while (card_word_value & 1);
28097 end_card = (last_card_word - &card_table [0])* card_word_width + bit_position;
28099 //dprintf (3, ("find_card: [%Ix, %Ix[ set", card, end_card));
28100 dprintf (3, ("fc: [%Ix, %Ix[", card, end_card));
28105 //because of heap expansion, computing end is complicated.
28106 uint8_t* compute_next_end (heap_segment* seg, uint8_t* low)
28108 if ((low >= heap_segment_mem (seg)) &&
28109 (low < heap_segment_allocated (seg)))
28112 return heap_segment_allocated (seg);
28116 gc_heap::compute_next_boundary (uint8_t* low, int gen_number,
28119 UNREFERENCED_PARAMETER(low);
28121 //when relocating, the fault line is the plan start of the younger
28122 //generation because the generation is promoted.
28123 if (relocating && (gen_number == (settings.condemned_generation + 1)))
28125 generation* gen = generation_of (gen_number - 1);
28126 uint8_t* gen_alloc = generation_plan_allocation_start (gen);
28127 assert (gen_alloc);
28132 assert (gen_number > settings.condemned_generation);
28133 return generation_allocation_start (generation_of (gen_number - 1 ));
28139 gc_heap::keep_card_live (uint8_t* o, size_t& n_gen,
28140 size_t& cg_pointers_found)
28143 if ((gc_low <= o) && (gc_high > o))
28147 #ifdef MULTIPLE_HEAPS
28150 gc_heap* hp = heap_of (o);
28153 if ((hp->gc_low <= o) &&
28160 #endif //MULTIPLE_HEAPS
28161 cg_pointers_found ++;
28162 dprintf (4, ("keep card live for %Ix", o));
28166 gc_heap::mark_through_cards_helper (uint8_t** poo, size_t& n_gen,
28167 size_t& cg_pointers_found,
28168 card_fn fn, uint8_t* nhigh,
28169 uint8_t* next_boundary)
28172 if ((gc_low <= *poo) && (gc_high > *poo))
28175 call_fn(fn) (poo THREAD_NUMBER_ARG);
28177 #ifdef MULTIPLE_HEAPS
28180 gc_heap* hp = heap_of_gc (*poo);
28183 if ((hp->gc_low <= *poo) &&
28184 (hp->gc_high > *poo))
28187 call_fn(fn) (poo THREAD_NUMBER_ARG);
28189 if ((fn == &gc_heap::relocate_address) ||
28190 ((hp->ephemeral_low <= *poo) &&
28191 (hp->ephemeral_high > *poo)))
28193 cg_pointers_found++;
28197 #endif //MULTIPLE_HEAPS
28198 if ((next_boundary <= *poo) && (nhigh > *poo))
28200 cg_pointers_found ++;
28201 dprintf (4, ("cg pointer %Ix found, %Id so far",
28202 (size_t)*poo, cg_pointers_found ));
28207 BOOL gc_heap::card_transition (uint8_t* po, uint8_t* end, size_t card_word_end,
28208 size_t& cg_pointers_found,
28209 size_t& n_eph, size_t& n_card_set,
28210 size_t& card, size_t& end_card,
28211 BOOL& foundp, uint8_t*& start_address,
28212 uint8_t*& limit, size_t& n_cards_cleared)
28214 dprintf (3, ("pointer %Ix past card %Ix", (size_t)po, (size_t)card));
28215 dprintf (3, ("ct: %Id cg", cg_pointers_found));
28216 BOOL passed_end_card_p = FALSE;
28219 if (cg_pointers_found == 0)
28221 //dprintf(3,(" Clearing cards [%Ix, %Ix[ ",
28222 dprintf(3,(" CC [%Ix, %Ix[ ",
28223 (size_t)card_address(card), (size_t)po));
28224 clear_cards (card, card_of(po));
28225 n_card_set -= (card_of (po) - card);
28226 n_cards_cleared += (card_of (po) - card);
28229 n_eph +=cg_pointers_found;
28230 cg_pointers_found = 0;
28231 card = card_of (po);
28232 if (card >= end_card)
28234 passed_end_card_p = TRUE;
28235 dprintf (3, ("card %Ix exceeding end_card %Ix",
28236 (size_t)card, (size_t)end_card));
28237 foundp = find_card (card_table, card, card_word_end, end_card);
28240 n_card_set+= end_card - card;
28241 start_address = card_address (card);
28242 dprintf (3, ("NewC: %Ix, start: %Ix, end: %Ix",
28243 (size_t)card, (size_t)start_address,
28244 (size_t)card_address (end_card)));
28246 limit = min (end, card_address (end_card));
28248 assert (!((limit == card_address (end_card))&&
28249 card_set_p (end_card)));
28252 return passed_end_card_p;
28255 void gc_heap::mark_through_cards_for_segments (card_fn fn, BOOL relocating)
28257 #ifdef BACKGROUND_GC
28258 dprintf (3, ("current_sweep_pos is %Ix, saved_sweep_ephemeral_seg is %Ix(%Ix)",
28259 current_sweep_pos, saved_sweep_ephemeral_seg, saved_sweep_ephemeral_start));
28261 heap_segment* soh_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
28262 PREFIX_ASSUME(soh_seg != NULL);
28266 dprintf (3, ("seg %Ix, bgc_alloc: %Ix, alloc: %Ix",
28268 heap_segment_background_allocated (soh_seg),
28269 heap_segment_allocated (soh_seg)));
28271 soh_seg = heap_segment_next_rw (soh_seg);
28273 #endif //BACKGROUND_GC
28275 uint8_t* low = gc_low;
28276 uint8_t* high = gc_high;
28277 size_t end_card = 0;
28279 generation* oldest_gen = generation_of (max_generation);
28280 int curr_gen_number = max_generation;
28281 uint8_t* gen_boundary = generation_allocation_start(generation_of(curr_gen_number - 1));
28282 uint8_t* next_boundary = compute_next_boundary(gc_low, curr_gen_number, relocating);
28284 heap_segment* seg = heap_segment_rw (generation_start_segment (oldest_gen));
28285 PREFIX_ASSUME(seg != NULL);
28287 uint8_t* beg = generation_allocation_start (oldest_gen);
28288 uint8_t* end = compute_next_end (seg, low);
28289 uint8_t* last_object = beg;
28291 size_t cg_pointers_found = 0;
28293 size_t card_word_end = (card_of (align_on_card_word (end)) / card_word_width);
28297 size_t n_card_set = 0;
28298 uint8_t* nhigh = (relocating ?
28299 heap_segment_plan_allocated (ephemeral_heap_segment) : high);
28301 BOOL foundp = FALSE;
28302 uint8_t* start_address = 0;
28303 uint8_t* limit = 0;
28304 size_t card = card_of (beg);
28305 #ifdef BACKGROUND_GC
28306 BOOL consider_bgc_mark_p = FALSE;
28307 BOOL check_current_sweep_p = FALSE;
28308 BOOL check_saved_sweep_p = FALSE;
28309 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
28310 #endif //BACKGROUND_GC
28312 dprintf(3, ("CMs: %Ix->%Ix", (size_t)beg, (size_t)end));
28313 size_t total_cards_cleared = 0;
28317 if (card_of(last_object) > card)
28319 dprintf (3, ("Found %Id cg pointers", cg_pointers_found));
28320 if (cg_pointers_found == 0)
28322 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card), (size_t)last_object));
28323 clear_cards (card, card_of(last_object));
28324 n_card_set -= (card_of (last_object) - card);
28325 total_cards_cleared += (card_of (last_object) - card);
28328 n_eph += cg_pointers_found;
28329 cg_pointers_found = 0;
28330 card = card_of (last_object);
28333 if (card >= end_card)
28335 foundp = find_card (card_table, card, card_word_end, end_card);
28338 n_card_set += end_card - card;
28339 start_address = max (beg, card_address (card));
28341 limit = min (end, card_address (end_card));
28343 if (!foundp || (last_object >= end) || (card_address (card) >= end))
28345 if (foundp && (cg_pointers_found == 0))
28347 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card),
28349 clear_cards (card, card_of (end));
28350 n_card_set -= (card_of (end) - card);
28351 total_cards_cleared += (card_of (end) - card);
28353 n_eph += cg_pointers_found;
28354 cg_pointers_found = 0;
28355 if ((seg = heap_segment_next_in_range (seg)) != 0)
28357 #ifdef BACKGROUND_GC
28358 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
28359 #endif //BACKGROUND_GC
28360 beg = heap_segment_mem (seg);
28361 end = compute_next_end (seg, low);
28362 card_word_end = card_of (align_on_card_word (end)) / card_word_width;
28363 card = card_of (beg);
28374 assert (card_set_p (card));
28376 uint8_t* o = last_object;
28378 o = find_first_object (start_address, last_object);
28379 // Never visit an object twice.
28380 assert (o >= last_object);
28382 //dprintf(3,("Considering card %Ix start object: %Ix, %Ix[ boundary: %Ix",
28383 dprintf(3, ("c: %Ix, o: %Ix, l: %Ix[ boundary: %Ix",
28384 card, (size_t)o, (size_t)limit, (size_t)gen_boundary));
28388 assert (Align (size (o)) >= Align (min_obj_size));
28389 size_t s = size (o);
28391 uint8_t* next_o = o + Align (s);
28394 if ((o >= gen_boundary) &&
28395 (seg == ephemeral_heap_segment))
28397 dprintf (3, ("switching gen boundary %Ix", (size_t)gen_boundary));
28399 assert ((curr_gen_number > 0));
28400 gen_boundary = generation_allocation_start
28401 (generation_of (curr_gen_number - 1));
28402 next_boundary = (compute_next_boundary
28403 (low, curr_gen_number, relocating));
28406 dprintf (4, ("|%Ix|", (size_t)o));
28408 if (next_o < start_address)
28413 #ifdef BACKGROUND_GC
28414 if (!fgc_should_consider_object (o, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p))
28418 #endif //BACKGROUND_GC
28420 #ifdef COLLECTIBLE_CLASS
28421 if (is_collectible(o))
28423 BOOL passed_end_card_p = FALSE;
28425 if (card_of (o) > card)
28427 passed_end_card_p = card_transition (o, end, card_word_end,
28431 foundp, start_address,
28432 limit, total_cards_cleared);
28435 if ((!passed_end_card_p || foundp) && (card_of (o) == card))
28437 // card is valid and it covers the head of the object
28438 if (fn == &gc_heap::relocate_address)
28440 keep_card_live (o, n_gen, cg_pointers_found);
28444 uint8_t* class_obj = get_class_object (o);
28445 mark_through_cards_helper (&class_obj, n_gen,
28446 cg_pointers_found, fn,
28447 nhigh, next_boundary);
28451 if (passed_end_card_p)
28453 if (foundp && (card_address (card) < next_o))
28455 goto go_through_refs;
28457 else if (foundp && (start_address < limit))
28459 next_o = find_first_object (start_address, o);
28468 #endif //COLLECTIBLE_CLASS
28470 if (contain_pointers (o))
28472 dprintf(3,("Going through %Ix start_address: %Ix", (size_t)o, (size_t)start_address));
28475 dprintf (4, ("normal object path"));
28477 (method_table(o), o, s, poo,
28478 start_address, use_start, (o + s),
28480 dprintf (4, ("<%Ix>:%Ix", (size_t)poo, (size_t)*poo));
28481 if (card_of ((uint8_t*)poo) > card)
28483 BOOL passed_end_card_p = card_transition ((uint8_t*)poo, end,
28488 foundp, start_address,
28489 limit, total_cards_cleared);
28491 if (passed_end_card_p)
28493 if (foundp && (card_address (card) < next_o))
28497 if (ppstop <= (uint8_t**)start_address)
28499 else if (poo < (uint8_t**)start_address)
28500 {poo = (uint8_t**)start_address;}
28503 else if (foundp && (start_address < limit))
28505 next_o = find_first_object (start_address, o);
28513 mark_through_cards_helper (poo, n_gen,
28514 cg_pointers_found, fn,
28515 nhigh, next_boundary);
28522 if (((size_t)next_o / brick_size) != ((size_t) o / brick_size))
28524 if (brick_table [brick_of (o)] <0)
28525 fix_brick_to_highest (o, next_o);
28533 // compute the efficiency ratio of the card table
28536 generation_skip_ratio = ((n_eph > 400)? (int)(((float)n_gen / (float)n_eph) * 100) : 100);
28537 dprintf (3, ("Msoh: cross: %Id, useful: %Id, cards set: %Id, cards cleared: %Id, ratio: %d",
28538 n_eph, n_gen , n_card_set, total_cards_cleared, generation_skip_ratio));
28542 dprintf (3, ("R: Msoh: cross: %Id, useful: %Id, cards set: %Id, cards cleared: %Id, ratio: %d",
28543 n_gen, n_eph, n_card_set, total_cards_cleared, generation_skip_ratio));
28547 #ifdef SEG_REUSE_STATS
28548 size_t gc_heap::dump_buckets (size_t* ordered_indices, int count, size_t* total_size)
28550 size_t total_items = 0;
28552 for (int i = 0; i < count; i++)
28554 total_items += ordered_indices[i];
28555 *total_size += ordered_indices[i] << (MIN_INDEX_POWER2 + i);
28556 dprintf (SEG_REUSE_LOG_0, ("[%d]%4d 2^%2d", heap_number, ordered_indices[i], (MIN_INDEX_POWER2 + i)));
28558 dprintf (SEG_REUSE_LOG_0, ("[%d]Total %d items, total size is 0x%Ix", heap_number, total_items, *total_size));
28559 return total_items;
28561 #endif // SEG_REUSE_STATS
28563 void gc_heap::count_plug (size_t last_plug_size, uint8_t*& last_plug)
28565 // detect pinned plugs
28566 if (!pinned_plug_que_empty_p() && (last_plug == pinned_plug (oldest_pin())))
28568 deque_pinned_plug();
28569 update_oldest_pinned_plug();
28570 dprintf (3, ("dequed pin,now oldest pin is %Ix", pinned_plug (oldest_pin())));
28574 size_t plug_size = last_plug_size + Align(min_obj_size);
28575 BOOL is_padded = FALSE;
28578 plug_size += Align (min_obj_size);
28580 #endif //SHORT_PLUGS
28582 #ifdef RESPECT_LARGE_ALIGNMENT
28583 plug_size += switch_alignment_size (is_padded);
28584 #endif //RESPECT_LARGE_ALIGNMENT
28586 total_ephemeral_plugs += plug_size;
28587 size_t plug_size_power2 = round_up_power2 (plug_size);
28588 ordered_plug_indices[relative_index_power2_plug (plug_size_power2)]++;
28589 dprintf (SEG_REUSE_LOG_1, ("[%d]count_plug: adding 0x%Ix - %Id (2^%d) to ordered plug array",
28593 (relative_index_power2_plug (plug_size_power2) + MIN_INDEX_POWER2)));
28597 void gc_heap::count_plugs_in_brick (uint8_t* tree, uint8_t*& last_plug)
28599 assert ((tree != NULL));
28600 if (node_left_child (tree))
28602 count_plugs_in_brick (tree + node_left_child (tree), last_plug);
28605 if (last_plug != 0)
28607 uint8_t* plug = tree;
28608 size_t gap_size = node_gap_size (plug);
28609 uint8_t* gap = (plug - gap_size);
28610 uint8_t* last_plug_end = gap;
28611 size_t last_plug_size = (last_plug_end - last_plug);
28612 dprintf (3, ("tree: %Ix, last plug: %Ix, gap size: %Ix, gap: %Ix, last plug size: %Ix",
28613 tree, last_plug, gap_size, gap, last_plug_size));
28615 if (tree == oldest_pinned_plug)
28617 dprintf (3, ("tree %Ix is pinned, last plug is %Ix, size is %Ix",
28618 tree, last_plug, last_plug_size));
28619 mark* m = oldest_pin();
28620 if (m->has_pre_plug_info())
28622 last_plug_size += sizeof (gap_reloc_pair);
28623 dprintf (3, ("pin %Ix has pre plug, adjusting plug size to %Ix", tree, last_plug_size));
28626 // Can't assert here - if it's a pinned plug it can be less.
28627 //assert (last_plug_size >= Align (min_obj_size));
28629 count_plug (last_plug_size, last_plug);
28634 if (node_right_child (tree))
28636 count_plugs_in_brick (tree + node_right_child (tree), last_plug);
28640 void gc_heap::build_ordered_plug_indices ()
28642 memset (ordered_plug_indices, 0, sizeof(ordered_plug_indices));
28643 memset (saved_ordered_plug_indices, 0, sizeof(saved_ordered_plug_indices));
28645 uint8_t* start_address = generation_limit (max_generation);
28646 uint8_t* end_address = heap_segment_allocated (ephemeral_heap_segment);
28647 size_t current_brick = brick_of (start_address);
28648 size_t end_brick = brick_of (end_address - 1);
28649 uint8_t* last_plug = 0;
28651 //Look for the right pinned plug to start from.
28652 reset_pinned_queue_bos();
28653 while (!pinned_plug_que_empty_p())
28655 mark* m = oldest_pin();
28656 if ((m->first >= start_address) && (m->first < end_address))
28658 dprintf (3, ("found a pin %Ix between %Ix and %Ix", m->first, start_address, end_address));
28663 deque_pinned_plug();
28666 update_oldest_pinned_plug();
28668 while (current_brick <= end_brick)
28670 int brick_entry = brick_table [ current_brick ];
28671 if (brick_entry >= 0)
28673 count_plugs_in_brick (brick_address (current_brick) + brick_entry -1, last_plug);
28681 count_plug (end_address - last_plug, last_plug);
28684 // we need to make sure that after fitting all the existing plugs, we
28685 // have big enough free space left to guarantee that the next allocation
28687 size_t extra_size = END_SPACE_AFTER_GC + Align (min_obj_size);
28688 total_ephemeral_plugs += extra_size;
28689 dprintf (SEG_REUSE_LOG_0, ("Making sure we can fit a large object after fitting all plugs"));
28690 ordered_plug_indices[relative_index_power2_plug (round_up_power2 (extra_size))]++;
28692 memcpy (saved_ordered_plug_indices, ordered_plug_indices, sizeof(ordered_plug_indices));
28694 #ifdef SEG_REUSE_STATS
28695 dprintf (SEG_REUSE_LOG_0, ("Plugs:"));
28696 size_t total_plug_power2 = 0;
28697 dump_buckets (ordered_plug_indices, MAX_NUM_BUCKETS, &total_plug_power2);
28698 dprintf (SEG_REUSE_LOG_0, ("plugs: 0x%Ix (rounded up to 0x%Ix (%d%%))",
28699 total_ephemeral_plugs,
28701 (total_ephemeral_plugs ?
28702 (total_plug_power2 * 100 / total_ephemeral_plugs) :
28704 dprintf (SEG_REUSE_LOG_0, ("-------------------"));
28705 #endif // SEG_REUSE_STATS
28708 void gc_heap::init_ordered_free_space_indices ()
28710 memset (ordered_free_space_indices, 0, sizeof(ordered_free_space_indices));
28711 memset (saved_ordered_free_space_indices, 0, sizeof(saved_ordered_free_space_indices));
28714 void gc_heap::trim_free_spaces_indices ()
28716 trimmed_free_space_index = -1;
28717 size_t max_count = max_free_space_items - 1;
28720 for (i = (MAX_NUM_BUCKETS - 1); i >= 0; i--)
28722 count += ordered_free_space_indices[i];
28724 if (count >= max_count)
28730 ptrdiff_t extra_free_space_items = count - max_count;
28732 if (extra_free_space_items > 0)
28734 ordered_free_space_indices[i] -= extra_free_space_items;
28735 free_space_items = max_count;
28736 trimmed_free_space_index = i;
28740 free_space_items = count;
28748 free_space_buckets = MAX_NUM_BUCKETS - i;
28750 for (--i; i >= 0; i--)
28752 ordered_free_space_indices[i] = 0;
28755 memcpy (saved_ordered_free_space_indices,
28756 ordered_free_space_indices,
28757 sizeof(ordered_free_space_indices));
28760 // We fit as many plugs as we can and update the number of plugs left and the number
28761 // of free spaces left.
28762 BOOL gc_heap::can_fit_in_spaces_p (size_t* ordered_blocks, int small_index, size_t* ordered_spaces, int big_index)
28764 assert (small_index <= big_index);
28765 assert (big_index < MAX_NUM_BUCKETS);
28767 size_t small_blocks = ordered_blocks[small_index];
28769 if (small_blocks == 0)
28774 size_t big_spaces = ordered_spaces[big_index];
28776 if (big_spaces == 0)
28781 dprintf (SEG_REUSE_LOG_1, ("[%d]Fitting %Id 2^%d plugs into %Id 2^%d free spaces",
28783 small_blocks, (small_index + MIN_INDEX_POWER2),
28784 big_spaces, (big_index + MIN_INDEX_POWER2)));
28786 size_t big_to_small = big_spaces << (big_index - small_index);
28788 ptrdiff_t extra_small_spaces = big_to_small - small_blocks;
28789 dprintf (SEG_REUSE_LOG_1, ("[%d]%d 2^%d spaces can fit %d 2^%d blocks",
28791 big_spaces, (big_index + MIN_INDEX_POWER2), big_to_small, (small_index + MIN_INDEX_POWER2)));
28792 BOOL can_fit = (extra_small_spaces >= 0);
28796 dprintf (SEG_REUSE_LOG_1, ("[%d]Can fit with %d 2^%d extras blocks",
28798 extra_small_spaces, (small_index + MIN_INDEX_POWER2)));
28803 dprintf (SEG_REUSE_LOG_1, ("[%d]Setting # of 2^%d spaces to 0", heap_number, (big_index + MIN_INDEX_POWER2)));
28804 ordered_spaces[big_index] = 0;
28805 if (extra_small_spaces > 0)
28807 dprintf (SEG_REUSE_LOG_1, ("[%d]Setting # of 2^%d blocks to 0", heap_number, (small_index + MIN_INDEX_POWER2)));
28808 ordered_blocks[small_index] = 0;
28809 for (i = small_index; i < big_index; i++)
28811 if (extra_small_spaces & 1)
28813 dprintf (SEG_REUSE_LOG_1, ("[%d]Increasing # of 2^%d spaces from %d to %d",
28815 (i + MIN_INDEX_POWER2), ordered_spaces[i], (ordered_spaces[i] + 1)));
28816 ordered_spaces[i] += 1;
28818 extra_small_spaces >>= 1;
28821 dprintf (SEG_REUSE_LOG_1, ("[%d]Finally increasing # of 2^%d spaces from %d to %d",
28823 (i + MIN_INDEX_POWER2), ordered_spaces[i], (ordered_spaces[i] + extra_small_spaces)));
28824 ordered_spaces[i] += extra_small_spaces;
28828 dprintf (SEG_REUSE_LOG_1, ("[%d]Decreasing # of 2^%d blocks from %d to %d",
28830 (small_index + MIN_INDEX_POWER2),
28831 ordered_blocks[small_index],
28832 (ordered_blocks[small_index] - big_to_small)));
28833 ordered_blocks[small_index] -= big_to_small;
28836 #ifdef SEG_REUSE_STATS
28838 dprintf (SEG_REUSE_LOG_1, ("[%d]Plugs became:", heap_number));
28839 dump_buckets (ordered_blocks, MAX_NUM_BUCKETS, &temp);
28841 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces became:", heap_number));
28842 dump_buckets (ordered_spaces, MAX_NUM_BUCKETS, &temp);
28843 #endif //SEG_REUSE_STATS
28848 // space_index gets updated to the biggest available space index.
28849 BOOL gc_heap::can_fit_blocks_p (size_t* ordered_blocks, int block_index, size_t* ordered_spaces, int* space_index)
28851 assert (*space_index >= block_index);
28853 while (!can_fit_in_spaces_p (ordered_blocks, block_index, ordered_spaces, *space_index))
28856 if (*space_index < block_index)
28865 BOOL gc_heap::can_fit_all_blocks_p (size_t* ordered_blocks, size_t* ordered_spaces, int count)
28867 #ifdef FEATURE_STRUCTALIGN
28868 // BARTOKTODO (4841): reenable when can_fit_in_spaces_p takes alignment requirements into account
28870 #endif // FEATURE_STRUCTALIGN
28871 int space_index = count - 1;
28872 for (int block_index = (count - 1); block_index >= 0; block_index--)
28874 if (!can_fit_blocks_p (ordered_blocks, block_index, ordered_spaces, &space_index))
28883 void gc_heap::build_ordered_free_spaces (heap_segment* seg)
28885 assert (bestfit_seg);
28887 //bestfit_seg->add_buckets (MAX_NUM_BUCKETS - free_space_buckets + MIN_INDEX_POWER2,
28888 // ordered_free_space_indices + (MAX_NUM_BUCKETS - free_space_buckets),
28889 // free_space_buckets,
28890 // free_space_items);
28892 bestfit_seg->add_buckets (MIN_INDEX_POWER2,
28893 ordered_free_space_indices,
28897 assert (settings.condemned_generation == max_generation);
28899 uint8_t* first_address = heap_segment_mem (seg);
28900 uint8_t* end_address = heap_segment_reserved (seg);
28901 //look through the pinned plugs for relevant ones.
28902 //Look for the right pinned plug to start from.
28903 reset_pinned_queue_bos();
28905 // See comment in can_expand_into_p why we need (max_generation + 1).
28906 size_t eph_gen_starts = (Align (min_obj_size)) * (max_generation + 1);
28907 BOOL has_fit_gen_starts = FALSE;
28909 while (!pinned_plug_que_empty_p())
28912 if ((pinned_plug (m) >= first_address) &&
28913 (pinned_plug (m) < end_address) &&
28914 (pinned_len (m) >= eph_gen_starts))
28917 assert ((pinned_plug (m) - pinned_len (m)) == bestfit_first_pin);
28922 deque_pinned_plug();
28926 if (!pinned_plug_que_empty_p())
28928 bestfit_seg->add ((void*)m, TRUE, TRUE);
28929 deque_pinned_plug();
28931 has_fit_gen_starts = TRUE;
28934 while (!pinned_plug_que_empty_p() &&
28935 ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address)))
28937 bestfit_seg->add ((void*)m, TRUE, FALSE);
28938 deque_pinned_plug();
28942 if (commit_end_of_seg)
28944 if (!has_fit_gen_starts)
28946 assert (bestfit_first_pin == heap_segment_plan_allocated (seg));
28948 bestfit_seg->add ((void*)seg, FALSE, (!has_fit_gen_starts));
28952 bestfit_seg->check();
28956 BOOL gc_heap::try_best_fit (BOOL end_of_segment_p)
28958 if (!end_of_segment_p)
28960 trim_free_spaces_indices ();
28963 BOOL can_bestfit = can_fit_all_blocks_p (ordered_plug_indices,
28964 ordered_free_space_indices,
28967 return can_bestfit;
28970 BOOL gc_heap::best_fit (size_t free_space,
28971 size_t largest_free_space,
28972 size_t additional_space,
28973 BOOL* use_additional_space)
28975 dprintf (SEG_REUSE_LOG_0, ("gen%d: trying best fit mechanism", settings.condemned_generation));
28977 assert (!additional_space || (additional_space && use_additional_space));
28978 if (use_additional_space)
28980 *use_additional_space = FALSE;
28983 if (ordered_plug_indices_init == FALSE)
28985 total_ephemeral_plugs = 0;
28986 build_ordered_plug_indices();
28987 ordered_plug_indices_init = TRUE;
28991 memcpy (ordered_plug_indices, saved_ordered_plug_indices, sizeof(ordered_plug_indices));
28994 if (total_ephemeral_plugs == (END_SPACE_AFTER_GC + Align (min_obj_size)))
28996 dprintf (SEG_REUSE_LOG_0, ("No ephemeral plugs to realloc, done"));
28997 size_t empty_eph = (END_SPACE_AFTER_GC + Align (min_obj_size) + (Align (min_obj_size)) * (max_generation + 1));
28998 BOOL can_fit_empty_eph = (largest_free_space >= empty_eph);
28999 if (!can_fit_empty_eph)
29001 can_fit_empty_eph = (additional_space >= empty_eph);
29003 if (can_fit_empty_eph)
29005 *use_additional_space = TRUE;
29009 return can_fit_empty_eph;
29012 if ((total_ephemeral_plugs + approximate_new_allocation()) >= (free_space + additional_space))
29014 dprintf (SEG_REUSE_LOG_0, ("We won't have enough free space left in this segment after fitting, done"));
29018 if ((free_space + additional_space) == 0)
29020 dprintf (SEG_REUSE_LOG_0, ("No free space in this segment, done"));
29024 #ifdef SEG_REUSE_STATS
29025 dprintf (SEG_REUSE_LOG_0, ("Free spaces:"));
29026 size_t total_free_space_power2 = 0;
29027 size_t total_free_space_items =
29028 dump_buckets (ordered_free_space_indices,
29030 &total_free_space_power2);
29031 dprintf (SEG_REUSE_LOG_0, ("currently max free spaces is %Id", max_free_space_items));
29033 dprintf (SEG_REUSE_LOG_0, ("Ephemeral plugs: 0x%Ix, free space: 0x%Ix (rounded down to 0x%Ix (%Id%%)), additional free_space: 0x%Ix",
29034 total_ephemeral_plugs,
29036 total_free_space_power2,
29037 (free_space ? (total_free_space_power2 * 100 / free_space) : 0),
29038 additional_space));
29040 size_t saved_all_free_space_indices[MAX_NUM_BUCKETS];
29041 memcpy (saved_all_free_space_indices,
29042 ordered_free_space_indices,
29043 sizeof(saved_all_free_space_indices));
29045 #endif // SEG_REUSE_STATS
29047 if (total_ephemeral_plugs > (free_space + additional_space))
29052 use_bestfit = try_best_fit(FALSE);
29054 if (!use_bestfit && additional_space)
29056 int relative_free_space_index = relative_index_power2_free_space (round_down_power2 (additional_space));
29058 if (relative_free_space_index != -1)
29060 int relative_plug_index = 0;
29061 size_t plugs_to_fit = 0;
29063 for (relative_plug_index = (MAX_NUM_BUCKETS - 1); relative_plug_index >= 0; relative_plug_index--)
29065 plugs_to_fit = ordered_plug_indices[relative_plug_index];
29066 if (plugs_to_fit != 0)
29072 if ((relative_plug_index > relative_free_space_index) ||
29073 ((relative_plug_index == relative_free_space_index) &&
29074 (plugs_to_fit > 1)))
29076 #ifdef SEG_REUSE_STATS
29077 dprintf (SEG_REUSE_LOG_0, ("additional space is 2^%d but we stopped at %d 2^%d plug(s)",
29078 (relative_free_space_index + MIN_INDEX_POWER2),
29080 (relative_plug_index + MIN_INDEX_POWER2)));
29081 #endif // SEG_REUSE_STATS
29085 dprintf (SEG_REUSE_LOG_0, ("Adding end of segment (2^%d)", (relative_free_space_index + MIN_INDEX_POWER2)));
29086 ordered_free_space_indices[relative_free_space_index]++;
29087 use_bestfit = try_best_fit(TRUE);
29090 free_space_items++;
29091 // Since we might've trimmed away some of the free spaces we had, we should see
29092 // if we really need to use end of seg space - if it's the same or smaller than
29093 // the largest space we trimmed we can just add that one back instead of
29094 // using end of seg.
29095 if (relative_free_space_index > trimmed_free_space_index)
29097 *use_additional_space = TRUE;
29101 // If the addition space is <= than the last trimmed space, we
29102 // should just use that last trimmed space instead.
29103 saved_ordered_free_space_indices[trimmed_free_space_index]++;
29113 dprintf (SEG_REUSE_LOG_0, ("couldn't fit..."));
29115 #ifdef SEG_REUSE_STATS
29116 size_t saved_max = max_free_space_items;
29117 BOOL temp_bestfit = FALSE;
29119 dprintf (SEG_REUSE_LOG_0, ("----Starting experiment process----"));
29120 dprintf (SEG_REUSE_LOG_0, ("----Couldn't fit with max free items %Id", max_free_space_items));
29122 // TODO: need to take the end of segment into consideration.
29123 while (max_free_space_items <= total_free_space_items)
29125 max_free_space_items += max_free_space_items / 2;
29126 dprintf (SEG_REUSE_LOG_0, ("----Temporarily increasing max free spaces to %Id", max_free_space_items));
29127 memcpy (ordered_free_space_indices,
29128 saved_all_free_space_indices,
29129 sizeof(ordered_free_space_indices));
29130 if (try_best_fit(FALSE))
29132 temp_bestfit = TRUE;
29139 dprintf (SEG_REUSE_LOG_0, ("----With %Id max free spaces we could fit", max_free_space_items));
29143 dprintf (SEG_REUSE_LOG_0, ("----Tried all free spaces and still couldn't fit, lost too much space"));
29146 dprintf (SEG_REUSE_LOG_0, ("----Restoring max free spaces to %Id", saved_max));
29147 max_free_space_items = saved_max;
29148 #endif // SEG_REUSE_STATS
29149 if (free_space_items)
29151 max_free_space_items = min (MAX_NUM_FREE_SPACES, free_space_items * 2);
29152 max_free_space_items = max (max_free_space_items, MIN_NUM_FREE_SPACES);
29156 max_free_space_items = MAX_NUM_FREE_SPACES;
29160 dprintf (SEG_REUSE_LOG_0, ("Adjusted number of max free spaces to %Id", max_free_space_items));
29161 dprintf (SEG_REUSE_LOG_0, ("------End of best fitting process------\n"));
29163 return use_bestfit;
29166 BOOL gc_heap::process_free_space (heap_segment* seg,
29168 size_t min_free_size,
29169 size_t min_cont_size,
29170 size_t* total_free_space,
29171 size_t* largest_free_space)
29173 *total_free_space += free_space;
29174 *largest_free_space = max (*largest_free_space, free_space);
29176 #ifdef SIMPLE_DPRINTF
29177 dprintf (SEG_REUSE_LOG_1, ("free space len: %Ix, total free space: %Ix, largest free space: %Ix",
29178 free_space, *total_free_space, *largest_free_space));
29179 #endif //SIMPLE_DPRINTF
29181 if ((*total_free_space >= min_free_size) && (*largest_free_space >= min_cont_size))
29183 #ifdef SIMPLE_DPRINTF
29184 dprintf (SEG_REUSE_LOG_0, ("(gen%d)total free: %Ix(min: %Ix), largest free: %Ix(min: %Ix). Found segment %Ix to reuse without bestfit",
29185 settings.condemned_generation,
29186 *total_free_space, min_free_size, *largest_free_space, min_cont_size,
29189 UNREFERENCED_PARAMETER(seg);
29190 #endif //SIMPLE_DPRINTF
29194 int free_space_index = relative_index_power2_free_space (round_down_power2 (free_space));
29195 if (free_space_index != -1)
29197 ordered_free_space_indices[free_space_index]++;
29202 BOOL gc_heap::expand_reused_seg_p()
29204 BOOL reused_seg = FALSE;
29205 int heap_expand_mechanism = gc_data_per_heap.get_mechanism (gc_heap_expand);
29206 if ((heap_expand_mechanism == expand_reuse_bestfit) ||
29207 (heap_expand_mechanism == expand_reuse_normal))
29215 BOOL gc_heap::can_expand_into_p (heap_segment* seg, size_t min_free_size, size_t min_cont_size,
29216 allocator* gen_allocator)
29218 min_cont_size += END_SPACE_AFTER_GC;
29219 use_bestfit = FALSE;
29220 commit_end_of_seg = FALSE;
29221 bestfit_first_pin = 0;
29222 uint8_t* first_address = heap_segment_mem (seg);
29223 uint8_t* end_address = heap_segment_reserved (seg);
29224 size_t end_extra_space = end_space_after_gc();
29226 if ((heap_segment_reserved (seg) - end_extra_space) <= heap_segment_plan_allocated (seg))
29228 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: can't use segment [%Ix %Ix, has less than %d bytes at the end",
29229 first_address, end_address, end_extra_space));
29233 end_address -= end_extra_space;
29235 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p(gen%d): min free: %Ix, min continuous: %Ix",
29236 settings.condemned_generation, min_free_size, min_cont_size));
29237 size_t eph_gen_starts = eph_gen_starts_size;
29239 if (settings.condemned_generation == max_generation)
29241 size_t free_space = 0;
29242 size_t largest_free_space = free_space;
29243 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: gen2: testing segment [%Ix %Ix", first_address, end_address));
29244 //Look through the pinned plugs for relevant ones and Look for the right pinned plug to start from.
29245 //We are going to allocate the generation starts in the 1st free space,
29246 //so start from the first free space that's big enough for gen starts and a min object size.
29247 // If we see a free space that is >= gen starts but < gen starts + min obj size we just don't use it -
29248 // we could use it by allocating the last generation start a bit bigger but
29249 // the complexity isn't worth the effort (those plugs are from gen2
29250 // already anyway).
29251 reset_pinned_queue_bos();
29253 BOOL has_fit_gen_starts = FALSE;
29255 init_ordered_free_space_indices ();
29256 while (!pinned_plug_que_empty_p())
29259 if ((pinned_plug (m) >= first_address) &&
29260 (pinned_plug (m) < end_address) &&
29261 (pinned_len (m) >= (eph_gen_starts + Align (min_obj_size))))
29267 deque_pinned_plug();
29271 if (!pinned_plug_que_empty_p())
29273 bestfit_first_pin = pinned_plug (m) - pinned_len (m);
29275 if (process_free_space (seg,
29276 pinned_len (m) - eph_gen_starts,
29277 min_free_size, min_cont_size,
29278 &free_space, &largest_free_space))
29283 deque_pinned_plug();
29285 has_fit_gen_starts = TRUE;
29288 dprintf (3, ("first pin is %Ix", pinned_plug (m)));
29290 //tally up free space
29291 while (!pinned_plug_que_empty_p() &&
29292 ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address)))
29294 dprintf (3, ("looking at pin %Ix", pinned_plug (m)));
29295 if (process_free_space (seg,
29297 min_free_size, min_cont_size,
29298 &free_space, &largest_free_space))
29303 deque_pinned_plug();
29307 //try to find space at the end of the segment.
29308 size_t end_space = (end_address - heap_segment_plan_allocated (seg));
29309 size_t additional_space = ((min_free_size > free_space) ? (min_free_size - free_space) : 0);
29310 dprintf (SEG_REUSE_LOG_0, ("end space: %Ix; additional: %Ix", end_space, additional_space));
29311 if (end_space >= additional_space)
29313 BOOL can_fit = TRUE;
29314 commit_end_of_seg = TRUE;
29316 if (largest_free_space < min_cont_size)
29318 if (end_space >= min_cont_size)
29320 additional_space = max (min_cont_size, additional_space);
29321 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse without bestfit, with committing end of seg for eph",
29326 if (settings.concurrent)
29329 commit_end_of_seg = FALSE;
29333 size_t additional_space_bestfit = additional_space;
29334 if (!has_fit_gen_starts)
29336 if (additional_space_bestfit < (eph_gen_starts + Align (min_obj_size)))
29338 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, gen starts not allocated yet and end space is too small: %Id",
29339 additional_space_bestfit));
29343 bestfit_first_pin = heap_segment_plan_allocated (seg);
29344 additional_space_bestfit -= eph_gen_starts;
29347 can_fit = best_fit (free_space,
29348 largest_free_space,
29349 additional_space_bestfit,
29350 &commit_end_of_seg);
29354 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse with bestfit, %s committing end of seg",
29355 seg, (commit_end_of_seg ? "with" : "without")));
29359 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, total free space is %Ix", (free_space + end_space)));
29366 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse without bestfit, with committing end of seg", seg));
29369 assert (additional_space <= end_space);
29370 if (commit_end_of_seg)
29372 if (!grow_heap_segment (seg, heap_segment_plan_allocated (seg) + additional_space))
29374 dprintf (2, ("Couldn't commit end of segment?!"));
29375 use_bestfit = FALSE;
29382 // We increase the index here because growing heap segment could create a discrepency with
29383 // the additional space we used (could be bigger).
29384 size_t free_space_end_of_seg =
29385 heap_segment_committed (seg) - heap_segment_plan_allocated (seg);
29386 int relative_free_space_index = relative_index_power2_free_space (round_down_power2 (free_space_end_of_seg));
29387 saved_ordered_free_space_indices[relative_free_space_index]++;
29393 memcpy (ordered_free_space_indices,
29394 saved_ordered_free_space_indices,
29395 sizeof(ordered_free_space_indices));
29396 max_free_space_items = max (MIN_NUM_FREE_SPACES, free_space_items * 3 / 2);
29397 max_free_space_items = min (MAX_NUM_FREE_SPACES, max_free_space_items);
29398 dprintf (SEG_REUSE_LOG_0, ("could fit! %Id free spaces, %Id max", free_space_items, max_free_space_items));
29404 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, total free space is %Ix", (free_space + end_space)));
29409 assert (settings.condemned_generation == (max_generation-1));
29410 size_t free_space = (end_address - heap_segment_plan_allocated (seg));
29411 size_t largest_free_space = free_space;
29412 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: gen1: testing segment [%Ix %Ix", first_address, end_address));
29413 //find the first free list in range of the current segment
29414 size_t sz_list = gen_allocator->first_bucket_size();
29415 unsigned int a_l_idx = 0;
29416 uint8_t* free_list = 0;
29417 for (; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
29419 if ((eph_gen_starts <= sz_list) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
29421 free_list = gen_allocator->alloc_list_head_of (a_l_idx);
29424 if ((free_list >= first_address) &&
29425 (free_list < end_address) &&
29426 (unused_array_size (free_list) >= eph_gen_starts))
29432 free_list = free_list_slot (free_list);
29440 init_ordered_free_space_indices ();
29441 if (process_free_space (seg,
29442 unused_array_size (free_list) - eph_gen_starts + Align (min_obj_size),
29443 min_free_size, min_cont_size,
29444 &free_space, &largest_free_space))
29449 free_list = free_list_slot (free_list);
29453 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, no free list"));
29457 //tally up free space
29463 if ((free_list >= first_address) && (free_list < end_address) &&
29464 process_free_space (seg,
29465 unused_array_size (free_list),
29466 min_free_size, min_cont_size,
29467 &free_space, &largest_free_space))
29472 free_list = free_list_slot (free_list);
29475 if (a_l_idx < gen_allocator->number_of_buckets())
29477 free_list = gen_allocator->alloc_list_head_of (a_l_idx);
29483 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, total free space is %Ix", free_space));
29487 BOOL can_fit = best_fit (free_space, 0, NULL);
29490 dprintf (SEG_REUSE_LOG_0, ("(gen1)Found segment %Ix to reuse with bestfit", seg));
29494 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, total free space is %Ix", free_space));
29502 void gc_heap::realloc_plug (size_t last_plug_size, uint8_t*& last_plug,
29503 generation* gen, uint8_t* start_address,
29504 unsigned int& active_new_gen_number,
29505 uint8_t*& last_pinned_gap, BOOL& leftp,
29508 , mark* pinned_plug_entry
29509 #endif //SHORT_PLUGS
29512 // detect generation boundaries
29513 // make sure that active_new_gen_number is not the youngest generation.
29514 // because the generation_limit wouldn't return the right thing in this case.
29517 if ((active_new_gen_number > 1) &&
29518 (last_plug >= generation_limit (active_new_gen_number)))
29520 assert (last_plug >= start_address);
29521 active_new_gen_number--;
29522 realloc_plan_generation_start (generation_of (active_new_gen_number), gen);
29523 assert (generation_plan_allocation_start (generation_of (active_new_gen_number)));
29528 // detect pinned plugs
29529 if (!pinned_plug_que_empty_p() && (last_plug == pinned_plug (oldest_pin())))
29531 size_t entry = deque_pinned_plug();
29532 mark* m = pinned_plug_of (entry);
29534 size_t saved_pinned_len = pinned_len(m);
29535 pinned_len(m) = last_plug - last_pinned_gap;
29536 //dprintf (3,("Adjusting pinned gap: [%Ix, %Ix[", (size_t)last_pinned_gap, (size_t)last_plug));
29538 if (m->has_post_plug_info())
29540 last_plug_size += sizeof (gap_reloc_pair);
29541 dprintf (3, ("ra pinned %Ix was shortened, adjusting plug size to %Ix", last_plug, last_plug_size))
29544 last_pinned_gap = last_plug + last_plug_size;
29545 dprintf (3, ("ra found pin %Ix, len: %Ix->%Ix, last_p: %Ix, last_p_size: %Ix",
29546 pinned_plug (m), saved_pinned_len, pinned_len (m), last_plug, last_plug_size));
29549 //we are creating a generation fault. set the cards.
29551 size_t end_card = card_of (align_on_card (last_plug + last_plug_size));
29552 size_t card = card_of (last_plug);
29553 while (card != end_card)
29560 else if (last_plug >= start_address)
29562 #ifdef FEATURE_STRUCTALIGN
29563 int requiredAlignment;
29565 node_aligninfo (last_plug, requiredAlignment, pad);
29567 // from how we previously aligned the plug's destination address,
29568 // compute the actual alignment offset.
29569 uint8_t* reloc_plug = last_plug + node_relocation_distance (last_plug);
29570 ptrdiff_t alignmentOffset = ComputeStructAlignPad(reloc_plug, requiredAlignment, 0);
29571 if (!alignmentOffset)
29573 // allocate_in_expanded_heap doesn't expect alignmentOffset to be zero.
29574 alignmentOffset = requiredAlignment;
29577 //clear the alignment info because we are reallocating
29578 clear_node_aligninfo (last_plug);
29579 #else // FEATURE_STRUCTALIGN
29580 //clear the realignment flag because we are reallocating
29581 clear_node_realigned (last_plug);
29582 #endif // FEATURE_STRUCTALIGN
29583 BOOL adjacentp = FALSE;
29584 BOOL set_padding_on_saved_p = FALSE;
29588 last_plug_size += sizeof (gap_reloc_pair);
29591 assert (pinned_plug_entry != NULL);
29592 if (last_plug_size <= sizeof (plug_and_gap))
29594 set_padding_on_saved_p = TRUE;
29596 #endif //SHORT_PLUGS
29598 dprintf (3, ("ra plug %Ix was shortened, adjusting plug size to %Ix", last_plug, last_plug_size))
29602 clear_padding_in_expand (last_plug, set_padding_on_saved_p, pinned_plug_entry);
29603 #endif //SHORT_PLUGS
29605 uint8_t* new_address = allocate_in_expanded_heap(gen, last_plug_size, adjacentp, last_plug,
29607 set_padding_on_saved_p,
29609 #endif //SHORT_PLUGS
29610 TRUE, active_new_gen_number REQD_ALIGN_AND_OFFSET_ARG);
29612 dprintf (3, ("ra NA: [%Ix, %Ix[: %Ix", new_address, (new_address + last_plug_size), last_plug_size));
29613 assert (new_address);
29614 set_node_relocation_distance (last_plug, new_address - last_plug);
29615 #ifdef FEATURE_STRUCTALIGN
29616 if (leftp && node_alignpad (last_plug) == 0)
29617 #else // FEATURE_STRUCTALIGN
29618 if (leftp && !node_realigned (last_plug))
29619 #endif // FEATURE_STRUCTALIGN
29621 // TODO - temporarily disable L optimization because of a bug in it.
29622 //set_node_left (last_plug);
29624 dprintf (3,(" Re-allocating %Ix->%Ix len %Id", (size_t)last_plug, (size_t)new_address, last_plug_size));
29629 void gc_heap::realloc_in_brick (uint8_t* tree, uint8_t*& last_plug,
29630 uint8_t* start_address,
29632 unsigned int& active_new_gen_number,
29633 uint8_t*& last_pinned_gap, BOOL& leftp)
29635 assert (tree != NULL);
29636 int left_node = node_left_child (tree);
29637 int right_node = node_right_child (tree);
29639 dprintf (3, ("ra: tree: %Ix, last_pin_gap: %Ix, last_p: %Ix, L: %d, R: %d",
29640 tree, last_pinned_gap, last_plug, left_node, right_node));
29644 dprintf (3, ("LN: realloc %Ix(%Ix)", (tree + left_node), last_plug));
29645 realloc_in_brick ((tree + left_node), last_plug, start_address,
29646 gen, active_new_gen_number, last_pinned_gap,
29650 if (last_plug != 0)
29652 uint8_t* plug = tree;
29654 BOOL has_pre_plug_info_p = FALSE;
29655 BOOL has_post_plug_info_p = FALSE;
29656 mark* pinned_plug_entry = get_next_pinned_entry (tree,
29657 &has_pre_plug_info_p,
29658 &has_post_plug_info_p,
29661 // We only care about the pre plug info 'cause that's what decides if the last plug is shortened.
29662 // The pinned plugs are handled in realloc_plug.
29663 size_t gap_size = node_gap_size (plug);
29664 uint8_t* gap = (plug - gap_size);
29665 uint8_t* last_plug_end = gap;
29666 size_t last_plug_size = (last_plug_end - last_plug);
29667 // Cannot assert this - a plug could be less than that due to the shortened ones.
29668 //assert (last_plug_size >= Align (min_obj_size));
29669 dprintf (3, ("ra: plug %Ix, gap size: %Ix, last_pin_gap: %Ix, last_p: %Ix, last_p_end: %Ix, shortened: %d",
29670 plug, gap_size, last_pinned_gap, last_plug, last_plug_end, (has_pre_plug_info_p ? 1 : 0)));
29671 realloc_plug (last_plug_size, last_plug, gen, start_address,
29672 active_new_gen_number, last_pinned_gap,
29673 leftp, has_pre_plug_info_p
29675 , pinned_plug_entry
29676 #endif //SHORT_PLUGS
29684 dprintf (3, ("RN: realloc %Ix(%Ix)", (tree + right_node), last_plug));
29685 realloc_in_brick ((tree + right_node), last_plug, start_address,
29686 gen, active_new_gen_number, last_pinned_gap,
29692 gc_heap::realloc_plugs (generation* consing_gen, heap_segment* seg,
29693 uint8_t* start_address, uint8_t* end_address,
29694 unsigned active_new_gen_number)
29696 dprintf (3, ("--- Reallocing ---"));
29700 //make sure that every generation has a planned allocation start
29701 int gen_number = max_generation - 1;
29702 while (gen_number >= 0)
29704 generation* gen = generation_of (gen_number);
29705 if (0 == generation_plan_allocation_start (gen))
29707 generation_plan_allocation_start (gen) =
29708 bestfit_first_pin + (max_generation - gen_number - 1) * Align (min_obj_size);
29709 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
29710 assert (generation_plan_allocation_start (gen));
29716 uint8_t* first_address = start_address;
29717 //Look for the right pinned plug to start from.
29718 reset_pinned_queue_bos();
29719 uint8_t* planned_ephemeral_seg_end = heap_segment_plan_allocated (seg);
29720 while (!pinned_plug_que_empty_p())
29722 mark* m = oldest_pin();
29723 if ((pinned_plug (m) >= planned_ephemeral_seg_end) && (pinned_plug (m) < end_address))
29725 if (pinned_plug (m) < first_address)
29727 first_address = pinned_plug (m);
29732 deque_pinned_plug();
29735 size_t current_brick = brick_of (first_address);
29736 size_t end_brick = brick_of (end_address-1);
29737 uint8_t* last_plug = 0;
29739 uint8_t* last_pinned_gap = heap_segment_plan_allocated (seg);
29740 BOOL leftp = FALSE;
29742 dprintf (3, ("start addr: %Ix, first addr: %Ix, current oldest pin: %Ix",
29743 start_address, first_address, pinned_plug (oldest_pin())));
29745 while (current_brick <= end_brick)
29747 int brick_entry = brick_table [ current_brick ];
29748 if (brick_entry >= 0)
29750 realloc_in_brick ((brick_address (current_brick) + brick_entry - 1),
29751 last_plug, start_address, consing_gen,
29752 active_new_gen_number, last_pinned_gap,
29758 if (last_plug != 0)
29760 realloc_plug (end_address - last_plug, last_plug, consing_gen,
29762 active_new_gen_number, last_pinned_gap,
29766 #endif //SHORT_PLUGS
29770 //Fix the old segment allocated size
29771 assert (last_pinned_gap >= heap_segment_mem (seg));
29772 assert (last_pinned_gap <= heap_segment_committed (seg));
29773 heap_segment_plan_allocated (seg) = last_pinned_gap;
29776 void gc_heap::verify_no_pins (uint8_t* start, uint8_t* end)
29779 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
29781 BOOL contains_pinned_plugs = FALSE;
29784 while (mi != mark_stack_tos)
29786 m = pinned_plug_of (mi);
29787 if ((pinned_plug (m) >= start) && (pinned_plug (m) < end))
29789 contains_pinned_plugs = TRUE;
29796 if (contains_pinned_plugs)
29801 #endif //VERIFY_HEAP
29804 void gc_heap::set_expand_in_full_gc (int condemned_gen_number)
29806 if (!should_expand_in_full_gc)
29808 if ((condemned_gen_number != max_generation) &&
29809 (settings.pause_mode != pause_low_latency) &&
29810 (settings.pause_mode != pause_sustained_low_latency))
29812 should_expand_in_full_gc = TRUE;
29817 void gc_heap::save_ephemeral_generation_starts()
29819 for (int ephemeral_generation = 0; ephemeral_generation < max_generation; ephemeral_generation++)
29821 saved_ephemeral_plan_start[ephemeral_generation] =
29822 generation_plan_allocation_start (generation_of (ephemeral_generation));
29823 saved_ephemeral_plan_start_size[ephemeral_generation] =
29824 generation_plan_allocation_start_size (generation_of (ephemeral_generation));
29828 generation* gc_heap::expand_heap (int condemned_generation,
29829 generation* consing_gen,
29830 heap_segment* new_heap_segment)
29832 UNREFERENCED_PARAMETER(condemned_generation);
29833 assert (condemned_generation >= (max_generation -1));
29834 unsigned int active_new_gen_number = max_generation; //Set one too high to get generation gap
29835 uint8_t* start_address = generation_limit (max_generation);
29836 uint8_t* end_address = heap_segment_allocated (ephemeral_heap_segment);
29837 BOOL should_promote_ephemeral = FALSE;
29838 ptrdiff_t eph_size = total_ephemeral_size;
29839 #ifdef BACKGROUND_GC
29840 dprintf(2,("%s: ---- Heap Expansion ----", (recursive_gc_sync::background_running_p() ? "FGC" : "NGC")));
29841 #endif //BACKGROUND_GC
29842 settings.heap_expansion = TRUE;
29844 #ifdef BACKGROUND_GC
29845 if (cm_in_progress)
29847 if (!expanded_in_fgc)
29849 expanded_in_fgc = TRUE;
29852 #endif //BACKGROUND_GC
29854 //reset the elevation state for next time.
29855 dprintf (2, ("Elevation: elevation = el_none"));
29856 if (settings.should_lock_elevation && !expand_reused_seg_p())
29857 settings.should_lock_elevation = FALSE;
29859 heap_segment* new_seg = new_heap_segment;
29862 return consing_gen;
29864 //copy the card and brick tables
29865 if (g_gc_card_table!= card_table)
29866 copy_brick_card_table();
29868 BOOL new_segment_p = (heap_segment_next (new_seg) == 0);
29869 dprintf (2, ("new_segment_p %Ix", (size_t)new_segment_p));
29871 assert (generation_plan_allocation_start (generation_of (max_generation-1)));
29872 assert (generation_plan_allocation_start (generation_of (max_generation-1)) >=
29873 heap_segment_mem (ephemeral_heap_segment));
29874 assert (generation_plan_allocation_start (generation_of (max_generation-1)) <=
29875 heap_segment_committed (ephemeral_heap_segment));
29877 assert (generation_plan_allocation_start (youngest_generation));
29878 assert (generation_plan_allocation_start (youngest_generation) <
29879 heap_segment_plan_allocated (ephemeral_heap_segment));
29881 if (settings.pause_mode == pause_no_gc)
29883 // We don't reuse for no gc, so the size used on the new eph seg is eph_size.
29884 if ((size_t)(heap_segment_reserved (new_seg) - heap_segment_mem (new_seg)) < (eph_size + soh_allocation_no_gc))
29885 should_promote_ephemeral = TRUE;
29891 should_promote_ephemeral = dt_low_ephemeral_space_p (tuning_deciding_promote_ephemeral);
29895 if (should_promote_ephemeral)
29897 ephemeral_promotion = TRUE;
29898 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, expand_new_seg_ep);
29899 dprintf (2, ("promoting ephemeral"));
29900 save_ephemeral_generation_starts();
29904 // commit the new ephemeral segment all at once if it is a new one.
29905 if ((eph_size > 0) && new_segment_p)
29907 #ifdef FEATURE_STRUCTALIGN
29908 // The destination may require a larger alignment padding than the source.
29909 // Assume the worst possible alignment padding.
29910 eph_size += ComputeStructAlignPad(heap_segment_mem (new_seg), MAX_STRUCTALIGN, OBJECT_ALIGNMENT_OFFSET);
29911 #endif // FEATURE_STRUCTALIGN
29912 #ifdef RESPECT_LARGE_ALIGNMENT
29913 //Since the generation start can be larger than min_obj_size
29914 //The alignment could be switched.
29915 eph_size += switch_alignment_size(FALSE);
29916 #endif //RESPECT_LARGE_ALIGNMENT
29917 //Since the generation start can be larger than min_obj_size
29918 //Compare the alignment of the first object in gen1
29919 if (grow_heap_segment (new_seg, heap_segment_mem (new_seg) + eph_size) == 0)
29921 fgm_result.set_fgm (fgm_commit_eph_segment, eph_size, FALSE);
29922 return consing_gen;
29924 heap_segment_used (new_seg) = heap_segment_committed (new_seg);
29927 //Fix the end of the old ephemeral heap segment
29928 heap_segment_plan_allocated (ephemeral_heap_segment) =
29929 generation_plan_allocation_start (generation_of (max_generation-1));
29931 dprintf (3, ("Old ephemeral allocated set to %Ix",
29932 (size_t)heap_segment_plan_allocated (ephemeral_heap_segment)));
29937 // TODO - Is this really necessary? We should think about it.
29938 //initialize the first brick
29939 size_t first_brick = brick_of (heap_segment_mem (new_seg));
29940 set_brick (first_brick,
29941 heap_segment_mem (new_seg) - brick_address (first_brick));
29944 //From this point on, we cannot run out of memory
29946 //reset the allocation of the consing generation back to the end of the
29947 //old ephemeral segment
29948 generation_allocation_limit (consing_gen) =
29949 heap_segment_plan_allocated (ephemeral_heap_segment);
29950 generation_allocation_pointer (consing_gen) = generation_allocation_limit (consing_gen);
29951 generation_allocation_segment (consing_gen) = ephemeral_heap_segment;
29953 //clear the generation gap for all of the ephemeral generations
29955 int generation_num = max_generation-1;
29956 while (generation_num >= 0)
29958 generation* gen = generation_of (generation_num);
29959 generation_plan_allocation_start (gen) = 0;
29964 heap_segment* old_seg = ephemeral_heap_segment;
29965 ephemeral_heap_segment = new_seg;
29967 //Note: the ephemeral segment shouldn't be threaded onto the segment chain
29968 //because the relocation and compact phases shouldn't see it
29970 // set the generation members used by allocate_in_expanded_heap
29971 // and switch to ephemeral generation
29972 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
29974 if (!should_promote_ephemeral)
29976 realloc_plugs (consing_gen, old_seg, start_address, end_address,
29977 active_new_gen_number);
29982 repair_allocation_in_expanded_heap (consing_gen);
29985 // assert that the generation gap for all of the ephemeral generations were allocated.
29988 int generation_num = max_generation-1;
29989 while (generation_num >= 0)
29991 generation* gen = generation_of (generation_num);
29992 assert (generation_plan_allocation_start (gen));
29998 if (!new_segment_p)
30000 dprintf (2, ("Demoting ephemeral segment"));
30001 //demote the entire segment.
30002 settings.demotion = TRUE;
30003 get_gc_data_per_heap()->set_mechanism_bit (gc_demotion_bit);
30004 demotion_low = heap_segment_mem (ephemeral_heap_segment);
30005 demotion_high = heap_segment_reserved (ephemeral_heap_segment);
30009 demotion_low = MAX_PTR;
30011 #ifndef MULTIPLE_HEAPS
30012 settings.demotion = FALSE;
30013 get_gc_data_per_heap()->clear_mechanism_bit (gc_demotion_bit);
30014 #endif //!MULTIPLE_HEAPS
30016 ptrdiff_t eph_size1 = total_ephemeral_size;
30017 MAYBE_UNUSED_VAR(eph_size1);
30019 if (!should_promote_ephemeral && new_segment_p)
30021 assert (eph_size1 <= eph_size);
30024 if (heap_segment_mem (old_seg) == heap_segment_plan_allocated (old_seg))
30026 // This is to catch when we accidently delete a segment that has pins.
30027 verify_no_pins (heap_segment_mem (old_seg), heap_segment_reserved (old_seg));
30030 verify_no_pins (heap_segment_plan_allocated (old_seg), heap_segment_reserved(old_seg));
30032 dprintf(2,("---- End of Heap Expansion ----"));
30033 return consing_gen;
30036 void gc_heap::set_static_data()
30038 static_data* pause_mode_sdata = static_data_table[latency_level];
30039 for (int i = 0; i < NUMBERGENERATIONS; i++)
30041 dynamic_data* dd = dynamic_data_of (i);
30042 static_data* sdata = &pause_mode_sdata[i];
30045 dd->min_size = sdata->min_size;
30047 dprintf (GTC_LOG, ("PM: %d, gen%d: min: %Id, max: %Id, fr_l: %Id, fr_b: %d%%",
30048 settings.pause_mode,i,
30049 dd->min_size, dd_max_size (dd),
30050 sdata->fragmentation_limit, (int)(sdata->fragmentation_burden_limit * 100)));
30054 // Initialize the values that are not const.
30055 void gc_heap::init_static_data()
30057 size_t gen0_min_size = get_gen0_min_size();
30059 size_t gen0_max_size =
30060 #ifdef MULTIPLE_HEAPS
30061 max (6*1024*1024, min ( Align(soh_segment_size/2), 200*1024*1024));
30062 #else //MULTIPLE_HEAPS
30063 (gc_can_use_concurrent ?
30065 max (6*1024*1024, min ( Align(soh_segment_size/2), 200*1024*1024)));
30066 #endif //MULTIPLE_HEAPS
30068 if (heap_hard_limit)
30070 size_t gen0_max_size_seg = soh_segment_size / 4;
30071 dprintf (GTC_LOG, ("limit gen0 max %Id->%Id", gen0_max_size, gen0_max_size_seg));
30072 gen0_max_size = min (gen0_max_size, gen0_max_size_seg);
30075 size_t gen0_max_size_config = (size_t)GCConfig::GetGCGen0MaxBudget();
30077 if (gen0_max_size_config)
30079 gen0_max_size = min (gen0_max_size, gen0_max_size_config);
30082 gen0_max_size = Align (gen0_max_size);
30084 gen0_min_size = min (gen0_min_size, gen0_max_size);
30086 // TODO: gen0_max_size has a 200mb cap; gen1_max_size should also have a cap.
30087 size_t gen1_max_size = (size_t)
30088 #ifdef MULTIPLE_HEAPS
30089 max (6*1024*1024, Align(soh_segment_size/2));
30090 #else //MULTIPLE_HEAPS
30091 (gc_can_use_concurrent ?
30093 max (6*1024*1024, Align(soh_segment_size/2)));
30094 #endif //MULTIPLE_HEAPS
30096 dprintf (GTC_LOG, ("gen0 min: %Id, max: %Id, gen1 max: %Id",
30097 gen0_min_size, gen0_max_size, gen1_max_size));
30099 for (int i = latency_level_first; i <= latency_level_last; i++)
30101 static_data_table[i][0].min_size = gen0_min_size;
30102 static_data_table[i][0].max_size = gen0_max_size;
30103 static_data_table[i][1].max_size = gen1_max_size;
30107 bool gc_heap::init_dynamic_data()
30109 qpf = GCToOSInterface::QueryPerformanceFrequency();
30111 uint32_t now = (uint32_t)GetHighPrecisionTimeStamp();
30115 for (int i = 0; i <= max_generation+1; i++)
30117 dynamic_data* dd = dynamic_data_of (i);
30119 dd->time_clock = now;
30120 dd->current_size = 0;
30121 dd->promoted_size = 0;
30122 dd->collection_count = 0;
30123 dd->new_allocation = dd->min_size;
30124 dd->gc_new_allocation = dd->new_allocation;
30125 dd->desired_allocation = dd->new_allocation;
30126 dd->fragmentation = 0;
30129 #ifdef GC_CONFIG_DRIVEN
30130 if (heap_number == 0)
30132 #endif //GC_CONFIG_DRIVEN
30137 float gc_heap::surv_to_growth (float cst, float limit, float max_limit)
30139 if (cst < ((max_limit - limit ) / (limit * (max_limit-1.0f))))
30140 return ((limit - limit*cst) / (1.0f - (cst * limit)));
30146 //if the allocation budget wasn't exhausted, the new budget may be wrong because the survival may
30147 //not be correct (collection happened too soon). Correct with a linear estimation based on the previous
30148 //value of the budget
30149 static size_t linear_allocation_model (float allocation_fraction, size_t new_allocation,
30150 size_t previous_desired_allocation, size_t collection_count)
30152 if ((allocation_fraction < 0.95) && (allocation_fraction > 0.0))
30154 dprintf (2, ("allocation fraction: %d", (int)(allocation_fraction/100.0)));
30155 new_allocation = (size_t)(allocation_fraction*new_allocation + (1.0-allocation_fraction)*previous_desired_allocation);
30158 size_t smoothing = 3; // exponential smoothing factor
30159 if (smoothing > collection_count)
30160 smoothing = collection_count;
30161 new_allocation = new_allocation / smoothing + ((previous_desired_allocation / smoothing) * (smoothing-1));
30163 UNREFERENCED_PARAMETER(collection_count);
30165 return new_allocation;
30168 size_t gc_heap::desired_new_allocation (dynamic_data* dd,
30169 size_t out, int gen_number,
30172 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
30174 if (dd_begin_data_size (dd) == 0)
30176 size_t new_allocation = dd_min_size (dd);
30177 current_gc_data_per_heap->gen_data[gen_number].new_allocation = new_allocation;
30178 return new_allocation;
30183 size_t previous_desired_allocation = dd_desired_allocation (dd);
30184 size_t current_size = dd_current_size (dd);
30185 float max_limit = dd_max_limit (dd);
30186 float limit = dd_limit (dd);
30187 size_t min_gc_size = dd_min_size (dd);
30189 size_t max_size = dd_max_size (dd);
30190 size_t new_allocation = 0;
30191 float allocation_fraction = (float) (dd_desired_allocation (dd) - dd_gc_new_allocation (dd)) / (float) (dd_desired_allocation (dd));
30192 if (gen_number >= max_generation)
30194 size_t new_size = 0;
30196 cst = min (1.0f, float (out) / float (dd_begin_data_size (dd)));
30198 f = surv_to_growth (cst, limit, max_limit);
30199 size_t max_growth_size = (size_t)(max_size / f);
30200 if (current_size >= max_growth_size)
30202 new_size = max_size;
30206 new_size = (size_t) min (max ( (f * current_size), min_gc_size), max_size);
30209 assert ((new_size >= current_size) || (new_size == max_size));
30211 if (gen_number == max_generation)
30213 new_allocation = max((new_size - current_size), min_gc_size);
30215 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
30216 dd_desired_allocation (dd), dd_collection_count (dd));
30218 if ((dd_fragmentation (dd) > ((size_t)((f-1)*current_size))))
30220 //reducing allocation in case of fragmentation
30221 size_t new_allocation1 = max (min_gc_size,
30223 (size_t)((float)new_allocation * current_size /
30224 ((float)current_size + 2*dd_fragmentation (dd))));
30225 dprintf (2, ("Reducing max_gen allocation due to fragmentation from %Id to %Id",
30226 new_allocation, new_allocation1));
30227 new_allocation = new_allocation1;
30230 else //large object heap
30232 uint32_t memory_load = 0;
30233 uint64_t available_physical = 0;
30234 get_memory_info (&memory_load, &available_physical);
30236 if (heap_hard_limit)
30238 size_t loh_allocated = 0;
30239 size_t loh_committed = committed_size (true, &loh_allocated);
30240 dprintf (1, ("GC#%Id h%d, GMI: LOH budget, LOH commit %Id (obj %Id, frag %Id), total commit: %Id (recorded: %Id)",
30241 (size_t)settings.gc_index, heap_number,
30242 loh_committed, loh_allocated,
30243 dd_fragmentation (dynamic_data_of (max_generation + 1)),
30244 get_total_committed_size(), (current_total_committed - current_total_committed_bookkeeping)));
30247 if (heap_number == 0)
30248 settings.exit_memory_load = memory_load;
30249 if (available_physical > 1024*1024)
30250 available_physical -= 1024*1024;
30252 uint64_t available_free = available_physical + (uint64_t)generation_free_list_space (generation_of (gen_number));
30253 if (available_free > (uint64_t)MAX_PTR)
30255 available_free = (uint64_t)MAX_PTR;
30258 //try to avoid OOM during large object allocation
30259 new_allocation = max (min(max((new_size - current_size), dd_desired_allocation (dynamic_data_of (max_generation))),
30260 (size_t)available_free),
30261 max ((current_size/4), min_gc_size));
30263 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
30264 dd_desired_allocation (dd), dd_collection_count (dd));
30270 size_t survivors = out;
30271 cst = float (survivors) / float (dd_begin_data_size (dd));
30272 f = surv_to_growth (cst, limit, max_limit);
30273 new_allocation = (size_t) min (max ((f * (survivors)), min_gc_size), max_size);
30275 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
30276 dd_desired_allocation (dd), dd_collection_count (dd));
30278 if (gen_number == 0)
30283 //printf ("%f, %Id\n", cst, new_allocation);
30284 size_t free_space = generation_free_list_space (generation_of (gen_number));
30285 // DTREVIEW - is min_gc_size really a good choice?
30286 // on 64-bit this will almost always be true.
30287 dprintf (GTC_LOG, ("frag: %Id, min: %Id", free_space, min_gc_size));
30288 if (free_space > min_gc_size)
30290 settings.gen0_reduction_count = 2;
30294 if (settings.gen0_reduction_count > 0)
30295 settings.gen0_reduction_count--;
30298 if (settings.gen0_reduction_count > 0)
30300 dprintf (2, ("Reducing new allocation based on fragmentation"));
30301 new_allocation = min (new_allocation,
30302 max (min_gc_size, (max_size/3)));
30307 size_t new_allocation_ret =
30308 Align (new_allocation, get_alignment_constant (!(gen_number == (max_generation+1))));
30309 int gen_data_index = gen_number;
30310 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_data_index]);
30311 gen_data->new_allocation = new_allocation_ret;
30313 dd_surv (dd) = cst;
30315 #ifdef SIMPLE_DPRINTF
30316 dprintf (1, ("h%d g%d surv: %Id current: %Id alloc: %Id (%d%%) f: %d%% new-size: %Id new-alloc: %Id",
30317 heap_number, gen_number, out, current_size, (dd_desired_allocation (dd) - dd_gc_new_allocation (dd)),
30318 (int)(cst*100), (int)(f*100), current_size + new_allocation, new_allocation));
30320 dprintf (1,("gen: %d in: %Id out: %Id ", gen_number, generation_allocation_size (generation_of (gen_number)), out));
30321 dprintf (1,("current: %Id alloc: %Id ", current_size, (dd_desired_allocation (dd) - dd_gc_new_allocation (dd))));
30322 dprintf (1,(" surv: %d%% f: %d%% new-size: %Id new-alloc: %Id",
30323 (int)(cst*100), (int)(f*100), current_size + new_allocation, new_allocation));
30324 #endif //SIMPLE_DPRINTF
30326 return new_allocation_ret;
30330 //returns the planned size of a generation (including free list element)
30331 size_t gc_heap::generation_plan_size (int gen_number)
30333 if (0 == gen_number)
30334 return max((heap_segment_plan_allocated (ephemeral_heap_segment) -
30335 generation_plan_allocation_start (generation_of (gen_number))),
30336 (int)Align (min_obj_size));
30339 generation* gen = generation_of (gen_number);
30340 if (heap_segment_rw (generation_start_segment (gen)) == ephemeral_heap_segment)
30341 return (generation_plan_allocation_start (generation_of (gen_number - 1)) -
30342 generation_plan_allocation_start (generation_of (gen_number)));
30345 size_t gensize = 0;
30346 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
30348 PREFIX_ASSUME(seg != NULL);
30350 while (seg && (seg != ephemeral_heap_segment))
30352 gensize += heap_segment_plan_allocated (seg) -
30353 heap_segment_mem (seg);
30354 seg = heap_segment_next_rw (seg);
30358 gensize += (generation_plan_allocation_start (generation_of (gen_number - 1)) -
30359 heap_segment_mem (ephemeral_heap_segment));
30367 //returns the size of a generation (including free list element)
30368 size_t gc_heap::generation_size (int gen_number)
30370 if (0 == gen_number)
30371 return max((heap_segment_allocated (ephemeral_heap_segment) -
30372 generation_allocation_start (generation_of (gen_number))),
30373 (int)Align (min_obj_size));
30376 generation* gen = generation_of (gen_number);
30377 if (heap_segment_rw (generation_start_segment (gen)) == ephemeral_heap_segment)
30378 return (generation_allocation_start (generation_of (gen_number - 1)) -
30379 generation_allocation_start (generation_of (gen_number)));
30382 size_t gensize = 0;
30383 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
30385 PREFIX_ASSUME(seg != NULL);
30387 while (seg && (seg != ephemeral_heap_segment))
30389 gensize += heap_segment_allocated (seg) -
30390 heap_segment_mem (seg);
30391 seg = heap_segment_next_rw (seg);
30395 gensize += (generation_allocation_start (generation_of (gen_number - 1)) -
30396 heap_segment_mem (ephemeral_heap_segment));
30405 size_t gc_heap::compute_in (int gen_number)
30407 assert (gen_number != 0);
30408 dynamic_data* dd = dynamic_data_of (gen_number);
30410 size_t in = generation_allocation_size (generation_of (gen_number));
30412 if (gen_number == max_generation && ephemeral_promotion)
30415 for (int i = 0; i <= max_generation; i++)
30417 dynamic_data* dd = dynamic_data_of (i);
30418 in += dd_survived_size (dd);
30419 if (i != max_generation)
30421 generation_condemned_allocated (generation_of (gen_number)) += dd_survived_size (dd);
30426 dd_gc_new_allocation (dd) -= in;
30427 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
30429 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
30430 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_number]);
30433 generation_allocation_size (generation_of (gen_number)) = 0;
30437 void gc_heap::compute_promoted_allocation (int gen_number)
30439 compute_in (gen_number);
30444 size_t gc_heap::trim_youngest_desired (uint32_t memory_load,
30445 size_t total_new_allocation,
30446 size_t total_min_allocation)
30448 if (memory_load < MAX_ALLOWED_MEM_LOAD)
30450 // If the total of memory load and gen0 budget exceeds
30451 // our max memory load limit, trim the gen0 budget so the total
30452 // is the max memory load limit.
30453 size_t remain_memory_load = (MAX_ALLOWED_MEM_LOAD - memory_load) * mem_one_percent;
30454 return min (total_new_allocation, remain_memory_load);
30458 return max (mem_one_percent, total_min_allocation);
30462 size_t gc_heap::joined_youngest_desired (size_t new_allocation)
30464 dprintf (2, ("Entry memory load: %d; gen0 new_alloc: %Id", settings.entry_memory_load, new_allocation));
30466 size_t final_new_allocation = new_allocation;
30467 if (new_allocation > MIN_YOUNGEST_GEN_DESIRED)
30469 uint32_t num_heaps = 1;
30471 #ifdef MULTIPLE_HEAPS
30472 num_heaps = gc_heap::n_heaps;
30473 #endif //MULTIPLE_HEAPS
30475 size_t total_new_allocation = new_allocation * num_heaps;
30476 size_t total_min_allocation = MIN_YOUNGEST_GEN_DESIRED * num_heaps;
30478 if ((settings.entry_memory_load >= MAX_ALLOWED_MEM_LOAD) ||
30479 (total_new_allocation > max (youngest_gen_desired_th, total_min_allocation)))
30481 uint32_t memory_load = 0;
30482 get_memory_info (&memory_load);
30483 settings.exit_memory_load = memory_load;
30484 dprintf (2, ("Current emory load: %d", memory_load));
30486 size_t final_total =
30487 trim_youngest_desired (memory_load, total_new_allocation, total_min_allocation);
30488 size_t max_new_allocation =
30489 #ifdef MULTIPLE_HEAPS
30490 dd_max_size (g_heaps[0]->dynamic_data_of (0));
30491 #else //MULTIPLE_HEAPS
30492 dd_max_size (dynamic_data_of (0));
30493 #endif //MULTIPLE_HEAPS
30495 final_new_allocation = min (Align ((final_total / num_heaps), get_alignment_constant (TRUE)), max_new_allocation);
30499 if (final_new_allocation < new_allocation)
30501 settings.gen0_reduction_count = 2;
30504 return final_new_allocation;
30509 gc_history_per_heap* gc_heap::get_gc_data_per_heap()
30511 #ifdef BACKGROUND_GC
30512 return (settings.concurrent ? &bgc_data_per_heap : &gc_data_per_heap);
30514 return &gc_data_per_heap;
30515 #endif //BACKGROUND_GC
30518 void gc_heap::compute_new_dynamic_data (int gen_number)
30520 PREFIX_ASSUME(gen_number >= 0);
30521 PREFIX_ASSUME(gen_number <= max_generation);
30523 dynamic_data* dd = dynamic_data_of (gen_number);
30524 generation* gen = generation_of (gen_number);
30525 size_t in = (gen_number==0) ? 0 : compute_in (gen_number);
30527 size_t total_gen_size = generation_size (gen_number);
30528 //keep track of fragmentation
30529 dd_fragmentation (dd) = generation_free_list_space (gen) + generation_free_obj_space (gen);
30530 dd_current_size (dd) = total_gen_size - dd_fragmentation (dd);
30532 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
30534 size_t out = dd_survived_size (dd);
30536 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_number]);
30537 gen_data->size_after = total_gen_size;
30538 gen_data->free_list_space_after = generation_free_list_space (gen);
30539 gen_data->free_obj_space_after = generation_free_obj_space (gen);
30541 if ((settings.pause_mode == pause_low_latency) && (gen_number <= 1))
30543 // When we are in the low latency mode, we can still be
30544 // condemning more than gen1's 'cause of induced GCs.
30545 dd_desired_allocation (dd) = low_latency_alloc;
30549 if (gen_number == 0)
30551 //compensate for dead finalizable objects promotion.
30552 //they shoudn't be counted for growth.
30553 size_t final_promoted = 0;
30554 final_promoted = min (promoted_bytes (heap_number), out);
30555 // Prefast: this is clear from above but prefast needs to be told explicitly
30556 PREFIX_ASSUME(final_promoted <= out);
30558 dprintf (2, ("gen: %d final promoted: %Id", gen_number, final_promoted));
30559 dd_freach_previous_promotion (dd) = final_promoted;
30560 size_t lower_bound = desired_new_allocation (dd, out-final_promoted, gen_number, 0);
30562 if (settings.condemned_generation == 0)
30564 //there is no noise.
30565 dd_desired_allocation (dd) = lower_bound;
30569 size_t higher_bound = desired_new_allocation (dd, out, gen_number, 1);
30571 // <TODO>This assert was causing AppDomains\unload\test1n\test1nrun.bat to fail</TODO>
30572 //assert ( lower_bound <= higher_bound);
30574 //discount the noise. Change the desired allocation
30575 //only if the previous value is outside of the range.
30576 if (dd_desired_allocation (dd) < lower_bound)
30578 dd_desired_allocation (dd) = lower_bound;
30580 else if (dd_desired_allocation (dd) > higher_bound)
30582 dd_desired_allocation (dd) = higher_bound;
30584 #if defined (BIT64) && !defined (MULTIPLE_HEAPS)
30585 dd_desired_allocation (dd) = joined_youngest_desired (dd_desired_allocation (dd));
30586 #endif // BIT64 && !MULTIPLE_HEAPS
30587 trim_youngest_desired_low_memory();
30588 dprintf (2, ("final gen0 new_alloc: %Id", dd_desired_allocation (dd)));
30593 dd_desired_allocation (dd) = desired_new_allocation (dd, out, gen_number, 0);
30597 gen_data->pinned_surv = dd_pinned_survived_size (dd);
30598 gen_data->npinned_surv = dd_survived_size (dd) - dd_pinned_survived_size (dd);
30600 dd_gc_new_allocation (dd) = dd_desired_allocation (dd);
30601 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
30604 dd_promoted_size (dd) = out;
30605 if (gen_number == max_generation)
30607 dd = dynamic_data_of (max_generation+1);
30608 total_gen_size = generation_size (max_generation + 1);
30609 dd_fragmentation (dd) = generation_free_list_space (large_object_generation) +
30610 generation_free_obj_space (large_object_generation);
30611 dd_current_size (dd) = total_gen_size - dd_fragmentation (dd);
30612 dd_survived_size (dd) = dd_current_size (dd);
30614 out = dd_current_size (dd);
30615 dd_desired_allocation (dd) = desired_new_allocation (dd, out, max_generation+1, 0);
30616 dd_gc_new_allocation (dd) = Align (dd_desired_allocation (dd),
30617 get_alignment_constant (FALSE));
30618 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
30620 gen_data = &(current_gc_data_per_heap->gen_data[max_generation+1]);
30621 gen_data->size_after = total_gen_size;
30622 gen_data->free_list_space_after = generation_free_list_space (large_object_generation);
30623 gen_data->free_obj_space_after = generation_free_obj_space (large_object_generation);
30624 gen_data->npinned_surv = out;
30625 #ifdef BACKGROUND_GC
30626 end_loh_size = total_gen_size;
30627 #endif //BACKGROUND_GC
30629 dd_promoted_size (dd) = out;
30633 void gc_heap::trim_youngest_desired_low_memory()
30635 if (g_low_memory_status)
30637 size_t committed_mem = 0;
30638 heap_segment* seg = generation_start_segment (generation_of (max_generation));
30641 committed_mem += heap_segment_committed (seg) - heap_segment_mem (seg);
30642 seg = heap_segment_next (seg);
30644 seg = generation_start_segment (generation_of (max_generation + 1));
30647 committed_mem += heap_segment_committed (seg) - heap_segment_mem (seg);
30648 seg = heap_segment_next (seg);
30651 dynamic_data* dd = dynamic_data_of (0);
30652 size_t current = dd_desired_allocation (dd);
30653 size_t candidate = max (Align ((committed_mem / 10), get_alignment_constant(FALSE)), dd_min_size (dd));
30655 dd_desired_allocation (dd) = min (current, candidate);
30659 void gc_heap::decommit_ephemeral_segment_pages()
30661 if (settings.concurrent)
30666 size_t slack_space = heap_segment_committed (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment);
30668 dynamic_data* dd = dynamic_data_of (0);
30670 #ifndef MULTIPLE_HEAPS
30671 size_t extra_space = (g_low_memory_status ? 0 : (512 * 1024));
30672 size_t decommit_timeout = (g_low_memory_status ? 0 : GC_EPHEMERAL_DECOMMIT_TIMEOUT);
30673 size_t ephemeral_elapsed = dd_time_clock(dd) - gc_last_ephemeral_decommit_time;
30675 if (dd_desired_allocation (dd) > gc_gen0_desired_high)
30677 gc_gen0_desired_high = dd_desired_allocation (dd) + extra_space;
30680 if (ephemeral_elapsed >= decommit_timeout)
30682 slack_space = min (slack_space, gc_gen0_desired_high);
30684 gc_last_ephemeral_decommit_time = dd_time_clock(dd);
30685 gc_gen0_desired_high = 0;
30687 #endif //!MULTIPLE_HEAPS
30689 if (settings.condemned_generation >= (max_generation-1))
30691 size_t new_slack_space =
30693 max(min(min(soh_segment_size/32, dd_max_size(dd)), (generation_size (max_generation) / 10)), dd_desired_allocation(dd));
30695 #ifdef FEATURE_CORECLR
30696 dd_desired_allocation (dd);
30699 #endif //FEATURE_CORECLR
30702 slack_space = min (slack_space, new_slack_space);
30705 decommit_heap_segment_pages (ephemeral_heap_segment, slack_space);
30707 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
30708 current_gc_data_per_heap->extra_gen0_committed = heap_segment_committed (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment);
30711 //This is meant to be called by decide_on_compacting.
30713 size_t gc_heap::generation_fragmentation (generation* gen,
30714 generation* consing_gen,
30718 uint8_t* alloc = generation_allocation_pointer (consing_gen);
30719 // If the allocation pointer has reached the ephemeral segment
30720 // fine, otherwise the whole ephemeral segment is considered
30722 if (in_range_for_segment (alloc, ephemeral_heap_segment))
30724 if (alloc <= heap_segment_allocated(ephemeral_heap_segment))
30725 frag = end - alloc;
30728 // case when no survivors, allocated set to beginning
30731 dprintf (3, ("ephemeral frag: %Id", frag));
30734 frag = (heap_segment_allocated (ephemeral_heap_segment) -
30735 heap_segment_mem (ephemeral_heap_segment));
30736 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
30738 PREFIX_ASSUME(seg != NULL);
30740 while (seg != ephemeral_heap_segment)
30742 frag += (heap_segment_allocated (seg) -
30743 heap_segment_plan_allocated (seg));
30744 dprintf (3, ("seg: %Ix, frag: %Id", (size_t)seg,
30745 (heap_segment_allocated (seg) -
30746 heap_segment_plan_allocated (seg))));
30748 seg = heap_segment_next_rw (seg);
30751 dprintf (3, ("frag: %Id discounting pinned plugs", frag));
30752 //add the length of the dequeued plug free space
30754 while (bos < mark_stack_bos)
30756 frag += (pinned_len (pinned_plug_of (bos)));
30763 // for SOH this returns the total sizes of the generation and its
30764 // younger generation(s).
30765 // for LOH this returns just LOH size.
30766 size_t gc_heap::generation_sizes (generation* gen)
30769 if (generation_start_segment (gen ) == ephemeral_heap_segment)
30770 result = (heap_segment_allocated (ephemeral_heap_segment) -
30771 generation_allocation_start (gen));
30774 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
30776 PREFIX_ASSUME(seg != NULL);
30780 result += (heap_segment_allocated (seg) -
30781 heap_segment_mem (seg));
30782 seg = heap_segment_next_in_range (seg);
30789 size_t gc_heap::estimated_reclaim (int gen_number)
30791 dynamic_data* dd = dynamic_data_of (gen_number);
30792 size_t gen_allocated = (dd_desired_allocation (dd) - dd_new_allocation (dd));
30793 size_t gen_total_size = gen_allocated + dd_current_size (dd);
30794 size_t est_gen_surv = (size_t)((float) (gen_total_size) * dd_surv (dd));
30795 size_t est_gen_free = gen_total_size - est_gen_surv + dd_fragmentation (dd);
30797 dprintf (GTC_LOG, ("h%d gen%d total size: %Id, est dead space: %Id (s: %d, allocated: %Id), frag: %Id",
30798 heap_number, gen_number,
30801 (int)(dd_surv (dd) * 100),
30803 dd_fragmentation (dd)));
30805 return est_gen_free;
30808 BOOL gc_heap::decide_on_compacting (int condemned_gen_number,
30809 size_t fragmentation,
30810 BOOL& should_expand)
30812 BOOL should_compact = FALSE;
30813 should_expand = FALSE;
30814 generation* gen = generation_of (condemned_gen_number);
30815 dynamic_data* dd = dynamic_data_of (condemned_gen_number);
30816 size_t gen_sizes = generation_sizes(gen);
30817 float fragmentation_burden = ( ((0 == fragmentation) || (0 == gen_sizes)) ? (0.0f) :
30818 (float (fragmentation) / gen_sizes) );
30820 dprintf (GTC_LOG, ("h%d g%d fragmentation: %Id (%d%%)",
30821 heap_number, settings.condemned_generation,
30822 fragmentation, (int)(fragmentation_burden * 100.0)));
30825 // for pure GC stress runs we need compaction, for GC stress "mix"
30826 // we need to ensure a better mix of compacting and sweeping collections
30827 if (GCStress<cfg_any>::IsEnabled() && !settings.concurrent
30828 && !g_pConfig->IsGCStressMix())
30829 should_compact = TRUE;
30832 // in GC stress "mix" mode, for stress induced collections make sure we
30833 // keep sweeps and compactions relatively balanced. do not (yet) force sweeps
30834 // against the GC's determination, as it may lead to premature OOMs.
30835 if (g_pConfig->IsGCStressMix() && settings.stress_induced)
30837 int compactions = g_GCStatistics.cntCompactFGC+g_GCStatistics.cntCompactNGC;
30838 int sweeps = g_GCStatistics.cntFGC + g_GCStatistics.cntNGC - compactions;
30839 if (compactions < sweeps / 10)
30841 should_compact = TRUE;
30845 #endif //STRESS_HEAP
30847 if (GCConfig::GetForceCompact())
30848 should_compact = TRUE;
30850 if ((condemned_gen_number == max_generation) && last_gc_before_oom)
30852 should_compact = TRUE;
30853 last_gc_before_oom = FALSE;
30854 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_last_gc);
30857 if (settings.reason == reason_induced_compacting)
30859 dprintf (2, ("induced compacting GC"));
30860 should_compact = TRUE;
30861 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_induced_compacting);
30864 if (settings.reason == reason_pm_full_gc)
30866 assert (condemned_gen_number == max_generation);
30867 if (heap_number == 0)
30869 dprintf (GTC_LOG, ("PM doing compacting full GC after a gen1"));
30871 should_compact = TRUE;
30874 dprintf (2, ("Fragmentation: %d Fragmentation burden %d%%",
30875 fragmentation, (int) (100*fragmentation_burden)));
30877 if (provisional_mode_triggered && (condemned_gen_number == (max_generation - 1)))
30879 dprintf (GTC_LOG, ("gen1 in PM always compact"));
30880 should_compact = TRUE;
30883 if (!should_compact)
30885 if (dt_low_ephemeral_space_p (tuning_deciding_compaction))
30887 dprintf(GTC_LOG, ("compacting due to low ephemeral"));
30888 should_compact = TRUE;
30889 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_low_ephemeral);
30893 if (should_compact)
30895 if ((condemned_gen_number >= (max_generation - 1)))
30897 if (dt_low_ephemeral_space_p (tuning_deciding_expansion))
30899 dprintf (GTC_LOG,("Not enough space for all ephemeral generations with compaction"));
30900 should_expand = TRUE;
30906 BOOL high_memory = FALSE;
30909 if (!should_compact)
30911 // We are not putting this in dt_high_frag_p because it's not exactly
30912 // high fragmentation - it's just enough planned fragmentation for us to
30913 // want to compact. Also the "fragmentation" we are talking about here
30914 // is different from anywhere else.
30915 BOOL frag_exceeded = ((fragmentation >= dd_fragmentation_limit (dd)) &&
30916 (fragmentation_burden >= dd_fragmentation_burden_limit (dd)));
30920 #ifdef BACKGROUND_GC
30921 // do not force compaction if this was a stress-induced GC
30922 IN_STRESS_HEAP(if (!settings.stress_induced))
30924 #endif // BACKGROUND_GC
30925 assert (settings.concurrent == FALSE);
30926 should_compact = TRUE;
30927 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_high_frag);
30928 #ifdef BACKGROUND_GC
30930 #endif // BACKGROUND_GC
30934 // check for high memory situation
30935 if(!should_compact)
30937 uint32_t num_heaps = 1;
30938 #ifdef MULTIPLE_HEAPS
30939 num_heaps = gc_heap::n_heaps;
30940 #endif // MULTIPLE_HEAPS
30942 ptrdiff_t reclaim_space = generation_size(max_generation) - generation_plan_size(max_generation);
30944 if((settings.entry_memory_load >= high_memory_load_th) && (settings.entry_memory_load < v_high_memory_load_th))
30946 if(reclaim_space > (int64_t)(min_high_fragmentation_threshold (entry_available_physical_mem, num_heaps)))
30948 dprintf(GTC_LOG,("compacting due to fragmentation in high memory"));
30949 should_compact = TRUE;
30950 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_high_mem_frag);
30952 high_memory = TRUE;
30954 else if(settings.entry_memory_load >= v_high_memory_load_th)
30956 if(reclaim_space > (ptrdiff_t)(min_reclaim_fragmentation_threshold (num_heaps)))
30958 dprintf(GTC_LOG,("compacting due to fragmentation in very high memory"));
30959 should_compact = TRUE;
30960 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_vhigh_mem_frag);
30962 high_memory = TRUE;
30968 // The purpose of calling ensure_gap_allocation here is to make sure
30969 // that we actually are able to commit the memory to allocate generation
30971 if ((should_compact == FALSE) &&
30972 (ensure_gap_allocation (condemned_gen_number) == FALSE))
30974 should_compact = TRUE;
30975 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_no_gaps);
30978 if (settings.condemned_generation == max_generation)
30980 //check the progress
30983 (high_memory && !should_compact) ||
30985 (generation_plan_allocation_start (generation_of (max_generation - 1)) >=
30986 generation_allocation_start (generation_of (max_generation - 1))))
30988 dprintf (1, ("gen1 start %Ix->%Ix, gen2 size %Id->%Id, lock elevation",
30989 generation_allocation_start (generation_of (max_generation - 1)),
30990 generation_plan_allocation_start (generation_of (max_generation - 1)),
30991 generation_size (max_generation),
30992 generation_plan_size (max_generation)));
30993 //no progress -> lock
30994 settings.should_lock_elevation = TRUE;
30998 if (settings.pause_mode == pause_no_gc)
31000 should_compact = TRUE;
31001 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_plan_allocated (ephemeral_heap_segment))
31002 < soh_allocation_no_gc)
31004 should_expand = TRUE;
31008 dprintf (2, ("will %s(%s)", (should_compact ? "compact" : "sweep"), (should_expand ? "ex" : "")));
31009 return should_compact;
31012 size_t align_lower_good_size_allocation (size_t size)
31014 return (size/64)*64;
31017 size_t gc_heap::approximate_new_allocation()
31019 dynamic_data* dd0 = dynamic_data_of (0);
31020 return max (2*dd_min_size (dd0), ((dd_desired_allocation (dd0)*2)/3));
31023 BOOL gc_heap::sufficient_space_end_seg (uint8_t* start, uint8_t* seg_end, size_t end_space_required, gc_tuning_point tp)
31025 BOOL can_fit = FALSE;
31026 size_t end_seg_space = (size_t)(seg_end - start);
31027 if (end_seg_space > end_space_required)
31029 // If hard limit is specified, and if we attributed all that's left in commit to the ephemeral seg
31030 // so we treat that as segment end, do we have enough space.
31031 if (heap_hard_limit)
31033 size_t left_in_commit = heap_hard_limit - current_total_committed;
31035 #ifdef MULTIPLE_HEAPS
31036 num_heaps = n_heaps;
31037 #endif //MULTIPLE_HEAPS
31038 left_in_commit /= num_heaps;
31039 if (left_in_commit > end_space_required)
31044 dprintf (2, ("h%d end seg %Id, but only %Id left in HARD LIMIT commit, required: %Id %s on eph (%d)",
31045 heap_number, end_seg_space,
31046 left_in_commit, end_space_required,
31047 (can_fit ? "ok" : "short"), (int)tp));
31056 // After we did a GC we expect to have at least this
31057 // much space at the end of the segment to satisfy
31058 // a reasonable amount of allocation requests.
31059 size_t gc_heap::end_space_after_gc()
31061 return max ((dd_min_size (dynamic_data_of (0))/2), (END_SPACE_AFTER_GC + Align (min_obj_size)));
31064 BOOL gc_heap::ephemeral_gen_fit_p (gc_tuning_point tp)
31066 uint8_t* start = 0;
31068 if ((tp == tuning_deciding_condemned_gen) ||
31069 (tp == tuning_deciding_compaction))
31071 start = (settings.concurrent ? alloc_allocated : heap_segment_allocated (ephemeral_heap_segment));
31072 if (settings.concurrent)
31074 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment (alloc_allocated)",
31075 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated)));
31079 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment (allocated)",
31080 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment))));
31083 else if (tp == tuning_deciding_expansion)
31085 start = heap_segment_plan_allocated (ephemeral_heap_segment);
31086 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment based on plan",
31087 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - start)));
31091 assert (tp == tuning_deciding_full_gc);
31092 dprintf (GTC_LOG, ("FGC: %Id left at the end of ephemeral segment (alloc_allocated)",
31093 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated)));
31094 start = alloc_allocated;
31097 if (start == 0) // empty ephemeral generations
31099 assert (tp == tuning_deciding_expansion);
31100 // if there are no survivors in the ephemeral segment,
31101 // this should be the beginning of ephemeral segment.
31102 start = generation_allocation_pointer (generation_of (max_generation));
31103 assert (start == heap_segment_mem (ephemeral_heap_segment));
31106 if (tp == tuning_deciding_expansion)
31108 assert (settings.condemned_generation >= (max_generation-1));
31109 size_t gen0size = approximate_new_allocation();
31110 size_t eph_size = gen0size;
31111 size_t gen_min_sizes = 0;
31113 for (int j = 1; j <= max_generation-1; j++)
31115 gen_min_sizes += 2*dd_min_size (dynamic_data_of(j));
31118 eph_size += gen_min_sizes;
31120 dprintf (3, ("h%d deciding on expansion, need %Id (gen0: %Id, 2*min: %Id)",
31121 heap_number, gen0size, gen_min_sizes, eph_size));
31123 // We must find room for one large object and enough room for gen0size
31124 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - start) > eph_size)
31126 dprintf (3, ("Enough room before end of segment"));
31131 size_t room = align_lower_good_size_allocation
31132 (heap_segment_reserved (ephemeral_heap_segment) - start);
31133 size_t end_seg = room;
31135 //look at the plug free space
31136 size_t largest_alloc = END_SPACE_AFTER_GC + Align (min_obj_size);
31137 bool large_chunk_found = FALSE;
31139 uint8_t* gen0start = generation_plan_allocation_start (youngest_generation);
31140 dprintf (3, ("ephemeral_gen_fit_p: gen0 plan start: %Ix", (size_t)gen0start));
31141 if (gen0start == 0)
31143 dprintf (3, ("ephemeral_gen_fit_p: room before free list search %Id, needed: %Id",
31145 while ((bos < mark_stack_bos) &&
31146 !((room >= gen0size) && large_chunk_found))
31148 uint8_t* plug = pinned_plug (pinned_plug_of (bos));
31149 if (in_range_for_segment (plug, ephemeral_heap_segment))
31151 if (plug >= gen0start)
31153 size_t chunk = align_lower_good_size_allocation (pinned_len (pinned_plug_of (bos)));
31155 if (!large_chunk_found)
31157 large_chunk_found = (chunk >= largest_alloc);
31159 dprintf (3, ("ephemeral_gen_fit_p: room now %Id, large chunk: %Id",
31160 room, large_chunk_found));
31166 if (room >= gen0size)
31168 if (large_chunk_found)
31170 sufficient_gen0_space_p = TRUE;
31172 dprintf (3, ("Enough room"));
31177 // now we need to find largest_alloc at the end of the segment.
31178 if (end_seg >= end_space_after_gc())
31180 dprintf (3, ("Enough room (may need end of seg)"));
31186 dprintf (3, ("Not enough room"));
31192 size_t end_space = 0;
31193 dynamic_data* dd = dynamic_data_of (0);
31194 if ((tp == tuning_deciding_condemned_gen) ||
31195 (tp == tuning_deciding_full_gc))
31197 end_space = max (2*dd_min_size (dd), end_space_after_gc());
31201 assert (tp == tuning_deciding_compaction);
31202 end_space = approximate_new_allocation();
31205 BOOL can_fit = sufficient_space_end_seg (start, heap_segment_reserved (ephemeral_heap_segment), end_space, tp);
31211 CObjectHeader* gc_heap::allocate_large_object (size_t jsize, int64_t& alloc_bytes)
31213 //create a new alloc context because gen3context is shared.
31214 alloc_context acontext;
31215 acontext.alloc_ptr = 0;
31216 acontext.alloc_limit = 0;
31217 acontext.alloc_bytes = 0;
31218 #ifdef MULTIPLE_HEAPS
31219 acontext.set_alloc_heap(vm_heap);
31220 #endif //MULTIPLE_HEAPS
31223 size_t maxObjectSize = (INT64_MAX - 7 - Align(min_obj_size));
31225 size_t maxObjectSize = (INT32_MAX - 7 - Align(min_obj_size));
31228 if (jsize >= maxObjectSize)
31230 if (GCConfig::GetBreakOnOOM())
31232 GCToOSInterface::DebugBreak();
31237 size_t size = AlignQword (jsize);
31238 int align_const = get_alignment_constant (FALSE);
31239 #ifdef FEATURE_LOH_COMPACTION
31240 size_t pad = Align (loh_padding_obj_size, align_const);
31243 #endif //FEATURE_LOH_COMPACTION
31245 assert (size >= Align (min_obj_size, align_const));
31247 #pragma inline_depth(0)
31249 if (! allocate_more_space (&acontext, (size + pad), max_generation+1))
31255 #pragma inline_depth(20)
31259 uint8_t* current_lowest_address = lowest_address;
31260 uint8_t* current_highest_address = highest_address;
31261 #ifdef BACKGROUND_GC
31262 if (recursive_gc_sync::background_running_p())
31264 current_lowest_address = background_saved_lowest_address;
31265 current_highest_address = background_saved_highest_address;
31267 #endif //BACKGROUND_GC
31268 #endif // MARK_ARRAY
31270 #ifdef FEATURE_LOH_COMPACTION
31271 // The GC allocator made a free object already in this alloc context and
31272 // adjusted the alloc_ptr accordingly.
31273 #endif //FEATURE_LOH_COMPACTION
31275 uint8_t* result = acontext.alloc_ptr;
31277 assert ((size_t)(acontext.alloc_limit - acontext.alloc_ptr) == size);
31278 alloc_bytes += size;
31280 CObjectHeader* obj = (CObjectHeader*)result;
31283 if (recursive_gc_sync::background_running_p())
31285 if ((result < current_highest_address) && (result >= current_lowest_address))
31287 dprintf (3, ("Clearing mark bit at address %Ix",
31288 (size_t)(&mark_array [mark_word_of (result)])));
31290 mark_array_clear_marked (result);
31292 #ifdef BACKGROUND_GC
31293 //the object has to cover one full mark uint32_t
31294 assert (size > mark_word_size);
31295 if (current_c_gc_state != c_gc_state_free)
31297 dprintf (3, ("Concurrent allocation of a large object %Ix",
31299 //mark the new block specially so we know it is a new object
31300 if ((result < current_highest_address) && (result >= current_lowest_address))
31302 dprintf (3, ("Setting mark bit at address %Ix",
31303 (size_t)(&mark_array [mark_word_of (result)])));
31305 mark_array_set_marked (result);
31308 #endif //BACKGROUND_GC
31310 #endif //MARK_ARRAY
31313 assert ((size_t)obj == Align ((size_t)obj, align_const));
31318 void reset_memory (uint8_t* o, size_t sizeo)
31320 if (sizeo > 128 * 1024)
31322 // We cannot reset the memory for the useful part of a free object.
31323 size_t size_to_skip = min_free_list - plug_skew;
31325 size_t page_start = align_on_page ((size_t)(o + size_to_skip));
31326 size_t size = align_lower_page ((size_t)o + sizeo - size_to_skip - plug_skew) - page_start;
31327 // Note we need to compensate for an OS bug here. This bug would cause the MEM_RESET to fail
31328 // on write watched memory.
31331 #ifdef MULTIPLE_HEAPS
31332 bool unlock_p = true;
31334 // We don't do unlock because there could be many processes using workstation GC and it's
31335 // bad perf to have many threads doing unlock at the same time.
31336 bool unlock_p = false;
31337 #endif //MULTIPLE_HEAPS
31339 reset_mm_p = GCToOSInterface::VirtualReset((void*)page_start, size, unlock_p);
31344 void gc_heap::reset_large_object (uint8_t* o)
31346 // If it's a large object, allow the O/S to discard the backing store for these pages.
31347 reset_memory (o, size(o));
31350 BOOL gc_heap::large_object_marked (uint8_t* o, BOOL clearp)
31353 // It shouldn't be necessary to do these comparisons because this is only used for blocking
31354 // GCs and LOH segments cannot be out of range.
31355 if ((o >= lowest_address) && (o < highest_address))
31375 void gc_heap::walk_survivors_relocation (void* profiling_context, record_surv_fn fn)
31377 // Now walk the portion of memory that is actually being relocated.
31378 walk_relocation (profiling_context, fn);
31380 #ifdef FEATURE_LOH_COMPACTION
31381 if (loh_compacted_p)
31383 walk_relocation_for_loh (profiling_context, fn);
31385 #endif //FEATURE_LOH_COMPACTION
31388 void gc_heap::walk_survivors_for_loh (void* profiling_context, record_surv_fn fn)
31390 generation* gen = large_object_generation;
31391 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));;
31393 PREFIX_ASSUME(seg != NULL);
31395 uint8_t* o = generation_allocation_start (gen);
31396 uint8_t* plug_end = o;
31397 uint8_t* plug_start = o;
31401 if (o >= heap_segment_allocated (seg))
31403 seg = heap_segment_next (seg);
31407 o = heap_segment_mem (seg);
31409 if (large_object_marked(o, FALSE))
31416 o = o + AlignQword (size (o));
31417 if (o >= heap_segment_allocated (seg))
31421 m = large_object_marked (o, FALSE);
31426 fn (plug_start, plug_end, 0, profiling_context, false, false);
31430 while (o < heap_segment_allocated (seg) && !large_object_marked(o, FALSE))
31432 o = o + AlignQword (size (o));
31438 #ifdef BACKGROUND_GC
31440 BOOL gc_heap::background_object_marked (uint8_t* o, BOOL clearp)
31443 if ((o >= background_saved_lowest_address) && (o < background_saved_highest_address))
31445 if (mark_array_marked (o))
31449 mark_array_clear_marked (o);
31450 //dprintf (3, ("mark array bit for object %Ix is cleared", o));
31451 dprintf (3, ("CM: %Ix", o));
31461 dprintf (3, ("o %Ix(%d) %s", o, size(o), (m ? "was bm" : "was NOT bm")));
31465 void gc_heap::background_delay_delete_loh_segments()
31467 generation* gen = large_object_generation;
31468 heap_segment* seg = heap_segment_rw (generation_start_segment (large_object_generation));
31469 heap_segment* prev_seg = 0;
31473 heap_segment* next_seg = heap_segment_next (seg);
31474 if (seg->flags & heap_segment_flags_loh_delete)
31476 dprintf (3, ("deleting %Ix-%Ix-%Ix", (size_t)seg, heap_segment_allocated (seg), heap_segment_reserved (seg)));
31477 delete_heap_segment (seg, (GCConfig::GetRetainVM() != 0));
31478 heap_segment_next (prev_seg) = next_seg;
31489 uint8_t* gc_heap::background_next_end (heap_segment* seg, BOOL large_objects_p)
31492 (large_objects_p ? heap_segment_allocated (seg) : heap_segment_background_allocated (seg));
31495 void gc_heap::set_mem_verify (uint8_t* start, uint8_t* end, uint8_t b)
31500 if ((GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC) &&
31501 !(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_NO_MEM_FILL))
31503 dprintf (3, ("setting mem to %c [%Ix, [%Ix", b, start, end));
31504 memset (start, b, (end - start));
31507 #endif //VERIFY_HEAP
31510 void gc_heap::generation_delete_heap_segment (generation* gen,
31512 heap_segment* prev_seg,
31513 heap_segment* next_seg)
31515 dprintf (3, ("bgc sweep: deleting seg %Ix", seg));
31516 if (gen == large_object_generation)
31518 dprintf (3, ("Preparing empty large segment %Ix for deletion", (size_t)seg));
31520 // We cannot thread segs in here onto freeable_large_heap_segment because
31521 // grow_brick_card_tables could be committing mark array which needs to read
31522 // the seg list. So we delay it till next time we suspend EE.
31523 seg->flags |= heap_segment_flags_loh_delete;
31524 // Since we will be decommitting the seg, we need to prevent heap verification
31525 // to verify this segment.
31526 heap_segment_allocated (seg) = heap_segment_mem (seg);
31530 if (seg == ephemeral_heap_segment)
31535 heap_segment_next (next_seg) = prev_seg;
31537 dprintf (3, ("Preparing empty small segment %Ix for deletion", (size_t)seg));
31538 heap_segment_next (seg) = freeable_small_heap_segment;
31539 freeable_small_heap_segment = seg;
31542 decommit_heap_segment (seg);
31543 seg->flags |= heap_segment_flags_decommitted;
31545 set_mem_verify (heap_segment_allocated (seg) - plug_skew, heap_segment_used (seg), 0xbb);
31548 void gc_heap::process_background_segment_end (heap_segment* seg,
31550 uint8_t* last_plug_end,
31551 heap_segment* start_seg,
31555 uint8_t* allocated = heap_segment_allocated (seg);
31556 uint8_t* background_allocated = heap_segment_background_allocated (seg);
31557 BOOL loh_p = heap_segment_loh_p (seg);
31559 dprintf (3, ("Processing end of background segment [%Ix, %Ix[(%Ix[)",
31560 (size_t)heap_segment_mem (seg), background_allocated, allocated));
31562 if (!loh_p && (allocated != background_allocated))
31564 assert (gen != large_object_generation);
31566 dprintf (3, ("Make a free object before newly promoted objects [%Ix, %Ix[",
31567 (size_t)last_plug_end, background_allocated));
31568 thread_gap (last_plug_end, background_allocated - last_plug_end, generation_of (max_generation));
31571 fix_brick_to_highest (last_plug_end, background_allocated);
31573 // When we allowed fgc's during going through gaps, we could have erased the brick
31574 // that corresponds to bgc_allocated 'cause we had to update the brick there,
31575 // recover it here.
31576 fix_brick_to_highest (background_allocated, background_allocated);
31580 // by default, if allocated == background_allocated, it can't
31581 // be the ephemeral segment.
31582 if (seg == ephemeral_heap_segment)
31587 if (allocated == heap_segment_mem (seg))
31589 // this can happen with LOH segments when multiple threads
31590 // allocate new segments and not all of them were needed to
31591 // satisfy allocation requests.
31592 assert (gen == large_object_generation);
31595 if (last_plug_end == heap_segment_mem (seg))
31597 dprintf (3, ("Segment allocated is %Ix (beginning of this seg) - %s be deleted",
31598 (size_t)allocated, (*delete_p ? "should" : "should not")));
31600 if (seg != start_seg)
31607 dprintf (3, ("Trimming seg to %Ix[", (size_t)last_plug_end));
31608 heap_segment_allocated (seg) = last_plug_end;
31609 set_mem_verify (heap_segment_allocated (seg) - plug_skew, heap_segment_used (seg), 0xbb);
31611 decommit_heap_segment_pages (seg, 0);
31615 dprintf (3, ("verifying seg %Ix's mark array was completely cleared", seg));
31616 bgc_verify_mark_array_cleared (seg);
31619 void gc_heap::process_n_background_segments (heap_segment* seg,
31620 heap_segment* prev_seg,
31623 assert (gen != large_object_generation);
31627 dprintf (2, ("processing seg %Ix (not seen by bgc mark)", seg));
31628 heap_segment* next_seg = heap_segment_next (seg);
31630 if (heap_segment_read_only_p (seg))
31636 if (heap_segment_allocated (seg) == heap_segment_mem (seg))
31638 // This can happen - if we have a LOH segment where nothing survived
31639 // or a SOH segment allocated by a gen1 GC when BGC was going where
31640 // nothing survived last time we did a gen1 GC.
31641 generation_delete_heap_segment (gen, seg, prev_seg, next_seg);
31649 verify_soh_segment_list();
31655 BOOL gc_heap::fgc_should_consider_object (uint8_t* o,
31657 BOOL consider_bgc_mark_p,
31658 BOOL check_current_sweep_p,
31659 BOOL check_saved_sweep_p)
31661 // the logic for this function must be kept in sync with the analogous function
31662 // in ToolBox\SOS\Strike\gc.cpp
31664 // TRUE means we don't need to check the bgc mark bit
31665 // FALSE means we do.
31666 BOOL no_bgc_mark_p = FALSE;
31668 if (consider_bgc_mark_p)
31670 if (check_current_sweep_p && (o < current_sweep_pos))
31672 dprintf (3, ("no bgc mark - o: %Ix < cs: %Ix", o, current_sweep_pos));
31673 no_bgc_mark_p = TRUE;
31676 if (!no_bgc_mark_p)
31678 if(check_saved_sweep_p && (o >= saved_sweep_ephemeral_start))
31680 dprintf (3, ("no bgc mark - o: %Ix >= ss: %Ix", o, saved_sweep_ephemeral_start));
31681 no_bgc_mark_p = TRUE;
31684 if (!check_saved_sweep_p)
31686 uint8_t* background_allocated = heap_segment_background_allocated (seg);
31687 // if this was the saved ephemeral segment, check_saved_sweep_p
31688 // would've been true.
31689 assert (heap_segment_background_allocated (seg) != saved_sweep_ephemeral_start);
31690 // background_allocated could be 0 for the new segments acquired during bgc
31691 // sweep and we still want no_bgc_mark_p to be true.
31692 if (o >= background_allocated)
31694 dprintf (3, ("no bgc mark - o: %Ix >= ba: %Ix", o, background_allocated));
31695 no_bgc_mark_p = TRUE;
31702 no_bgc_mark_p = TRUE;
31705 dprintf (3, ("bgc mark %Ix: %s (bm: %s)", o, (no_bgc_mark_p ? "no" : "yes"), (background_object_marked (o, FALSE) ? "yes" : "no")));
31706 return (no_bgc_mark_p ? TRUE : background_object_marked (o, FALSE));
31709 // consider_bgc_mark_p tells you if you need to care about the bgc mark bit at all
31710 // if it's TRUE, check_current_sweep_p tells you if you should consider the
31711 // current sweep position or not.
31712 void gc_heap::should_check_bgc_mark (heap_segment* seg,
31713 BOOL* consider_bgc_mark_p,
31714 BOOL* check_current_sweep_p,
31715 BOOL* check_saved_sweep_p)
31717 // the logic for this function must be kept in sync with the analogous function
31718 // in ToolBox\SOS\Strike\gc.cpp
31719 *consider_bgc_mark_p = FALSE;
31720 *check_current_sweep_p = FALSE;
31721 *check_saved_sweep_p = FALSE;
31723 if (current_c_gc_state == c_gc_state_planning)
31725 // We are doing the current_sweep_pos comparison here because we have yet to
31726 // turn on the swept flag for the segment but in_range_for_segment will return
31727 // FALSE if the address is the same as reserved.
31728 if ((seg->flags & heap_segment_flags_swept) || (current_sweep_pos == heap_segment_reserved (seg)))
31730 dprintf (3, ("seg %Ix is already swept by bgc", seg));
31734 *consider_bgc_mark_p = TRUE;
31736 dprintf (3, ("seg %Ix hasn't been swept by bgc", seg));
31738 if (seg == saved_sweep_ephemeral_seg)
31740 dprintf (3, ("seg %Ix is the saved ephemeral seg", seg));
31741 *check_saved_sweep_p = TRUE;
31744 if (in_range_for_segment (current_sweep_pos, seg))
31746 dprintf (3, ("current sweep pos is %Ix and within seg %Ix",
31747 current_sweep_pos, seg));
31748 *check_current_sweep_p = TRUE;
31754 void gc_heap::background_ephemeral_sweep()
31756 dprintf (3, ("bgc ephemeral sweep"));
31758 int align_const = get_alignment_constant (TRUE);
31760 saved_sweep_ephemeral_seg = ephemeral_heap_segment;
31761 saved_sweep_ephemeral_start = generation_allocation_start (generation_of (max_generation - 1));
31763 // Since we don't want to interfere with gen0 allocation while we are threading gen0 free list,
31764 // we thread onto a list first then publish it when we are done.
31765 allocator youngest_free_list;
31766 size_t youngest_free_list_space = 0;
31767 size_t youngest_free_obj_space = 0;
31769 youngest_free_list.clear();
31771 for (int i = 0; i <= (max_generation - 1); i++)
31773 generation* gen_to_reset = generation_of (i);
31774 assert (generation_free_list_space (gen_to_reset) == 0);
31775 // Can only assert free_list_space is 0, not free_obj_space as the allocator could have added
31776 // something there.
31779 for (int i = (max_generation - 1); i >= 0; i--)
31781 generation* current_gen = generation_of (i);
31782 uint8_t* o = generation_allocation_start (current_gen);
31783 //Skip the generation gap object
31784 o = o + Align(size (o), align_const);
31785 uint8_t* end = ((i > 0) ?
31786 generation_allocation_start (generation_of (i - 1)) :
31787 heap_segment_allocated (ephemeral_heap_segment));
31789 uint8_t* plug_end = o;
31790 uint8_t* plug_start = o;
31791 BOOL marked_p = FALSE;
31795 marked_p = background_object_marked (o, TRUE);
31799 size_t plug_size = plug_start - plug_end;
31803 thread_gap (plug_end, plug_size, current_gen);
31809 make_unused_array (plug_end, plug_size);
31810 if (plug_size >= min_free_list)
31812 youngest_free_list_space += plug_size;
31813 youngest_free_list.thread_item (plug_end, plug_size);
31817 youngest_free_obj_space += plug_size;
31822 fix_brick_to_highest (plug_end, plug_start);
31823 fix_brick_to_highest (plug_start, plug_start);
31828 o = o + Align (size (o), align_const);
31834 m = background_object_marked (o, TRUE);
31837 dprintf (3, ("bgs: plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
31841 while ((o < end) && !background_object_marked (o, FALSE))
31843 o = o + Align (size (o), align_const);
31848 if (plug_end != end)
31852 thread_gap (plug_end, end - plug_end, current_gen);
31853 fix_brick_to_highest (plug_end, end);
31857 heap_segment_allocated (ephemeral_heap_segment) = plug_end;
31858 // the following line is temporary.
31859 heap_segment_saved_bg_allocated (ephemeral_heap_segment) = plug_end;
31861 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
31863 make_unused_array (plug_end, (end - plug_end));
31865 #endif //VERIFY_HEAP
31869 dd_fragmentation (dynamic_data_of (i)) =
31870 generation_free_list_space (current_gen) + generation_free_obj_space (current_gen);
31873 generation* youngest_gen = generation_of (0);
31874 generation_free_list_space (youngest_gen) = youngest_free_list_space;
31875 generation_free_obj_space (youngest_gen) = youngest_free_obj_space;
31876 dd_fragmentation (dynamic_data_of (0)) = youngest_free_list_space + youngest_free_obj_space;
31877 generation_allocator (youngest_gen)->copy_with_no_repair (&youngest_free_list);
31880 void gc_heap::background_sweep()
31882 generation* gen = generation_of (max_generation);
31883 dynamic_data* dd = dynamic_data_of (max_generation);
31884 // For SOH segments we go backwards.
31885 heap_segment* start_seg = ephemeral_heap_segment;
31886 PREFIX_ASSUME(start_seg != NULL);
31887 heap_segment* fseg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
31888 heap_segment* seg = start_seg;
31889 uint8_t* o = heap_segment_mem (seg);
31891 heap_segment* prev_seg = heap_segment_next (seg);
31892 int align_const = get_alignment_constant (TRUE);
31895 assert (o == generation_allocation_start (generation_of (max_generation)));
31896 o = o + Align(size (o), align_const);
31899 uint8_t* plug_end = o;
31900 uint8_t* plug_start = o;
31901 next_sweep_obj = o;
31902 current_sweep_pos = o;
31904 //uint8_t* end = background_next_end (seg, (gen == large_object_generation));
31905 uint8_t* end = heap_segment_background_allocated (seg);
31906 BOOL delete_p = FALSE;
31908 //concurrent_print_time_delta ("finished with mark and start with sweep");
31909 concurrent_print_time_delta ("Sw");
31910 dprintf (2, ("---- (GC%d)Background Sweep Phase ----", VolatileLoad(&settings.gc_index)));
31912 //block concurrent allocation for large objects
31913 dprintf (3, ("lh state: planning"));
31914 if (gc_lh_block_event.IsValid())
31916 gc_lh_block_event.Reset();
31919 for (int i = 0; i <= (max_generation + 1); i++)
31921 generation* gen_to_reset = generation_of (i);
31922 generation_allocator (gen_to_reset)->clear();
31923 generation_free_list_space (gen_to_reset) = 0;
31924 generation_free_obj_space (gen_to_reset) = 0;
31925 generation_free_list_allocated (gen_to_reset) = 0;
31926 generation_end_seg_allocated (gen_to_reset) = 0;
31927 generation_condemned_allocated (gen_to_reset) = 0;
31928 //reset the allocation so foreground gc can allocate into older generation
31929 generation_allocation_pointer (gen_to_reset)= 0;
31930 generation_allocation_limit (gen_to_reset) = 0;
31931 generation_allocation_segment (gen_to_reset) = heap_segment_rw (generation_start_segment (gen_to_reset));
31934 FIRE_EVENT(BGC2ndNonConEnd);
31936 loh_alloc_thread_count = 0;
31937 current_bgc_state = bgc_sweep_soh;
31938 verify_soh_segment_list();
31940 #ifdef FEATURE_BASICFREEZE
31941 if ((generation_start_segment (gen) != ephemeral_heap_segment) &&
31942 ro_segments_in_range)
31944 sweep_ro_segments (generation_start_segment (gen));
31946 #endif // FEATURE_BASICFREEZE
31948 //TODO BACKGROUND_GC: can we move this to where we switch to the LOH?
31949 if (current_c_gc_state != c_gc_state_planning)
31951 current_c_gc_state = c_gc_state_planning;
31954 concurrent_print_time_delta ("Swe");
31956 heap_segment* loh_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation + 1)));
31957 PREFIX_ASSUME(loh_seg != NULL);
31960 loh_seg->flags &= ~heap_segment_flags_swept;
31961 heap_segment_background_allocated (loh_seg) = heap_segment_allocated (loh_seg);
31962 loh_seg = heap_segment_next_rw (loh_seg);
31965 #ifdef MULTIPLE_HEAPS
31966 bgc_t_join.join(this, gc_join_restart_ee);
31967 if (bgc_t_join.joined())
31968 #endif //MULTIPLE_HEAPS
31970 #ifdef MULTIPLE_HEAPS
31971 dprintf(2, ("Starting BGC threads for resuming EE"));
31972 bgc_t_join.restart();
31973 #endif //MULTIPLE_HEAPS
31976 if (heap_number == 0)
31981 FIRE_EVENT(BGC2ndConBegin);
31983 background_ephemeral_sweep();
31985 concurrent_print_time_delta ("Swe eph");
31987 #ifdef MULTIPLE_HEAPS
31988 bgc_t_join.join(this, gc_join_after_ephemeral_sweep);
31989 if (bgc_t_join.joined())
31990 #endif //MULTIPLE_HEAPS
31992 #ifdef FEATURE_EVENT_TRACE
31993 bgc_heap_walk_for_etw_p = GCEventStatus::IsEnabled(GCEventProvider_Default,
31994 GCEventKeyword_GCHeapSurvivalAndMovement,
31995 GCEventLevel_Information);
31996 #endif //FEATURE_EVENT_TRACE
31998 leave_spin_lock (&gc_lock);
32000 #ifdef MULTIPLE_HEAPS
32001 dprintf(2, ("Starting BGC threads for BGC sweeping"));
32002 bgc_t_join.restart();
32003 #endif //MULTIPLE_HEAPS
32006 disable_preemptive (true);
32008 dprintf (2, ("bgs: sweeping gen2 objects"));
32009 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
32010 (size_t)heap_segment_mem (seg),
32011 (size_t)heap_segment_allocated (seg),
32012 (size_t)heap_segment_background_allocated (seg)));
32014 int num_objs = 256;
32015 int current_num_objs = 0;
32016 heap_segment* next_seg = 0;
32022 if (gen == large_object_generation)
32024 next_seg = heap_segment_next (seg);
32028 next_seg = heap_segment_prev (fseg, seg);
32033 if (!heap_segment_read_only_p (seg))
32035 if (gen == large_object_generation)
32037 // we can treat all LOH segments as in the bgc domain
32038 // regardless of whether we saw in bgc mark or not
32039 // because we don't allow LOH allocations during bgc
32040 // sweep anyway - the LOH segments can't change.
32041 process_background_segment_end (seg, gen, plug_end,
32042 start_seg, &delete_p);
32046 assert (heap_segment_background_allocated (seg) != 0);
32047 process_background_segment_end (seg, gen, plug_end,
32048 start_seg, &delete_p);
32050 assert (next_seg || !delete_p);
32056 generation_delete_heap_segment (gen, seg, prev_seg, next_seg);
32061 dprintf (2, ("seg %Ix has been swept", seg));
32062 seg->flags |= heap_segment_flags_swept;
32065 verify_soh_segment_list();
32069 dprintf (GTC_LOG, ("seg: %Ix, next_seg: %Ix, prev_seg: %Ix", seg, next_seg, prev_seg));
32073 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
32075 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
32077 if (gen != large_object_generation)
32079 dprintf (2, ("bgs: sweeping gen3 objects"));
32080 concurrent_print_time_delta ("Swe SOH");
32081 FIRE_EVENT(BGC1stSweepEnd, 0);
32083 enter_spin_lock (&more_space_lock_loh);
32084 add_saved_spinlock_info (true, me_acquire, mt_bgc_loh_sweep);
32086 concurrent_print_time_delta ("Swe LOH took msl");
32088 // We wait till all allocating threads are completely done.
32089 int spin_count = yp_spin_count_unit;
32090 while (loh_alloc_thread_count)
32092 spin_and_switch (spin_count, (loh_alloc_thread_count == 0));
32095 current_bgc_state = bgc_sweep_loh;
32096 gen = generation_of (max_generation+1);
32097 start_seg = heap_segment_rw (generation_start_segment (gen));
32099 PREFIX_ASSUME(start_seg != NULL);
32103 o = generation_allocation_start (gen);
32104 assert (method_table (o) == g_gc_pFreeObjectMethodTable);
32105 align_const = get_alignment_constant (FALSE);
32106 o = o + Align(size (o), align_const);
32108 end = heap_segment_allocated (seg);
32109 dprintf (2, ("sweeping gen3 objects"));
32110 generation_free_obj_space (gen) = 0;
32111 generation_allocator (gen)->clear();
32112 generation_free_list_space (gen) = 0;
32114 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
32115 (size_t)heap_segment_mem (seg),
32116 (size_t)heap_segment_allocated (seg),
32117 (size_t)heap_segment_background_allocated (seg)));
32124 o = heap_segment_mem (seg);
32127 assert (gen != large_object_generation);
32128 assert (o == generation_allocation_start (generation_of (max_generation)));
32129 align_const = get_alignment_constant (TRUE);
32130 o = o + Align(size (o), align_const);
32134 current_sweep_pos = o;
32135 next_sweep_obj = o;
32138 end = background_next_end (seg, (gen == large_object_generation));
32139 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
32140 (size_t)heap_segment_mem (seg),
32141 (size_t)heap_segment_allocated (seg),
32142 (size_t)heap_segment_background_allocated (seg)));
32146 if ((o < end) && background_object_marked (o, TRUE))
32149 if (gen == large_object_generation)
32151 dprintf (2, ("loh fr: [%Ix-%Ix[(%Id)", plug_end, plug_start, plug_start-plug_end));
32154 thread_gap (plug_end, plug_start-plug_end, gen);
32155 if (gen != large_object_generation)
32157 add_gen_free (max_generation, plug_start-plug_end);
32158 fix_brick_to_highest (plug_end, plug_start);
32159 // we need to fix the brick for the next plug here 'cause an FGC can
32160 // happen and can't read a stale brick.
32161 fix_brick_to_highest (plug_start, plug_start);
32168 next_sweep_obj = o + Align(size (o), align_const);
32169 current_num_objs++;
32170 if (current_num_objs >= num_objs)
32172 current_sweep_pos = next_sweep_obj;
32175 current_num_objs = 0;
32178 o = next_sweep_obj;
32184 m = background_object_marked (o, TRUE);
32187 if (gen != large_object_generation)
32189 add_gen_plug (max_generation, plug_end-plug_start);
32190 dd_survived_size (dd) += (plug_end - plug_start);
32192 dprintf (3, ("bgs: plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
32196 while ((o < end) && !background_object_marked (o, FALSE))
32198 next_sweep_obj = o + Align(size (o), align_const);;
32199 current_num_objs++;
32200 if (current_num_objs >= num_objs)
32202 current_sweep_pos = plug_end;
32203 dprintf (1234, ("f: swept till %Ix", current_sweep_pos));
32205 current_num_objs = 0;
32208 o = next_sweep_obj;
32213 size_t total_loh_size = generation_size (max_generation + 1);
32214 size_t total_soh_size = generation_sizes (generation_of (max_generation));
32216 dprintf (GTC_LOG, ("h%d: S: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
32218 dprintf (GTC_LOG, ("end of bgc sweep: gen2 FL: %Id, FO: %Id",
32219 generation_free_list_space (generation_of (max_generation)),
32220 generation_free_obj_space (generation_of (max_generation))));
32221 dprintf (GTC_LOG, ("h%d: end of bgc sweep: gen3 FL: %Id, FO: %Id",
32223 generation_free_list_space (generation_of (max_generation + 1)),
32224 generation_free_obj_space (generation_of (max_generation + 1))));
32226 FIRE_EVENT(BGC2ndConEnd);
32227 concurrent_print_time_delta ("background sweep");
32229 heap_segment* reset_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
32230 PREFIX_ASSUME(reset_seg != NULL);
32234 heap_segment_saved_bg_allocated (reset_seg) = heap_segment_background_allocated (reset_seg);
32235 heap_segment_background_allocated (reset_seg) = 0;
32236 reset_seg = heap_segment_next_rw (reset_seg);
32239 generation* loh_gen = generation_of (max_generation + 1);
32240 generation_allocation_segment (loh_gen) = heap_segment_rw (generation_start_segment (loh_gen));
32242 // We calculate dynamic data here because if we wait till we signal the lh event,
32243 // the allocation thread can change the fragmentation and we may read an intermediate
32244 // value (which can be greater than the generation size). Plus by that time it won't
32246 compute_new_dynamic_data (max_generation);
32248 enable_preemptive ();
32250 #ifdef MULTIPLE_HEAPS
32251 bgc_t_join.join(this, gc_join_set_state_free);
32252 if (bgc_t_join.joined())
32253 #endif //MULTIPLE_HEAPS
32255 // TODO: We are using this join just to set the state. Should
32256 // look into eliminating it - check to make sure things that use
32257 // this state can live with per heap state like should_check_bgc_mark.
32258 current_c_gc_state = c_gc_state_free;
32260 #ifdef MULTIPLE_HEAPS
32261 dprintf(2, ("Starting BGC threads after background sweep phase"));
32262 bgc_t_join.restart();
32263 #endif //MULTIPLE_HEAPS
32266 disable_preemptive (true);
32268 if (gc_lh_block_event.IsValid())
32270 gc_lh_block_event.Set();
32273 add_saved_spinlock_info (true, me_release, mt_bgc_loh_sweep);
32274 leave_spin_lock (&more_space_lock_loh);
32276 //dprintf (GTC_LOG, ("---- (GC%d)End Background Sweep Phase ----", VolatileLoad(&settings.gc_index)));
32277 dprintf (GTC_LOG, ("---- (GC%d)ESw ----", VolatileLoad(&settings.gc_index)));
32279 #endif //BACKGROUND_GC
32281 void gc_heap::sweep_large_objects ()
32283 //this min value is for the sake of the dynamic tuning.
32284 //so we know that we are not starting even if we have no
32286 generation* gen = large_object_generation;
32287 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
32289 PREFIX_ASSUME(start_seg != NULL);
32291 heap_segment* seg = start_seg;
32292 heap_segment* prev_seg = 0;
32293 uint8_t* o = generation_allocation_start (gen);
32294 int align_const = get_alignment_constant (FALSE);
32296 //Skip the generation gap object
32297 o = o + Align(size (o), align_const);
32299 uint8_t* plug_end = o;
32300 uint8_t* plug_start = o;
32302 generation_allocator (gen)->clear();
32303 generation_free_list_space (gen) = 0;
32304 generation_free_obj_space (gen) = 0;
32307 dprintf (3, ("sweeping large objects"));
32308 dprintf (3, ("seg: %Ix, [%Ix, %Ix[, starting from %Ix",
32310 (size_t)heap_segment_mem (seg),
32311 (size_t)heap_segment_allocated (seg),
32316 if (o >= heap_segment_allocated (seg))
32318 heap_segment* next_seg = heap_segment_next (seg);
32319 //delete the empty segment if not the only one
32320 if ((plug_end == heap_segment_mem (seg)) &&
32321 (seg != start_seg) && !heap_segment_read_only_p (seg))
32323 //prepare for deletion
32324 dprintf (3, ("Preparing empty large segment %Ix", (size_t)seg));
32326 heap_segment_next (prev_seg) = next_seg;
32327 heap_segment_next (seg) = freeable_large_heap_segment;
32328 freeable_large_heap_segment = seg;
32332 if (!heap_segment_read_only_p (seg))
32334 dprintf (3, ("Trimming seg to %Ix[", (size_t)plug_end));
32335 heap_segment_allocated (seg) = plug_end;
32336 decommit_heap_segment_pages (seg, 0);
32345 o = heap_segment_mem (seg);
32347 dprintf (3, ("seg: %Ix, [%Ix, %Ix[", (size_t)seg,
32348 (size_t)heap_segment_mem (seg),
32349 (size_t)heap_segment_allocated (seg)));
32352 if (large_object_marked(o, TRUE))
32355 //everything between plug_end and plug_start is free
32356 thread_gap (plug_end, plug_start-plug_end, gen);
32361 o = o + AlignQword (size (o));
32362 if (o >= heap_segment_allocated (seg))
32366 m = large_object_marked (o, TRUE);
32369 dprintf (3, ("plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
32373 while (o < heap_segment_allocated (seg) && !large_object_marked(o, FALSE))
32375 o = o + AlignQword (size (o));
32380 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
32382 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
32385 void gc_heap::relocate_in_large_objects ()
32387 relocate_args args;
32389 args.high = gc_high;
32390 args.last_plug = 0;
32392 generation* gen = large_object_generation;
32394 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32396 PREFIX_ASSUME(seg != NULL);
32398 uint8_t* o = generation_allocation_start (gen);
32402 if (o >= heap_segment_allocated (seg))
32404 seg = heap_segment_next_rw (seg);
32409 o = heap_segment_mem (seg);
32412 while (o < heap_segment_allocated (seg))
32414 check_class_object_demotion (o);
32415 if (contain_pointers (o))
32417 dprintf(3, ("Relocating through large object %Ix", (size_t)o));
32418 go_through_object_nostart (method_table (o), o, size(o), pval,
32420 reloc_survivor_helper (pval);
32423 o = o + AlignQword (size (o));
32428 void gc_heap::mark_through_cards_for_large_objects (card_fn fn,
32431 uint8_t* low = gc_low;
32432 size_t end_card = 0;
32433 generation* oldest_gen = generation_of (max_generation+1);
32434 heap_segment* seg = heap_segment_rw (generation_start_segment (oldest_gen));
32436 PREFIX_ASSUME(seg != NULL);
32438 uint8_t* beg = generation_allocation_start (oldest_gen);
32439 uint8_t* end = heap_segment_allocated (seg);
32441 size_t cg_pointers_found = 0;
32443 size_t card_word_end = (card_of (align_on_card_word (end)) /
32448 size_t n_card_set = 0;
32449 uint8_t* next_boundary = (relocating ?
32450 generation_plan_allocation_start (generation_of (max_generation -1)) :
32453 uint8_t* nhigh = (relocating ?
32454 heap_segment_plan_allocated (ephemeral_heap_segment) :
32457 BOOL foundp = FALSE;
32458 uint8_t* start_address = 0;
32459 uint8_t* limit = 0;
32460 size_t card = card_of (beg);
32462 #ifdef BACKGROUND_GC
32463 BOOL consider_bgc_mark_p = FALSE;
32464 BOOL check_current_sweep_p = FALSE;
32465 BOOL check_saved_sweep_p = FALSE;
32466 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
32467 #endif //BACKGROUND_GC
32469 size_t total_cards_cleared = 0;
32471 //dprintf(3,( "scanning large objects from %Ix to %Ix", (size_t)beg, (size_t)end));
32472 dprintf(3, ("CMl: %Ix->%Ix", (size_t)beg, (size_t)end));
32475 if ((o < end) && (card_of(o) > card))
32477 dprintf (3, ("Found %Id cg pointers", cg_pointers_found));
32478 if (cg_pointers_found == 0)
32480 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card), (size_t)o));
32481 clear_cards (card, card_of((uint8_t*)o));
32482 total_cards_cleared += (card_of((uint8_t*)o) - card);
32484 n_eph +=cg_pointers_found;
32485 cg_pointers_found = 0;
32486 card = card_of ((uint8_t*)o);
32488 if ((o < end) &&(card >= end_card))
32490 foundp = find_card (card_table, card, card_word_end, end_card);
32493 n_card_set+= end_card - card;
32494 start_address = max (beg, card_address (card));
32496 limit = min (end, card_address (end_card));
32498 if ((!foundp) || (o >= end) || (card_address (card) >= end))
32500 if ((foundp) && (cg_pointers_found == 0))
32502 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card),
32503 (size_t)card_address(card+1)));
32504 clear_cards (card, card+1);
32505 total_cards_cleared += 1;
32507 n_eph +=cg_pointers_found;
32508 cg_pointers_found = 0;
32509 if ((seg = heap_segment_next_rw (seg)) != 0)
32511 #ifdef BACKGROUND_GC
32512 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
32513 #endif //BACKGROUND_GC
32514 beg = heap_segment_mem (seg);
32515 end = compute_next_end (seg, low);
32516 card_word_end = card_of (align_on_card_word (end)) / card_word_width;
32517 card = card_of (beg);
32528 assert (card_set_p (card));
32530 dprintf(3,("card %Ix: o: %Ix, l: %Ix[ ",
32531 card, (size_t)o, (size_t)limit));
32533 assert (Align (size (o)) >= Align (min_obj_size));
32534 size_t s = size (o);
32535 uint8_t* next_o = o + AlignQword (s);
32541 assert (Align (s) >= Align (min_obj_size));
32542 next_o = o + AlignQword (s);
32545 dprintf (4, ("|%Ix|", (size_t)o));
32546 if (next_o < start_address)
32551 #ifdef BACKGROUND_GC
32552 if (!fgc_should_consider_object (o, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p))
32556 #endif //BACKGROUND_GC
32558 #ifdef COLLECTIBLE_CLASS
32559 if (is_collectible(o))
32561 BOOL passed_end_card_p = FALSE;
32563 if (card_of (o) > card)
32565 passed_end_card_p = card_transition (o, end, card_word_end,
32569 foundp, start_address,
32570 limit, total_cards_cleared);
32573 if ((!passed_end_card_p || foundp) && (card_of (o) == card))
32575 // card is valid and it covers the head of the object
32576 if (fn == &gc_heap::relocate_address)
32578 keep_card_live (o, n_gen, cg_pointers_found);
32582 uint8_t* class_obj = get_class_object (o);
32583 mark_through_cards_helper (&class_obj, n_gen,
32584 cg_pointers_found, fn,
32585 nhigh, next_boundary);
32589 if (passed_end_card_p)
32591 if (foundp && (card_address (card) < next_o))
32593 goto go_through_refs;
32603 #endif //COLLECTIBLE_CLASS
32605 if (contain_pointers (o))
32607 dprintf(3,("Going through %Ix", (size_t)o));
32609 go_through_object (method_table(o), o, s, poo,
32610 start_address, use_start, (o + s),
32612 if (card_of ((uint8_t*)poo) > card)
32614 BOOL passed_end_card_p = card_transition ((uint8_t*)poo, end,
32619 foundp, start_address,
32620 limit, total_cards_cleared);
32622 if (passed_end_card_p)
32624 if (foundp && (card_address (card) < next_o))
32628 if (ppstop <= (uint8_t**)start_address)
32630 else if (poo < (uint8_t**)start_address)
32631 {poo = (uint8_t**)start_address;}
32641 mark_through_cards_helper (poo, n_gen,
32642 cg_pointers_found, fn,
32643 nhigh, next_boundary);
32655 // compute the efficiency ratio of the card table
32658 generation_skip_ratio = min (((n_eph > 800) ?
32659 (int)(((float)n_gen / (float)n_eph) * 100) : 100),
32660 generation_skip_ratio);
32662 dprintf (3, ("Mloh: cross: %Id, useful: %Id, cards cleared: %Id, cards set: %Id, ratio: %d",
32663 n_eph, n_gen, total_cards_cleared, n_card_set, generation_skip_ratio));
32667 dprintf (3, ("R: Mloh: cross: %Id, useful: %Id, cards set: %Id, ratio: %d",
32668 n_eph, n_gen, n_card_set, generation_skip_ratio));
32672 void gc_heap::descr_segment (heap_segment* seg )
32675 uint8_t* x = heap_segment_mem (seg);
32676 while (x < heap_segment_allocated (seg))
32678 dprintf(2, ( "%Ix: %d ", (size_t)x, size (x)));
32679 x = x + Align(size (x));
32682 UNREFERENCED_PARAMETER(seg);
32686 void gc_heap::descr_generations_to_profiler (gen_walk_fn fn, void *context)
32688 #ifdef MULTIPLE_HEAPS
32689 int n_heaps = g_theGCHeap->GetNumberOfHeaps ();
32690 for (int i = 0; i < n_heaps; i++)
32692 gc_heap* hp = GCHeap::GetHeap(i)->pGenGCHeap;
32693 #else //MULTIPLE_HEAPS
32695 gc_heap* hp = NULL;
32697 // prefix complains about us dereferencing hp in wks build even though we only access static members
32698 // this way. not sure how to shut it up except for this ugly workaround:
32699 PREFIX_ASSUME(hp != NULL);
32700 #endif // _PREFAST_
32701 #endif //MULTIPLE_HEAPS
32703 int curr_gen_number0 = max_generation+1;
32704 while (curr_gen_number0 >= 0)
32706 generation* gen = hp->generation_of (curr_gen_number0);
32707 heap_segment* seg = generation_start_segment (gen);
32708 while (seg && (seg != hp->ephemeral_heap_segment))
32710 assert (curr_gen_number0 > 0);
32712 // report bounds from heap_segment_mem (seg) to
32713 // heap_segment_allocated (seg);
32714 // for generation # curr_gen_number0
32715 // for heap # heap_no
32717 fn(context, curr_gen_number0, heap_segment_mem (seg),
32718 heap_segment_allocated (seg),
32719 curr_gen_number0 == max_generation+1 ? heap_segment_reserved (seg) : heap_segment_allocated (seg));
32721 seg = heap_segment_next (seg);
32725 assert (seg == hp->ephemeral_heap_segment);
32726 assert (curr_gen_number0 <= max_generation);
32728 if (curr_gen_number0 == max_generation)
32730 if (heap_segment_mem (seg) < generation_allocation_start (hp->generation_of (max_generation-1)))
32732 // report bounds from heap_segment_mem (seg) to
32733 // generation_allocation_start (generation_of (max_generation-1))
32734 // for heap # heap_number
32736 fn(context, curr_gen_number0, heap_segment_mem (seg),
32737 generation_allocation_start (hp->generation_of (max_generation-1)),
32738 generation_allocation_start (hp->generation_of (max_generation-1)) );
32741 else if (curr_gen_number0 != 0)
32743 //report bounds from generation_allocation_start (generation_of (curr_gen_number0))
32744 // to generation_allocation_start (generation_of (curr_gen_number0-1))
32745 // for heap # heap_number
32747 fn(context, curr_gen_number0, generation_allocation_start (hp->generation_of (curr_gen_number0)),
32748 generation_allocation_start (hp->generation_of (curr_gen_number0-1)),
32749 generation_allocation_start (hp->generation_of (curr_gen_number0-1)));
32753 //report bounds from generation_allocation_start (generation_of (curr_gen_number0))
32754 // to heap_segment_allocated (ephemeral_heap_segment);
32755 // for heap # heap_number
32757 fn(context, curr_gen_number0, generation_allocation_start (hp->generation_of (curr_gen_number0)),
32758 heap_segment_allocated (hp->ephemeral_heap_segment),
32759 heap_segment_reserved (hp->ephemeral_heap_segment) );
32762 curr_gen_number0--;
32768 // Note that when logging is on it can take a long time to go through the free items.
32769 void gc_heap::print_free_list (int gen, heap_segment* seg)
32771 UNREFERENCED_PARAMETER(gen);
32772 UNREFERENCED_PARAMETER(seg);
32774 if (settings.concurrent == FALSE)
32776 uint8_t* seg_start = heap_segment_mem (seg);
32777 uint8_t* seg_end = heap_segment_allocated (seg);
32779 dprintf (3, ("Free list in seg %Ix:", seg_start));
32781 size_t total_free_item = 0;
32783 allocator* gen_allocator = generation_allocator (generation_of (gen));
32784 for (unsigned int b = 0; b < gen_allocator->number_of_buckets(); b++)
32786 uint8_t* fo = gen_allocator->alloc_list_head_of (b);
32789 if (fo >= seg_start && fo < seg_end)
32793 size_t free_item_len = size(fo);
32795 dprintf (3, ("[%Ix, %Ix[:%Id",
32797 (size_t)(fo + free_item_len),
32801 fo = free_list_slot (fo);
32805 dprintf (3, ("total %Id free items", total_free_item));
32811 void gc_heap::descr_generations (BOOL begin_gc_p)
32813 UNREFERENCED_PARAMETER(begin_gc_p);
32815 if (StressLog::StressLogOn(LF_GC, LL_INFO10))
32818 #ifdef MULTIPLE_HEAPS
32820 #endif //MULTIPLE_HEAPS
32822 STRESS_LOG1(LF_GC, LL_INFO10, "GC Heap %p\n", hp);
32823 for (int n = max_generation; n >= 0; --n)
32825 STRESS_LOG4(LF_GC, LL_INFO10, " Generation %d [%p, %p] cur = %p\n",
32827 generation_allocation_start(generation_of(n)),
32828 generation_allocation_limit(generation_of(n)),
32829 generation_allocation_pointer(generation_of(n)));
32831 heap_segment* seg = generation_start_segment(generation_of(n));
32834 STRESS_LOG4(LF_GC, LL_INFO10, " Segment mem %p alloc = %p used %p committed %p\n",
32835 heap_segment_mem(seg),
32836 heap_segment_allocated(seg),
32837 heap_segment_used(seg),
32838 heap_segment_committed(seg));
32839 seg = heap_segment_next(seg);
32843 #endif // STRESS_LOG
32846 dprintf (2, ("lowest_address: %Ix highest_address: %Ix",
32847 (size_t) lowest_address, (size_t) highest_address));
32848 #ifdef BACKGROUND_GC
32849 dprintf (2, ("bgc lowest_address: %Ix bgc highest_address: %Ix",
32850 (size_t) background_saved_lowest_address, (size_t) background_saved_highest_address));
32851 #endif //BACKGROUND_GC
32853 if (heap_number == 0)
32855 dprintf (1, ("total heap size: %Id, commit size: %Id", get_total_heap_size(), get_total_committed_size()));
32858 int curr_gen_number = max_generation+1;
32859 while (curr_gen_number >= 0)
32861 size_t total_gen_size = generation_size (curr_gen_number);
32862 #ifdef SIMPLE_DPRINTF
32863 dprintf (GTC_LOG, ("[%s][g%d]gen %d:, size: %Id, frag: %Id(L: %Id, O: %Id), f: %d%% %s %s %s",
32864 (begin_gc_p ? "BEG" : "END"),
32865 settings.condemned_generation,
32868 dd_fragmentation (dynamic_data_of (curr_gen_number)),
32869 generation_free_list_space (generation_of (curr_gen_number)),
32870 generation_free_obj_space (generation_of (curr_gen_number)),
32872 (int)(((double)dd_fragmentation (dynamic_data_of (curr_gen_number)) / (double)total_gen_size) * 100) :
32874 (begin_gc_p ? ("") : (settings.compaction ? "(compact)" : "(sweep)")),
32875 (settings.heap_expansion ? "(EX)" : " "),
32876 (settings.promotion ? "Promotion" : "NoPromotion")));
32878 dprintf (2, ( "Generation %d: gap size: %d, generation size: %Id, fragmentation: %Id",
32880 size (generation_allocation_start (generation_of (curr_gen_number))),
32882 dd_fragmentation (dynamic_data_of (curr_gen_number))));
32883 #endif //SIMPLE_DPRINTF
32885 generation* gen = generation_of (curr_gen_number);
32886 heap_segment* seg = generation_start_segment (gen);
32887 while (seg && (seg != ephemeral_heap_segment))
32889 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[-%Ix[ (%Id) (%Id)",
32891 (size_t)heap_segment_mem (seg),
32892 (size_t)heap_segment_allocated (seg),
32893 (size_t)heap_segment_committed (seg),
32894 (size_t)(heap_segment_allocated (seg) - heap_segment_mem (seg)),
32895 (size_t)(heap_segment_committed (seg) - heap_segment_allocated (seg))));
32896 print_free_list (curr_gen_number, seg);
32897 seg = heap_segment_next (seg);
32899 if (seg && (seg != generation_start_segment (gen)))
32901 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[",
32903 (size_t)heap_segment_mem (seg),
32904 (size_t)generation_allocation_start (generation_of (curr_gen_number-1))));
32905 print_free_list (curr_gen_number, seg);
32910 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[",
32912 (size_t)generation_allocation_start (generation_of (curr_gen_number)),
32913 (size_t)(((curr_gen_number == 0)) ?
32914 (heap_segment_allocated
32915 (generation_start_segment
32916 (generation_of (curr_gen_number)))) :
32917 (generation_allocation_start
32918 (generation_of (curr_gen_number - 1))))
32920 print_free_list (curr_gen_number, seg);
32932 //-----------------------------------------------------------------------------
32934 // VM Specific support
32936 //-----------------------------------------------------------------------------
32941 unsigned int PromotedObjectCount = 0;
32942 unsigned int CreatedObjectCount = 0;
32943 unsigned int AllocDuration = 0;
32944 unsigned int AllocCount = 0;
32945 unsigned int AllocBigCount = 0;
32946 unsigned int AllocSmallCount = 0;
32947 unsigned int AllocStart = 0;
32950 //Static member variables.
32951 VOLATILE(BOOL) GCHeap::GcInProgress = FALSE;
32953 //CMCSafeLock* GCHeap::fGcLock;
32954 GCEvent *GCHeap::WaitForGCEvent = NULL;
32957 unsigned int GCHeap::GcDuration;
32959 unsigned GCHeap::GcCondemnedGeneration = 0;
32960 size_t GCHeap::totalSurvivedSize = 0;
32961 #ifdef FEATURE_PREMORTEM_FINALIZATION
32962 CFinalize* GCHeap::m_Finalize = 0;
32963 BOOL GCHeap::GcCollectClasses = FALSE;
32964 VOLATILE(int32_t) GCHeap::m_GCFLock = 0;
32966 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
32968 #ifdef BACKGROUND_GC
32969 int GCHeap::gc_stress_fgcs_in_bgc = 0;
32970 #endif // BACKGROUND_GC
32971 #ifndef MULTIPLE_HEAPS
32972 OBJECTHANDLE GCHeap::m_StressObjs[NUM_HEAP_STRESS_OBJS];
32973 int GCHeap::m_CurStressObj = 0;
32974 #endif // !MULTIPLE_HEAPS
32975 #endif // STRESS_HEAP
32976 #endif // FEATURE_REDHAWK
32978 #endif //FEATURE_PREMORTEM_FINALIZATION
32980 class NoGCRegionLockHolder
32983 NoGCRegionLockHolder()
32985 enter_spin_lock_noinstru(&g_no_gc_lock);
32988 ~NoGCRegionLockHolder()
32990 leave_spin_lock_noinstru(&g_no_gc_lock);
32994 // An explanation of locking for finalization:
32996 // Multiple threads allocate objects. During the allocation, they are serialized by
32997 // the AllocLock above. But they release that lock before they register the object
32998 // for finalization. That's because there is much contention for the alloc lock, but
32999 // finalization is presumed to be a rare case.
33001 // So registering an object for finalization must be protected by the FinalizeLock.
33003 // There is another logical queue that involves finalization. When objects registered
33004 // for finalization become unreachable, they are moved from the "registered" queue to
33005 // the "unreachable" queue. Note that this only happens inside a GC, so no other
33006 // threads can be manipulating either queue at that time. Once the GC is over and
33007 // threads are resumed, the Finalizer thread will dequeue objects from the "unreachable"
33008 // queue and call their finalizers. This dequeue operation is also protected with
33009 // the finalize lock.
33011 // At first, this seems unnecessary. Only one thread is ever enqueuing or dequeuing
33012 // on the unreachable queue (either the GC thread during a GC or the finalizer thread
33013 // when a GC is not in progress). The reason we share a lock with threads enqueuing
33014 // on the "registered" queue is that the "registered" and "unreachable" queues are
33017 // They are actually two regions of a longer list, which can only grow at one end.
33018 // So to enqueue an object to the "registered" list, you actually rotate an unreachable
33019 // object at the boundary between the logical queues, out to the other end of the
33020 // unreachable queue -- where all growing takes place. Then you move the boundary
33021 // pointer so that the gap we created at the boundary is now on the "registered"
33022 // side rather than the "unreachable" side. Now the object can be placed into the
33023 // "registered" side at that point. This is much more efficient than doing moves
33024 // of arbitrarily long regions, but it causes the two queues to require a shared lock.
33026 // Notice that Enter/LeaveFinalizeLock is not a GC-aware spin lock. Instead, it relies
33027 // on the fact that the lock will only be taken for a brief period and that it will
33028 // never provoke or allow a GC while the lock is held. This is critical. If the
33029 // FinalizeLock used enter_spin_lock (and thus sometimes enters preemptive mode to
33030 // allow a GC), then the Alloc client would have to GC protect a finalizable object
33031 // to protect against that eventuality. That is too slow!
33035 BOOL IsValidObject99(uint8_t *pObject)
33038 if (!((CObjectHeader*)pObject)->IsFree())
33039 ((CObjectHeader *) pObject)->Validate();
33040 #endif //VERIFY_HEAP
33044 #ifdef BACKGROUND_GC
33045 BOOL gc_heap::bgc_mark_array_range (heap_segment* seg,
33047 uint8_t** range_beg,
33048 uint8_t** range_end)
33050 uint8_t* seg_start = heap_segment_mem (seg);
33051 uint8_t* seg_end = (whole_seg_p ? heap_segment_reserved (seg) : align_on_mark_word (heap_segment_allocated (seg)));
33053 if ((seg_start < background_saved_highest_address) &&
33054 (seg_end > background_saved_lowest_address))
33056 *range_beg = max (seg_start, background_saved_lowest_address);
33057 *range_end = min (seg_end, background_saved_highest_address);
33066 void gc_heap::bgc_verify_mark_array_cleared (heap_segment* seg)
33068 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
33069 if (recursive_gc_sync::background_running_p() && GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
33071 uint8_t* range_beg = 0;
33072 uint8_t* range_end = 0;
33074 if (bgc_mark_array_range (seg, TRUE, &range_beg, &range_end))
33076 size_t markw = mark_word_of (range_beg);
33077 size_t markw_end = mark_word_of (range_end);
33078 while (markw < markw_end)
33080 if (mark_array [markw])
33082 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
33083 markw, mark_array [markw], mark_word_address (markw)));
33088 uint8_t* p = mark_word_address (markw_end);
33089 while (p < range_end)
33091 assert (!(mark_array_marked (p)));
33096 #endif //VERIFY_HEAP && MARK_ARRAY
33099 void gc_heap::verify_mark_bits_cleared (uint8_t* obj, size_t s)
33101 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
33102 size_t start_mark_bit = mark_bit_of (obj) + 1;
33103 size_t end_mark_bit = mark_bit_of (obj + s);
33104 unsigned int startbit = mark_bit_bit (start_mark_bit);
33105 unsigned int endbit = mark_bit_bit (end_mark_bit);
33106 size_t startwrd = mark_bit_word (start_mark_bit);
33107 size_t endwrd = mark_bit_word (end_mark_bit);
33108 unsigned int result = 0;
33110 unsigned int firstwrd = ~(lowbits (~0, startbit));
33111 unsigned int lastwrd = ~(highbits (~0, endbit));
33113 if (startwrd == endwrd)
33115 unsigned int wrd = firstwrd & lastwrd;
33116 result = mark_array[startwrd] & wrd;
33124 // verify the first mark word is cleared.
33127 result = mark_array[startwrd] & firstwrd;
33135 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
33137 result = mark_array[wrdtmp];
33144 // set the last mark word.
33147 result = mark_array[endwrd] & lastwrd;
33153 #endif //VERIFY_HEAP && MARK_ARRAY
33156 void gc_heap::clear_all_mark_array()
33159 //size_t num_dwords_written = 0;
33160 //size_t begin_time = GetHighPrecisionTimeStamp();
33162 generation* gen = generation_of (max_generation);
33163 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
33169 if (gen != large_object_generation)
33171 gen = generation_of (max_generation+1);
33172 seg = heap_segment_rw (generation_start_segment (gen));
33180 uint8_t* range_beg = 0;
33181 uint8_t* range_end = 0;
33183 if (bgc_mark_array_range (seg, (seg == ephemeral_heap_segment), &range_beg, &range_end))
33185 size_t markw = mark_word_of (range_beg);
33186 size_t markw_end = mark_word_of (range_end);
33187 size_t size_total = (markw_end - markw) * sizeof (uint32_t);
33188 //num_dwords_written = markw_end - markw;
33190 size_t size_left = 0;
33192 assert (((size_t)&mark_array[markw] & (sizeof(PTR_PTR)-1)) == 0);
33194 if ((size_total & (sizeof(PTR_PTR) - 1)) != 0)
33196 size = (size_total & ~(sizeof(PTR_PTR) - 1));
33197 size_left = size_total - size;
33198 assert ((size_left & (sizeof (uint32_t) - 1)) == 0);
33205 memclr ((uint8_t*)&mark_array[markw], size);
33207 if (size_left != 0)
33209 uint32_t* markw_to_clear = &mark_array[markw + size / sizeof (uint32_t)];
33210 for (size_t i = 0; i < (size_left / sizeof (uint32_t)); i++)
33212 *markw_to_clear = 0;
33218 seg = heap_segment_next_rw (seg);
33221 //size_t end_time = GetHighPrecisionTimeStamp() - begin_time;
33223 //printf ("took %Id ms to clear %Id bytes\n", end_time, num_dwords_written*sizeof(uint32_t));
33225 #endif //MARK_ARRAY
33228 #endif //BACKGROUND_GC
33230 void gc_heap::verify_mark_array_cleared (heap_segment* seg)
33232 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
33233 assert (card_table == g_gc_card_table);
33234 size_t markw = mark_word_of (heap_segment_mem (seg));
33235 size_t markw_end = mark_word_of (heap_segment_reserved (seg));
33237 while (markw < markw_end)
33239 if (mark_array [markw])
33241 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
33242 markw, mark_array [markw], mark_word_address (markw)));
33247 #endif //VERIFY_HEAP && MARK_ARRAY
33250 void gc_heap::verify_mark_array_cleared ()
33252 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
33253 if (recursive_gc_sync::background_running_p() && GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
33255 generation* gen = generation_of (max_generation);
33256 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
33262 if (gen != large_object_generation)
33264 gen = generation_of (max_generation+1);
33265 seg = heap_segment_rw (generation_start_segment (gen));
33273 bgc_verify_mark_array_cleared (seg);
33274 seg = heap_segment_next_rw (seg);
33277 #endif //VERIFY_HEAP && MARK_ARRAY
33280 void gc_heap::verify_seg_end_mark_array_cleared()
33282 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
33283 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
33285 generation* gen = generation_of (max_generation);
33286 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
33292 if (gen != large_object_generation)
33294 gen = generation_of (max_generation+1);
33295 seg = heap_segment_rw (generation_start_segment (gen));
33303 // We already cleared all mark array bits for ephemeral generations
33304 // at the beginning of bgc sweep
33305 uint8_t* from = ((seg == ephemeral_heap_segment) ?
33306 generation_allocation_start (generation_of (max_generation - 1)) :
33307 heap_segment_allocated (seg));
33308 size_t markw = mark_word_of (align_on_mark_word (from));
33309 size_t markw_end = mark_word_of (heap_segment_reserved (seg));
33311 while (from < mark_word_address (markw))
33313 if (is_mark_bit_set (from))
33315 dprintf (3, ("mark bit for %Ix was not cleared", from));
33319 from += mark_bit_pitch;
33322 while (markw < markw_end)
33324 if (mark_array [markw])
33326 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
33327 markw, mark_array [markw], mark_word_address (markw)));
33332 seg = heap_segment_next_rw (seg);
33335 #endif //VERIFY_HEAP && MARK_ARRAY
33338 // This function is called to make sure we don't mess up the segment list
33339 // in SOH. It's called by:
33340 // 1) begin and end of ephemeral GCs
33341 // 2) during bgc sweep when we switch segments.
33342 void gc_heap::verify_soh_segment_list()
33345 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
33347 generation* gen = generation_of (max_generation);
33348 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
33349 heap_segment* last_seg = 0;
33353 seg = heap_segment_next_rw (seg);
33355 if (last_seg != ephemeral_heap_segment)
33360 #endif //VERIFY_HEAP
33363 // This function can be called at any foreground GCs or blocking GCs. For background GCs,
33364 // it can be called at the end of the final marking; and at any point during background
33366 // NOTE - to be able to call this function during background sweep, we need to temporarily
33367 // NOT clear the mark array bits as we go.
33368 void gc_heap::verify_partial ()
33370 #ifdef BACKGROUND_GC
33371 //printf ("GC#%d: Verifying loh during sweep\n", settings.gc_index);
33372 //generation* gen = large_object_generation;
33373 generation* gen = generation_of (max_generation);
33374 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
33375 int align_const = get_alignment_constant (gen != large_object_generation);
33381 // Different ways to fail.
33382 BOOL mark_missed_p = FALSE;
33383 BOOL bad_ref_p = FALSE;
33384 BOOL free_ref_p = FALSE;
33390 if (gen != large_object_generation)
33393 gen = large_object_generation;
33394 align_const = get_alignment_constant (gen != large_object_generation);
33395 seg = heap_segment_rw (generation_start_segment (gen));
33404 o = heap_segment_mem (seg);
33405 end = heap_segment_allocated (seg);
33406 //printf ("validating [%Ix-[%Ix\n", o, end);
33411 BOOL marked_p = background_object_marked (o, FALSE);
33415 go_through_object_cl (method_table (o), o, s, oo,
33419 //dprintf (3, ("VOM: verifying member %Ix in obj %Ix", (size_t)*oo, o));
33420 MethodTable *pMT = method_table (*oo);
33422 if (pMT == g_gc_pFreeObjectMethodTable)
33428 if (!pMT->SanityCheck())
33431 dprintf (3, ("Bad member of %Ix %Ix",
33432 (size_t)oo, (size_t)*oo));
33436 if (current_bgc_state == bgc_final_marking)
33438 if (marked_p && !background_object_marked (*oo, FALSE))
33440 mark_missed_p = TRUE;
33449 o = o + Align(s, align_const);
33451 seg = heap_segment_next_rw (seg);
33454 //printf ("didn't find any large object large enough...\n");
33455 //printf ("finished verifying loh\n");
33456 #endif //BACKGROUND_GC
33462 gc_heap::verify_free_lists ()
33464 for (int gen_num = 0; gen_num <= max_generation+1; gen_num++)
33466 dprintf (3, ("Verifying free list for gen:%d", gen_num));
33467 allocator* gen_alloc = generation_allocator (generation_of (gen_num));
33468 size_t sz = gen_alloc->first_bucket_size();
33469 bool verify_undo_slot = (gen_num != 0) && (gen_num != max_generation+1) && !gen_alloc->discard_if_no_fit_p();
33471 for (unsigned int a_l_number = 0; a_l_number < gen_alloc->number_of_buckets(); a_l_number++)
33473 uint8_t* free_list = gen_alloc->alloc_list_head_of (a_l_number);
33477 if (!((CObjectHeader*)free_list)->IsFree())
33479 dprintf (3, ("Verifiying Heap: curr free list item %Ix isn't a free object)",
33480 (size_t)free_list));
33483 if (((a_l_number < (gen_alloc->number_of_buckets()-1))&& (unused_array_size (free_list) >= sz))
33484 || ((a_l_number != 0) && (unused_array_size (free_list) < sz/2)))
33486 dprintf (3, ("Verifiying Heap: curr free list item %Ix isn't in the right bucket",
33487 (size_t)free_list));
33490 if (verify_undo_slot && (free_list_undo (free_list) != UNDO_EMPTY))
33492 dprintf (3, ("Verifiying Heap: curr free list item %Ix has non empty undo slot",
33493 (size_t)free_list));
33496 if ((gen_num != max_generation+1)&&(object_gennum (free_list)!= gen_num))
33498 dprintf (3, ("Verifiying Heap: curr free list item %Ix is in the wrong generation free list",
33499 (size_t)free_list));
33504 free_list = free_list_slot (free_list);
33506 //verify the sanity of the tail
33507 uint8_t* tail = gen_alloc->alloc_list_tail_of (a_l_number);
33508 if (!((tail == 0) || (tail == prev)))
33510 dprintf (3, ("Verifying Heap: tail of free list is not correct"));
33515 uint8_t* head = gen_alloc->alloc_list_head_of (a_l_number);
33516 if ((head != 0) && (free_list_slot (head) != 0))
33518 dprintf (3, ("Verifying Heap: tail of free list is not correct"));
33529 gc_heap::verify_heap (BOOL begin_gc_p)
33531 int heap_verify_level = static_cast<int>(GCConfig::GetHeapVerifyLevel());
33532 size_t last_valid_brick = 0;
33533 BOOL bCurrentBrickInvalid = FALSE;
33534 BOOL large_brick_p = TRUE;
33535 size_t curr_brick = 0;
33536 size_t prev_brick = (size_t)-1;
33537 int curr_gen_num = max_generation+1;
33538 heap_segment* seg = heap_segment_in_range (generation_start_segment (generation_of (curr_gen_num ) ));
33540 PREFIX_ASSUME(seg != NULL);
33542 uint8_t* curr_object = heap_segment_mem (seg);
33543 uint8_t* prev_object = 0;
33544 uint8_t* begin_youngest = generation_allocation_start(generation_of(0));
33545 uint8_t* end_youngest = heap_segment_allocated (ephemeral_heap_segment);
33546 uint8_t* next_boundary = generation_allocation_start (generation_of (max_generation - 1));
33547 int align_const = get_alignment_constant (FALSE);
33548 size_t total_objects_verified = 0;
33549 size_t total_objects_verified_deep = 0;
33551 #ifdef BACKGROUND_GC
33552 BOOL consider_bgc_mark_p = FALSE;
33553 BOOL check_current_sweep_p = FALSE;
33554 BOOL check_saved_sweep_p = FALSE;
33555 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
33556 #endif //BACKGROUND_GC
33558 #ifdef MULTIPLE_HEAPS
33559 t_join* current_join = &gc_t_join;
33560 #ifdef BACKGROUND_GC
33561 if (settings.concurrent && (bgc_thread_id.IsCurrentThread()))
33563 // We always call verify_heap on entry of GC on the SVR GC threads.
33564 current_join = &bgc_t_join;
33566 #endif //BACKGROUND_GC
33567 #endif //MULTIPLE_HEAPS
33569 UNREFERENCED_PARAMETER(begin_gc_p);
33570 #ifdef BACKGROUND_GC
33571 dprintf (2,("[%s]GC#%d(%s): Verifying heap - begin",
33572 (begin_gc_p ? "BEG" : "END"),
33573 VolatileLoad(&settings.gc_index),
33574 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC"))));
33576 dprintf (2,("[%s]GC#%d: Verifying heap - begin",
33577 (begin_gc_p ? "BEG" : "END"), VolatileLoad(&settings.gc_index)));
33578 #endif //BACKGROUND_GC
33580 #ifndef MULTIPLE_HEAPS
33581 if ((ephemeral_low != generation_allocation_start (generation_of (max_generation - 1))) ||
33582 (ephemeral_high != heap_segment_reserved (ephemeral_heap_segment)))
33586 #endif //MULTIPLE_HEAPS
33588 #ifdef BACKGROUND_GC
33589 //don't touch the memory because the program is allocating from it.
33590 if (!settings.concurrent)
33591 #endif //BACKGROUND_GC
33593 if (!(heap_verify_level & GCConfig::HEAPVERIFY_NO_MEM_FILL))
33595 //uninit the unused portions of segments.
33596 generation* gen1 = large_object_generation;
33597 heap_segment* seg1 = heap_segment_rw (generation_start_segment (gen1));
33598 PREFIX_ASSUME(seg1 != NULL);
33604 uint8_t* clear_start = heap_segment_allocated (seg1) - plug_skew;
33605 if (heap_segment_used (seg1) > clear_start)
33607 dprintf (3, ("setting end of seg %Ix: [%Ix-[%Ix to 0xaa",
33608 heap_segment_mem (seg1),
33610 heap_segment_used (seg1)));
33611 memset (heap_segment_allocated (seg1) - plug_skew, 0xaa,
33612 (heap_segment_used (seg1) - clear_start));
33614 seg1 = heap_segment_next_rw (seg1);
33618 if (gen1 == large_object_generation)
33620 gen1 = generation_of (max_generation);
33621 seg1 = heap_segment_rw (generation_start_segment (gen1));
33622 PREFIX_ASSUME(seg1 != NULL);
33633 #ifdef MULTIPLE_HEAPS
33634 current_join->join(this, gc_join_verify_copy_table);
33635 if (current_join->joined())
33637 // in concurrent GC, new segment could be allocated when GC is working so the card brick table might not be updated at this point
33638 for (int i = 0; i < n_heaps; i++)
33640 //copy the card and brick tables
33641 if (g_gc_card_table != g_heaps[i]->card_table)
33643 g_heaps[i]->copy_brick_card_table();
33647 current_join->restart();
33650 if (g_gc_card_table != card_table)
33651 copy_brick_card_table();
33652 #endif //MULTIPLE_HEAPS
33654 //verify that the generation structures makes sense
33656 generation* gen = generation_of (max_generation);
33658 assert (generation_allocation_start (gen) ==
33659 heap_segment_mem (heap_segment_rw (generation_start_segment (gen))));
33660 int gen_num = max_generation-1;
33661 generation* prev_gen = gen;
33662 while (gen_num >= 0)
33664 gen = generation_of (gen_num);
33665 assert (generation_allocation_segment (gen) == ephemeral_heap_segment);
33666 assert (generation_allocation_start (gen) >= heap_segment_mem (ephemeral_heap_segment));
33667 assert (generation_allocation_start (gen) < heap_segment_allocated (ephemeral_heap_segment));
33669 if (generation_start_segment (prev_gen ) ==
33670 generation_start_segment (gen))
33672 assert (generation_allocation_start (prev_gen) <
33673 generation_allocation_start (gen));
33682 // Handle segment transitions
33683 if (curr_object >= heap_segment_allocated (seg))
33685 if (curr_object > heap_segment_allocated(seg))
33687 dprintf (3, ("Verifiying Heap: curr_object: %Ix > heap_segment_allocated (seg: %Ix)",
33688 (size_t)curr_object, (size_t)seg));
33691 seg = heap_segment_next_in_range (seg);
33694 #ifdef BACKGROUND_GC
33695 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
33696 #endif //BACKGROUND_GC
33697 curr_object = heap_segment_mem(seg);
33703 if (curr_gen_num == (max_generation+1))
33706 seg = heap_segment_in_range (generation_start_segment (generation_of (curr_gen_num)));
33708 PREFIX_ASSUME(seg != NULL);
33710 #ifdef BACKGROUND_GC
33711 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
33712 #endif //BACKGROUND_GC
33713 curr_object = heap_segment_mem (seg);
33715 large_brick_p = FALSE;
33716 align_const = get_alignment_constant (TRUE);
33719 break; // Done Verifying Heap -- no more segments
33723 // Are we at the end of the youngest_generation?
33724 if (seg == ephemeral_heap_segment)
33726 if (curr_object >= end_youngest)
33728 // prev_object length is too long if we hit this int3
33729 if (curr_object > end_youngest)
33731 dprintf (3, ("Verifiying Heap: curr_object: %Ix > end_youngest: %Ix",
33732 (size_t)curr_object, (size_t)end_youngest));
33738 if ((curr_object >= next_boundary) && (curr_gen_num > 0))
33741 if (curr_gen_num > 0)
33743 next_boundary = generation_allocation_start (generation_of (curr_gen_num - 1));
33748 //if (is_mark_set (curr_object))
33750 // printf ("curr_object: %Ix is marked!",(size_t)curr_object);
33751 // FATAL_GC_ERROR();
33754 size_t s = size (curr_object);
33755 dprintf (3, ("o: %Ix, s: %d", (size_t)curr_object, s));
33758 dprintf (3, ("Verifying Heap: size of current object %Ix == 0", curr_object));
33762 // If object is not in the youngest generation, then lets
33763 // verify that the brick table is correct....
33764 if (((seg != ephemeral_heap_segment) ||
33765 (brick_of(curr_object) < brick_of(begin_youngest))))
33767 curr_brick = brick_of(curr_object);
33769 // Brick Table Verification...
33771 // On brick transition
33772 // if brick is negative
33773 // verify that brick indirects to previous valid brick
33775 // set current brick invalid flag to be flipped if we
33776 // encounter an object at the correct place
33778 if (curr_brick != prev_brick)
33780 // If the last brick we were examining had positive
33781 // entry but we never found the matching object, then
33782 // we have a problem
33783 // If prev_brick was the last one of the segment
33784 // it's ok for it to be invalid because it is never looked at
33785 if (bCurrentBrickInvalid &&
33786 (curr_brick != brick_of (heap_segment_mem (seg))) &&
33787 !heap_segment_read_only_p (seg))
33789 dprintf (3, ("curr brick %Ix invalid", curr_brick));
33795 //large objects verify the table only if they are in
33797 if ((heap_segment_reserved (seg) <= highest_address) &&
33798 (heap_segment_mem (seg) >= lowest_address) &&
33799 brick_table [curr_brick] != 0)
33801 dprintf (3, ("curr_brick %Ix for large object %Ix not set to -32768",
33802 curr_brick, (size_t)curr_object));
33807 bCurrentBrickInvalid = FALSE;
33812 // If the current brick contains a negative value make sure
33813 // that the indirection terminates at the last valid brick
33814 if (brick_table [curr_brick] <= 0)
33816 if (brick_table [curr_brick] == 0)
33818 dprintf(3, ("curr_brick %Ix for object %Ix set to 0",
33819 curr_brick, (size_t)curr_object));
33822 ptrdiff_t i = curr_brick;
33823 while ((i >= ((ptrdiff_t) brick_of (heap_segment_mem (seg)))) &&
33824 (brick_table[i] < 0))
33826 i = i + brick_table[i];
33828 if (i < ((ptrdiff_t)(brick_of (heap_segment_mem (seg))) - 1))
33830 dprintf (3, ("ptrdiff i: %Ix < brick_of (heap_segment_mem (seg)):%Ix - 1. curr_brick: %Ix",
33831 i, brick_of (heap_segment_mem (seg)),
33835 // if (i != last_valid_brick)
33836 // FATAL_GC_ERROR();
33837 bCurrentBrickInvalid = FALSE;
33839 else if (!heap_segment_read_only_p (seg))
33841 bCurrentBrickInvalid = TRUE;
33846 if (bCurrentBrickInvalid)
33848 if (curr_object == (brick_address(curr_brick) + brick_table[curr_brick] - 1))
33850 bCurrentBrickInvalid = FALSE;
33851 last_valid_brick = curr_brick;
33856 if (*((uint8_t**)curr_object) != (uint8_t *) g_gc_pFreeObjectMethodTable)
33858 #ifdef FEATURE_LOH_COMPACTION
33859 if ((curr_gen_num == (max_generation+1)) && (prev_object != 0))
33861 assert (method_table (prev_object) == g_gc_pFreeObjectMethodTable);
33863 #endif //FEATURE_LOH_COMPACTION
33865 total_objects_verified++;
33867 BOOL can_verify_deep = TRUE;
33868 #ifdef BACKGROUND_GC
33869 can_verify_deep = fgc_should_consider_object (curr_object, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p);
33870 #endif //BACKGROUND_GC
33872 BOOL deep_verify_obj = can_verify_deep;
33873 if ((heap_verify_level & GCConfig::HEAPVERIFY_DEEP_ON_COMPACT) && !settings.compaction)
33874 deep_verify_obj = FALSE;
33876 ((CObjectHeader*)curr_object)->ValidateHeap((Object*)curr_object, deep_verify_obj);
33878 if (can_verify_deep)
33880 if (curr_gen_num > 0)
33882 BOOL need_card_p = FALSE;
33883 if (contain_pointers_or_collectible (curr_object))
33885 dprintf (4, ("curr_object: %Ix", (size_t)curr_object));
33886 size_t crd = card_of (curr_object);
33887 BOOL found_card_p = card_set_p (crd);
33889 #ifdef COLLECTIBLE_CLASS
33890 if (is_collectible(curr_object))
33892 uint8_t* class_obj = get_class_object (curr_object);
33893 if ((class_obj < ephemeral_high) && (class_obj >= next_boundary))
33897 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix pointing to class object %Ix",
33898 card_of (curr_object), (size_t)curr_object, class_obj));
33904 #endif //COLLECTIBLE_CLASS
33906 if (contain_pointers(curr_object))
33908 go_through_object_nostart
33909 (method_table(curr_object), curr_object, s, oo,
33911 if ((crd != card_of ((uint8_t*)oo)) && !found_card_p)
33913 crd = card_of ((uint8_t*)oo);
33914 found_card_p = card_set_p (crd);
33915 need_card_p = FALSE;
33917 if ((*oo < ephemeral_high) && (*oo >= next_boundary))
33919 need_card_p = TRUE;
33922 if (need_card_p && !found_card_p)
33925 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix, %Ix:%Ix[",
33926 card_of (curr_object), (size_t)curr_object,
33927 card_of (curr_object+Align(s, align_const)), (size_t)curr_object+Align(s, align_const)));
33933 if (need_card_p && !found_card_p)
33935 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix, %Ix:%Ix[",
33936 card_of (curr_object), (size_t)curr_object,
33937 card_of (curr_object+Align(s, align_const)), (size_t)curr_object+Align(s, align_const)));
33942 total_objects_verified_deep++;
33946 prev_object = curr_object;
33947 prev_brick = curr_brick;
33948 curr_object = curr_object + Align(s, align_const);
33949 if (curr_object < prev_object)
33951 dprintf (3, ("overflow because of a bad object size: %Ix size %Ix", prev_object, s));
33956 #ifdef BACKGROUND_GC
33957 dprintf (2, ("(%s)(%s)(%s) total_objects_verified is %Id, total_objects_verified_deep is %Id",
33958 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p () ? "FGC" : "NGC")),
33959 (begin_gc_p ? "BEG" : "END"),
33960 ((current_c_gc_state == c_gc_state_planning) ? "in plan" : "not in plan"),
33961 total_objects_verified, total_objects_verified_deep));
33962 if (current_c_gc_state != c_gc_state_planning)
33964 assert (total_objects_verified == total_objects_verified_deep);
33966 #endif //BACKGROUND_GC
33968 verify_free_lists();
33970 #ifdef FEATURE_PREMORTEM_FINALIZATION
33971 finalize_queue->CheckFinalizerObjects();
33972 #endif // FEATURE_PREMORTEM_FINALIZATION
33975 // to be consistent with handle table APIs pass a ScanContext*
33976 // to provide the heap number. the SC isn't complete though so
33977 // limit its scope to handle table verification.
33979 sc.thread_number = heap_number;
33980 GCScan::VerifyHandleTable(max_generation, max_generation, &sc);
33983 #ifdef MULTIPLE_HEAPS
33984 current_join->join(this, gc_join_verify_objects_done);
33985 if (current_join->joined())
33986 #endif //MULTIPLE_HEAPS
33988 GCToEEInterface::VerifySyncTableEntry();
33989 #ifdef MULTIPLE_HEAPS
33990 current_join->restart();
33991 #endif //MULTIPLE_HEAPS
33994 #ifdef BACKGROUND_GC
33995 if (!settings.concurrent)
33997 if (current_c_gc_state == c_gc_state_planning)
33999 // temporarily commenting this out 'cause an FGC
34000 // could be triggered before we sweep ephemeral.
34001 //verify_seg_end_mark_array_cleared();
34005 if (settings.concurrent)
34007 verify_mark_array_cleared();
34009 dprintf (2,("GC%d(%s): Verifying heap - end",
34010 VolatileLoad(&settings.gc_index),
34011 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC"))));
34013 dprintf (2,("GC#d: Verifying heap - end", VolatileLoad(&settings.gc_index)));
34014 #endif //BACKGROUND_GC
34017 #endif //VERIFY_HEAP
34020 void GCHeap::ValidateObjectMember (Object* obj)
34023 size_t s = size (obj);
34024 uint8_t* o = (uint8_t*)obj;
34026 go_through_object_cl (method_table (obj), o, s, oo,
34028 uint8_t* child_o = *oo;
34031 dprintf (3, ("VOM: m: %Ix obj %Ix", (size_t)child_o, o));
34032 MethodTable *pMT = method_table (child_o);
34034 if (!pMT->SanityCheck()) {
34035 dprintf (3, ("Bad member of %Ix %Ix",
34036 (size_t)oo, (size_t)child_o));
34041 #endif // VERIFY_HEAP
34044 void DestructObject (CObjectHeader* hdr)
34046 UNREFERENCED_PARAMETER(hdr); // compiler bug? -- this *is*, indeed, referenced
34047 hdr->~CObjectHeader();
34050 HRESULT GCHeap::Shutdown ()
34054 GCScan::GcRuntimeStructuresValid (FALSE);
34056 // Cannot assert this, since we use SuspendEE as the mechanism to quiesce all
34057 // threads except the one performing the shutdown.
34058 // ASSERT( !GcInProgress );
34060 // Guard against any more GC occurring and against any threads blocking
34061 // for GC to complete when the GC heap is gone. This fixes a race condition
34062 // where a thread in GC is destroyed as part of process destruction and
34063 // the remaining threads block for GC complete.
34066 //EnterAllocLock();
34068 //EnterFinalizeLock();
34071 // during shutdown lot of threads are suspended
34072 // on this even, we don't want to wake them up just yet
34073 //CloseHandle (WaitForGCEvent);
34075 //find out if the global card table hasn't been used yet
34076 uint32_t* ct = &g_gc_card_table[card_word (gcard_of (g_gc_lowest_address))];
34077 if (card_table_refcount (ct) == 0)
34079 destroy_card_table (ct);
34080 g_gc_card_table = nullptr;
34082 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
34083 g_gc_card_bundle_table = nullptr;
34085 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
34086 SoftwareWriteWatch::StaticClose();
34087 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
34090 //destroy all segments on the standby list
34091 while(gc_heap::segment_standby_list != 0)
34093 heap_segment* next_seg = heap_segment_next (gc_heap::segment_standby_list);
34094 #ifdef MULTIPLE_HEAPS
34095 (gc_heap::g_heaps[0])->delete_heap_segment (gc_heap::segment_standby_list, FALSE);
34096 #else //MULTIPLE_HEAPS
34097 pGenGCHeap->delete_heap_segment (gc_heap::segment_standby_list, FALSE);
34098 #endif //MULTIPLE_HEAPS
34099 gc_heap::segment_standby_list = next_seg;
34103 #ifdef MULTIPLE_HEAPS
34105 for (int i = 0; i < gc_heap::n_heaps; i ++)
34107 delete gc_heap::g_heaps[i]->vm_heap;
34108 //destroy pure GC stuff
34109 gc_heap::destroy_gc_heap (gc_heap::g_heaps[i]);
34112 gc_heap::destroy_gc_heap (pGenGCHeap);
34114 #endif //MULTIPLE_HEAPS
34115 gc_heap::shutdown_gc();
34120 // Wait until a garbage collection is complete
34121 // returns NOERROR if wait was OK, other error code if failure.
34122 // WARNING: This will not undo the must complete state. If you are
34123 // in a must complete when you call this, you'd better know what you're
34126 #ifdef FEATURE_PREMORTEM_FINALIZATION
34128 HRESULT AllocateCFinalize(CFinalize **pCFinalize)
34130 *pCFinalize = new (nothrow) CFinalize();
34131 if (*pCFinalize == NULL || !(*pCFinalize)->Initialize())
34132 return E_OUTOFMEMORY;
34136 #endif // FEATURE_PREMORTEM_FINALIZATION
34138 // init the instance heap
34139 HRESULT GCHeap::Init(size_t hn)
34141 HRESULT hres = S_OK;
34143 #ifdef MULTIPLE_HEAPS
34144 if ((pGenGCHeap = gc_heap::make_gc_heap(this, (int)hn)) == 0)
34145 hres = E_OUTOFMEMORY;
34147 UNREFERENCED_PARAMETER(hn);
34148 if (!gc_heap::make_gc_heap())
34149 hres = E_OUTOFMEMORY;
34150 #endif //MULTIPLE_HEAPS
34156 //System wide initialization
34157 HRESULT GCHeap::Initialize()
34161 g_gc_pFreeObjectMethodTable = GCToEEInterface::GetFreeObjectMethodTable();
34162 g_num_processors = GCToOSInterface::GetTotalProcessorCount();
34163 assert(g_num_processors != 0);
34165 //Initialize the static members.
34168 CreatedObjectCount = 0;
34171 bool is_restricted;
34172 gc_heap::total_physical_mem = GCToOSInterface::GetPhysicalMemoryLimit (&is_restricted);
34175 gc_heap::heap_hard_limit = (size_t)GCConfig::GetGCHeapHardLimit();
34177 if (!(gc_heap::heap_hard_limit))
34179 uint32_t percent_of_mem = (uint32_t)GCConfig::GetGCHeapHardLimitPercent();
34180 if ((percent_of_mem > 0) && (percent_of_mem < 100))
34182 gc_heap::heap_hard_limit = (size_t)(gc_heap::total_physical_mem * (uint64_t)percent_of_mem / (uint64_t)100);
34186 // If the hard limit is specified, the user is saying even if the process is already
34187 // running in a container, use this limit for the GC heap.
34188 if (!(gc_heap::heap_hard_limit))
34192 uint64_t physical_mem_for_gc = gc_heap::total_physical_mem * (uint64_t)75 / (uint64_t)100;
34193 //printf ("returned physical mem %I64d, setting it to max (75%%: %I64d, 20mb)\n",
34194 // gc_heap::total_physical_mem, physical_mem_for_gc);
34195 gc_heap::heap_hard_limit = (size_t)max ((20 * 1024 * 1024), physical_mem_for_gc);
34199 //printf ("heap_hard_limit is %Id, total physical mem: %Id, %s restricted\n",
34200 // gc_heap::heap_hard_limit, gc_heap::total_physical_mem, (is_restricted ? "is" : "is not"));
34204 uint32_t nhp_from_config = 0;
34206 #ifdef MULTIPLE_HEAPS
34207 nhp_from_config = static_cast<uint32_t>(GCConfig::GetHeapCount());
34209 uint32_t nhp_from_process = GCToOSInterface::GetCurrentProcessCpuCount();
34211 if (nhp_from_config)
34213 // Even when the user specifies a heap count, it should not be more
34214 // than the number of procs this process can use.
34215 nhp_from_config = min (nhp_from_config, nhp_from_process);
34218 nhp = ((nhp_from_config == 0) ? nhp_from_process : nhp_from_config);
34220 nhp = min (nhp, MAX_SUPPORTED_CPUS);
34221 #ifndef FEATURE_REDHAWK
34222 gc_heap::gc_thread_no_affinitize_p = (gc_heap::heap_hard_limit ? false : (GCConfig::GetNoAffinitize() != 0));
34224 size_t gc_thread_affinity_mask = static_cast<size_t>(GCConfig::GetGCHeapAffinitizeMask());
34226 if (gc_heap::heap_hard_limit)
34228 gc_heap::gc_thread_no_affinitize_p = (gc_thread_affinity_mask == 0);
34231 if (!(gc_heap::gc_thread_no_affinitize_p))
34233 if (!(GCToOSInterface::CanEnableGCCPUGroups()))
34235 uintptr_t pmask, smask;
34236 if (GCToOSInterface::GetCurrentProcessAffinityMask(&pmask, &smask))
34240 if (gc_thread_affinity_mask)
34242 pmask &= gc_thread_affinity_mask;
34245 process_mask = pmask;
34247 unsigned int set_bits_in_pmask = 0;
34251 set_bits_in_pmask++;
34255 nhp = min (nhp, set_bits_in_pmask);
34259 gc_heap::gc_thread_no_affinitize_p = true;
34263 #endif //!FEATURE_REDHAWK
34264 #endif //MULTIPLE_HEAPS
34266 size_t seg_size = 0;
34267 size_t large_seg_size = 0;
34269 if (gc_heap::heap_hard_limit)
34271 seg_size = gc_heap::get_segment_size_hard_limit (&nhp, (nhp_from_config == 0));
34272 gc_heap::soh_segment_size = seg_size;
34273 large_seg_size = seg_size * 2;
34277 seg_size = get_valid_segment_size();
34278 gc_heap::soh_segment_size = seg_size;
34279 large_seg_size = get_valid_segment_size (TRUE);
34282 dprintf (1, ("%d heaps, soh seg size: %Id mb, loh: %Id mb\n",
34284 (seg_size / (size_t)1024 / 1024),
34285 (large_seg_size / 1024 / 1024)));
34287 gc_heap::min_loh_segment_size = large_seg_size;
34288 gc_heap::min_segment_size = min (seg_size, large_seg_size);
34289 #ifdef SEG_MAPPING_TABLE
34290 gc_heap::min_segment_size_shr = index_of_highest_set_bit (gc_heap::min_segment_size);
34291 #endif //SEG_MAPPING_TABLE
34293 #ifdef MULTIPLE_HEAPS
34294 gc_heap::n_heaps = nhp;
34295 hr = gc_heap::initialize_gc (seg_size, large_seg_size /*LHEAP_ALLOC*/, nhp);
34297 hr = gc_heap::initialize_gc (seg_size, large_seg_size /*LHEAP_ALLOC*/);
34298 #endif //MULTIPLE_HEAPS
34303 gc_heap::mem_one_percent = gc_heap::total_physical_mem / 100;
34304 #ifndef MULTIPLE_HEAPS
34305 gc_heap::mem_one_percent /= g_num_processors;
34306 #endif //!MULTIPLE_HEAPS
34308 uint32_t highmem_th_from_config = (uint32_t)GCConfig::GetGCHighMemPercent();
34309 if (highmem_th_from_config)
34311 gc_heap::high_memory_load_th = min (99, highmem_th_from_config);
34312 gc_heap::v_high_memory_load_th = min (99, (highmem_th_from_config + 7));
34316 // We should only use this if we are in the "many process" mode which really is only applicable
34317 // to very powerful machines - before that's implemented, temporarily I am only enabling this for 80GB+ memory.
34318 // For now I am using an estimate to calculate these numbers but this should really be obtained
34319 // programmatically going forward.
34320 // I am assuming 47 processes using WKS GC and 3 using SVR GC.
34321 // I am assuming 3 in part due to the "very high memory load" is 97%.
34322 int available_mem_th = 10;
34323 if (gc_heap::total_physical_mem >= ((uint64_t)80 * 1024 * 1024 * 1024))
34325 int adjusted_available_mem_th = 3 + (int)((float)47 / (float)(GCToOSInterface::GetTotalProcessorCount()));
34326 available_mem_th = min (available_mem_th, adjusted_available_mem_th);
34329 gc_heap::high_memory_load_th = 100 - available_mem_th;
34330 gc_heap::v_high_memory_load_th = 97;
34333 gc_heap::m_high_memory_load_th = min ((gc_heap::high_memory_load_th + 5), gc_heap::v_high_memory_load_th);
34335 gc_heap::pm_stress_on = (GCConfig::GetGCProvModeStress() != 0);
34338 gc_heap::youngest_gen_desired_th = gc_heap::mem_one_percent;
34341 WaitForGCEvent = new (nothrow) GCEvent;
34343 if (!WaitForGCEvent)
34345 return E_OUTOFMEMORY;
34348 if (!WaitForGCEvent->CreateManualEventNoThrow(TRUE))
34353 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
34354 #if defined (STRESS_HEAP) && !defined (MULTIPLE_HEAPS)
34355 if (GCStress<cfg_any>::IsEnabled()) {
34356 for(int i = 0; i < GCHeap::NUM_HEAP_STRESS_OBJS; i++)
34358 m_StressObjs[i] = CreateGlobalHandle(0);
34360 m_CurStressObj = 0;
34362 #endif //STRESS_HEAP && !MULTIPLE_HEAPS
34363 #endif // FEATURE_REDHAWK
34365 initGCShadow(); // If we are debugging write barriers, initialize heap shadow
34367 #ifdef MULTIPLE_HEAPS
34369 for (unsigned i = 0; i < nhp; i++)
34371 GCHeap* Hp = new (nothrow) GCHeap();
34373 return E_OUTOFMEMORY;
34375 if ((hr = Hp->Init (i))!= S_OK)
34380 // initialize numa node to heap map
34381 heap_select::init_numa_node_to_heap_map(nhp);
34384 #endif //MULTIPLE_HEAPS
34388 GCScan::GcRuntimeStructuresValid (TRUE);
34390 GCToEEInterface::DiagUpdateGenerationBounds();
34397 // GC callback functions
34398 bool GCHeap::IsPromoted(Object* object)
34401 ((CObjectHeader*)object)->Validate();
34404 uint8_t* o = (uint8_t*)object;
34406 if (gc_heap::settings.condemned_generation == max_generation)
34408 #ifdef MULTIPLE_HEAPS
34409 gc_heap* hp = gc_heap::g_heaps[0];
34411 gc_heap* hp = pGenGCHeap;
34412 #endif //MULTIPLE_HEAPS
34414 #ifdef BACKGROUND_GC
34415 if (gc_heap::settings.concurrent)
34417 bool is_marked = (!((o < hp->background_saved_highest_address) && (o >= hp->background_saved_lowest_address))||
34418 hp->background_marked (o));
34422 #endif //BACKGROUND_GC
34424 return (!((o < hp->highest_address) && (o >= hp->lowest_address))
34425 || hp->is_mark_set (o));
34430 gc_heap* hp = gc_heap::heap_of (o);
34431 return (!((o < hp->gc_high) && (o >= hp->gc_low))
34432 || hp->is_mark_set (o));
34436 size_t GCHeap::GetPromotedBytes(int heap_index)
34438 #ifdef BACKGROUND_GC
34439 if (gc_heap::settings.concurrent)
34441 return gc_heap::bpromoted_bytes (heap_index);
34444 #endif //BACKGROUND_GC
34446 return gc_heap::promoted_bytes (heap_index);
34450 void GCHeap::SetYieldProcessorScalingFactor (float scalingFactor)
34452 assert (yp_spin_count_unit != 0);
34453 int saved_yp_spin_count_unit = yp_spin_count_unit;
34454 yp_spin_count_unit = (int)((float)yp_spin_count_unit * scalingFactor / (float)9);
34456 // It's very suspicious if it becomes 0
34457 if (yp_spin_count_unit == 0)
34459 yp_spin_count_unit = saved_yp_spin_count_unit;
34463 unsigned int GCHeap::WhichGeneration (Object* object)
34465 gc_heap* hp = gc_heap::heap_of ((uint8_t*)object);
34466 unsigned int g = hp->object_gennum ((uint8_t*)object);
34467 dprintf (3, ("%Ix is in gen %d", (size_t)object, g));
34471 bool GCHeap::IsEphemeral (Object* object)
34473 uint8_t* o = (uint8_t*)object;
34474 gc_heap* hp = gc_heap::heap_of (o);
34475 return !!hp->ephemeral_pointer_p (o);
34478 // Return NULL if can't find next object. When EE is not suspended,
34479 // the result is not accurate: if the input arg is in gen0, the function could
34480 // return zeroed out memory as next object
34481 Object * GCHeap::NextObj (Object * object)
34484 uint8_t* o = (uint8_t*)object;
34486 #ifndef FEATURE_BASICFREEZE
34487 if (!((o < g_gc_highest_address) && (o >= g_gc_lowest_address)))
34491 #endif //!FEATURE_BASICFREEZE
34493 heap_segment * hs = gc_heap::find_segment (o, FALSE);
34499 BOOL large_object_p = heap_segment_loh_p (hs);
34500 if (large_object_p)
34501 return NULL; //could be racing with another core allocating.
34502 #ifdef MULTIPLE_HEAPS
34503 gc_heap* hp = heap_segment_heap (hs);
34504 #else //MULTIPLE_HEAPS
34506 #endif //MULTIPLE_HEAPS
34507 unsigned int g = hp->object_gennum ((uint8_t*)object);
34508 if ((g == 0) && hp->settings.demotion)
34509 return NULL;//could be racing with another core allocating.
34510 int align_const = get_alignment_constant (!large_object_p);
34511 uint8_t* nextobj = o + Align (size (o), align_const);
34512 if (nextobj <= o) // either overflow or 0 sized object.
34517 if ((nextobj < heap_segment_mem(hs)) ||
34518 (nextobj >= heap_segment_allocated(hs) && hs != hp->ephemeral_heap_segment) ||
34519 (nextobj >= hp->alloc_allocated))
34524 return (Object *)nextobj;
34527 #endif // VERIFY_HEAP
34530 // returns TRUE if the pointer is in one of the GC heaps.
34531 bool GCHeap::IsHeapPointer (void* vpObject, bool small_heap_only)
34533 // removed STATIC_CONTRACT_CAN_TAKE_LOCK here because find_segment
34534 // no longer calls GCEvent::Wait which eventually takes a lock.
34536 uint8_t* object = (uint8_t*) vpObject;
34537 #ifndef FEATURE_BASICFREEZE
34538 if (!((object < g_gc_highest_address) && (object >= g_gc_lowest_address)))
34540 #endif //!FEATURE_BASICFREEZE
34542 heap_segment * hs = gc_heap::find_segment (object, small_heap_only);
34546 #ifdef STRESS_PINNING
34547 static n_promote = 0;
34548 #endif //STRESS_PINNING
34549 // promote an object
34550 void GCHeap::Promote(Object** ppObject, ScanContext* sc, uint32_t flags)
34552 THREAD_NUMBER_FROM_CONTEXT;
34553 #ifndef MULTIPLE_HEAPS
34554 const int thread = 0;
34555 #endif //!MULTIPLE_HEAPS
34557 uint8_t* o = (uint8_t*)*ppObject;
34562 #ifdef DEBUG_DestroyedHandleValue
34563 // we can race with destroy handle during concurrent scan
34564 if (o == (uint8_t*)DEBUG_DestroyedHandleValue)
34566 #endif //DEBUG_DestroyedHandleValue
34570 gc_heap* hp = gc_heap::heap_of (o);
34572 dprintf (3, ("Promote %Ix", (size_t)o));
34574 #ifdef INTERIOR_POINTERS
34575 if (flags & GC_CALL_INTERIOR)
34577 if ((o < hp->gc_low) || (o >= hp->gc_high))
34581 if ( (o = hp->find_object (o, hp->gc_low)) == 0)
34587 #endif //INTERIOR_POINTERS
34589 #ifdef FEATURE_CONSERVATIVE_GC
34590 // For conservative GC, a value on stack may point to middle of a free object.
34591 // In this case, we don't need to promote the pointer.
34592 if (GCConfig::GetConservativeGC()
34593 && ((CObjectHeader*)o)->IsFree())
34600 ((CObjectHeader*)o)->ValidatePromote(sc, flags);
34602 UNREFERENCED_PARAMETER(sc);
34605 if (flags & GC_CALL_PINNED)
34606 hp->pin_object (o, (uint8_t**) ppObject, hp->gc_low, hp->gc_high);
34608 #ifdef STRESS_PINNING
34609 if ((++n_promote % 20) == 1)
34610 hp->pin_object (o, (uint8_t**) ppObject, hp->gc_low, hp->gc_high);
34611 #endif //STRESS_PINNING
34613 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
34614 size_t promoted_size_begin = hp->promoted_bytes (thread);
34615 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
34617 if ((o >= hp->gc_low) && (o < hp->gc_high))
34619 hpt->mark_object_simple (&o THREAD_NUMBER_ARG);
34622 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
34623 size_t promoted_size_end = hp->promoted_bytes (thread);
34624 if (g_fEnableAppDomainMonitoring)
34626 if (sc->pCurrentDomain)
34628 GCToEEInterface::RecordSurvivedBytesForHeap((promoted_size_end - promoted_size_begin), thread, sc->pCurrentDomain);
34631 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
34633 STRESS_LOG_ROOT_PROMOTE(ppObject, o, o ? header(o)->GetMethodTable() : NULL);
34636 void GCHeap::Relocate (Object** ppObject, ScanContext* sc,
34639 UNREFERENCED_PARAMETER(sc);
34641 uint8_t* object = (uint8_t*)(Object*)(*ppObject);
34643 THREAD_NUMBER_FROM_CONTEXT;
34645 //dprintf (3, ("Relocate location %Ix\n", (size_t)ppObject));
34646 dprintf (3, ("R: %Ix", (size_t)ppObject));
34651 gc_heap* hp = gc_heap::heap_of (object);
34654 if (!(flags & GC_CALL_INTERIOR))
34656 // We cannot validate this object if it's in the condemned gen because it could
34657 // be one of the objects that were overwritten by an artificial gap due to a pinned plug.
34658 if (!((object >= hp->gc_low) && (object < hp->gc_high)))
34660 ((CObjectHeader*)object)->Validate(FALSE);
34665 dprintf (3, ("Relocate %Ix\n", (size_t)object));
34669 if ((flags & GC_CALL_INTERIOR) && gc_heap::settings.loh_compaction)
34671 if (!((object >= hp->gc_low) && (object < hp->gc_high)))
34676 if (gc_heap::loh_object_p (object))
34678 pheader = hp->find_object (object, 0);
34684 ptrdiff_t ref_offset = object - pheader;
34685 hp->relocate_address(&pheader THREAD_NUMBER_ARG);
34686 *ppObject = (Object*)(pheader + ref_offset);
34693 hp->relocate_address(&pheader THREAD_NUMBER_ARG);
34694 *ppObject = (Object*)pheader;
34697 STRESS_LOG_ROOT_RELOCATE(ppObject, object, pheader, ((!(flags & GC_CALL_INTERIOR)) ? ((Object*)object)->GetGCSafeMethodTable() : 0));
34700 /*static*/ bool GCHeap::IsObjectInFixedHeap(Object *pObj)
34702 // For now we simply look at the size of the object to determine if it in the
34703 // fixed heap or not. If the bit indicating this gets set at some point
34704 // we should key off that instead.
34705 return size( pObj ) >= loh_size_threshold;
34708 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
34711 void StressHeapDummy ();
34713 static int32_t GCStressStartCount = -1;
34714 static int32_t GCStressCurCount = 0;
34715 static int32_t GCStressStartAtJit = -1;
34717 // the maximum number of foreground GCs we'll induce during one BGC
34718 // (this number does not include "naturally" occuring GCs).
34719 static int32_t GCStressMaxFGCsPerBGC = -1;
34721 // CLRRandom implementation can produce FPU exceptions if
34722 // the test/application run by CLR is enabling any FPU exceptions.
34723 // We want to avoid any unexpected exception coming from stress
34724 // infrastructure, so CLRRandom is not an option.
34725 // The code below is a replicate of CRT rand() implementation.
34726 // Using CRT rand() is not an option because we will interfere with the user application
34727 // that may also use it.
34728 int StressRNG(int iMaxValue)
34730 static BOOL bisRandInit = FALSE;
34731 static int lHoldrand = 1L;
34735 lHoldrand = (int)time(NULL);
34736 bisRandInit = TRUE;
34738 int randValue = (((lHoldrand = lHoldrand * 214013L + 2531011L) >> 16) & 0x7fff);
34739 return randValue % iMaxValue;
34741 #endif // STRESS_HEAP
34742 #endif // !FEATURE_REDHAWK
34744 // free up object so that things will move and then do a GC
34745 //return TRUE if GC actually happens, otherwise FALSE
34746 bool GCHeap::StressHeap(gc_alloc_context * context)
34748 #if defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
34749 alloc_context* acontext = static_cast<alloc_context*>(context);
34750 assert(context != nullptr);
34752 // if GC stress was dynamically disabled during this run we return FALSE
34753 if (!GCStressPolicy::IsEnabled())
34757 if (g_pConfig->FastGCStressLevel() && !GCToEEInterface::GetThread()->StressHeapIsEnabled()) {
34763 if ((g_pConfig->GetGCStressLevel() & EEConfig::GCSTRESS_UNIQUE)
34765 || g_pConfig->FastGCStressLevel() > 1
34768 if (!Thread::UniqueStack(&acontext)) {
34773 #ifdef BACKGROUND_GC
34774 // don't trigger a GC from the GC threads but still trigger GCs from user threads.
34775 if (GCToEEInterface::WasCurrentThreadCreatedByGC())
34779 #endif //BACKGROUND_GC
34781 if (GCStressStartAtJit == -1 || GCStressStartCount == -1)
34783 GCStressStartCount = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_GCStressStart);
34784 GCStressStartAtJit = CLRConfig::GetConfigValue(CLRConfig::INTERNAL_GCStressStartAtJit);
34787 if (GCStressMaxFGCsPerBGC == -1)
34789 GCStressMaxFGCsPerBGC = CLRConfig::GetConfigValue(CLRConfig::INTERNAL_GCStressMaxFGCsPerBGC);
34790 if (g_pConfig->IsGCStressMix() && GCStressMaxFGCsPerBGC == -1)
34791 GCStressMaxFGCsPerBGC = 6;
34795 if (g_JitCount < GCStressStartAtJit)
34799 // Allow programmer to skip the first N Stress GCs so that you can
34800 // get to the interesting ones faster.
34801 Interlocked::Increment(&GCStressCurCount);
34802 if (GCStressCurCount < GCStressStartCount)
34805 // throttle the number of stress-induced GCs by a factor given by GCStressStep
34806 if ((GCStressCurCount % g_pConfig->GetGCStressStep()) != 0)
34811 #ifdef BACKGROUND_GC
34812 if (IsConcurrentGCEnabled() && IsConcurrentGCInProgress())
34814 // allow a maximum number of stress induced FGCs during one BGC
34815 if (gc_stress_fgcs_in_bgc >= GCStressMaxFGCsPerBGC)
34817 ++gc_stress_fgcs_in_bgc;
34819 #endif // BACKGROUND_GC
34821 if (g_pStringClass == 0)
34823 // If the String class has not been loaded, dont do any stressing. This should
34824 // be kept to a minimum to get as complete coverage as possible.
34825 _ASSERTE(g_fEEInit);
34829 #ifndef MULTIPLE_HEAPS
34830 static int32_t OneAtATime = -1;
34832 // Only bother with this if the stress level is big enough and if nobody else is
34833 // doing it right now. Note that some callers are inside the AllocLock and are
34834 // guaranteed synchronized. But others are using AllocationContexts and have no
34835 // particular synchronization.
34837 // For this latter case, we want a very high-speed way of limiting this to one
34838 // at a time. A secondary advantage is that we release part of our StressObjs
34839 // buffer sparingly but just as effectively.
34841 if (Interlocked::Increment(&OneAtATime) == 0 &&
34842 !TrackAllocations()) // Messing with object sizes can confuse the profiler (see ICorProfilerInfo::GetObjectSize)
34846 // If the current string is used up
34847 if (HndFetchHandle(m_StressObjs[m_CurStressObj]) == 0)
34849 // Populate handles with strings
34850 int i = m_CurStressObj;
34851 while(HndFetchHandle(m_StressObjs[i]) == 0)
34853 _ASSERTE(m_StressObjs[i] != 0);
34854 unsigned strLen = ((unsigned)loh_size_threshold - 32) / sizeof(WCHAR);
34855 unsigned strSize = PtrAlign(StringObject::GetSize(strLen));
34857 // update the cached type handle before allocating
34858 SetTypeHandleOnThreadForAlloc(TypeHandle(g_pStringClass));
34859 str = (StringObject*) pGenGCHeap->allocate (strSize, acontext);
34862 str->SetMethodTable (g_pStringClass);
34863 str->SetStringLength (strLen);
34864 HndAssignHandle(m_StressObjs[i], ObjectToOBJECTREF(str));
34866 i = (i + 1) % NUM_HEAP_STRESS_OBJS;
34867 if (i == m_CurStressObj) break;
34870 // advance the current handle to the next string
34871 m_CurStressObj = (m_CurStressObj + 1) % NUM_HEAP_STRESS_OBJS;
34874 // Get the current string
34875 str = (StringObject*) OBJECTREFToObject(HndFetchHandle(m_StressObjs[m_CurStressObj]));
34878 // Chop off the end of the string and form a new object out of it.
34879 // This will 'free' an object at the begining of the heap, which will
34880 // force data movement. Note that we can only do this so many times.
34881 // before we have to move on to the next string.
34882 unsigned sizeOfNewObj = (unsigned)Align(min_obj_size * 31);
34883 if (str->GetStringLength() > sizeOfNewObj / sizeof(WCHAR))
34885 unsigned sizeToNextObj = (unsigned)Align(size(str));
34886 uint8_t* freeObj = ((uint8_t*) str) + sizeToNextObj - sizeOfNewObj;
34887 pGenGCHeap->make_unused_array (freeObj, sizeOfNewObj);
34888 str->SetStringLength(str->GetStringLength() - (sizeOfNewObj / sizeof(WCHAR)));
34892 // Let the string itself become garbage.
34893 // will be realloced next time around
34894 HndAssignHandle(m_StressObjs[m_CurStressObj], 0);
34898 Interlocked::Decrement(&OneAtATime);
34899 #endif // !MULTIPLE_HEAPS
34900 if (IsConcurrentGCEnabled())
34902 int rgen = StressRNG(10);
34904 // gen0:gen1:gen2 distribution: 40:40:20
34907 else if (rgen >= 4)
34912 GarbageCollectTry (rgen, FALSE, collection_gcstress);
34916 GarbageCollect(max_generation, FALSE, collection_gcstress);
34921 UNREFERENCED_PARAMETER(context);
34923 #endif // defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
34927 #ifdef FEATURE_PREMORTEM_FINALIZATION
34928 #define REGISTER_FOR_FINALIZATION(_object, _size) \
34929 hp->finalize_queue->RegisterForFinalization (0, (_object), (_size))
34930 #else // FEATURE_PREMORTEM_FINALIZATION
34931 #define REGISTER_FOR_FINALIZATION(_object, _size) true
34932 #endif // FEATURE_PREMORTEM_FINALIZATION
34934 #define CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(_object, _size, _register) do { \
34935 if ((_object) == NULL || ((_register) && !REGISTER_FOR_FINALIZATION(_object, _size))) \
34937 STRESS_LOG_OOM_STACK(_size); \
34943 // Small Object Allocator
34946 // Allocate small object with an alignment requirement of 8-bytes.
34948 GCHeap::AllocAlign8(gc_alloc_context* ctx, size_t size, uint32_t flags )
34950 #ifdef FEATURE_64BIT_ALIGNMENT
34956 alloc_context* acontext = static_cast<alloc_context*>(ctx);
34958 #ifdef MULTIPLE_HEAPS
34959 if (acontext->get_alloc_heap() == 0)
34961 AssignHeap (acontext);
34962 assert (acontext->get_alloc_heap());
34965 gc_heap* hp = acontext->get_alloc_heap()->pGenGCHeap;
34967 gc_heap* hp = pGenGCHeap;
34968 #endif //MULTIPLE_HEAPS
34970 return AllocAlign8Common(hp, acontext, size, flags);
34972 UNREFERENCED_PARAMETER(ctx);
34973 UNREFERENCED_PARAMETER(size);
34974 UNREFERENCED_PARAMETER(flags);
34975 assert(!"should not call GCHeap::AllocAlign8 without FEATURE_64BIT_ALIGNMENT defined!");
34977 #endif //FEATURE_64BIT_ALIGNMENT
34980 // Common code used by both variants of AllocAlign8 above.
34982 GCHeap::AllocAlign8Common(void* _hp, alloc_context* acontext, size_t size, uint32_t flags)
34984 #ifdef FEATURE_64BIT_ALIGNMENT
34990 gc_heap* hp = (gc_heap*)_hp;
34994 Object* newAlloc = NULL;
34997 #ifdef COUNT_CYCLES
34998 AllocStart = GetCycleCount32();
35000 #elif defined(ENABLE_INSTRUMENTATION)
35001 unsigned AllocStart = GetInstLogTime();
35003 #endif //COUNT_CYCLES
35006 if (size < loh_size_threshold)
35012 // Depending on where in the object the payload requiring 8-byte alignment resides we might have to
35013 // align the object header on an 8-byte boundary or midway between two such boundaries. The unaligned
35014 // case is indicated to the GC via the GC_ALLOC_ALIGN8_BIAS flag.
35015 size_t desiredAlignment = (flags & GC_ALLOC_ALIGN8_BIAS) ? 4 : 0;
35017 // Retrieve the address of the next allocation from the context (note that we're inside the alloc
35018 // lock at this point).
35019 uint8_t* result = acontext->alloc_ptr;
35021 // Will an allocation at this point yield the correct alignment and fit into the remainder of the
35023 if ((((size_t)result & 7) == desiredAlignment) && ((result + size) <= acontext->alloc_limit))
35025 // Yes, we can just go ahead and make the allocation.
35026 newAlloc = (Object*) hp->allocate (size, acontext);
35027 ASSERT(((size_t)newAlloc & 7) == desiredAlignment);
35031 // No, either the next available address is not aligned in the way we require it or there's
35032 // not enough space to allocate an object of the required size. In both cases we allocate a
35033 // padding object (marked as a free object). This object's size is such that it will reverse
35034 // the alignment of the next header (asserted below).
35036 // We allocate both together then decide based on the result whether we'll format the space as
35037 // free object + real object or real object + free object.
35038 ASSERT((Align(min_obj_size) & 7) == 4);
35039 CObjectHeader *freeobj = (CObjectHeader*) hp->allocate (Align(size) + Align(min_obj_size), acontext);
35042 if (((size_t)freeobj & 7) == desiredAlignment)
35044 // New allocation has desired alignment, return this one and place the free object at the
35045 // end of the allocated space.
35046 newAlloc = (Object*)freeobj;
35047 freeobj = (CObjectHeader*)((uint8_t*)freeobj + Align(size));
35051 // New allocation is still mis-aligned, format the initial space as a free object and the
35052 // rest of the space should be correctly aligned for the real object.
35053 newAlloc = (Object*)((uint8_t*)freeobj + Align(min_obj_size));
35054 ASSERT(((size_t)newAlloc & 7) == desiredAlignment);
35056 freeobj->SetFree(min_obj_size);
35062 // The LOH always guarantees at least 8-byte alignment, regardless of platform. Moreover it doesn't
35063 // support mis-aligned object headers so we can't support biased headers as above. Luckily for us
35064 // we've managed to arrange things so the only case where we see a bias is for boxed value types and
35065 // these can never get large enough to be allocated on the LOH.
35066 ASSERT(65536 < loh_size_threshold);
35067 ASSERT((flags & GC_ALLOC_ALIGN8_BIAS) == 0);
35069 alloc_context* acontext = generation_alloc_context (hp->generation_of (max_generation+1));
35071 newAlloc = (Object*) hp->allocate_large_object (size, acontext->alloc_bytes_loh);
35072 ASSERT(((size_t)newAlloc & 7) == 0);
35075 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
35078 #ifdef COUNT_CYCLES
35079 finish = GetCycleCount32();
35080 #elif defined(ENABLE_INSTRUMENTATION)
35081 finish = GetInstLogTime();
35082 #endif //COUNT_CYCLES
35083 AllocDuration += finish - AllocStart;
35088 UNREFERENCED_PARAMETER(_hp);
35089 UNREFERENCED_PARAMETER(acontext);
35090 UNREFERENCED_PARAMETER(size);
35091 UNREFERENCED_PARAMETER(flags);
35092 assert(!"Should not call GCHeap::AllocAlign8Common without FEATURE_64BIT_ALIGNMENT defined!");
35094 #endif // FEATURE_64BIT_ALIGNMENT
35098 GCHeap::AllocLHeap( size_t size, uint32_t flags REQD_ALIGN_DCL)
35107 Object* newAlloc = NULL;
35110 #ifdef COUNT_CYCLES
35111 AllocStart = GetCycleCount32();
35113 #elif defined(ENABLE_INSTRUMENTATION)
35114 unsigned AllocStart = GetInstLogTime();
35116 #endif //COUNT_CYCLES
35119 #ifdef MULTIPLE_HEAPS
35120 //take the first heap....
35121 gc_heap* hp = gc_heap::g_heaps[0];
35123 gc_heap* hp = pGenGCHeap;
35125 // prefix complains about us dereferencing hp in wks build even though we only access static members
35126 // this way. not sure how to shut it up except for this ugly workaround:
35127 PREFIX_ASSUME(hp != NULL);
35129 #endif //MULTIPLE_HEAPS
35131 alloc_context* acontext = generation_alloc_context (hp->generation_of (max_generation+1));
35133 newAlloc = (Object*) hp->allocate_large_object (size + ComputeMaxStructAlignPadLarge(requiredAlignment), acontext->alloc_bytes_loh);
35134 #ifdef FEATURE_STRUCTALIGN
35135 newAlloc = (Object*) hp->pad_for_alignment_large ((uint8_t*) newAlloc, requiredAlignment, size);
35136 #endif // FEATURE_STRUCTALIGN
35137 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
35140 #ifdef COUNT_CYCLES
35141 finish = GetCycleCount32();
35142 #elif defined(ENABLE_INSTRUMENTATION)
35143 finish = GetInstLogTime();
35144 #endif //COUNT_CYCLES
35145 AllocDuration += finish - AllocStart;
35152 GCHeap::Alloc(gc_alloc_context* context, size_t size, uint32_t flags REQD_ALIGN_DCL)
35161 Object* newAlloc = NULL;
35162 alloc_context* acontext = static_cast<alloc_context*>(context);
35165 #ifdef COUNT_CYCLES
35166 AllocStart = GetCycleCount32();
35168 #elif defined(ENABLE_INSTRUMENTATION)
35169 unsigned AllocStart = GetInstLogTime();
35171 #endif //COUNT_CYCLES
35174 #ifdef MULTIPLE_HEAPS
35175 if (acontext->get_alloc_heap() == 0)
35177 AssignHeap (acontext);
35178 assert (acontext->get_alloc_heap());
35180 #endif //MULTIPLE_HEAPS
35182 #ifdef MULTIPLE_HEAPS
35183 gc_heap* hp = acontext->get_alloc_heap()->pGenGCHeap;
35185 gc_heap* hp = pGenGCHeap;
35187 // prefix complains about us dereferencing hp in wks build even though we only access static members
35188 // this way. not sure how to shut it up except for this ugly workaround:
35189 PREFIX_ASSUME(hp != NULL);
35191 #endif //MULTIPLE_HEAPS
35193 if (size < loh_size_threshold)
35199 newAlloc = (Object*) hp->allocate (size + ComputeMaxStructAlignPad(requiredAlignment), acontext);
35200 #ifdef FEATURE_STRUCTALIGN
35201 newAlloc = (Object*) hp->pad_for_alignment ((uint8_t*) newAlloc, requiredAlignment, size, acontext);
35202 #endif // FEATURE_STRUCTALIGN
35203 // ASSERT (newAlloc);
35207 newAlloc = (Object*) hp->allocate_large_object (size + ComputeMaxStructAlignPadLarge(requiredAlignment), acontext->alloc_bytes_loh);
35208 #ifdef FEATURE_STRUCTALIGN
35209 newAlloc = (Object*) hp->pad_for_alignment_large ((uint8_t*) newAlloc, requiredAlignment, size);
35210 #endif // FEATURE_STRUCTALIGN
35213 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
35216 #ifdef COUNT_CYCLES
35217 finish = GetCycleCount32();
35218 #elif defined(ENABLE_INSTRUMENTATION)
35219 finish = GetInstLogTime();
35220 #endif //COUNT_CYCLES
35221 AllocDuration += finish - AllocStart;
35228 GCHeap::FixAllocContext (gc_alloc_context* context, void* arg, void *heap)
35230 alloc_context* acontext = static_cast<alloc_context*>(context);
35231 #ifdef MULTIPLE_HEAPS
35234 acontext->alloc_count = 0;
35236 uint8_t * alloc_ptr = acontext->alloc_ptr;
35241 // The acontext->alloc_heap can be out of sync with the ptrs because
35242 // of heap re-assignment in allocate
35243 gc_heap* hp = gc_heap::heap_of (alloc_ptr);
35245 gc_heap* hp = pGenGCHeap;
35246 #endif //MULTIPLE_HEAPS
35248 if (heap == NULL || heap == hp)
35250 hp->fix_allocation_context (acontext, ((arg != 0)? TRUE : FALSE),
35251 get_alignment_constant(TRUE));
35256 GCHeap::GetContainingObject (void *pInteriorPtr, bool fCollectedGenOnly)
35258 uint8_t *o = (uint8_t*)pInteriorPtr;
35260 gc_heap* hp = gc_heap::heap_of (o);
35262 uint8_t* lowest = (fCollectedGenOnly ? hp->gc_low : hp->lowest_address);
35263 uint8_t* highest = (fCollectedGenOnly ? hp->gc_high : hp->highest_address);
35265 if (o >= lowest && o < highest)
35267 o = hp->find_object (o, lowest);
35274 return (Object *)o;
35277 BOOL should_collect_optimized (dynamic_data* dd, BOOL low_memory_p)
35279 if (dd_new_allocation (dd) < 0)
35284 if (((float)(dd_new_allocation (dd)) / (float)dd_desired_allocation (dd)) < (low_memory_p ? 0.7 : 0.3))
35292 //----------------------------------------------------------------------------
35293 // #GarbageCollector
35295 // API to ensure that a complete new garbage collection takes place
35298 GCHeap::GarbageCollect (int generation, bool low_memory_p, int mode)
35303 size_t total_allocated = 0;
35304 size_t total_desired = 0;
35305 #ifdef MULTIPLE_HEAPS
35307 for (hn = 0; hn < gc_heap::n_heaps; hn++)
35309 gc_heap* hp = gc_heap::g_heaps [hn];
35310 total_desired += dd_desired_allocation (hp->dynamic_data_of (0));
35311 total_allocated += dd_desired_allocation (hp->dynamic_data_of (0))-
35312 dd_new_allocation (hp->dynamic_data_of (0));
35315 gc_heap* hp = pGenGCHeap;
35316 total_desired = dd_desired_allocation (hp->dynamic_data_of (0));
35317 total_allocated = dd_desired_allocation (hp->dynamic_data_of (0))-
35318 dd_new_allocation (hp->dynamic_data_of (0));
35319 #endif //MULTIPLE_HEAPS
35321 if ((total_desired > gc_heap::mem_one_percent) && (total_allocated < gc_heap::mem_one_percent))
35323 dprintf (2, ("Async low mem but we've only allocated %d (< 10%% of physical mem) out of %d, returning",
35324 total_allocated, total_desired));
35331 #ifdef MULTIPLE_HEAPS
35332 gc_heap* hpt = gc_heap::g_heaps[0];
35335 #endif //MULTIPLE_HEAPS
35337 generation = (generation < 0) ? max_generation : min (generation, max_generation);
35338 dynamic_data* dd = hpt->dynamic_data_of (generation);
35340 #ifdef BACKGROUND_GC
35341 if (recursive_gc_sync::background_running_p())
35343 if ((mode == collection_optimized) || (mode & collection_non_blocking))
35347 if (mode & collection_blocking)
35349 pGenGCHeap->background_gc_wait();
35350 if (mode & collection_optimized)
35356 #endif //BACKGROUND_GC
35358 if (mode & collection_optimized)
35360 if (pGenGCHeap->gc_started)
35366 BOOL should_collect = FALSE;
35367 BOOL should_check_loh = (generation == max_generation);
35368 #ifdef MULTIPLE_HEAPS
35369 for (int i = 0; i < gc_heap::n_heaps; i++)
35371 dynamic_data* dd1 = gc_heap::g_heaps [i]->dynamic_data_of (generation);
35372 dynamic_data* dd2 = (should_check_loh ?
35373 (gc_heap::g_heaps [i]->dynamic_data_of (max_generation + 1)) :
35376 if (should_collect_optimized (dd1, low_memory_p))
35378 should_collect = TRUE;
35381 if (dd2 && should_collect_optimized (dd2, low_memory_p))
35383 should_collect = TRUE;
35388 should_collect = should_collect_optimized (dd, low_memory_p);
35389 if (!should_collect && should_check_loh)
35392 should_collect_optimized (hpt->dynamic_data_of (max_generation + 1), low_memory_p);
35394 #endif //MULTIPLE_HEAPS
35395 if (!should_collect)
35402 size_t CollectionCountAtEntry = dd_collection_count (dd);
35403 size_t BlockingCollectionCountAtEntry = gc_heap::full_gc_counts[gc_type_blocking];
35404 size_t CurrentCollectionCount = 0;
35408 CurrentCollectionCount = GarbageCollectTry(generation, low_memory_p, mode);
35410 if ((mode & collection_blocking) &&
35411 (generation == max_generation) &&
35412 (gc_heap::full_gc_counts[gc_type_blocking] == BlockingCollectionCountAtEntry))
35414 #ifdef BACKGROUND_GC
35415 if (recursive_gc_sync::background_running_p())
35417 pGenGCHeap->background_gc_wait();
35419 #endif //BACKGROUND_GC
35424 if (CollectionCountAtEntry == CurrentCollectionCount)
35433 GCHeap::GarbageCollectTry (int generation, BOOL low_memory_p, int mode)
35435 int gen = (generation < 0) ?
35436 max_generation : min (generation, max_generation);
35438 gc_reason reason = reason_empty;
35442 if (mode & collection_blocking)
35444 reason = reason_lowmemory_blocking;
35448 reason = reason_lowmemory;
35453 reason = reason_induced;
35456 if (reason == reason_induced)
35458 if (mode & collection_compacting)
35460 reason = reason_induced_compacting;
35462 else if (mode & collection_non_blocking)
35464 reason = reason_induced_noforce;
35467 else if (mode & collection_gcstress)
35469 reason = reason_gcstress;
35474 return GarbageCollectGeneration (gen, reason);
35477 void gc_heap::do_pre_gc()
35479 STRESS_LOG_GC_STACK;
35482 STRESS_LOG_GC_START(VolatileLoad(&settings.gc_index),
35483 (uint32_t)settings.condemned_generation,
35484 (uint32_t)settings.reason);
35485 #endif // STRESS_LOG
35487 #ifdef MULTIPLE_HEAPS
35488 gc_heap* hp = g_heaps[0];
35491 #endif //MULTIPLE_HEAPS
35493 #ifdef BACKGROUND_GC
35494 settings.b_state = hp->current_bgc_state;
35495 #endif //BACKGROUND_GC
35498 size_t total_allocated_since_last_gc = get_total_allocated_since_last_gc();
35499 #ifdef BACKGROUND_GC
35500 dprintf (1, ("*GC* %d(gen0:%d)(%d)(alloc: %Id)(%s)(%d)",
35501 VolatileLoad(&settings.gc_index),
35502 dd_collection_count (hp->dynamic_data_of (0)),
35503 settings.condemned_generation,
35504 total_allocated_since_last_gc,
35505 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC")),
35506 settings.b_state));
35508 dprintf (1, ("*GC* %d(gen0:%d)(%d)(alloc: %Id)",
35509 VolatileLoad(&settings.gc_index),
35510 dd_collection_count(hp->dynamic_data_of(0)),
35511 settings.condemned_generation,
35512 total_allocated_since_last_gc));
35513 #endif //BACKGROUND_GC
35515 if (heap_hard_limit)
35517 size_t total_heap_committed = get_total_committed_size();
35518 size_t total_heap_committed_recorded = current_total_committed - current_total_committed_bookkeeping;
35519 dprintf (1, ("(%d)GC commit BEG #%Id: %Id (recorded: %Id)",
35520 settings.condemned_generation,
35521 (size_t)settings.gc_index, total_heap_committed, total_heap_committed_recorded));
35525 // TODO: this can happen...it's because of the way we are calling
35526 // do_pre_gc, will fix later.
35527 //if (last_gc_index > VolatileLoad(&settings.gc_index))
35529 // FATAL_GC_ERROR();
35532 last_gc_index = VolatileLoad(&settings.gc_index);
35533 GCHeap::UpdatePreGCCounters();
35535 if (settings.concurrent)
35537 #ifdef BACKGROUND_GC
35538 full_gc_counts[gc_type_background]++;
35539 #if defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
35540 GCHeap::gc_stress_fgcs_in_bgc = 0;
35541 #endif // STRESS_HEAP && !FEATURE_REDHAWK
35542 #endif // BACKGROUND_GC
35546 if (settings.condemned_generation == max_generation)
35548 full_gc_counts[gc_type_blocking]++;
35552 #ifdef BACKGROUND_GC
35553 if (settings.background_p)
35555 ephemeral_fgc_counts[settings.condemned_generation]++;
35557 #endif //BACKGROUND_GC
35561 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
35562 if (g_fEnableAppDomainMonitoring)
35564 GCToEEInterface::ResetTotalSurvivedBytes();
35566 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
35569 #ifdef GC_CONFIG_DRIVEN
35570 void gc_heap::record_interesting_info_per_heap()
35572 // datapoints are always from the last blocking GC so don't record again
35574 if (!(settings.concurrent))
35576 for (int i = 0; i < max_idp_count; i++)
35578 interesting_data_per_heap[i] += interesting_data_per_gc[i];
35582 int compact_reason = get_gc_data_per_heap()->get_mechanism (gc_heap_compact);
35583 if (compact_reason >= 0)
35584 (compact_reasons_per_heap[compact_reason])++;
35585 int expand_mechanism = get_gc_data_per_heap()->get_mechanism (gc_heap_expand);
35586 if (expand_mechanism >= 0)
35587 (expand_mechanisms_per_heap[expand_mechanism])++;
35589 for (int i = 0; i < max_gc_mechanism_bits_count; i++)
35591 if (get_gc_data_per_heap()->is_mechanism_bit_set ((gc_mechanism_bit_per_heap)i))
35592 (interesting_mechanism_bits_per_heap[i])++;
35595 // h# | GC | gen | C | EX | NF | BF | ML | DM || PreS | PostS | Merge | Conv | Pre | Post | PrPo | PreP | PostP |
35596 cprintf (("%2d | %6d | %1d | %1s | %2s | %2s | %2s | %2s | %2s || %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id |",
35598 (size_t)settings.gc_index,
35599 settings.condemned_generation,
35600 // TEMP - I am just doing this for wks GC 'cuase I wanna see the pattern of doing C/S GCs.
35601 (settings.compaction ? (((compact_reason >= 0) && gc_heap_compact_reason_mandatory_p[compact_reason]) ? "M" : "W") : ""), // compaction
35602 ((expand_mechanism >= 0)? "X" : ""), // EX
35603 ((expand_mechanism == expand_reuse_normal) ? "X" : ""), // NF
35604 ((expand_mechanism == expand_reuse_bestfit) ? "X" : ""), // BF
35605 (get_gc_data_per_heap()->is_mechanism_bit_set (gc_mark_list_bit) ? "X" : ""), // ML
35606 (get_gc_data_per_heap()->is_mechanism_bit_set (gc_demotion_bit) ? "X" : ""), // DM
35607 interesting_data_per_gc[idp_pre_short],
35608 interesting_data_per_gc[idp_post_short],
35609 interesting_data_per_gc[idp_merged_pin],
35610 interesting_data_per_gc[idp_converted_pin],
35611 interesting_data_per_gc[idp_pre_pin],
35612 interesting_data_per_gc[idp_post_pin],
35613 interesting_data_per_gc[idp_pre_and_post_pin],
35614 interesting_data_per_gc[idp_pre_short_padded],
35615 interesting_data_per_gc[idp_post_short_padded]));
35618 void gc_heap::record_global_mechanisms()
35620 for (int i = 0; i < max_global_mechanisms_count; i++)
35622 if (gc_data_global.get_mechanism_p ((gc_global_mechanism_p)i))
35624 ::record_global_mechanism (i);
35629 BOOL gc_heap::should_do_sweeping_gc (BOOL compact_p)
35631 if (!compact_ratio)
35632 return (!compact_p);
35634 size_t compact_count = compact_or_sweep_gcs[0];
35635 size_t sweep_count = compact_or_sweep_gcs[1];
35637 size_t total_count = compact_count + sweep_count;
35638 BOOL should_compact = compact_p;
35639 if (total_count > 3)
35643 int temp_ratio = (int)((compact_count + 1) * 100 / (total_count + 1));
35644 if (temp_ratio > compact_ratio)
35646 // cprintf (("compact would be: %d, total_count: %d, ratio would be %d%% > target\n",
35647 // (compact_count + 1), (total_count + 1), temp_ratio));
35648 should_compact = FALSE;
35653 int temp_ratio = (int)((sweep_count + 1) * 100 / (total_count + 1));
35654 if (temp_ratio > (100 - compact_ratio))
35656 // cprintf (("sweep would be: %d, total_count: %d, ratio would be %d%% > target\n",
35657 // (sweep_count + 1), (total_count + 1), temp_ratio));
35658 should_compact = TRUE;
35663 return !should_compact;
35665 #endif //GC_CONFIG_DRIVEN
35667 bool gc_heap::is_pm_ratio_exceeded()
35669 size_t maxgen_frag = 0;
35670 size_t maxgen_size = 0;
35671 size_t total_heap_size = get_total_heap_size();
35673 #ifdef MULTIPLE_HEAPS
35674 for (int i = 0; i < gc_heap::n_heaps; i++)
35676 gc_heap* hp = gc_heap::g_heaps[i];
35677 #else //MULTIPLE_HEAPS
35679 gc_heap* hp = pGenGCHeap;
35680 #endif //MULTIPLE_HEAPS
35682 maxgen_frag += dd_fragmentation (hp->dynamic_data_of (max_generation));
35683 maxgen_size += hp->generation_size (max_generation);
35686 double maxgen_ratio = (double)maxgen_size / (double)total_heap_size;
35687 double maxgen_frag_ratio = (double)maxgen_frag / (double)maxgen_size;
35688 dprintf (GTC_LOG, ("maxgen %Id(%d%% total heap), frag: %Id (%d%% maxgen)",
35689 maxgen_size, (int)(maxgen_ratio * 100.0),
35690 maxgen_frag, (int)(maxgen_frag_ratio * 100.0)));
35692 bool maxgen_highfrag_p = ((maxgen_ratio > 0.5) && (maxgen_frag_ratio > 0.1));
35694 // We need to adjust elevation here because if there's enough fragmentation it's not
35696 if (maxgen_highfrag_p)
35698 settings.should_lock_elevation = FALSE;
35699 dprintf (GTC_LOG, ("high frag gen2, turn off elevation"));
35702 return maxgen_highfrag_p;
35705 void gc_heap::do_post_gc()
35707 if (!settings.concurrent)
35713 #ifdef COUNT_CYCLES
35714 AllocStart = GetCycleCount32();
35716 AllocStart = clock();
35717 #endif //COUNT_CYCLES
35720 #ifdef MULTIPLE_HEAPS
35721 gc_heap* hp = g_heaps[0];
35724 #endif //MULTIPLE_HEAPS
35726 GCToEEInterface::GcDone(settings.condemned_generation);
35728 GCToEEInterface::DiagGCEnd(VolatileLoad(&settings.gc_index),
35729 (uint32_t)settings.condemned_generation,
35730 (uint32_t)settings.reason,
35731 !!settings.concurrent);
35733 //dprintf (1, (" ****end of Garbage Collection**** %d(gen0:%d)(%d)",
35734 dprintf (1, ("*EGC* %d(gen0:%d)(%d)(%s)",
35735 VolatileLoad(&settings.gc_index),
35736 dd_collection_count(hp->dynamic_data_of(0)),
35737 settings.condemned_generation,
35738 (settings.concurrent ? "BGC" : "GC")));
35740 if (settings.exit_memory_load != 0)
35741 last_gc_memory_load = settings.exit_memory_load;
35742 else if (settings.entry_memory_load != 0)
35743 last_gc_memory_load = settings.entry_memory_load;
35745 last_gc_heap_size = get_total_heap_size();
35746 last_gc_fragmentation = get_total_fragmentation();
35749 if (heap_hard_limit)
35751 size_t total_heap_committed = get_total_committed_size();
35752 size_t total_heap_committed_recorded = current_total_committed - current_total_committed_bookkeeping;
35753 dprintf (1, ("(%d)GC commit END #%Id: %Id (recorded: %Id), heap %Id, frag: %Id",
35754 settings.condemned_generation,
35755 (size_t)settings.gc_index, total_heap_committed, total_heap_committed_recorded,
35756 last_gc_heap_size, last_gc_fragmentation));
35760 // Note we only do this at the end of full blocking GCs because we do not want
35761 // to turn on this provisional mode during the middle of a BGC.
35762 if ((settings.condemned_generation == max_generation) && (!settings.concurrent))
35766 size_t full_compacting_gc_count = full_gc_counts[gc_type_compacting];
35767 if (provisional_mode_triggered)
35769 uint64_t r = gc_rand::get_rand(10);
35770 if ((full_compacting_gc_count - provisional_triggered_gc_count) >= r)
35772 provisional_mode_triggered = false;
35773 provisional_off_gc_count = full_compacting_gc_count;
35774 dprintf (GTC_LOG, ("%Id NGC2s when turned on, %Id NGCs since(%Id)",
35775 provisional_triggered_gc_count, (full_compacting_gc_count - provisional_triggered_gc_count),
35776 num_provisional_triggered));
35781 uint64_t r = gc_rand::get_rand(5);
35782 if ((full_compacting_gc_count - provisional_off_gc_count) >= r)
35784 provisional_mode_triggered = true;
35785 provisional_triggered_gc_count = full_compacting_gc_count;
35786 num_provisional_triggered++;
35787 dprintf (GTC_LOG, ("%Id NGC2s when turned off, %Id NGCs since(%Id)",
35788 provisional_off_gc_count, (full_compacting_gc_count - provisional_off_gc_count),
35789 num_provisional_triggered));
35795 if (provisional_mode_triggered)
35797 if ((settings.entry_memory_load < high_memory_load_th) ||
35798 !is_pm_ratio_exceeded())
35800 dprintf (GTC_LOG, ("turning off PM"));
35801 provisional_mode_triggered = false;
35804 else if ((settings.entry_memory_load >= high_memory_load_th) && is_pm_ratio_exceeded())
35806 dprintf (GTC_LOG, ("highmem && highfrag - turning on PM"));
35807 provisional_mode_triggered = true;
35808 num_provisional_triggered++;
35813 GCHeap::UpdatePostGCCounters();
35814 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
35815 //if (g_fEnableARM)
35817 // SystemDomain::GetADSurvivedBytes();
35819 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
35822 STRESS_LOG_GC_END(VolatileLoad(&settings.gc_index),
35823 (uint32_t)settings.condemned_generation,
35824 (uint32_t)settings.reason);
35825 #endif // STRESS_LOG
35827 #ifdef GC_CONFIG_DRIVEN
35828 if (!settings.concurrent)
35830 if (settings.compaction)
35831 (compact_or_sweep_gcs[0])++;
35833 (compact_or_sweep_gcs[1])++;
35836 #ifdef MULTIPLE_HEAPS
35837 for (int i = 0; i < n_heaps; i++)
35838 g_heaps[i]->record_interesting_info_per_heap();
35840 record_interesting_info_per_heap();
35841 #endif //MULTIPLE_HEAPS
35842 record_global_mechanisms();
35843 #endif //GC_CONFIG_DRIVEN
35846 unsigned GCHeap::GetGcCount()
35848 return (unsigned int)VolatileLoad(&pGenGCHeap->settings.gc_index);
35852 GCHeap::GarbageCollectGeneration (unsigned int gen, gc_reason reason)
35854 dprintf (2, ("triggered a GC!"));
35856 #ifdef MULTIPLE_HEAPS
35857 gc_heap* hpt = gc_heap::g_heaps[0];
35860 #endif //MULTIPLE_HEAPS
35861 bool cooperative_mode = true;
35862 dynamic_data* dd = hpt->dynamic_data_of (gen);
35863 size_t localCount = dd_collection_count (dd);
35865 enter_spin_lock (&gc_heap::gc_lock);
35866 dprintf (SPINLOCK_LOG, ("GC Egc"));
35867 ASSERT_HOLDING_SPIN_LOCK(&gc_heap::gc_lock);
35869 //don't trigger another GC if one was already in progress
35870 //while waiting for the lock
35872 size_t col_count = dd_collection_count (dd);
35874 if (localCount != col_count)
35876 #ifdef SYNCHRONIZATION_STATS
35877 gc_lock_contended++;
35878 #endif //SYNCHRONIZATION_STATS
35879 dprintf (SPINLOCK_LOG, ("no need GC Lgc"));
35880 leave_spin_lock (&gc_heap::gc_lock);
35882 // We don't need to release msl here 'cause this means a GC
35883 // has happened and would have release all msl's.
35888 #ifdef COUNT_CYCLES
35889 int gc_start = GetCycleCount32();
35890 #endif //COUNT_CYCLES
35893 #ifdef COUNT_CYCLES
35894 AllocDuration += GetCycleCount32() - AllocStart;
35896 AllocDuration += clock() - AllocStart;
35897 #endif //COUNT_CYCLES
35900 gc_heap::g_low_memory_status = (reason == reason_lowmemory) ||
35901 (reason == reason_lowmemory_blocking) ||
35902 (gc_heap::latency_level == latency_level_memory_footprint);
35904 gc_trigger_reason = reason;
35906 #ifdef MULTIPLE_HEAPS
35907 for (int i = 0; i < gc_heap::n_heaps; i++)
35909 gc_heap::g_heaps[i]->reset_gc_done();
35912 gc_heap::reset_gc_done();
35913 #endif //MULTIPLE_HEAPS
35915 gc_heap::gc_started = TRUE;
35918 init_sync_log_stats();
35920 #ifndef MULTIPLE_HEAPS
35921 cooperative_mode = gc_heap::enable_preemptive ();
35923 dprintf (2, ("Suspending EE"));
35924 BEGIN_TIMING(suspend_ee_during_log);
35925 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC);
35926 END_TIMING(suspend_ee_during_log);
35927 gc_heap::proceed_with_gc_p = gc_heap::should_proceed_with_gc();
35928 gc_heap::disable_preemptive (cooperative_mode);
35929 if (gc_heap::proceed_with_gc_p)
35930 pGenGCHeap->settings.init_mechanisms();
35932 gc_heap::update_collection_counts_for_no_gc();
35934 #endif //!MULTIPLE_HEAPS
35937 // MAP_EVENT_MONITORS(EE_MONITOR_GARBAGE_COLLECTIONS, NotifyEvent(EE_EVENT_TYPE_GC_STARTED, 0));
35940 #ifdef COUNT_CYCLES
35943 start = GetCycleCount32();
35948 #endif //COUNT_CYCLES
35949 PromotedObjectCount = 0;
35952 unsigned int condemned_generation_number = gen;
35954 // We want to get a stack from the user thread that triggered the GC
35955 // instead of on the GC thread which is the case for Server GC.
35956 // But we are doing it for Workstation GC as well to be uniform.
35957 FIRE_EVENT(GCTriggered, static_cast<uint32_t>(reason));
35959 #ifdef MULTIPLE_HEAPS
35960 GcCondemnedGeneration = condemned_generation_number;
35962 cooperative_mode = gc_heap::enable_preemptive ();
35964 BEGIN_TIMING(gc_during_log);
35965 gc_heap::ee_suspend_event.Set();
35966 gc_heap::wait_for_gc_done();
35967 END_TIMING(gc_during_log);
35969 gc_heap::disable_preemptive (cooperative_mode);
35971 condemned_generation_number = GcCondemnedGeneration;
35973 if (gc_heap::proceed_with_gc_p)
35975 BEGIN_TIMING(gc_during_log);
35976 pGenGCHeap->garbage_collect (condemned_generation_number);
35977 if (gc_heap::pm_trigger_full_gc)
35979 pGenGCHeap->garbage_collect_pm_full_gc();
35981 END_TIMING(gc_during_log);
35983 #endif //MULTIPLE_HEAPS
35986 #ifdef COUNT_CYCLES
35987 finish = GetCycleCount32();
35990 #endif //COUNT_CYCLES
35991 GcDuration += finish - start;
35993 ("<GC# %d> Condemned: %d, Duration: %d, total: %d Alloc Avg: %d, Small Objects:%d Large Objects:%d",
35994 VolatileLoad(&pGenGCHeap->settings.gc_index), condemned_generation_number,
35995 finish - start, GcDuration,
35996 AllocCount ? (AllocDuration / AllocCount) : 0,
35997 AllocSmallCount, AllocBigCount));
36002 #ifdef BACKGROUND_GC
36003 // We are deciding whether we should fire the alloc wait end event here
36004 // because in begin_foreground we could be calling end_foreground
36005 // if we need to retry.
36006 if (gc_heap::alloc_wait_event_p)
36008 hpt->fire_alloc_wait_event_end (awr_fgc_wait_for_bgc);
36009 gc_heap::alloc_wait_event_p = FALSE;
36011 #endif //BACKGROUND_GC
36013 #ifndef MULTIPLE_HEAPS
36014 #ifdef BACKGROUND_GC
36015 if (!gc_heap::dont_restart_ee_p)
36017 #endif //BACKGROUND_GC
36018 BEGIN_TIMING(restart_ee_during_log);
36019 GCToEEInterface::RestartEE(TRUE);
36020 END_TIMING(restart_ee_during_log);
36021 #ifdef BACKGROUND_GC
36023 #endif //BACKGROUND_GC
36024 #endif //!MULTIPLE_HEAPS
36026 #ifdef COUNT_CYCLES
36027 printf ("GC: %d Time: %d\n", GcCondemnedGeneration,
36028 GetCycleCount32() - gc_start);
36029 #endif //COUNT_CYCLES
36031 #ifndef MULTIPLE_HEAPS
36032 process_sync_log_stats();
36033 gc_heap::gc_started = FALSE;
36034 gc_heap::set_gc_done();
36035 dprintf (SPINLOCK_LOG, ("GC Lgc"));
36036 leave_spin_lock (&gc_heap::gc_lock);
36037 #endif //!MULTIPLE_HEAPS
36039 #ifdef FEATURE_PREMORTEM_FINALIZATION
36040 GCToEEInterface::EnableFinalization(!pGenGCHeap->settings.concurrent && pGenGCHeap->settings.found_finalizers);
36041 #endif // FEATURE_PREMORTEM_FINALIZATION
36043 return dd_collection_count (dd);
36046 size_t GCHeap::GetTotalBytesInUse ()
36048 #ifdef MULTIPLE_HEAPS
36049 //enumarate all the heaps and get their size.
36050 size_t tot_size = 0;
36051 for (int i = 0; i < gc_heap::n_heaps; i++)
36053 GCHeap* Hp = gc_heap::g_heaps [i]->vm_heap;
36054 tot_size += Hp->ApproxTotalBytesInUse (FALSE);
36058 return ApproxTotalBytesInUse ();
36059 #endif //MULTIPLE_HEAPS
36062 int GCHeap::CollectionCount (int generation, int get_bgc_fgc_count)
36064 if (get_bgc_fgc_count != 0)
36066 #ifdef BACKGROUND_GC
36067 if (generation == max_generation)
36069 return (int)(gc_heap::full_gc_counts[gc_type_background]);
36073 return (int)(gc_heap::ephemeral_fgc_counts[generation]);
36077 #endif //BACKGROUND_GC
36080 #ifdef MULTIPLE_HEAPS
36081 gc_heap* hp = gc_heap::g_heaps [0];
36082 #else //MULTIPLE_HEAPS
36083 gc_heap* hp = pGenGCHeap;
36084 #endif //MULTIPLE_HEAPS
36085 if (generation > max_generation)
36088 return (int)dd_collection_count (hp->dynamic_data_of (generation));
36091 size_t GCHeap::ApproxTotalBytesInUse(BOOL small_heap_only)
36093 size_t totsize = 0;
36095 //ASSERT(InMustComplete());
36096 enter_spin_lock (&pGenGCHeap->gc_lock);
36098 heap_segment* eph_seg = generation_allocation_segment (pGenGCHeap->generation_of (0));
36099 // Get small block heap size info
36100 totsize = (pGenGCHeap->alloc_allocated - heap_segment_mem (eph_seg));
36101 heap_segment* seg1 = generation_start_segment (pGenGCHeap->generation_of (max_generation));
36102 while (seg1 != eph_seg)
36104 totsize += heap_segment_allocated (seg1) -
36105 heap_segment_mem (seg1);
36106 seg1 = heap_segment_next (seg1);
36109 //discount the fragmentation
36110 for (int i = 0; i <= max_generation; i++)
36112 generation* gen = pGenGCHeap->generation_of (i);
36113 totsize -= (generation_free_list_space (gen) + generation_free_obj_space (gen));
36116 if (!small_heap_only)
36118 heap_segment* seg2 = generation_start_segment (pGenGCHeap->generation_of (max_generation+1));
36122 totsize += heap_segment_allocated (seg2) -
36123 heap_segment_mem (seg2);
36124 seg2 = heap_segment_next (seg2);
36127 //discount the fragmentation
36128 generation* loh_gen = pGenGCHeap->generation_of (max_generation+1);
36129 size_t frag = generation_free_list_space (loh_gen) + generation_free_obj_space (loh_gen);
36132 leave_spin_lock (&pGenGCHeap->gc_lock);
36136 #ifdef MULTIPLE_HEAPS
36137 void GCHeap::AssignHeap (alloc_context* acontext)
36139 // Assign heap based on processor
36140 acontext->set_alloc_heap(GetHeap(heap_select::select_heap(acontext, 0)));
36141 acontext->set_home_heap(acontext->get_alloc_heap());
36143 GCHeap* GCHeap::GetHeap (int n)
36145 assert (n < gc_heap::n_heaps);
36146 return gc_heap::g_heaps [n]->vm_heap;
36148 #endif //MULTIPLE_HEAPS
36150 bool GCHeap::IsThreadUsingAllocationContextHeap(gc_alloc_context* context, int thread_number)
36152 alloc_context* acontext = static_cast<alloc_context*>(context);
36153 #ifdef MULTIPLE_HEAPS
36154 return ((acontext->get_home_heap() == GetHeap(thread_number)) ||
36155 ((acontext->get_home_heap() == 0) && (thread_number == 0)));
36157 UNREFERENCED_PARAMETER(acontext);
36158 UNREFERENCED_PARAMETER(thread_number);
36160 #endif //MULTIPLE_HEAPS
36163 // Returns the number of processors required to trigger the use of thread based allocation contexts
36164 int GCHeap::GetNumberOfHeaps ()
36166 #ifdef MULTIPLE_HEAPS
36167 return gc_heap::n_heaps;
36170 #endif //MULTIPLE_HEAPS
36174 in this way we spend extra time cycling through all the heaps while create the handle
36175 it ought to be changed by keeping alloc_context.home_heap as number (equals heap_number)
36177 int GCHeap::GetHomeHeapNumber ()
36179 #ifdef MULTIPLE_HEAPS
36180 gc_alloc_context* ctx = GCToEEInterface::GetAllocContext();
36186 GCHeap *hp = static_cast<alloc_context*>(ctx)->get_home_heap();
36187 return (hp ? hp->pGenGCHeap->heap_number : 0);
36190 #endif //MULTIPLE_HEAPS
36193 unsigned int GCHeap::GetCondemnedGeneration()
36195 return gc_heap::settings.condemned_generation;
36198 void GCHeap::GetMemoryInfo(uint32_t* highMemLoadThreshold,
36199 uint64_t* totalPhysicalMem,
36200 uint32_t* lastRecordedMemLoad,
36201 size_t* lastRecordedHeapSize,
36202 size_t* lastRecordedFragmentation)
36204 *highMemLoadThreshold = gc_heap::high_memory_load_th;
36205 *totalPhysicalMem = gc_heap::total_physical_mem;
36206 *lastRecordedMemLoad = gc_heap::last_gc_memory_load;
36207 *lastRecordedHeapSize = gc_heap::last_gc_heap_size;
36208 *lastRecordedFragmentation = gc_heap::last_gc_fragmentation;
36211 int GCHeap::GetGcLatencyMode()
36213 return (int)(pGenGCHeap->settings.pause_mode);
36216 int GCHeap::SetGcLatencyMode (int newLatencyMode)
36218 if (gc_heap::settings.pause_mode == pause_no_gc)
36219 return (int)set_pause_mode_no_gc;
36221 gc_pause_mode new_mode = (gc_pause_mode)newLatencyMode;
36223 if (new_mode == pause_low_latency)
36225 #ifndef MULTIPLE_HEAPS
36226 pGenGCHeap->settings.pause_mode = new_mode;
36227 #endif //!MULTIPLE_HEAPS
36229 else if (new_mode == pause_sustained_low_latency)
36231 #ifdef BACKGROUND_GC
36232 if (gc_heap::gc_can_use_concurrent)
36234 pGenGCHeap->settings.pause_mode = new_mode;
36236 #endif //BACKGROUND_GC
36240 pGenGCHeap->settings.pause_mode = new_mode;
36243 #ifdef BACKGROUND_GC
36244 if (recursive_gc_sync::background_running_p())
36246 // If we get here, it means we are doing an FGC. If the pause
36247 // mode was altered we will need to save it in the BGC settings.
36248 if (gc_heap::saved_bgc_settings.pause_mode != new_mode)
36250 gc_heap::saved_bgc_settings.pause_mode = new_mode;
36253 #endif //BACKGROUND_GC
36255 return (int)set_pause_mode_success;
36258 int GCHeap::GetLOHCompactionMode()
36260 return pGenGCHeap->loh_compaction_mode;
36263 void GCHeap::SetLOHCompactionMode (int newLOHCompactionyMode)
36265 #ifdef FEATURE_LOH_COMPACTION
36266 pGenGCHeap->loh_compaction_mode = (gc_loh_compaction_mode)newLOHCompactionyMode;
36267 #endif //FEATURE_LOH_COMPACTION
36270 bool GCHeap::RegisterForFullGCNotification(uint32_t gen2Percentage,
36271 uint32_t lohPercentage)
36273 #ifdef MULTIPLE_HEAPS
36274 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36276 gc_heap* hp = gc_heap::g_heaps [hn];
36277 hp->fgn_last_alloc = dd_new_allocation (hp->dynamic_data_of (0));
36279 #else //MULTIPLE_HEAPS
36280 pGenGCHeap->fgn_last_alloc = dd_new_allocation (pGenGCHeap->dynamic_data_of (0));
36281 #endif //MULTIPLE_HEAPS
36283 pGenGCHeap->full_gc_approach_event.Reset();
36284 pGenGCHeap->full_gc_end_event.Reset();
36285 pGenGCHeap->full_gc_approach_event_set = false;
36287 pGenGCHeap->fgn_maxgen_percent = gen2Percentage;
36288 pGenGCHeap->fgn_loh_percent = lohPercentage;
36293 bool GCHeap::CancelFullGCNotification()
36295 pGenGCHeap->fgn_maxgen_percent = 0;
36296 pGenGCHeap->fgn_loh_percent = 0;
36298 pGenGCHeap->full_gc_approach_event.Set();
36299 pGenGCHeap->full_gc_end_event.Set();
36304 int GCHeap::WaitForFullGCApproach(int millisecondsTimeout)
36306 dprintf (2, ("WFGA: Begin wait"));
36307 int result = gc_heap::full_gc_wait (&(pGenGCHeap->full_gc_approach_event), millisecondsTimeout);
36308 dprintf (2, ("WFGA: End wait"));
36312 int GCHeap::WaitForFullGCComplete(int millisecondsTimeout)
36314 dprintf (2, ("WFGE: Begin wait"));
36315 int result = gc_heap::full_gc_wait (&(pGenGCHeap->full_gc_end_event), millisecondsTimeout);
36316 dprintf (2, ("WFGE: End wait"));
36320 int GCHeap::StartNoGCRegion(uint64_t totalSize, bool lohSizeKnown, uint64_t lohSize, bool disallowFullBlockingGC)
36322 NoGCRegionLockHolder lh;
36324 dprintf (1, ("begin no gc called"));
36325 start_no_gc_region_status status = gc_heap::prepare_for_no_gc_region (totalSize, lohSizeKnown, lohSize, disallowFullBlockingGC);
36326 if (status == start_no_gc_success)
36328 GarbageCollect (max_generation);
36329 status = gc_heap::get_start_no_gc_region_status();
36332 if (status != start_no_gc_success)
36333 gc_heap::handle_failure_for_no_gc();
36335 return (int)status;
36338 int GCHeap::EndNoGCRegion()
36340 NoGCRegionLockHolder lh;
36341 return (int)gc_heap::end_no_gc_region();
36344 void GCHeap::PublishObject (uint8_t* Obj)
36346 #ifdef BACKGROUND_GC
36347 gc_heap* hp = gc_heap::heap_of (Obj);
36348 hp->bgc_alloc_lock->loh_alloc_done (Obj);
36349 hp->bgc_untrack_loh_alloc();
36350 #endif //BACKGROUND_GC
36353 // The spec for this one isn't clear. This function
36354 // returns the size that can be allocated without
36355 // triggering a GC of any kind.
36356 size_t GCHeap::ApproxFreeBytes()
36359 //ASSERT(InMustComplete());
36360 enter_spin_lock (&pGenGCHeap->gc_lock);
36362 generation* gen = pGenGCHeap->generation_of (0);
36363 size_t res = generation_allocation_limit (gen) - generation_allocation_pointer (gen);
36365 leave_spin_lock (&pGenGCHeap->gc_lock);
36370 HRESULT GCHeap::GetGcCounters(int gen, gc_counters* counters)
36372 if ((gen < 0) || (gen > max_generation))
36374 #ifdef MULTIPLE_HEAPS
36375 counters->current_size = 0;
36376 counters->promoted_size = 0;
36377 counters->collection_count = 0;
36379 //enumarate all the heaps and get their counters.
36380 for (int i = 0; i < gc_heap::n_heaps; i++)
36382 dynamic_data* dd = gc_heap::g_heaps [i]->dynamic_data_of (gen);
36384 counters->current_size += dd_current_size (dd);
36385 counters->promoted_size += dd_promoted_size (dd);
36387 counters->collection_count += dd_collection_count (dd);
36390 dynamic_data* dd = pGenGCHeap->dynamic_data_of (gen);
36391 counters->current_size = dd_current_size (dd);
36392 counters->promoted_size = dd_promoted_size (dd);
36393 counters->collection_count = dd_collection_count (dd);
36394 #endif //MULTIPLE_HEAPS
36398 // Get the segment size to use, making sure it conforms.
36399 size_t GCHeap::GetValidSegmentSize(bool large_seg)
36401 return (large_seg ? gc_heap::min_loh_segment_size : gc_heap::soh_segment_size);
36404 // Get the max gen0 heap size, making sure it conforms.
36405 size_t gc_heap::get_gen0_min_size()
36407 size_t gen0size = static_cast<size_t>(GCConfig::GetGen0Size());
36408 bool is_config_invalid = ((gen0size == 0) || !g_theGCHeap->IsValidGen0MaxSize(gen0size));
36409 if (is_config_invalid)
36412 // performance data seems to indicate halving the size results
36413 // in optimal perf. Ask for adjusted gen0 size.
36414 gen0size = max(GCToOSInterface::GetCacheSizePerLogicalCpu(FALSE),(256*1024));
36416 // if gen0 size is too large given the available memory, reduce it.
36417 // Get true cache size, as we don't want to reduce below this.
36418 size_t trueSize = max(GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE),(256*1024));
36419 dprintf (1, ("cache: %Id-%Id",
36420 GCToOSInterface::GetCacheSizePerLogicalCpu(FALSE),
36421 GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE)));
36423 int n_heaps = gc_heap::n_heaps;
36425 size_t trueSize = GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE);
36426 gen0size = max((4*trueSize/5),(256*1024));
36427 trueSize = max(trueSize, (256*1024));
36431 dprintf (1, ("gen0size: %Id * %d = %Id, physical mem: %Id / 6 = %Id",
36432 gen0size, n_heaps, (gen0size * n_heaps),
36433 gc_heap::total_physical_mem,
36434 gc_heap::total_physical_mem / 6));
36436 // if the total min GC across heaps will exceed 1/6th of available memory,
36437 // then reduce the min GC size until it either fits or has been reduced to cache size.
36438 while ((gen0size * n_heaps) > (gc_heap::total_physical_mem / 6))
36440 gen0size = gen0size / 2;
36441 if (gen0size <= trueSize)
36443 gen0size = trueSize;
36449 size_t seg_size = gc_heap::soh_segment_size;
36452 // Generation 0 must never be more than 1/2 the segment size.
36453 if (gen0size >= (seg_size / 2))
36454 gen0size = seg_size / 2;
36456 // If the value from config is valid we use it as is without this adjustment.
36457 if (is_config_invalid)
36459 if (heap_hard_limit)
36461 size_t gen0size_seg = seg_size / 8;
36462 if (gen0size >= gen0size_seg)
36464 dprintf (1, ("gen0 limited by seg size %Id->%Id", gen0size, gen0size_seg));
36465 gen0size = gen0size_seg;
36469 gen0size = gen0size / 8 * 5;
36472 gen0size = Align (gen0size);
36477 void GCHeap::SetReservedVMLimit (size_t vmlimit)
36479 gc_heap::reserved_memory_limit = vmlimit;
36482 //versions of same method on each heap
36484 #ifdef FEATURE_PREMORTEM_FINALIZATION
36486 Object* GCHeap::GetNextFinalizableObject()
36489 #ifdef MULTIPLE_HEAPS
36491 //return the first non critical one in the first queue.
36492 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36494 gc_heap* hp = gc_heap::g_heaps [hn];
36495 Object* O = hp->finalize_queue->GetNextFinalizableObject(TRUE);
36499 //return the first non crtitical/critical one in the first queue.
36500 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36502 gc_heap* hp = gc_heap::g_heaps [hn];
36503 Object* O = hp->finalize_queue->GetNextFinalizableObject(FALSE);
36510 #else //MULTIPLE_HEAPS
36511 return pGenGCHeap->finalize_queue->GetNextFinalizableObject();
36512 #endif //MULTIPLE_HEAPS
36516 size_t GCHeap::GetNumberFinalizableObjects()
36518 #ifdef MULTIPLE_HEAPS
36520 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36522 gc_heap* hp = gc_heap::g_heaps [hn];
36523 cnt += hp->finalize_queue->GetNumberFinalizableObjects();
36528 #else //MULTIPLE_HEAPS
36529 return pGenGCHeap->finalize_queue->GetNumberFinalizableObjects();
36530 #endif //MULTIPLE_HEAPS
36533 size_t GCHeap::GetFinalizablePromotedCount()
36535 #ifdef MULTIPLE_HEAPS
36538 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36540 gc_heap* hp = gc_heap::g_heaps [hn];
36541 cnt += hp->finalize_queue->GetPromotedCount();
36545 #else //MULTIPLE_HEAPS
36546 return pGenGCHeap->finalize_queue->GetPromotedCount();
36547 #endif //MULTIPLE_HEAPS
36550 bool GCHeap::FinalizeAppDomain(void *pDomain, bool fRunFinalizers)
36552 #ifdef MULTIPLE_HEAPS
36553 bool foundp = false;
36554 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36556 gc_heap* hp = gc_heap::g_heaps [hn];
36557 if (hp->finalize_queue->FinalizeAppDomain (pDomain, fRunFinalizers))
36562 #else //MULTIPLE_HEAPS
36563 return pGenGCHeap->finalize_queue->FinalizeAppDomain (pDomain, fRunFinalizers);
36564 #endif //MULTIPLE_HEAPS
36567 bool GCHeap::ShouldRestartFinalizerWatchDog()
36569 // This condition was historically used as part of the condition to detect finalizer thread timeouts
36570 return gc_heap::gc_lock.lock != -1;
36573 void GCHeap::SetFinalizeQueueForShutdown(bool fHasLock)
36575 #ifdef MULTIPLE_HEAPS
36576 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36578 gc_heap* hp = gc_heap::g_heaps [hn];
36579 hp->finalize_queue->SetSegForShutDown(fHasLock);
36582 #else //MULTIPLE_HEAPS
36583 pGenGCHeap->finalize_queue->SetSegForShutDown(fHasLock);
36584 #endif //MULTIPLE_HEAPS
36587 //---------------------------------------------------------------------------
36588 // Finalized class tracking
36589 //---------------------------------------------------------------------------
36591 bool GCHeap::RegisterForFinalization (int gen, Object* obj)
36595 if (((((CObjectHeader*)obj)->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN))
36597 //just reset the bit
36598 ((CObjectHeader*)obj)->GetHeader()->ClrBit(BIT_SBLK_FINALIZER_RUN);
36603 gc_heap* hp = gc_heap::heap_of ((uint8_t*)obj);
36604 return hp->finalize_queue->RegisterForFinalization (gen, obj);
36608 void GCHeap::SetFinalizationRun (Object* obj)
36610 ((CObjectHeader*)obj)->GetHeader()->SetBit(BIT_SBLK_FINALIZER_RUN);
36614 //--------------------------------------------------------------------
36616 // Support for finalization
36618 //--------------------------------------------------------------------
36621 unsigned int gen_segment (int gen)
36623 assert (((signed)NUMBERGENERATIONS - gen - 1)>=0);
36624 return (NUMBERGENERATIONS - gen - 1);
36627 bool CFinalize::Initialize()
36634 m_Array = new (nothrow)(Object*[100]);
36639 STRESS_LOG_OOM_STACK(sizeof(Object*[100]));
36640 if (GCConfig::GetBreakOnOOM())
36642 GCToOSInterface::DebugBreak();
36646 m_EndArray = &m_Array[100];
36648 for (int i =0; i < FreeList; i++)
36650 SegQueueLimit (i) = m_Array;
36652 m_PromotedCount = 0;
36655 lockowner_threadid.Clear();
36661 CFinalize::~CFinalize()
36666 size_t CFinalize::GetPromotedCount ()
36668 return m_PromotedCount;
36672 void CFinalize::EnterFinalizeLock()
36674 _ASSERTE(dbgOnly_IsSpecialEEThread() ||
36675 GCToEEInterface::GetThread() == 0 ||
36676 GCToEEInterface::IsPreemptiveGCDisabled());
36679 if (Interlocked::CompareExchange(&lock, 0, -1) >= 0)
36681 unsigned int i = 0;
36684 YieldProcessor(); // indicate to the processor that we are spinning
36686 GCToOSInterface::YieldThread (0);
36688 GCToOSInterface::Sleep (5);
36694 lockowner_threadid.SetToCurrentThread();
36699 void CFinalize::LeaveFinalizeLock()
36701 _ASSERTE(dbgOnly_IsSpecialEEThread() ||
36702 GCToEEInterface::GetThread() == 0 ||
36703 GCToEEInterface::IsPreemptiveGCDisabled());
36706 lockowner_threadid.Clear();
36712 CFinalize::RegisterForFinalization (int gen, Object* obj, size_t size)
36719 EnterFinalizeLock();
36721 unsigned int dest = 0;
36723 if (g_fFinalizerRunOnShutDown)
36725 //no method table available yet,
36726 //put it in the finalizer queue and sort out when
36728 dest = FinalizerListSeg;
36732 dest = gen_segment (gen);
36734 // Adjust boundary for segments so that GC will keep objects alive.
36735 Object*** s_i = &SegQueue (FreeList);
36736 if ((*s_i) == m_EndArray)
36740 LeaveFinalizeLock();
36741 if (method_table(obj) == NULL)
36743 // If the object is uninitialized, a valid size should have been passed.
36744 assert (size >= Align (min_obj_size));
36745 dprintf (3, ("Making unused array [%Ix, %Ix[", (size_t)obj, (size_t)(obj+size)));
36746 ((CObjectHeader*)obj)->SetFree(size);
36748 STRESS_LOG_OOM_STACK(0);
36749 if (GCConfig::GetBreakOnOOM())
36751 GCToOSInterface::DebugBreak();
36756 Object*** end_si = &SegQueueLimit (dest);
36759 //is the segment empty?
36760 if (!(*s_i == *(s_i-1)))
36762 //no, swap the end elements.
36763 *(*s_i) = *(*(s_i-1));
36765 //increment the fill pointer
36767 //go to the next segment.
36769 } while (s_i > end_si);
36771 // We have reached the destination segment
36772 // store the object
36774 // increment the fill pointer
36777 LeaveFinalizeLock();
36783 CFinalize::GetNextFinalizableObject (BOOL only_non_critical)
36787 EnterFinalizeLock();
36790 if (!IsSegEmpty(FinalizerListSeg))
36792 if (g_fFinalizerRunOnShutDown)
36794 obj = *(SegQueueLimit (FinalizerListSeg)-1);
36795 if (method_table(obj)->HasCriticalFinalizer())
36797 MoveItem ((SegQueueLimit (FinalizerListSeg)-1),
36798 FinalizerListSeg, CriticalFinalizerListSeg);
36802 --SegQueueLimit (FinalizerListSeg);
36805 obj = *(--SegQueueLimit (FinalizerListSeg));
36808 else if (!only_non_critical && !IsSegEmpty(CriticalFinalizerListSeg))
36810 //the FinalizerList is empty, we can adjust both
36811 // limit instead of moving the object to the free list
36812 obj = *(--SegQueueLimit (CriticalFinalizerListSeg));
36813 --SegQueueLimit (FinalizerListSeg);
36817 dprintf (3, ("running finalizer for %Ix (mt: %Ix)", obj, method_table (obj)));
36819 LeaveFinalizeLock();
36824 CFinalize::SetSegForShutDown(BOOL fHasLock)
36829 EnterFinalizeLock();
36830 for (i = 0; i <= max_generation; i++)
36832 unsigned int seg = gen_segment (i);
36833 Object** startIndex = SegQueueLimit (seg)-1;
36834 Object** stopIndex = SegQueue (seg);
36835 for (Object** po = startIndex; po >= stopIndex; po--)
36838 if (method_table(obj)->HasCriticalFinalizer())
36840 MoveItem (po, seg, CriticalFinalizerListSeg);
36844 MoveItem (po, seg, FinalizerListSeg);
36849 LeaveFinalizeLock();
36853 CFinalize::DiscardNonCriticalObjects()
36855 //empty the finalization queue
36856 Object** startIndex = SegQueueLimit (FinalizerListSeg)-1;
36857 Object** stopIndex = SegQueue (FinalizerListSeg);
36858 for (Object** po = startIndex; po >= stopIndex; po--)
36860 MoveItem (po, FinalizerListSeg, FreeList);
36865 CFinalize::GetNumberFinalizableObjects()
36867 return SegQueueLimit (FinalizerListSeg) -
36868 (g_fFinalizerRunOnShutDown ? m_Array : SegQueue(FinalizerListSeg));
36872 CFinalize::FinalizeSegForAppDomain (void *pDomain,
36873 BOOL fRunFinalizers,
36876 BOOL finalizedFound = FALSE;
36877 Object** endIndex = SegQueue (Seg);
36878 for (Object** i = SegQueueLimit (Seg)-1; i >= endIndex ;i--)
36880 CObjectHeader* obj = (CObjectHeader*)*i;
36882 // Objects are put into the finalization queue before they are complete (ie their methodtable
36883 // may be null) so we must check that the object we found has a method table before checking
36884 // if it has the index we are looking for. If the methodtable is null, it can't be from the
36885 // unloading domain, so skip it.
36886 if (method_table(obj) == NULL)
36891 // does the EE actually want us to finalize this object?
36892 if (!GCToEEInterface::ShouldFinalizeObjectForUnload(pDomain, obj))
36897 if (!fRunFinalizers || (obj->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN)
36899 //remove the object because we don't want to
36900 //run the finalizer
36901 MoveItem (i, Seg, FreeList);
36902 //Reset the bit so it will be put back on the queue
36903 //if resurrected and re-registered.
36904 obj->GetHeader()->ClrBit (BIT_SBLK_FINALIZER_RUN);
36908 if (method_table(obj)->HasCriticalFinalizer())
36910 finalizedFound = TRUE;
36911 MoveItem (i, Seg, CriticalFinalizerListSeg);
36915 if (GCToEEInterface::AppDomainIsRudeUnload(pDomain))
36917 MoveItem (i, Seg, FreeList);
36921 finalizedFound = TRUE;
36922 MoveItem (i, Seg, FinalizerListSeg);
36928 return finalizedFound;
36932 CFinalize::FinalizeAppDomain (void *pDomain, bool fRunFinalizers)
36934 bool finalizedFound = false;
36936 unsigned int startSeg = gen_segment (max_generation);
36938 EnterFinalizeLock();
36940 for (unsigned int Seg = startSeg; Seg <= gen_segment (0); Seg++)
36942 if (FinalizeSegForAppDomain (pDomain, fRunFinalizers, Seg))
36944 finalizedFound = true;
36948 LeaveFinalizeLock();
36950 return finalizedFound;
36954 CFinalize::MoveItem (Object** fromIndex,
36955 unsigned int fromSeg,
36956 unsigned int toSeg)
36960 ASSERT (fromSeg != toSeg);
36961 if (fromSeg > toSeg)
36965 // Place the element at the boundary closest to dest
36966 Object** srcIndex = fromIndex;
36967 for (unsigned int i = fromSeg; i != toSeg; i+= step)
36969 Object**& destFill = m_FillPointers[i+(step - 1 )/2];
36970 Object** destIndex = destFill - (step + 1)/2;
36971 if (srcIndex != destIndex)
36973 Object* tmp = *srcIndex;
36974 *srcIndex = *destIndex;
36978 srcIndex = destIndex;
36983 CFinalize::GcScanRoots (promote_func* fn, int hn, ScanContext *pSC)
36989 pSC->thread_number = hn;
36991 //scan the finalization queue
36992 Object** startIndex = SegQueue (CriticalFinalizerListSeg);
36993 Object** stopIndex = SegQueueLimit (FinalizerListSeg);
36995 for (Object** po = startIndex; po < stopIndex; po++)
36998 //dprintf (3, ("scan freacheable %Ix", (size_t)o));
36999 dprintf (3, ("scan f %Ix", (size_t)o));
37000 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
37001 if (g_fEnableAppDomainMonitoring)
37003 pSC->pCurrentDomain = GCToEEInterface::GetAppDomainForObject(o);
37005 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
37011 void CFinalize::WalkFReachableObjects (fq_walk_fn fn)
37013 Object** startIndex = SegQueue (CriticalFinalizerListSeg);
37014 Object** stopCriticalIndex = SegQueueLimit (CriticalFinalizerListSeg);
37015 Object** stopIndex = SegQueueLimit (FinalizerListSeg);
37016 for (Object** po = startIndex; po < stopIndex; po++)
37019 fn(po < stopCriticalIndex, *po);
37024 CFinalize::ScanForFinalization (promote_func* pfn, int gen, BOOL mark_only_p,
37028 sc.promotion = TRUE;
37029 #ifdef MULTIPLE_HEAPS
37030 sc.thread_number = hp->heap_number;
37032 UNREFERENCED_PARAMETER(hp);
37033 #endif //MULTIPLE_HEAPS
37035 BOOL finalizedFound = FALSE;
37037 //start with gen and explore all the younger generations.
37038 unsigned int startSeg = gen_segment (gen);
37040 m_PromotedCount = 0;
37041 for (unsigned int Seg = startSeg; Seg <= gen_segment(0); Seg++)
37043 Object** endIndex = SegQueue (Seg);
37044 for (Object** i = SegQueueLimit (Seg)-1; i >= endIndex ;i--)
37046 CObjectHeader* obj = (CObjectHeader*)*i;
37047 dprintf (3, ("scanning: %Ix", (size_t)obj));
37048 if (!g_theGCHeap->IsPromoted (obj))
37050 dprintf (3, ("freacheable: %Ix", (size_t)obj));
37052 assert (method_table(obj)->HasFinalizer());
37054 if (GCToEEInterface::EagerFinalized(obj))
37056 MoveItem (i, Seg, FreeList);
37058 else if ((obj->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN)
37060 //remove the object because we don't want to
37061 //run the finalizer
37063 MoveItem (i, Seg, FreeList);
37065 //Reset the bit so it will be put back on the queue
37066 //if resurrected and re-registered.
37067 obj->GetHeader()->ClrBit (BIT_SBLK_FINALIZER_RUN);
37074 if (method_table(obj)->HasCriticalFinalizer())
37076 MoveItem (i, Seg, CriticalFinalizerListSeg);
37080 MoveItem (i, Seg, FinalizerListSeg);
37084 #ifdef BACKGROUND_GC
37087 if ((gen == max_generation) && (recursive_gc_sync::background_running_p()))
37089 // TODO - fix the following line.
37090 //assert (gc_heap::background_object_marked ((uint8_t*)obj, FALSE));
37091 dprintf (3, ("%Ix is marked", (size_t)obj));
37094 #endif //BACKGROUND_GC
37098 finalizedFound = !IsSegEmpty(FinalizerListSeg) ||
37099 !IsSegEmpty(CriticalFinalizerListSeg);
37101 if (finalizedFound)
37103 //Promote the f-reachable objects
37105 #ifdef MULTIPLE_HEAPS
37109 #endif //MULTIPLE_HEAPS
37112 hp->settings.found_finalizers = TRUE;
37114 #ifdef BACKGROUND_GC
37115 if (hp->settings.concurrent)
37117 hp->settings.found_finalizers = !(IsSegEmpty(FinalizerListSeg) && IsSegEmpty(CriticalFinalizerListSeg));
37119 #endif //BACKGROUND_GC
37120 if (hp->settings.concurrent && hp->settings.found_finalizers)
37123 GCToEEInterface::EnableFinalization(true);
37127 return finalizedFound;
37130 //Relocates all of the objects in the finalization array
37132 CFinalize::RelocateFinalizationData (int gen, gc_heap* hp)
37135 sc.promotion = FALSE;
37136 #ifdef MULTIPLE_HEAPS
37137 sc.thread_number = hp->heap_number;
37139 UNREFERENCED_PARAMETER(hp);
37140 #endif //MULTIPLE_HEAPS
37142 unsigned int Seg = gen_segment (gen);
37144 Object** startIndex = SegQueue (Seg);
37145 for (Object** po = startIndex; po < SegQueue (FreeList);po++)
37147 GCHeap::Relocate (po, &sc);
37152 CFinalize::UpdatePromotedGenerations (int gen, BOOL gen_0_empty_p)
37154 // update the generation fill pointers.
37155 // if gen_0_empty is FALSE, test each object to find out if
37156 // it was promoted or not
37159 for (int i = min (gen+1, max_generation); i > 0; i--)
37161 m_FillPointers [gen_segment(i)] = m_FillPointers [gen_segment(i-1)];
37166 //Look for demoted or promoted plugs
37168 for (int i = gen; i >= 0; i--)
37170 unsigned int Seg = gen_segment (i);
37171 Object** startIndex = SegQueue (Seg);
37173 for (Object** po = startIndex;
37174 po < SegQueueLimit (gen_segment(i)); po++)
37176 int new_gen = g_theGCHeap->WhichGeneration (*po);
37182 MoveItem (po, gen_segment (i), gen_segment (new_gen));
37187 MoveItem (po, gen_segment (i), gen_segment (new_gen));
37188 //back down in order to see all objects.
37199 CFinalize::GrowArray()
37201 size_t oldArraySize = (m_EndArray - m_Array);
37202 size_t newArraySize = (size_t)(((float)oldArraySize / 10) * 12);
37204 Object** newArray = new (nothrow) Object*[newArraySize];
37207 // It's not safe to throw here, because of the FinalizeLock. Tell our caller
37208 // to throw for us.
37209 // ASSERT (newArray);
37212 memcpy (newArray, m_Array, oldArraySize*sizeof(Object*));
37214 //adjust the fill pointers
37215 for (int i = 0; i < FreeList; i++)
37217 m_FillPointers [i] += (newArray - m_Array);
37220 m_Array = newArray;
37221 m_EndArray = &m_Array [newArraySize];
37227 void CFinalize::CheckFinalizerObjects()
37229 for (int i = 0; i <= max_generation; i++)
37231 Object **startIndex = SegQueue (gen_segment (i));
37232 Object **stopIndex = SegQueueLimit (gen_segment (i));
37234 for (Object **po = startIndex; po < stopIndex; po++)
37236 if ((int)g_theGCHeap->WhichGeneration (*po) < i)
37238 ((CObjectHeader*)*po)->Validate();
37242 #endif //VERIFY_HEAP
37244 #endif // FEATURE_PREMORTEM_FINALIZATION
37247 //------------------------------------------------------------------------------
37249 // End of VM specific support
37251 //------------------------------------------------------------------------------
37252 void gc_heap::walk_heap_per_heap (walk_fn fn, void* context, int gen_number, BOOL walk_large_object_heap_p)
37254 generation* gen = gc_heap::generation_of (gen_number);
37255 heap_segment* seg = generation_start_segment (gen);
37256 uint8_t* x = ((gen_number == max_generation) ? heap_segment_mem (seg) :
37257 generation_allocation_start (gen));
37259 uint8_t* end = heap_segment_allocated (seg);
37260 BOOL small_object_segments = TRUE;
37261 int align_const = get_alignment_constant (small_object_segments);
37268 if ((seg = heap_segment_next (seg)) != 0)
37270 x = heap_segment_mem (seg);
37271 end = heap_segment_allocated (seg);
37276 if (small_object_segments && walk_large_object_heap_p)
37279 small_object_segments = FALSE;
37280 align_const = get_alignment_constant (small_object_segments);
37281 seg = generation_start_segment (large_object_generation);
37282 x = heap_segment_mem (seg);
37283 end = heap_segment_allocated (seg);
37293 size_t s = size (x);
37294 CObjectHeader* o = (CObjectHeader*)x;
37299 _ASSERTE(((size_t)o & 0x3) == 0); // Last two bits should never be set at this point
37301 if (!fn (o->GetObjectBase(), context))
37304 x = x + Align (s, align_const);
37308 void gc_heap::walk_finalize_queue (fq_walk_fn fn)
37310 #ifdef FEATURE_PREMORTEM_FINALIZATION
37311 finalize_queue->WalkFReachableObjects (fn);
37312 #endif //FEATURE_PREMORTEM_FINALIZATION
37315 void gc_heap::walk_heap (walk_fn fn, void* context, int gen_number, BOOL walk_large_object_heap_p)
37317 #ifdef MULTIPLE_HEAPS
37318 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
37320 gc_heap* hp = gc_heap::g_heaps [hn];
37322 hp->walk_heap_per_heap (fn, context, gen_number, walk_large_object_heap_p);
37325 walk_heap_per_heap(fn, context, gen_number, walk_large_object_heap_p);
37326 #endif //MULTIPLE_HEAPS
37329 void GCHeap::DiagWalkObject (Object* obj, walk_fn fn, void* context)
37331 uint8_t* o = (uint8_t*)obj;
37334 go_through_object_cl (method_table (o), o, size(o), oo,
37338 Object *oh = (Object*)*oo;
37339 if (!fn (oh, context))
37347 void GCHeap::DiagWalkSurvivorsWithType (void* gc_context, record_surv_fn fn, void* diag_context, walk_surv_type type)
37349 gc_heap* hp = (gc_heap*)gc_context;
37350 hp->walk_survivors (fn, diag_context, type);
37353 void GCHeap::DiagWalkHeap (walk_fn fn, void* context, int gen_number, bool walk_large_object_heap_p)
37355 gc_heap::walk_heap (fn, context, gen_number, walk_large_object_heap_p);
37358 void GCHeap::DiagWalkFinalizeQueue (void* gc_context, fq_walk_fn fn)
37360 gc_heap* hp = (gc_heap*)gc_context;
37361 hp->walk_finalize_queue (fn);
37364 void GCHeap::DiagScanFinalizeQueue (fq_scan_fn fn, ScanContext* sc)
37366 #ifdef MULTIPLE_HEAPS
37367 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
37369 gc_heap* hp = gc_heap::g_heaps [hn];
37370 hp->finalize_queue->GcScanRoots(fn, hn, sc);
37373 pGenGCHeap->finalize_queue->GcScanRoots(fn, 0, sc);
37374 #endif //MULTIPLE_HEAPS
37377 void GCHeap::DiagScanHandles (handle_scan_fn fn, int gen_number, ScanContext* context)
37379 UNREFERENCED_PARAMETER(gen_number);
37380 GCScan::GcScanHandlesForProfilerAndETW (max_generation, context, fn);
37383 void GCHeap::DiagScanDependentHandles (handle_scan_fn fn, int gen_number, ScanContext* context)
37385 UNREFERENCED_PARAMETER(gen_number);
37386 GCScan::GcScanDependentHandlesForProfilerAndETW (max_generation, context, fn);
37389 // Go through and touch (read) each page straddled by a memory block.
37390 void TouchPages(void * pStart, size_t cb)
37392 const uint32_t pagesize = OS_PAGE_SIZE;
37393 _ASSERTE(0 == (pagesize & (pagesize-1))); // Must be a power of 2.
37396 VOLATILE(char)* pEnd = (VOLATILE(char)*)(cb + (char*)pStart);
37397 VOLATILE(char)* p = (VOLATILE(char)*)(((char*)pStart) - (((size_t)pStart) & (pagesize-1)));
37401 a = VolatileLoad(p);
37402 //printf("Touching page %lxh\n", (uint32_t)p);
37408 #if defined(WRITE_BARRIER_CHECK) && !defined (SERVER_GC)
37409 // This code is designed to catch the failure to update the write barrier
37410 // The way it works is to copy the whole heap right after every GC. The write
37411 // barrier code has been modified so that it updates the shadow as well as the
37412 // real GC heap. Before doing the next GC, we walk the heap, looking for pointers
37413 // that were updated in the real heap, but not the shadow. A mismatch indicates
37414 // an error. The offending code can be found by breaking after the correct GC,
37415 // and then placing a data breakpoint on the Heap location that was updated without
37416 // going through the write barrier.
37418 // Called at process shutdown
37419 void deleteGCShadow()
37421 if (g_GCShadow != 0)
37422 GCToOSInterface::VirtualRelease (g_GCShadow, g_GCShadowEnd - g_GCShadow);
37427 // Called at startup and right after a GC, get a snapshot of the GC Heap
37428 void initGCShadow()
37430 if (!(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_BARRIERCHECK))
37433 size_t len = g_gc_highest_address - g_gc_lowest_address;
37434 if (len > (size_t)(g_GCShadowEnd - g_GCShadow))
37437 g_GCShadowEnd = g_GCShadow = (uint8_t *)GCToOSInterface::VirtualReserve(len, 0, VirtualReserveFlags::None);
37438 if (g_GCShadow == NULL || !GCToOSInterface::VirtualCommit(g_GCShadow, len))
37440 _ASSERTE(!"Not enough memory to run HeapVerify level 2");
37441 // If after the assert we decide to allow the program to continue
37442 // running we need to be in a state that will not trigger any
37443 // additional AVs while we fail to allocate a shadow segment, i.e.
37444 // ensure calls to updateGCShadow() checkGCWriteBarrier() don't AV
37449 g_GCShadowEnd += len;
37452 // save the value of g_gc_lowest_address at this time. If this value changes before
37453 // the next call to checkGCWriteBarrier() it means we extended the heap (with a
37454 // large object segment most probably), and the whole shadow segment is inconsistent.
37455 g_shadow_lowest_address = g_gc_lowest_address;
37457 //****** Copy the whole GC heap ******
37459 // NOTE: This is the one situation where the combination of heap_segment_rw(gen_start_segment())
37460 // can produce a NULL result. This is because the initialization has not completed.
37462 generation* gen = gc_heap::generation_of (max_generation);
37463 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
37465 ptrdiff_t delta = g_GCShadow - g_gc_lowest_address;
37466 BOOL small_object_segments = TRUE;
37471 if (small_object_segments)
37473 small_object_segments = FALSE;
37474 seg = heap_segment_rw (generation_start_segment (gc_heap::generation_of (max_generation+1)));
37480 // Copy the segment
37481 uint8_t* start = heap_segment_mem(seg);
37482 uint8_t* end = heap_segment_allocated (seg);
37483 memcpy(start + delta, start, end - start);
37484 seg = heap_segment_next_rw (seg);
37488 #define INVALIDGCVALUE (void*)((size_t)0xcccccccd)
37490 // test to see if 'ptr' was only updated via the write barrier.
37491 inline void testGCShadow(Object** ptr)
37493 Object** shadow = (Object**) &g_GCShadow[((uint8_t*) ptr - g_gc_lowest_address)];
37494 if (*ptr != 0 && (uint8_t*) shadow < g_GCShadowEnd && *ptr != *shadow)
37497 // If you get this assertion, someone updated a GC pointer in the heap without
37498 // using the write barrier. To find out who, check the value of
37499 // dd_collection_count (dynamic_data_of (0)). Also
37500 // note the value of 'ptr'. Rerun the App that the previous GC just occurred.
37501 // Then put a data breakpoint for the value of 'ptr' Then check every write
37502 // to pointer between the two GCs. The last one is not using the write barrier.
37504 // If the memory of interest does not exist at system startup,
37505 // you need to set the data breakpoint right after the memory gets committed
37506 // Set a breakpoint at the end of grow_heap_segment, and put the value of 'ptr'
37507 // in the memory window. run until the memory gets mapped. Then you can set
37510 // Note a recent change, we've identified race conditions when updating the gc shadow.
37511 // Throughout the runtime, code will update an address in the gc heap, then erect the
37512 // write barrier, which calls updateGCShadow. With an app that pounds one heap location
37513 // from multiple threads, you can hit this assert even though all involved are using the
37514 // write barrier properly. Thusly, we detect the race and set this location to INVALIDGCVALUE.
37515 // TODO: the code in jithelp.asm doesn't call updateGCShadow, and hasn't been
37516 // TODO: fixed to detect the race. We've only seen this race from VolatileWritePtr,
37517 // TODO: so elect not to fix jithelp.asm at this time. It should be done if we start hitting
37518 // TODO: erroneous asserts in here.
37520 if(*shadow!=INVALIDGCVALUE)
37522 #ifdef FEATURE_BASICFREEZE
37523 // Write barriers for stores of references to frozen objects may be optimized away.
37524 if (!gc_heap::frozen_object_p(*ptr))
37525 #endif // FEATURE_BASICFREEZE
37527 _ASSERTE(!"Pointer updated without using write barrier");
37533 printf("saw a INVALIDGCVALUE. (just to let you know)\n");
37539 void testGCShadowHelper (uint8_t* x)
37541 size_t s = size (x);
37542 if (contain_pointers (x))
37544 go_through_object_nostart (method_table(x), x, s, oo,
37545 { testGCShadow((Object**) oo); });
37549 // Walk the whole heap, looking for pointers that were not updated with the write barrier.
37550 void checkGCWriteBarrier()
37552 // g_shadow_lowest_address != g_gc_lowest_address means the GC heap was extended by a segment
37553 // and the GC shadow segment did not track that change!
37554 if (g_GCShadowEnd <= g_GCShadow || g_shadow_lowest_address != g_gc_lowest_address)
37556 // No shadow stack, nothing to check.
37561 generation* gen = gc_heap::generation_of (max_generation);
37562 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
37564 PREFIX_ASSUME(seg != NULL);
37568 uint8_t* x = heap_segment_mem(seg);
37569 while (x < heap_segment_allocated (seg))
37571 size_t s = size (x);
37572 testGCShadowHelper (x);
37575 seg = heap_segment_next_rw (seg);
37580 // go through large object heap
37581 int alignment = get_alignment_constant(FALSE);
37582 generation* gen = gc_heap::generation_of (max_generation+1);
37583 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
37585 PREFIX_ASSUME(seg != NULL);
37589 uint8_t* x = heap_segment_mem(seg);
37590 while (x < heap_segment_allocated (seg))
37592 size_t s = size (x);
37593 testGCShadowHelper (x);
37594 x = x + Align (s, alignment);
37596 seg = heap_segment_next_rw (seg);
37600 #endif //WRITE_BARRIER_CHECK && !SERVER_GC
37602 #endif // !DACCESS_COMPILE
37604 #ifdef FEATURE_BASICFREEZE
37605 void gc_heap::walk_read_only_segment(heap_segment *seg, void *pvContext, object_callback_func pfnMethodTable, object_callback_func pfnObjRef)
37607 #ifdef DACCESS_COMPILE
37608 UNREFERENCED_PARAMETER(seg);
37609 UNREFERENCED_PARAMETER(pvContext);
37610 UNREFERENCED_PARAMETER(pfnMethodTable);
37611 UNREFERENCED_PARAMETER(pfnObjRef);
37613 uint8_t *o = heap_segment_mem(seg);
37615 // small heap alignment constant
37616 int alignment = get_alignment_constant(TRUE);
37618 while (o < heap_segment_allocated(seg))
37620 pfnMethodTable(pvContext, o);
37622 if (contain_pointers (o))
37624 go_through_object_nostart (method_table (o), o, size(o), oo,
37627 pfnObjRef(pvContext, oo);
37632 o += Align(size(o), alignment);
37634 #endif //!DACCESS_COMPILE
37636 #endif // FEATURE_BASICFREEZE
37638 #ifndef DACCESS_COMPILE
37639 HRESULT GCHeap::WaitUntilConcurrentGCCompleteAsync(int millisecondsTimeout)
37641 #ifdef BACKGROUND_GC
37642 if (recursive_gc_sync::background_running_p())
37644 uint32_t dwRet = pGenGCHeap->background_gc_wait(awr_ignored, millisecondsTimeout);
37645 if (dwRet == WAIT_OBJECT_0)
37647 else if (dwRet == WAIT_TIMEOUT)
37648 return HRESULT_FROM_WIN32(ERROR_TIMEOUT);
37650 return E_FAIL; // It is not clear if what the last error would be if the wait failed,
37651 // as there are too many layers in between. The best we can do is to return E_FAIL;
37657 #endif // !DACCESS_COMPILE
37659 void GCHeap::TemporaryEnableConcurrentGC()
37661 #ifdef BACKGROUND_GC
37662 gc_heap::temp_disable_concurrent_p = false;
37663 #endif //BACKGROUND_GC
37666 void GCHeap::TemporaryDisableConcurrentGC()
37668 #ifdef BACKGROUND_GC
37669 gc_heap::temp_disable_concurrent_p = true;
37670 #endif //BACKGROUND_GC
37673 bool GCHeap::IsConcurrentGCEnabled()
37675 #ifdef BACKGROUND_GC
37676 return (gc_heap::gc_can_use_concurrent && !(gc_heap::temp_disable_concurrent_p));
37679 #endif //BACKGROUND_GC
37682 void GCHeap::SetFinalizeRunOnShutdown(bool value)
37684 g_fFinalizerRunOnShutDown = value;
37687 void PopulateDacVars(GcDacVars *gcDacVars)
37689 #ifndef DACCESS_COMPILE
37690 assert(gcDacVars != nullptr);
37692 gcDacVars->major_version_number = 1;
37693 gcDacVars->minor_version_number = 0;
37694 gcDacVars->built_with_svr = &g_built_with_svr_gc;
37695 gcDacVars->build_variant = &g_build_variant;
37696 gcDacVars->gc_structures_invalid_cnt = const_cast<int32_t*>(&GCScan::m_GcStructuresInvalidCnt);
37697 gcDacVars->generation_size = sizeof(generation);
37698 gcDacVars->max_gen = &g_max_generation;
37699 #ifndef MULTIPLE_HEAPS
37700 gcDacVars->mark_array = &gc_heap::mark_array;
37701 gcDacVars->ephemeral_heap_segment = reinterpret_cast<dac_heap_segment**>(&gc_heap::ephemeral_heap_segment);
37702 gcDacVars->current_c_gc_state = const_cast<c_gc_state*>(&gc_heap::current_c_gc_state);
37703 gcDacVars->saved_sweep_ephemeral_seg = reinterpret_cast<dac_heap_segment**>(&gc_heap::saved_sweep_ephemeral_seg);
37704 gcDacVars->saved_sweep_ephemeral_start = &gc_heap::saved_sweep_ephemeral_start;
37705 gcDacVars->background_saved_lowest_address = &gc_heap::background_saved_lowest_address;
37706 gcDacVars->background_saved_highest_address = &gc_heap::background_saved_highest_address;
37707 gcDacVars->alloc_allocated = &gc_heap::alloc_allocated;
37708 gcDacVars->next_sweep_obj = &gc_heap::next_sweep_obj;
37709 gcDacVars->oom_info = &gc_heap::oom_info;
37710 gcDacVars->finalize_queue = reinterpret_cast<dac_finalize_queue**>(&gc_heap::finalize_queue);
37711 gcDacVars->generation_table = reinterpret_cast<dac_generation**>(&gc_heap::generation_table);
37712 #ifdef GC_CONFIG_DRIVEN
37713 gcDacVars->gc_global_mechanisms = reinterpret_cast<size_t**>(&gc_global_mechanisms);
37714 gcDacVars->interesting_data_per_heap = reinterpret_cast<size_t**>(&gc_heap::interesting_data_per_heap);
37715 gcDacVars->compact_reasons_per_heap = reinterpret_cast<size_t**>(&gc_heap::compact_reasons_per_heap);
37716 gcDacVars->expand_mechanisms_per_heap = reinterpret_cast<size_t**>(&gc_heap::expand_mechanisms_per_heap);
37717 gcDacVars->interesting_mechanism_bits_per_heap = reinterpret_cast<size_t**>(&gc_heap::interesting_mechanism_bits_per_heap);
37718 #endif // GC_CONFIG_DRIVEN
37719 #ifdef HEAP_ANALYZE
37720 gcDacVars->internal_root_array = &gc_heap::internal_root_array;
37721 gcDacVars->internal_root_array_index = &gc_heap::internal_root_array_index;
37722 gcDacVars->heap_analyze_success = &gc_heap::heap_analyze_success;
37723 #endif // HEAP_ANALYZE
37725 gcDacVars->n_heaps = &gc_heap::n_heaps;
37726 gcDacVars->g_heaps = reinterpret_cast<dac_gc_heap***>(&gc_heap::g_heaps);
37727 #endif // MULTIPLE_HEAPS
37728 #endif // DACCESS_COMPILE