1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
3 // See the LICENSE file in the project root for more information.
9 // GC automatically manages memory allocated by managed code.
10 // The design doc for GC can be found at Documentation/botr/garbage-collection.md
12 // This file includes both the code for GC and the allocator. The most common
13 // case for a GC to be triggered is from the allocator code. See
14 // code:#try_allocate_more_space where it calls GarbageCollectGeneration.
16 // Entry points for the allocator are GCHeap::Alloc* which are called by the
17 // allocation helpers in gcscan.cpp
21 #include "softwarewritewatch.h"
25 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
26 BOOL bgc_heap_walk_for_etw_p = FALSE;
27 #endif //BACKGROUND_GC && FEATURE_EVENT_TRACE
29 #if defined(FEATURE_REDHAWK)
30 #define MAYBE_UNUSED_VAR(v) v = v
32 #define MAYBE_UNUSED_VAR(v)
33 #endif // FEATURE_REDHAWK
35 #define MAX_PTR ((uint8_t*)(~(ptrdiff_t)0))
38 #define partial_size_th 100
39 #define num_partial_refs 64
41 #define partial_size_th 100
42 #define num_partial_refs 32
45 #define demotion_plug_len_th (6*1024*1024)
48 #define MARK_STACK_INITIAL_LENGTH 1024
50 #define MARK_STACK_INITIAL_LENGTH 128
53 #define LOH_PIN_QUEUE_LENGTH 100
54 #define LOH_PIN_DECAY 10
57 // Right now we support maximum 1024 procs - meaning that we will create at most
58 // that many GC threads and GC heaps.
59 #define MAX_SUPPORTED_CPUS 1024
61 #define MAX_SUPPORTED_CPUS 64
64 #ifdef GC_CONFIG_DRIVEN
65 int compact_ratio = 0;
66 #endif //GC_CONFIG_DRIVEN
68 // See comments in reset_memory.
69 BOOL reset_mm_p = TRUE;
71 bool g_fFinalizerRunOnShutDown = false;
74 bool g_built_with_svr_gc = true;
76 bool g_built_with_svr_gc = false;
77 #endif // FEATURE_SVR_GC
79 #if defined(BUILDENV_DEBUG)
80 uint8_t g_build_variant = 0;
81 #elif defined(BUILDENV_CHECKED)
82 uint8_t g_build_variant = 1;
84 uint8_t g_build_variant = 2;
85 #endif // defined(BUILDENV_DEBUG)
87 VOLATILE(int32_t) g_no_gc_lock = -1;
89 #if defined (TRACE_GC) && !defined (DACCESS_COMPILE)
90 const char * const allocation_state_str[] = {
96 "try_fit_new_seg_after_cg",
100 "try_free_full_seg_in_bgc",
101 "try_free_after_bgc",
104 "acquire_seg_after_cg",
105 "acquire_seg_after_bgc",
106 "check_and_wait_for_bgc",
107 "trigger_full_compact_gc",
108 "trigger_ephemeral_gc",
109 "trigger_2nd_ephemeral_gc",
112 #endif //TRACE_GC && !DACCESS_COMPILE
114 // Keep this in sync with the definition of gc_reason
115 #if (defined(DT_LOG) || defined(TRACE_GC)) && !defined (DACCESS_COMPILE)
116 static const char* const str_gc_reasons[] =
131 static const char* const str_gc_pause_modes[] =
136 "sustained_low_latency",
139 #endif // defined(DT_LOG) || defined(TRACE_GC)
142 BOOL is_induced (gc_reason reason)
144 return ((reason == reason_induced) ||
145 (reason == reason_induced_noforce) ||
146 (reason == reason_lowmemory) ||
147 (reason == reason_lowmemory_blocking) ||
148 (reason == reason_induced_compacting));
152 BOOL is_induced_blocking (gc_reason reason)
154 return ((reason == reason_induced) ||
155 (reason == reason_lowmemory_blocking) ||
156 (reason == reason_induced_compacting));
159 #ifndef DACCESS_COMPILE
162 size_t GetHighPrecisionTimeStamp()
164 int64_t ts = GCToOSInterface::QueryPerformanceCounter();
166 return (size_t)(ts / (qpf / 1000));
172 // There is a current and a prior copy of the statistics. This allows us to display deltas per reporting
173 // interval, as well as running totals. The 'min' and 'max' values require special treatment. They are
174 // Reset (zeroed) in the current statistics when we begin a new interval and they are updated via a
175 // comparison with the global min/max.
176 GCStatistics g_GCStatistics;
177 GCStatistics g_LastGCStatistics;
179 char* GCStatistics::logFileName = NULL;
180 FILE* GCStatistics::logFile = NULL;
182 void GCStatistics::AddGCStats(const gc_mechanisms& settings, size_t timeInMSec)
185 if (settings.concurrent)
187 bgc.Accumulate((uint32_t)timeInMSec*1000);
190 else if (settings.background_p)
192 fgc.Accumulate((uint32_t)timeInMSec*1000);
194 if (settings.compaction)
196 assert(settings.condemned_generation < max_generation);
197 cntFGCGen[settings.condemned_generation]++;
200 #endif // BACKGROUND_GC
202 ngc.Accumulate((uint32_t)timeInMSec*1000);
204 if (settings.compaction)
206 cntNGCGen[settings.condemned_generation]++;
209 if (is_induced (settings.reason))
210 cntReasons[(int)reason_induced]++;
212 else if (settings.stress_induced)
213 cntReasons[(int)reason_gcstress]++;
214 #endif // STRESS_HEAP
216 cntReasons[(int)settings.reason]++;
219 if (settings.concurrent || !settings.background_p)
221 #endif // BACKGROUND_GC
225 #endif // BACKGROUND_GC
228 void GCStatistics::Initialize()
230 LIMITED_METHOD_CONTRACT;
231 // for efficiency sake we're taking a dependency on the layout of a C++ object
232 // with a vtable. protect against violations of our premise:
233 static_assert(offsetof(GCStatistics, cntDisplay) == sizeof(void*),
234 "The first field of GCStatistics follows the pointer sized vtable");
236 int podOffs = offsetof(GCStatistics, cntDisplay); // offset of the first POD field
237 memset((uint8_t*)(&g_GCStatistics)+podOffs, 0, sizeof(g_GCStatistics)-podOffs);
238 memset((uint8_t*)(&g_LastGCStatistics)+podOffs, 0, sizeof(g_LastGCStatistics)-podOffs);
241 void GCStatistics::DisplayAndUpdate()
243 LIMITED_METHOD_CONTRACT;
245 if (logFileName == NULL || logFile == NULL)
250 fprintf(logFile, "\nGCMix **** Initialize *****\n\n");
252 fprintf(logFile, "GCMix **** Summary ***** %d\n", cntDisplay);
254 // NGC summary (total, timing info)
255 ngc.DisplayAndUpdate(logFile, "NGC ", &g_LastGCStatistics.ngc, cntNGC, g_LastGCStatistics.cntNGC, msec);
257 // FGC summary (total, timing info)
258 fgc.DisplayAndUpdate(logFile, "FGC ", &g_LastGCStatistics.fgc, cntFGC, g_LastGCStatistics.cntFGC, msec);
261 bgc.DisplayAndUpdate(logFile, "BGC ", &g_LastGCStatistics.bgc, cntBGC, g_LastGCStatistics.cntBGC, msec);
263 // NGC/FGC break out by generation & compacting vs. sweeping
264 fprintf(logFile, "NGC ");
265 for (int i = max_generation; i >= 0; --i)
266 fprintf(logFile, "gen%d %d (%d). ", i, cntNGCGen[i]-g_LastGCStatistics.cntNGCGen[i], cntNGCGen[i]);
267 fprintf(logFile, "\n");
269 fprintf(logFile, "FGC ");
270 for (int i = max_generation-1; i >= 0; --i)
271 fprintf(logFile, "gen%d %d (%d). ", i, cntFGCGen[i]-g_LastGCStatistics.cntFGCGen[i], cntFGCGen[i]);
272 fprintf(logFile, "\n");
274 // Compacting vs. Sweeping break out
275 int _cntSweep = cntNGC-cntCompactNGC;
276 int _cntLastSweep = g_LastGCStatistics.cntNGC-g_LastGCStatistics.cntCompactNGC;
277 fprintf(logFile, "NGC Sweeping %d (%d) Compacting %d (%d)\n",
278 _cntSweep - _cntLastSweep, _cntSweep,
279 cntCompactNGC - g_LastGCStatistics.cntCompactNGC, cntCompactNGC);
281 _cntSweep = cntFGC-cntCompactFGC;
282 _cntLastSweep = g_LastGCStatistics.cntFGC-g_LastGCStatistics.cntCompactFGC;
283 fprintf(logFile, "FGC Sweeping %d (%d) Compacting %d (%d)\n",
284 _cntSweep - _cntLastSweep, _cntSweep,
285 cntCompactFGC - g_LastGCStatistics.cntCompactFGC, cntCompactFGC);
289 for (int reason=(int)reason_alloc_soh; reason <= (int)reason_gcstress; ++reason)
291 if (cntReasons[reason] != 0)
292 fprintf(logFile, "%s %d (%d). ", str_gc_reasons[reason],
293 cntReasons[reason]-g_LastGCStatistics.cntReasons[reason], cntReasons[reason]);
296 fprintf(logFile, "\n\n");
298 // flush the log file...
302 g_LastGCStatistics = *this;
312 size_t round_up_power2 (size_t size)
314 // Get the 0-based index of the most-significant bit in size-1.
315 // If the call failed (because size-1 is zero), size must be 1,
316 // so return 1 (because 1 rounds up to itself).
317 DWORD highest_set_bit_index;
324 &highest_set_bit_index, size - 1)) { return 1; }
326 // The size == 0 case (which would have overflowed to SIZE_MAX when decremented)
327 // is handled below by relying on the fact that highest_set_bit_index is the maximum value
328 // (31 or 63, depending on sizeof(size_t)) and left-shifting a value >= 2 by that
329 // number of bits shifts in zeros from the right, resulting in an output of zero.
330 return static_cast<size_t>(2) << highest_set_bit_index;
334 size_t round_down_power2 (size_t size)
336 // Get the 0-based index of the most-significant bit in size.
337 // If the call failed, size must be zero so return zero.
338 DWORD highest_set_bit_index;
345 &highest_set_bit_index, size)) { return 0; }
347 // Left-shift 1 by highest_set_bit_index to get back a value containing only
348 // the most-significant set bit of size, i.e. size rounded down
349 // to the next power-of-two value.
350 return static_cast<size_t>(1) << highest_set_bit_index;
353 // Get the 0-based index of the most-significant bit in the value.
354 // Returns -1 if the input value is zero (i.e. has no set bits).
356 int index_of_highest_set_bit (size_t value)
358 // Get the 0-based index of the most-significant bit in the value.
359 // If the call failed (because value is zero), return -1.
360 DWORD highest_set_bit_index;
367 &highest_set_bit_index, value)) ? -1 : static_cast<int>(highest_set_bit_index);
371 int relative_index_power2_plug (size_t power2)
373 int index = index_of_highest_set_bit (power2);
374 assert (index <= MAX_INDEX_POWER2);
376 return ((index < MIN_INDEX_POWER2) ? 0 : (index - MIN_INDEX_POWER2));
380 int relative_index_power2_free_space (size_t power2)
382 int index = index_of_highest_set_bit (power2);
383 assert (index <= MAX_INDEX_POWER2);
385 return ((index < MIN_INDEX_POWER2) ? -1 : (index - MIN_INDEX_POWER2));
389 uint32_t bgc_alloc_spin_count = 140;
390 uint32_t bgc_alloc_spin_count_loh = 16;
391 uint32_t bgc_alloc_spin = 2;
395 void c_write (uint32_t& place, uint32_t value)
397 Interlocked::Exchange (&place, value);
401 #ifndef DACCESS_COMPILE
402 // If every heap's gen2 or gen3 size is less than this threshold we will do a blocking GC.
403 const size_t bgc_min_per_heap = 4*1024*1024;
405 int gc_heap::gchist_index = 0;
406 gc_mechanisms_store gc_heap::gchist[max_history_count];
408 #ifndef MULTIPLE_HEAPS
409 size_t gc_heap::total_promoted_bytes = 0;
410 VOLATILE(bgc_state) gc_heap::current_bgc_state = bgc_not_in_process;
411 int gc_heap::gchist_index_per_heap = 0;
412 gc_heap::gc_history gc_heap::gchist_per_heap[max_history_count];
413 #endif //MULTIPLE_HEAPS
415 void gc_heap::add_to_history_per_heap()
418 gc_history* current_hist = &gchist_per_heap[gchist_index_per_heap];
419 current_hist->gc_index = settings.gc_index;
420 current_hist->current_bgc_state = current_bgc_state;
421 size_t elapsed = dd_gc_elapsed_time (dynamic_data_of (0));
422 current_hist->gc_time_ms = (uint32_t)elapsed;
423 current_hist->gc_efficiency = (elapsed ? (total_promoted_bytes / elapsed) : total_promoted_bytes);
424 current_hist->eph_low = generation_allocation_start (generation_of (max_generation-1));
425 current_hist->gen0_start = generation_allocation_start (generation_of (0));
426 current_hist->eph_high = heap_segment_allocated (ephemeral_heap_segment);
428 current_hist->bgc_lowest = background_saved_lowest_address;
429 current_hist->bgc_highest = background_saved_highest_address;
430 #endif //BACKGROUND_GC
431 current_hist->fgc_lowest = lowest_address;
432 current_hist->fgc_highest = highest_address;
433 current_hist->g_lowest = g_gc_lowest_address;
434 current_hist->g_highest = g_gc_highest_address;
436 gchist_index_per_heap++;
437 if (gchist_index_per_heap == max_history_count)
439 gchist_index_per_heap = 0;
444 void gc_heap::add_to_history()
447 gc_mechanisms_store* current_settings = &gchist[gchist_index];
448 current_settings->store (&settings);
451 if (gchist_index == max_history_count)
458 #endif //DACCESS_COMPILE
459 #endif //BACKGROUND_GC
461 #if defined(TRACE_GC) && !defined(DACCESS_COMPILE)
462 BOOL gc_log_on = TRUE;
464 size_t gc_log_file_size = 0;
466 size_t gc_buffer_index = 0;
467 size_t max_gc_buffers = 0;
469 static CLRCriticalSection gc_log_lock;
471 // we keep this much in a buffer and only flush when the buffer is full
472 #define gc_log_buffer_size (1024*1024)
473 uint8_t* gc_log_buffer = 0;
474 size_t gc_log_buffer_offset = 0;
476 void log_va_msg(const char *fmt, va_list args)
480 const int BUFFERSIZE = 512;
481 static char rgchBuffer[BUFFERSIZE];
482 char * pBuffer = &rgchBuffer[0];
485 int buffer_start = 1;
486 int pid_len = sprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, "[%5d]", (uint32_t)GCToOSInterface::GetCurrentThreadIdForLogging());
487 buffer_start += pid_len;
488 memset(&pBuffer[buffer_start], '-', BUFFERSIZE - buffer_start);
489 int msg_len = _vsnprintf_s(&pBuffer[buffer_start], BUFFERSIZE - buffer_start, _TRUNCATE, fmt, args );
492 msg_len = BUFFERSIZE - buffer_start;
495 msg_len += buffer_start;
497 if ((gc_log_buffer_offset + msg_len) > (gc_log_buffer_size - 12))
500 memset (index_str, '-', 8);
501 sprintf_s (index_str, _countof(index_str), "%d", (int)gc_buffer_index);
502 gc_log_buffer[gc_log_buffer_offset] = '\n';
503 memcpy (gc_log_buffer + (gc_log_buffer_offset + 1), index_str, 8);
506 if (gc_buffer_index > max_gc_buffers)
508 fseek (gc_log, 0, SEEK_SET);
511 fwrite(gc_log_buffer, gc_log_buffer_size, 1, gc_log);
513 memset (gc_log_buffer, '*', gc_log_buffer_size);
514 gc_log_buffer_offset = 0;
517 memcpy (gc_log_buffer + gc_log_buffer_offset, pBuffer, msg_len);
518 gc_log_buffer_offset += msg_len;
523 void GCLog (const char *fmt, ... )
525 if (gc_log_on && (gc_log != NULL))
529 log_va_msg (fmt, args);
533 #endif // TRACE_GC && !DACCESS_COMPILE
535 #if defined(GC_CONFIG_DRIVEN) && !defined(DACCESS_COMPILE)
537 BOOL gc_config_log_on = FALSE;
538 FILE* gc_config_log = NULL;
540 // we keep this much in a buffer and only flush when the buffer is full
541 #define gc_config_log_buffer_size (1*1024) // TEMP
542 uint8_t* gc_config_log_buffer = 0;
543 size_t gc_config_log_buffer_offset = 0;
545 // For config since we log so little we keep the whole history. Also it's only
546 // ever logged by one thread so no need to synchronize.
547 void log_va_msg_config(const char *fmt, va_list args)
549 const int BUFFERSIZE = 256;
550 static char rgchBuffer[BUFFERSIZE];
551 char * pBuffer = &rgchBuffer[0];
554 int buffer_start = 1;
555 int msg_len = _vsnprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, _TRUNCATE, fmt, args );
556 assert (msg_len != -1);
557 msg_len += buffer_start;
559 if ((gc_config_log_buffer_offset + msg_len) > gc_config_log_buffer_size)
561 fwrite(gc_config_log_buffer, gc_config_log_buffer_offset, 1, gc_config_log);
562 fflush(gc_config_log);
563 gc_config_log_buffer_offset = 0;
566 memcpy (gc_config_log_buffer + gc_config_log_buffer_offset, pBuffer, msg_len);
567 gc_config_log_buffer_offset += msg_len;
570 void GCLogConfig (const char *fmt, ... )
572 if (gc_config_log_on && (gc_config_log != NULL))
575 va_start( args, fmt );
576 log_va_msg_config (fmt, args);
579 #endif // GC_CONFIG_DRIVEN && !DACCESS_COMPILE
581 #ifdef SYNCHRONIZATION_STATS
583 // Number of GCs have we done since we last logged.
584 static unsigned int gc_count_during_log;
585 // In ms. This is how often we print out stats.
586 static const unsigned int log_interval = 5000;
587 // Time (in ms) when we start a new log interval.
588 static unsigned int log_start_tick;
589 static unsigned int gc_lock_contended;
590 static int64_t log_start_hires;
591 // Cycles accumulated in SuspendEE during log_interval.
592 static uint64_t suspend_ee_during_log;
593 // Cycles accumulated in RestartEE during log_interval.
594 static uint64_t restart_ee_during_log;
595 static uint64_t gc_during_log;
597 #endif //SYNCHRONIZATION_STATS
600 init_sync_log_stats()
602 #ifdef SYNCHRONIZATION_STATS
603 if (gc_count_during_log == 0)
605 gc_heap::init_sync_stats();
606 suspend_ee_during_log = 0;
607 restart_ee_during_log = 0;
609 gc_lock_contended = 0;
611 log_start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
612 log_start_hires = GCToOSInterface::QueryPerformanceCounter();
614 gc_count_during_log++;
615 #endif //SYNCHRONIZATION_STATS
619 process_sync_log_stats()
621 #ifdef SYNCHRONIZATION_STATS
623 unsigned int log_elapsed = GCToOSInterface::GetLowPrecisionTimeStamp() - log_start_tick;
625 if (log_elapsed > log_interval)
627 uint64_t total = GCToOSInterface::QueryPerformanceCounter() - log_start_hires;
628 // Print out the cycles we spent on average in each suspend and restart.
629 printf("\n_________________________________________________________________________________\n"
630 "Past %d(s): #%3d GCs; Total gc_lock contended: %8u; GC: %12u\n"
631 "SuspendEE: %8u; RestartEE: %8u GC %.3f%%\n",
635 (unsigned int)(gc_during_log / gc_count_during_log),
636 (unsigned int)(suspend_ee_during_log / gc_count_during_log),
637 (unsigned int)(restart_ee_during_log / gc_count_during_log),
638 (double)(100.0f * gc_during_log / total));
639 gc_heap::print_sync_stats(gc_count_during_log);
641 gc_count_during_log = 0;
643 #endif //SYNCHRONIZATION_STATS
646 #ifdef MULTIPLE_HEAPS
650 gc_join_init_cpu_mapping = 0,
652 gc_join_generation_determined = 2,
653 gc_join_begin_mark_phase = 3,
654 gc_join_scan_dependent_handles = 4,
655 gc_join_rescan_dependent_handles = 5,
656 gc_join_scan_sizedref_done = 6,
657 gc_join_null_dead_short_weak = 7,
658 gc_join_scan_finalization = 8,
659 gc_join_null_dead_long_weak = 9,
660 gc_join_null_dead_syncblk = 10,
661 gc_join_decide_on_compaction = 11,
662 gc_join_rearrange_segs_compaction = 12,
663 gc_join_adjust_handle_age_compact = 13,
664 gc_join_adjust_handle_age_sweep = 14,
665 gc_join_begin_relocate_phase = 15,
666 gc_join_relocate_phase_done = 16,
667 gc_join_verify_objects_done = 17,
668 gc_join_start_bgc = 18,
669 gc_join_restart_ee = 19,
670 gc_join_concurrent_overflow = 20,
671 gc_join_suspend_ee = 21,
672 gc_join_bgc_after_ephemeral = 22,
673 gc_join_allow_fgc = 23,
674 gc_join_bgc_sweep = 24,
675 gc_join_suspend_ee_verify = 25,
676 gc_join_restart_ee_verify = 26,
677 gc_join_set_state_free = 27,
678 gc_r_join_update_card_bundle = 28,
679 gc_join_after_absorb = 29,
680 gc_join_verify_copy_table = 30,
681 gc_join_after_reset = 31,
682 gc_join_after_ephemeral_sweep = 32,
683 gc_join_after_profiler_heap_walk = 33,
684 gc_join_minimal_gc = 34,
685 gc_join_after_commit_soh_no_gc = 35,
686 gc_join_expand_loh_no_gc = 36,
687 gc_join_final_no_gc = 37,
688 gc_join_disable_software_write_watch = 38,
694 join_flavor_server_gc = 0,
698 #define first_thread_arrived 2
699 struct DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE) join_structure
701 // Shared non volatile keep on separate line to prevent eviction
704 // Keep polling/wait structures on separate line write once per join
705 DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE)
706 GCEvent joined_event[3]; // the last event in the array is only used for first_thread_arrived.
707 Volatile<int> lock_color;
708 VOLATILE(BOOL) wait_done;
709 VOLATILE(BOOL) joined_p;
711 // Keep volatile counted locks on separate cache line write many per join
712 DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE)
713 VOLATILE(int32_t) join_lock;
714 VOLATILE(int32_t) r_join_lock;
723 type_first_r_join = 3,
735 join_heap_restart = 100,
736 join_heap_r_restart = 200
748 join_structure join_struct;
751 gc_join_flavor flavor;
754 uint64_t start[MAX_SUPPORTED_CPUS], end[MAX_SUPPORTED_CPUS], start_seq;
755 // remember join id and last thread to arrive so restart can use these
757 // we want to print statistics every 10 seconds - this is to remember the start of the 10 sec interval
759 // counters for joins, in 1000's of clock cycles
760 uint64_t elapsed_total[gc_join_max], wake_total[gc_join_max], seq_loss_total[gc_join_max], par_loss_total[gc_join_max], in_join_total[gc_join_max];
764 BOOL init (int n_th, gc_join_flavor f)
766 dprintf (JOIN_LOG, ("Initializing join structure"));
767 join_struct.n_threads = n_th;
768 join_struct.lock_color = 0;
769 for (int i = 0; i < 3; i++)
771 if (!join_struct.joined_event[i].IsValid())
773 join_struct.joined_p = FALSE;
774 dprintf (JOIN_LOG, ("Creating join event %d", i));
775 // TODO - changing this to a non OS event
776 // because this is also used by BGC threads which are
777 // managed threads and WaitEx does not allow you to wait
778 // for an OS event on a managed thread.
779 // But we are not sure if this plays well in the hosting
781 //join_struct.joined_event[i].CreateOSManualEventNoThrow(FALSE);
782 if (!join_struct.joined_event[i].CreateManualEventNoThrow(FALSE))
786 join_struct.join_lock = join_struct.n_threads;
787 join_struct.r_join_lock = join_struct.n_threads;
788 join_struct.wait_done = FALSE;
792 start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
800 dprintf (JOIN_LOG, ("Destroying join structure"));
801 for (int i = 0; i < 3; i++)
803 if (join_struct.joined_event[i].IsValid())
804 join_struct.joined_event[i].CloseEvent();
808 inline void fire_event (int heap, join_time time, join_type type, int join_id)
810 FIRE_EVENT(GCJoin_V2, heap, time, type, join_id);
813 void join (gc_heap* gch, int join_id)
816 // parallel execution ends here
817 end[gch->heap_number] = get_ts();
820 assert (!join_struct.joined_p);
821 int color = join_struct.lock_color.LoadWithoutBarrier();
823 if (Interlocked::Decrement(&join_struct.join_lock) != 0)
825 dprintf (JOIN_LOG, ("join%d(%d): Join() Waiting...join_lock is now %d",
826 flavor, join_id, (int32_t)(join_struct.join_lock)));
828 fire_event (gch->heap_number, time_start, type_join, join_id);
830 //busy wait around the color
831 if (color == join_struct.lock_color.LoadWithoutBarrier())
834 int spin_count = 4096 * (gc_heap::n_heaps - 1);
835 for (int j = 0; j < spin_count; j++)
837 if (color != join_struct.lock_color.LoadWithoutBarrier())
841 YieldProcessor(); // indicate to the processor that we are spinning
844 // we've spun, and if color still hasn't changed, fall into hard wait
845 if (color == join_struct.lock_color.LoadWithoutBarrier())
847 dprintf (JOIN_LOG, ("join%d(%d): Join() hard wait on reset event %d, join_lock is now %d",
848 flavor, join_id, color, (int32_t)(join_struct.join_lock)));
850 //Thread* current_thread = GCToEEInterface::GetThread();
851 //BOOL cooperative_mode = gc_heap::enable_preemptive (current_thread);
852 uint32_t dwJoinWait = join_struct.joined_event[color].Wait(INFINITE, FALSE);
853 //gc_heap::disable_preemptive (current_thread, cooperative_mode);
855 if (dwJoinWait != WAIT_OBJECT_0)
857 STRESS_LOG1 (LF_GC, LL_FATALERROR, "joined event wait failed with code: %Ix", dwJoinWait);
862 // avoid race due to the thread about to reset the event (occasionally) being preempted before ResetEvent()
863 if (color == join_struct.lock_color.LoadWithoutBarrier())
868 dprintf (JOIN_LOG, ("join%d(%d): Join() done, join_lock is %d",
869 flavor, join_id, (int32_t)(join_struct.join_lock)));
872 fire_event (gch->heap_number, time_end, type_join, join_id);
875 // parallel execution starts here
876 start[gch->heap_number] = get_ts();
877 Interlocked::ExchangeAdd(&in_join_total[join_id], (start[gch->heap_number] - end[gch->heap_number]));
882 fire_event (gch->heap_number, time_start, type_last_join, join_id);
884 join_struct.joined_p = TRUE;
885 dprintf (JOIN_LOG, ("join%d(%d): Last thread to complete the join, setting id", flavor, join_id));
886 join_struct.joined_event[!color].Reset();
888 // this one is alone so it can proceed
890 // remember the join id, the last thread arriving, the start of the sequential phase,
891 // and keep track of the cycles spent waiting in the join
892 thd = gch->heap_number;
893 start_seq = get_ts();
894 Interlocked::ExchangeAdd(&in_join_total[join_id], (start_seq - end[gch->heap_number]));
899 // Reverse join - first thread gets here does the work; other threads will only proceed
900 // after the work is done.
901 // Note that you cannot call this twice in a row on the same thread. Plus there's no
902 // need to call it twice in row - you should just merge the work.
903 BOOL r_join (gc_heap* gch, int join_id)
906 if (join_struct.n_threads == 1)
911 if (Interlocked::CompareExchange(&join_struct.r_join_lock, 0, join_struct.n_threads) == 0)
913 if (!join_struct.wait_done)
915 dprintf (JOIN_LOG, ("r_join() Waiting..."));
917 fire_event (gch->heap_number, time_start, type_join, join_id);
919 //busy wait around the color
920 if (!join_struct.wait_done)
923 int spin_count = 2 * 4096 * (gc_heap::n_heaps - 1);
924 for (int j = 0; j < spin_count; j++)
926 if (join_struct.wait_done)
930 YieldProcessor(); // indicate to the processor that we are spinning
933 // we've spun, and if color still hasn't changed, fall into hard wait
934 if (!join_struct.wait_done)
936 dprintf (JOIN_LOG, ("Join() hard wait on reset event %d", first_thread_arrived));
937 uint32_t dwJoinWait = join_struct.joined_event[first_thread_arrived].Wait(INFINITE, FALSE);
938 if (dwJoinWait != WAIT_OBJECT_0)
940 STRESS_LOG1 (LF_GC, LL_FATALERROR, "joined event wait failed with code: %Ix", dwJoinWait);
945 // avoid race due to the thread about to reset the event (occasionally) being preempted before ResetEvent()
946 if (!join_struct.wait_done)
951 dprintf (JOIN_LOG, ("r_join() done"));
954 fire_event (gch->heap_number, time_end, type_join, join_id);
961 fire_event (gch->heap_number, time_start, type_first_r_join, join_id);
969 return GCToOSInterface::QueryPerformanceCounter();
972 void start_ts (gc_heap* gch)
974 // parallel execution ends here
975 start[gch->heap_number] = get_ts();
982 uint64_t elapsed_seq = get_ts() - start_seq;
983 uint64_t max = 0, sum = 0, wake = 0;
984 uint64_t min_ts = start[0];
985 for (int i = 1; i < join_struct.n_threads; i++)
987 if(min_ts > start[i]) min_ts = start[i];
990 for (int i = 0; i < join_struct.n_threads; i++)
992 uint64_t wake_delay = start[i] - min_ts;
993 uint64_t elapsed = end[i] - start[i];
999 uint64_t seq_loss = (join_struct.n_threads - 1)*elapsed_seq;
1000 uint64_t par_loss = join_struct.n_threads*max - sum;
1001 double efficiency = 0.0;
1003 efficiency = sum*100.0/(join_struct.n_threads*max);
1005 const double ts_scale = 1e-6;
1007 // enable this printf to get statistics on each individual join as it occurs
1008 // printf("join #%3d seq_loss = %5g par_loss = %5g efficiency = %3.0f%%\n", join_id, ts_scale*seq_loss, ts_scale*par_loss, efficiency);
1010 elapsed_total[id] += sum;
1011 wake_total[id] += wake;
1012 seq_loss_total[id] += seq_loss;
1013 par_loss_total[id] += par_loss;
1015 // every 10 seconds, print a summary of the time spent in each type of join
1016 if (GCToOSInterface::GetLowPrecisionTimeStamp() - start_tick > 10*1000)
1018 printf("**** summary *****\n");
1019 for (int i = 0; i < 16; i++)
1021 printf("join #%3d elapsed_total = %8g wake_loss = %8g seq_loss = %8g par_loss = %8g in_join_total = %8g\n",
1023 ts_scale*elapsed_total[i],
1024 ts_scale*wake_total[i],
1025 ts_scale*seq_loss_total[i],
1026 ts_scale*par_loss_total[i],
1027 ts_scale*in_join_total[i]);
1028 elapsed_total[i] = wake_total[i] = seq_loss_total[i] = par_loss_total[i] = in_join_total[i] = 0;
1030 start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
1034 fire_event (join_heap_restart, time_start, type_restart, -1);
1035 assert (join_struct.joined_p);
1036 join_struct.joined_p = FALSE;
1037 join_struct.join_lock = join_struct.n_threads;
1038 dprintf (JOIN_LOG, ("join%d(%d): Restarting from join: join_lock is %d", flavor, id, (int32_t)(join_struct.join_lock)));
1039 // printf("restart from join #%d at cycle %u from start of gc\n", join_id, GetCycleCount32() - gc_start);
1040 int color = join_struct.lock_color.LoadWithoutBarrier();
1041 join_struct.lock_color = !color;
1042 join_struct.joined_event[color].Set();
1044 // printf("Set joined_event %d\n", !join_struct.lock_color);
1046 fire_event (join_heap_restart, time_end, type_restart, -1);
1049 start[thd] = get_ts();
1055 dprintf (JOIN_LOG, ("join%d(%d): joined, join_lock is %d", flavor, id, (int32_t)(join_struct.join_lock)));
1056 return join_struct.joined_p;
1061 if (join_struct.n_threads != 1)
1063 fire_event (join_heap_r_restart, time_start, type_restart, -1);
1064 join_struct.wait_done = TRUE;
1065 join_struct.joined_event[first_thread_arrived].Set();
1066 fire_event (join_heap_r_restart, time_end, type_restart, -1);
1072 if (join_struct.n_threads != 1)
1074 join_struct.r_join_lock = join_struct.n_threads;
1075 join_struct.wait_done = FALSE;
1076 join_struct.joined_event[first_thread_arrived].Reset();
1083 #ifdef BACKGROUND_GC
1085 #endif //BACKGROUND_GC
1087 #endif //MULTIPLE_HEAPS
1089 #define spin_and_switch(count_to_spin, expr) \
1091 for (int j = 0; j < count_to_spin; j++) \
1101 GCToOSInterface::YieldThread(0); \
1105 #ifndef DACCESS_COMPILE
1106 #ifdef BACKGROUND_GC
1108 #define max_pending_allocs 64
1110 class exclusive_sync
1112 // TODO - verify that this is the right syntax for Volatile.
1113 VOLATILE(uint8_t*) rwp_object;
1114 VOLATILE(int32_t) needs_checking;
1118 uint8_t cache_separator[HS_CACHE_LINE_SIZE - sizeof (int) - sizeof (int32_t)];
1120 // TODO - perhaps each object should be on its own cache line...
1121 VOLATILE(uint8_t*) alloc_objects[max_pending_allocs];
1123 int find_free_index ()
1125 for (int i = 0; i < max_pending_allocs; i++)
1127 if (alloc_objects [i] == (uint8_t*)0)
1139 spin_count = 32 * (g_num_processors - 1);
1142 for (int i = 0; i < max_pending_allocs; i++)
1144 alloc_objects [i] = (uint8_t*)0;
1150 for (int i = 0; i < max_pending_allocs; i++)
1152 if (alloc_objects [i] != (uint8_t*)0)
1154 GCToOSInterface::DebugBreak();
1159 void bgc_mark_set (uint8_t* obj)
1161 dprintf (3, ("cm: probing %Ix", obj));
1163 if (Interlocked::CompareExchange(&needs_checking, 1, 0) == 0)
1165 // If we spend too much time spending all the allocs,
1166 // consider adding a high water mark and scan up
1167 // to that; we'll need to interlock in done when
1168 // we update the high watermark.
1169 for (int i = 0; i < max_pending_allocs; i++)
1171 if (obj == alloc_objects[i])
1174 dprintf (3, ("cm: will spin"));
1175 spin_and_switch (spin_count, (obj != alloc_objects[i]));
1182 dprintf (3, ("cm: set %Ix", obj));
1187 spin_and_switch (spin_count, (needs_checking == 0));
1192 int loh_alloc_set (uint8_t* obj)
1194 if (!gc_heap::cm_in_progress)
1200 dprintf (3, ("loh alloc: probing %Ix", obj));
1202 if (Interlocked::CompareExchange(&needs_checking, 1, 0) == 0)
1204 if (obj == rwp_object)
1207 spin_and_switch (spin_count, (obj != rwp_object));
1212 int cookie = find_free_index();
1216 alloc_objects[cookie] = obj;
1220 // GCToOSInterface::DebugBreak();
1223 dprintf (3, ("loh alloc: set %Ix at %d", obj, cookie));
1229 dprintf (3, ("loh alloc: setting %Ix will spin to acquire a free index", obj));
1230 spin_and_switch (spin_count, (find_free_index () != -1));
1237 dprintf (3, ("loh alloc: will spin on checking %Ix", obj));
1238 spin_and_switch (spin_count, (needs_checking == 0));
1243 void bgc_mark_done ()
1245 dprintf (3, ("cm: release lock on %Ix", (uint8_t *)rwp_object));
1249 void loh_alloc_done_with_index (int index)
1251 dprintf (3, ("loh alloc: release lock on %Ix based on %d", (uint8_t *)alloc_objects[index], index));
1252 assert ((index >= 0) && (index < max_pending_allocs));
1253 alloc_objects[index] = (uint8_t*)0;
1256 void loh_alloc_done (uint8_t* obj)
1258 #ifdef BACKGROUND_GC
1259 if (!gc_heap::cm_in_progress)
1264 for (int i = 0; i < max_pending_allocs; i++)
1266 if (alloc_objects [i] == obj)
1268 dprintf (3, ("loh alloc: release lock on %Ix at %d", (uint8_t *)alloc_objects[i], i));
1269 alloc_objects[i] = (uint8_t*)0;
1273 #endif //BACKGROUND_GC
1277 // Note that this class was written assuming just synchronization between
1278 // one background GC thread and multiple user threads that might request
1279 // an FGC - it does not take into account what kind of locks the multiple
1280 // user threads might be holding at the time (eg, there could only be one
1281 // user thread requesting an FGC because it needs to take gc_lock first)
1282 // so you'll see checks that may not be necessary if you take those conditions
1283 // into consideration.
1285 // With the introduction of Server Background GC we no longer use this
1286 // class to do synchronization between FGCs and BGC.
1287 class recursive_gc_sync
1289 static VOLATILE(int32_t) foreground_request_count;//initial state 0
1290 static VOLATILE(BOOL) gc_background_running; //initial state FALSE
1291 static VOLATILE(int32_t) foreground_count; // initial state 0;
1292 static VOLATILE(uint32_t) foreground_gate; // initial state FALSE;
1293 static GCEvent foreground_complete;//Auto Reset
1294 static GCEvent foreground_allowed;//Auto Reset
1296 static void begin_background();
1297 static void end_background();
1298 static void begin_foreground();
1299 static void end_foreground();
1300 BOOL allow_foreground ();
1302 static void shutdown();
1303 static BOOL background_running_p() {return gc_background_running;}
1306 VOLATILE(int32_t) recursive_gc_sync::foreground_request_count = 0;//initial state 0
1307 VOLATILE(int32_t) recursive_gc_sync::foreground_count = 0; // initial state 0;
1308 VOLATILE(BOOL) recursive_gc_sync::gc_background_running = FALSE; //initial state FALSE
1309 VOLATILE(uint32_t) recursive_gc_sync::foreground_gate = 0;
1310 GCEvent recursive_gc_sync::foreground_complete;//Auto Reset
1311 GCEvent recursive_gc_sync::foreground_allowed;//Manual Reset
1313 BOOL recursive_gc_sync::init ()
1315 foreground_request_count = 0;
1316 foreground_count = 0;
1317 gc_background_running = FALSE;
1318 foreground_gate = 0;
1320 if (!foreground_complete.CreateOSAutoEventNoThrow(FALSE))
1324 if (!foreground_allowed.CreateManualEventNoThrow(FALSE))
1336 void recursive_gc_sync::shutdown()
1338 if (foreground_complete.IsValid())
1339 foreground_complete.CloseEvent();
1340 if (foreground_allowed.IsValid())
1341 foreground_allowed.CloseEvent();
1344 void recursive_gc_sync::begin_background()
1346 dprintf (2, ("begin background"));
1347 foreground_request_count = 1;
1348 foreground_count = 1;
1349 foreground_allowed.Reset();
1350 gc_background_running = TRUE;
1352 void recursive_gc_sync::end_background()
1354 dprintf (2, ("end background"));
1355 gc_background_running = FALSE;
1356 foreground_gate = 1;
1357 foreground_allowed.Set();
1360 void recursive_gc_sync::begin_foreground()
1362 dprintf (2, ("begin_foreground"));
1364 bool cooperative_mode = false;
1365 if (gc_background_running)
1367 gc_heap::fire_alloc_wait_event_begin (awr_fgc_wait_for_bgc);
1368 gc_heap::alloc_wait_event_p = TRUE;
1372 Interlocked::Increment (&foreground_request_count);
1375 dprintf(2, ("Waiting sync gc point"));
1376 assert (foreground_allowed.IsValid());
1377 assert (foreground_complete.IsValid());
1379 cooperative_mode = gc_heap::enable_preemptive ();
1381 foreground_allowed.Wait(INFINITE, FALSE);
1383 dprintf(2, ("Waiting sync gc point is done"));
1385 gc_heap::disable_preemptive (cooperative_mode);
1387 if (foreground_gate)
1389 Interlocked::Increment (&foreground_count);
1390 dprintf (2, ("foreground_count: %d", (int32_t)foreground_count));
1391 if (foreground_gate)
1393 gc_heap::settings.concurrent = FALSE;
1404 goto try_again_no_inc;
1409 void recursive_gc_sync::end_foreground()
1411 dprintf (2, ("end_foreground"));
1412 if (gc_background_running)
1414 Interlocked::Decrement (&foreground_request_count);
1415 dprintf (2, ("foreground_count before decrement: %d", (int32_t)foreground_count));
1416 if (Interlocked::Decrement (&foreground_count) == 0)
1418 //c_write ((BOOL*)&foreground_gate, 0);
1419 // TODO - couldn't make the syntax work with Volatile<T>
1420 foreground_gate = 0;
1421 if (foreground_count == 0)
1423 foreground_allowed.Reset ();
1424 dprintf(2, ("setting foreground complete event"));
1425 foreground_complete.Set();
1432 BOOL recursive_gc_sync::allow_foreground()
1434 assert (gc_heap::settings.concurrent);
1435 dprintf (100, ("enter allow_foreground, f_req_count: %d, f_count: %d",
1436 (int32_t)foreground_request_count, (int32_t)foreground_count));
1438 BOOL did_fgc = FALSE;
1440 //if we have suspended the EE, just return because
1441 //some thread could be waiting on this to proceed.
1442 if (!GCHeap::GcInProgress)
1444 //TODO BACKGROUND_GC This is to stress the concurrency between
1445 //background and foreground
1446 // gc_heap::disallow_new_allocation (0);
1448 //GCToOSInterface::YieldThread(0);
1451 if (foreground_request_count != 0)
1453 //foreground wants to run
1454 //save the important settings
1455 //TODO BACKGROUND_GC be more selective about the important settings.
1456 gc_mechanisms saved_settings = gc_heap::settings;
1460 //c_write ((BOOL*)&foreground_gate, 1);
1461 // TODO - couldn't make the syntax work with Volatile<T>
1462 foreground_gate = 1;
1463 foreground_allowed.Set ();
1464 foreground_complete.Wait (INFINITE, FALSE);
1465 }while (/*foreground_request_count ||*/ foreground_gate);
1467 assert (!foreground_gate);
1469 //restore the important settings
1470 gc_heap::settings = saved_settings;
1471 GCHeap::GcCondemnedGeneration = gc_heap::settings.condemned_generation;
1472 //the background GC shouldn't be using gc_high and gc_low
1473 //gc_low = lowest_address;
1474 //gc_high = highest_address;
1477 //TODO BACKGROUND_GC This is to stress the concurrency between
1478 //background and foreground
1479 // gc_heap::allow_new_allocation (0);
1483 dprintf (100, ("leave allow_foreground"));
1484 assert (gc_heap::settings.concurrent);
1488 #endif //BACKGROUND_GC
1489 #endif //DACCESS_COMPILE
1492 #if defined(COUNT_CYCLES)
1494 #pragma warning(disable:4035)
1498 unsigned GetCycleCount32() // enough for about 40 seconds
1506 #pragma warning(default:4035)
1508 #endif //COUNT_CYCLES
1511 int mark_time, plan_time, sweep_time, reloc_time, compact_time;
1514 #ifndef MULTIPLE_HEAPS
1516 #endif // MULTIPLE_HEAPS
1518 void reset_memory (uint8_t* o, size_t sizeo);
1522 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1523 static bool virtual_alloc_hardware_write_watch = false;
1524 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1526 static bool hardware_write_watch_capability = false;
1528 #ifndef DACCESS_COMPILE
1530 //check if the write watch APIs are supported.
1532 void hardware_write_watch_api_supported()
1534 if (GCToOSInterface::SupportsWriteWatch())
1536 hardware_write_watch_capability = true;
1537 dprintf (2, ("WriteWatch supported"));
1541 dprintf (2,("WriteWatch not supported"));
1545 #endif //!DACCESS_COMPILE
1547 inline bool can_use_hardware_write_watch()
1549 return hardware_write_watch_capability;
1552 inline bool can_use_write_watch_for_gc_heap()
1554 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1556 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1557 return can_use_hardware_write_watch();
1558 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1561 inline bool can_use_write_watch_for_card_table()
1563 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
1566 return can_use_hardware_write_watch();
1571 #define mem_reserve (MEM_RESERVE)
1572 #endif //WRITE_WATCH
1574 //check if the low memory notification is supported
1576 #ifndef DACCESS_COMPILE
1578 void WaitLongerNoInstru (int i)
1580 // every 8th attempt:
1581 bool bToggleGC = GCToEEInterface::EnablePreemptiveGC();
1583 // if we're waiting for gc to finish, we should block immediately
1584 if (g_fSuspensionPending == 0)
1586 if (g_num_processors > 1)
1588 YieldProcessor(); // indicate to the processor that we are spining
1590 GCToOSInterface::YieldThread (0);
1592 GCToOSInterface::Sleep (5);
1595 GCToOSInterface::Sleep (5);
1598 // If CLR is hosted, a thread may reach here while it is in preemptive GC mode,
1599 // or it has no Thread object, in order to force a task to yield, or to triger a GC.
1600 // It is important that the thread is going to wait for GC. Otherwise the thread
1601 // is in a tight loop. If the thread has high priority, the perf is going to be very BAD.
1605 // In debug builds, all enter_spin_lock operations go through this code. If a GC has
1606 // started, it is important to block until the GC thread calls set_gc_done (since it is
1607 // guaranteed to have cleared g_TrapReturningThreads by this point). This avoids livelock
1608 // conditions which can otherwise occur if threads are allowed to spin in this function
1609 // (and therefore starve the GC thread) between the point when the GC thread sets the
1610 // WaitForGC event and the point when the GC thread clears g_TrapReturningThreads.
1611 if (gc_heap::gc_started)
1613 gc_heap::wait_for_gc_done();
1616 GCToEEInterface::DisablePreemptiveGC();
1618 else if (g_fSuspensionPending > 0)
1620 g_theGCHeap->WaitUntilGCComplete();
1625 static void safe_switch_to_thread()
1627 bool cooperative_mode = gc_heap::enable_preemptive();
1629 GCToOSInterface::YieldThread(0);
1631 gc_heap::disable_preemptive(cooperative_mode);
1635 // We need the following methods to have volatile arguments, so that they can accept
1636 // raw pointers in addition to the results of the & operator on Volatile<T>.
1639 static void enter_spin_lock_noinstru (RAW_KEYWORD(volatile) int32_t* lock)
1643 if (Interlocked::CompareExchange(lock, 0, -1) >= 0)
1646 while (VolatileLoad(lock) >= 0)
1648 if ((++i & 7) && !IsGCInProgress())
1650 if (g_num_processors > 1)
1652 #ifndef MULTIPLE_HEAPS
1653 int spin_count = 1024 * g_num_processors;
1654 #else //!MULTIPLE_HEAPS
1655 int spin_count = 32 * g_num_processors;
1656 #endif //!MULTIPLE_HEAPS
1657 for (int j = 0; j < spin_count; j++)
1659 if (VolatileLoad(lock) < 0 || IsGCInProgress())
1661 YieldProcessor(); // indicate to the processor that we are spining
1663 if (VolatileLoad(lock) >= 0 && !IsGCInProgress())
1665 safe_switch_to_thread();
1670 safe_switch_to_thread();
1675 WaitLongerNoInstru(i);
1683 static BOOL try_enter_spin_lock_noinstru(RAW_KEYWORD(volatile) int32_t* lock)
1685 return (Interlocked::CompareExchange(&*lock, 0, -1) < 0);
1689 static void leave_spin_lock_noinstru (RAW_KEYWORD(volatile) int32_t* lock)
1691 VolatileStore<int32_t>((int32_t*)lock, -1);
1697 static void enter_spin_lock(GCSpinLock *pSpinLock)
1699 enter_spin_lock_noinstru(&pSpinLock->lock);
1700 assert (pSpinLock->holding_thread == (Thread*)-1);
1701 pSpinLock->holding_thread = GCToEEInterface::GetThread();
1705 static BOOL try_enter_spin_lock(GCSpinLock *pSpinLock)
1707 BOOL ret = try_enter_spin_lock_noinstru(&pSpinLock->lock);
1709 pSpinLock->holding_thread = GCToEEInterface::GetThread();
1714 static void leave_spin_lock(GCSpinLock *pSpinLock)
1716 bool gc_thread_p = GCToEEInterface::WasCurrentThreadCreatedByGC();
1717 // _ASSERTE((pSpinLock->holding_thread == GCToEEInterface::GetThread()) || gc_thread_p || pSpinLock->released_by_gc_p);
1718 pSpinLock->released_by_gc_p = gc_thread_p;
1719 pSpinLock->holding_thread = (Thread*) -1;
1720 if (pSpinLock->lock != -1)
1721 leave_spin_lock_noinstru(&pSpinLock->lock);
1724 #define ASSERT_HOLDING_SPIN_LOCK(pSpinLock) \
1725 _ASSERTE((pSpinLock)->holding_thread == GCToEEInterface::GetThread());
1727 #define ASSERT_NOT_HOLDING_SPIN_LOCK(pSpinLock) \
1728 _ASSERTE((pSpinLock)->holding_thread != GCToEEInterface::GetThread());
1732 //In the concurrent version, the Enable/DisablePreemptiveGC is optional because
1733 //the gc thread call WaitLonger.
1734 void WaitLonger (int i
1735 #ifdef SYNCHRONIZATION_STATS
1736 , GCSpinLock* spin_lock
1737 #endif //SYNCHRONIZATION_STATS
1740 #ifdef SYNCHRONIZATION_STATS
1741 (spin_lock->num_wait_longer)++;
1742 #endif //SYNCHRONIZATION_STATS
1744 // every 8th attempt:
1745 bool bToggleGC = GCToEEInterface::EnablePreemptiveGC();
1748 // if we're waiting for gc to finish, we should block immediately
1749 if (!gc_heap::gc_started)
1751 #ifdef SYNCHRONIZATION_STATS
1752 (spin_lock->num_switch_thread_w)++;
1753 #endif //SYNCHRONIZATION_STATS
1754 if (g_num_processors > 1)
1756 YieldProcessor(); // indicate to the processor that we are spining
1758 GCToOSInterface::YieldThread (0);
1760 GCToOSInterface::Sleep (5);
1763 GCToOSInterface::Sleep (5);
1766 // If CLR is hosted, a thread may reach here while it is in preemptive GC mode,
1767 // or it has no Thread object, in order to force a task to yield, or to triger a GC.
1768 // It is important that the thread is going to wait for GC. Otherwise the thread
1769 // is in a tight loop. If the thread has high priority, the perf is going to be very BAD.
1770 if (gc_heap::gc_started)
1772 gc_heap::wait_for_gc_done();
1777 #ifdef SYNCHRONIZATION_STATS
1778 (spin_lock->num_disable_preemptive_w)++;
1779 #endif //SYNCHRONIZATION_STATS
1780 GCToEEInterface::DisablePreemptiveGC();
1785 static void enter_spin_lock (GCSpinLock* spin_lock)
1789 if (Interlocked::CompareExchange(&spin_lock->lock, 0, -1) >= 0)
1792 while (spin_lock->lock >= 0)
1794 if ((++i & 7) && !gc_heap::gc_started)
1796 if (g_num_processors > 1)
1798 #ifndef MULTIPLE_HEAPS
1799 int spin_count = 1024 * g_num_processors;
1800 #else //!MULTIPLE_HEAPS
1801 int spin_count = 32 * g_num_processors;
1802 #endif //!MULTIPLE_HEAPS
1803 for (int j = 0; j < spin_count; j++)
1805 if (spin_lock->lock < 0 || gc_heap::gc_started)
1807 YieldProcessor(); // indicate to the processor that we are spining
1809 if (spin_lock->lock >= 0 && !gc_heap::gc_started)
1811 #ifdef SYNCHRONIZATION_STATS
1812 (spin_lock->num_switch_thread)++;
1813 #endif //SYNCHRONIZATION_STATS
1814 bool cooperative_mode = gc_heap::enable_preemptive ();
1816 GCToOSInterface::YieldThread(0);
1818 gc_heap::disable_preemptive (cooperative_mode);
1822 GCToOSInterface::YieldThread(0);
1827 #ifdef SYNCHRONIZATION_STATS
1829 #endif //SYNCHRONIZATION_STATS
1837 inline BOOL try_enter_spin_lock(GCSpinLock* spin_lock)
1839 return (Interlocked::CompareExchange(&spin_lock->lock, 0, -1) < 0);
1843 static void leave_spin_lock (GCSpinLock * spin_lock)
1845 spin_lock->lock = -1;
1848 #define ASSERT_HOLDING_SPIN_LOCK(pSpinLock)
1852 bool gc_heap::enable_preemptive ()
1854 return GCToEEInterface::EnablePreemptiveGC();
1857 void gc_heap::disable_preemptive (bool restore_cooperative)
1859 if (restore_cooperative)
1861 GCToEEInterface::DisablePreemptiveGC();
1865 #endif // !DACCESS_COMPILE
1867 typedef void ** PTR_PTR;
1868 //This function clears a piece of memory
1869 // size has to be Dword aligned
1872 void memclr ( uint8_t* mem, size_t size)
1874 dprintf (3, ("MEMCLR: %Ix, %d", mem, size));
1875 assert ((size & (sizeof(PTR_PTR)-1)) == 0);
1876 assert (sizeof(PTR_PTR) == DATA_ALIGNMENT);
1879 // The compiler will recognize this pattern and replace it with memset call. We can as well just call
1880 // memset directly to make it obvious what's going on.
1881 PTR_PTR m = (PTR_PTR) mem;
1882 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
1886 memset (mem, 0, size);
1889 void memcopy (uint8_t* dmem, uint8_t* smem, size_t size)
1891 const size_t sz4ptr = sizeof(PTR_PTR)*4;
1892 const size_t sz2ptr = sizeof(PTR_PTR)*2;
1893 const size_t sz1ptr = sizeof(PTR_PTR)*1;
1895 // size must be a multiple of the pointer size
1896 assert ((size & (sizeof (PTR_PTR)-1)) == 0);
1897 assert (sizeof(PTR_PTR) == DATA_ALIGNMENT);
1899 // copy in groups of four pointer sized things at a time
1904 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1905 ((PTR_PTR)dmem)[1] = ((PTR_PTR)smem)[1];
1906 ((PTR_PTR)dmem)[2] = ((PTR_PTR)smem)[2];
1907 ((PTR_PTR)dmem)[3] = ((PTR_PTR)smem)[3];
1911 while ((size -= sz4ptr) >= sz4ptr);
1914 // still two pointer sized things or more left to copy?
1917 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1918 ((PTR_PTR)dmem)[1] = ((PTR_PTR)smem)[1];
1923 // still one pointer sized thing left to copy?
1926 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1934 ptrdiff_t round_down (ptrdiff_t add, int pitch)
1936 return ((add / pitch) * pitch);
1939 #if defined(FEATURE_STRUCTALIGN) && defined(RESPECT_LARGE_ALIGNMENT)
1940 // FEATURE_STRUCTALIGN allows the compiler to dictate the alignment,
1941 // i.e, if a larger alignment matters or is beneficial, the compiler
1942 // generated info tells us so. RESPECT_LARGE_ALIGNMENT is just the
1943 // converse - it's a heuristic for the GC to use a larger alignment.
1944 #error FEATURE_STRUCTALIGN should imply !RESPECT_LARGE_ALIGNMENT
1947 #if defined(FEATURE_STRUCTALIGN) && defined(FEATURE_LOH_COMPACTION)
1948 #error FEATURE_STRUCTALIGN and FEATURE_LOH_COMPACTION are mutually exclusive
1951 #if defined(GROWABLE_SEG_MAPPING_TABLE) && !defined(SEG_MAPPING_TABLE)
1952 #error if GROWABLE_SEG_MAPPING_TABLE is defined, SEG_MAPPING_TABLE must be defined
1956 BOOL same_large_alignment_p (uint8_t* p1, uint8_t* p2)
1958 #ifdef RESPECT_LARGE_ALIGNMENT
1959 return ((((size_t)p1 ^ (size_t)p2) & 7) == 0);
1961 UNREFERENCED_PARAMETER(p1);
1962 UNREFERENCED_PARAMETER(p2);
1964 #endif //RESPECT_LARGE_ALIGNMENT
1968 size_t switch_alignment_size (BOOL already_padded_p)
1970 if (already_padded_p)
1971 return DATA_ALIGNMENT;
1973 return (Align (min_obj_size) +((Align (min_obj_size)&DATA_ALIGNMENT)^DATA_ALIGNMENT));
1977 #ifdef FEATURE_STRUCTALIGN
1978 void set_node_aligninfo (uint8_t *node, int requiredAlignment, ptrdiff_t pad);
1979 void clear_node_aligninfo (uint8_t *node);
1980 #else // FEATURE_STRUCTALIGN
1981 #define node_realigned(node) (((plug_and_reloc*)(node))[-1].reloc & 1)
1982 void set_node_realigned (uint8_t* node);
1983 void clear_node_realigned(uint8_t* node);
1984 #endif // FEATURE_STRUCTALIGN
1987 size_t AlignQword (size_t nbytes)
1989 #ifdef FEATURE_STRUCTALIGN
1990 // This function is used to align everything on the large object
1991 // heap to an 8-byte boundary, to reduce the number of unaligned
1992 // accesses to (say) arrays of doubles. With FEATURE_STRUCTALIGN,
1993 // the compiler dictates the optimal alignment instead of having
1994 // a heuristic in the GC.
1995 return Align (nbytes);
1996 #else // FEATURE_STRUCTALIGN
1997 return (nbytes + 7) & ~7;
1998 #endif // FEATURE_STRUCTALIGN
2002 BOOL Aligned (size_t n)
2004 return (n & ALIGNCONST) == 0;
2007 #define OBJECT_ALIGNMENT_OFFSET (sizeof(MethodTable *))
2009 #ifdef FEATURE_STRUCTALIGN
2010 #define MAX_STRUCTALIGN OS_PAGE_SIZE
2011 #else // FEATURE_STRUCTALIGN
2012 #define MAX_STRUCTALIGN 0
2013 #endif // FEATURE_STRUCTALIGN
2015 #ifdef FEATURE_STRUCTALIGN
2017 ptrdiff_t AdjustmentForMinPadSize(ptrdiff_t pad, int requiredAlignment)
2019 // The resulting alignpad must be either 0 or at least min_obj_size.
2020 // Note that by computing the following difference on unsigned types,
2021 // we can do the range check 0 < alignpad < min_obj_size with a
2022 // single conditional branch.
2023 if ((size_t)(pad - DATA_ALIGNMENT) < Align (min_obj_size) - DATA_ALIGNMENT)
2025 return requiredAlignment;
2031 uint8_t* StructAlign (uint8_t* origPtr, int requiredAlignment, ptrdiff_t alignmentOffset=OBJECT_ALIGNMENT_OFFSET)
2033 // required alignment must be a power of two
2034 _ASSERTE(((size_t)origPtr & ALIGNCONST) == 0);
2035 _ASSERTE(((requiredAlignment - 1) & requiredAlignment) == 0);
2036 _ASSERTE(requiredAlignment >= sizeof(void *));
2037 _ASSERTE(requiredAlignment <= MAX_STRUCTALIGN);
2039 // When this method is invoked for individual objects (i.e., alignmentOffset
2040 // is just the size of the PostHeader), what needs to be aligned when
2041 // we're done is the pointer to the payload of the object (which means
2042 // the actual resulting object pointer is typically not aligned).
2044 uint8_t* result = (uint8_t*)Align ((size_t)origPtr + alignmentOffset, requiredAlignment-1) - alignmentOffset;
2045 ptrdiff_t alignpad = result - origPtr;
2047 return result + AdjustmentForMinPadSize (alignpad, requiredAlignment);
2051 ptrdiff_t ComputeStructAlignPad (uint8_t* plug, int requiredAlignment, size_t alignmentOffset=OBJECT_ALIGNMENT_OFFSET)
2053 return StructAlign (plug, requiredAlignment, alignmentOffset) - plug;
2056 BOOL IsStructAligned (uint8_t *ptr, int requiredAlignment)
2058 return StructAlign (ptr, requiredAlignment) == ptr;
2062 ptrdiff_t ComputeMaxStructAlignPad (int requiredAlignment)
2064 if (requiredAlignment == DATA_ALIGNMENT)
2066 // Since a non-zero alignment padding cannot be less than min_obj_size (so we can fit the
2067 // alignment padding object), the worst-case alignment padding is correspondingly larger
2068 // than the required alignment.
2069 return requiredAlignment + Align (min_obj_size) - DATA_ALIGNMENT;
2073 ptrdiff_t ComputeMaxStructAlignPadLarge (int requiredAlignment)
2075 if (requiredAlignment <= get_alignment_constant (TRUE)+1)
2077 // This is the same as ComputeMaxStructAlignPad, except that in addition to leaving space
2078 // for padding before the actual object, it also leaves space for filling a gap after the
2079 // actual object. This is needed on the large object heap, as the outer allocation functions
2080 // don't operate on an allocation context (which would have left space for the final gap).
2081 return requiredAlignment + Align (min_obj_size) * 2 - DATA_ALIGNMENT;
2084 uint8_t* gc_heap::pad_for_alignment (uint8_t* newAlloc, int requiredAlignment, size_t size, alloc_context* acontext)
2086 uint8_t* alignedPtr = StructAlign (newAlloc, requiredAlignment);
2087 if (alignedPtr != newAlloc) {
2088 make_unused_array (newAlloc, alignedPtr - newAlloc);
2090 acontext->alloc_ptr = alignedPtr + Align (size);
2094 uint8_t* gc_heap::pad_for_alignment_large (uint8_t* newAlloc, int requiredAlignment, size_t size)
2096 uint8_t* alignedPtr = StructAlign (newAlloc, requiredAlignment);
2097 if (alignedPtr != newAlloc) {
2098 make_unused_array (newAlloc, alignedPtr - newAlloc);
2100 if (alignedPtr < newAlloc + ComputeMaxStructAlignPadLarge (requiredAlignment)) {
2101 make_unused_array (alignedPtr + AlignQword (size), newAlloc + ComputeMaxStructAlignPadLarge (requiredAlignment) - alignedPtr);
2105 #else // FEATURE_STRUCTALIGN
2106 #define ComputeMaxStructAlignPad(requiredAlignment) 0
2107 #define ComputeMaxStructAlignPadLarge(requiredAlignment) 0
2108 #endif // FEATURE_STRUCTALIGN
2110 //CLR_SIZE is the max amount of bytes from gen0 that is set to 0 in one chunk
2112 #define CLR_SIZE ((size_t)(8*1024))
2114 #define CLR_SIZE ((size_t)(8*1024))
2117 #define END_SPACE_AFTER_GC (LARGE_OBJECT_SIZE + MAX_STRUCTALIGN)
2119 #ifdef BACKGROUND_GC
2120 #define SEGMENT_INITIAL_COMMIT (2*OS_PAGE_SIZE)
2122 #define SEGMENT_INITIAL_COMMIT (OS_PAGE_SIZE)
2123 #endif //BACKGROUND_GC
2129 #define INITIAL_ALLOC ((size_t)((size_t)4*1024*1024*1024))
2130 #define LHEAP_ALLOC ((size_t)(1024*1024*256))
2134 #define INITIAL_ALLOC ((size_t)(1024*1024*64))
2135 #define LHEAP_ALLOC ((size_t)(1024*1024*32))
2143 #define INITIAL_ALLOC ((size_t)(1024*1024*256))
2144 #define LHEAP_ALLOC ((size_t)(1024*1024*128))
2148 #define INITIAL_ALLOC ((size_t)(1024*1024*16))
2149 #define LHEAP_ALLOC ((size_t)(1024*1024*16))
2155 //amount in bytes of the etw allocation tick
2156 const size_t etw_allocation_tick = 100*1024;
2158 const size_t low_latency_alloc = 256*1024;
2160 const size_t fgn_check_quantum = 2*1024*1024;
2163 const int max_snoop_level = 128;
2168 //threshold of heap size to turn on card bundles.
2169 #define SH_TH_CARD_BUNDLE (40*1024*1024)
2170 #define MH_TH_CARD_BUNDLE (180*1024*1024)
2171 #endif //CARD_BUNDLE
2173 #define GC_EPHEMERAL_DECOMMIT_TIMEOUT 5000
2176 size_t align_on_page (size_t add)
2178 return ((add + OS_PAGE_SIZE - 1) & ~((size_t)OS_PAGE_SIZE - 1));
2182 uint8_t* align_on_page (uint8_t* add)
2184 return (uint8_t*)align_on_page ((size_t) add);
2188 size_t align_lower_page (size_t add)
2190 return (add & ~((size_t)OS_PAGE_SIZE - 1));
2194 uint8_t* align_lower_page (uint8_t* add)
2196 return (uint8_t*)align_lower_page ((size_t)add);
2200 size_t align_write_watch_lower_page (size_t add)
2202 return (add & ~(WRITE_WATCH_UNIT_SIZE - 1));
2206 uint8_t* align_write_watch_lower_page (uint8_t* add)
2208 return (uint8_t*)align_lower_page ((size_t)add);
2213 BOOL power_of_two_p (size_t integer)
2215 return !(integer & (integer-1));
2219 BOOL oddp (size_t integer)
2221 return (integer & 1) != 0;
2224 // we only ever use this for WORDs.
2225 size_t logcount (size_t word)
2227 //counts the number of high bits in a 16 bit word.
2228 assert (word < 0x10000);
2230 count = (word & 0x5555) + ( (word >> 1 ) & 0x5555);
2231 count = (count & 0x3333) + ( (count >> 2) & 0x3333);
2232 count = (count & 0x0F0F) + ( (count >> 4) & 0x0F0F);
2233 count = (count & 0x00FF) + ( (count >> 8) & 0x00FF);
2237 #ifndef DACCESS_COMPILE
2239 void stomp_write_barrier_resize(bool is_runtime_suspended, bool requires_upper_bounds_check)
2241 WriteBarrierParameters args = {};
2242 args.operation = WriteBarrierOp::StompResize;
2243 args.is_runtime_suspended = is_runtime_suspended;
2244 args.requires_upper_bounds_check = requires_upper_bounds_check;
2246 args.card_table = g_gc_card_table;
2247 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
2248 args.card_bundle_table = g_gc_card_bundle_table;
2251 args.lowest_address = g_gc_lowest_address;
2252 args.highest_address = g_gc_highest_address;
2254 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
2255 if (SoftwareWriteWatch::IsEnabledForGCHeap())
2257 args.write_watch_table = g_gc_sw_ww_table;
2259 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
2261 GCToEEInterface::StompWriteBarrier(&args);
2264 void stomp_write_barrier_ephemeral(uint8_t* ephemeral_low, uint8_t* ephemeral_high)
2266 WriteBarrierParameters args = {};
2267 args.operation = WriteBarrierOp::StompEphemeral;
2268 args.is_runtime_suspended = true;
2269 args.ephemeral_low = ephemeral_low;
2270 args.ephemeral_high = ephemeral_high;
2271 GCToEEInterface::StompWriteBarrier(&args);
2274 void stomp_write_barrier_initialize(uint8_t* ephemeral_low, uint8_t* ephemeral_high)
2276 WriteBarrierParameters args = {};
2277 args.operation = WriteBarrierOp::Initialize;
2278 args.is_runtime_suspended = true;
2279 args.requires_upper_bounds_check = false;
2280 args.card_table = g_gc_card_table;
2282 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
2283 args.card_bundle_table = g_gc_card_bundle_table;
2286 args.lowest_address = g_gc_lowest_address;
2287 args.highest_address = g_gc_highest_address;
2288 args.ephemeral_low = ephemeral_low;
2289 args.ephemeral_high = ephemeral_high;
2290 GCToEEInterface::StompWriteBarrier(&args);
2293 #endif // DACCESS_COMPILE
2295 //extract the low bits [0,low[ of a uint32_t
2296 #define lowbits(wrd, bits) ((wrd) & ((1 << (bits))-1))
2297 //extract the high bits [high, 32] of a uint32_t
2298 #define highbits(wrd, bits) ((wrd) & ~((1 << (bits))-1))
2300 // Things we need to manually initialize:
2301 // gen0 min_size - based on cache
2302 // gen0/1 max_size - based on segment size
2303 static static_data static_data_table[latency_level_last - latency_level_first + 1][NUMBERGENERATIONS] =
2305 // latency_level_memory_footprint
2308 {0, 0, 40000, 0.5f, 9.0f, 20.0f, 1000, 1},
2310 {163840, 0, 80000, 0.5f, 2.0f, 7.0f, 10000, 10},
2312 {256*1024, SSIZE_T_MAX, 200000, 0.25f, 1.2f, 1.8f, 100000, 100},
2314 {3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0}
2317 // latency_level_balanced
2321 #ifdef MULTIPLE_HEAPS
2325 #endif //MULTIPLE_HEAPS
2328 {9*32*1024, 0, 80000, 0.5f, 2.0f, 7.0f, 10000, 10},
2330 {256*1024, SSIZE_T_MAX, 200000, 0.25f, 1.2f, 1.8f, 100000, 100},
2332 {3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0}
2339 class CObjectHeader;
2343 class c_synchronize;
2345 #ifdef FEATURE_PREMORTEM_FINALIZATION
2346 #ifndef DACCESS_COMPILE
2348 HRESULT AllocateCFinalize(CFinalize **pCFinalize);
2349 #endif //!DACCESS_COMPILE
2350 #endif // FEATURE_PREMORTEM_FINALIZATION
2352 uint8_t* tree_search (uint8_t* tree, uint8_t* old_address);
2355 #ifdef USE_INTROSORT
2356 #define _sort introsort::sort
2357 #else //USE_INTROSORT
2358 #define _sort qsort1
2359 void qsort1(uint8_t** low, uint8_t** high, unsigned int depth);
2360 #endif //USE_INTROSORT
2362 void* virtual_alloc (size_t size);
2363 void virtual_free (void* add, size_t size);
2365 /* per heap static initialization */
2367 #ifndef MULTIPLE_HEAPS
2368 uint32_t* gc_heap::mark_array;
2369 #endif //MULTIPLE_HEAPS
2373 uint8_t** gc_heap::g_mark_list;
2375 #ifdef PARALLEL_MARK_LIST_SORT
2376 uint8_t** gc_heap::g_mark_list_copy;
2377 #endif //PARALLEL_MARK_LIST_SORT
2379 size_t gc_heap::mark_list_size;
2382 #ifdef SEG_MAPPING_TABLE
2383 seg_mapping* seg_mapping_table;
2384 #endif //SEG_MAPPING_TABLE
2386 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
2387 sorted_table* gc_heap::seg_table;
2388 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
2390 #ifdef MULTIPLE_HEAPS
2391 GCEvent gc_heap::ee_suspend_event;
2392 size_t gc_heap::min_balance_threshold = 0;
2393 #endif //MULTIPLE_HEAPS
2395 VOLATILE(BOOL) gc_heap::gc_started;
2397 #ifdef MULTIPLE_HEAPS
2399 GCEvent gc_heap::gc_start_event;
2401 bool gc_heap::gc_thread_no_affinitize_p = false;
2403 int gc_heap::n_heaps;
2405 gc_heap** gc_heap::g_heaps;
2407 size_t* gc_heap::g_promoted;
2410 int* gc_heap::g_mark_stack_busy;
2414 #ifdef BACKGROUND_GC
2415 size_t* gc_heap::g_bpromoted;
2416 #endif //BACKGROUND_GC
2418 #else //MULTIPLE_HEAPS
2420 size_t gc_heap::g_promoted;
2422 #ifdef BACKGROUND_GC
2423 size_t gc_heap::g_bpromoted;
2424 #endif //BACKGROUND_GC
2426 #endif //MULTIPLE_HEAPS
2428 size_t gc_heap::reserved_memory = 0;
2429 size_t gc_heap::reserved_memory_limit = 0;
2430 BOOL gc_heap::g_low_memory_status;
2432 #ifndef DACCESS_COMPILE
2433 static gc_reason gc_trigger_reason = reason_empty;
2434 #endif //DACCESS_COMPILE
2436 gc_latency_level gc_heap::latency_level = latency_level_default;
2438 gc_mechanisms gc_heap::settings;
2440 gc_history_global gc_heap::gc_data_global;
2442 size_t gc_heap::gc_last_ephemeral_decommit_time = 0;
2444 size_t gc_heap::gc_gen0_desired_high;
2447 double gc_heap::short_plugs_pad_ratio = 0;
2448 #endif //SHORT_PLUGS
2451 #define MAX_ALLOWED_MEM_LOAD 85
2453 // consider putting this in dynamic data -
2454 // we may want different values for workstation
2456 #define MIN_YOUNGEST_GEN_DESIRED (16*1024*1024)
2458 size_t gc_heap::youngest_gen_desired_th;
2461 uint32_t gc_heap::last_gc_memory_load = 0;
2463 size_t gc_heap::last_gc_heap_size = 0;
2465 size_t gc_heap::last_gc_fragmentation = 0;
2467 uint64_t gc_heap::mem_one_percent = 0;
2469 uint32_t gc_heap::high_memory_load_th = 0;
2471 uint64_t gc_heap::total_physical_mem = 0;
2473 uint64_t gc_heap::entry_available_physical_mem = 0;
2475 #ifdef BACKGROUND_GC
2476 GCEvent gc_heap::bgc_start_event;
2478 gc_mechanisms gc_heap::saved_bgc_settings;
2480 GCEvent gc_heap::background_gc_done_event;
2482 GCEvent gc_heap::ee_proceed_event;
2484 bool gc_heap::gc_can_use_concurrent = false;
2486 bool gc_heap::temp_disable_concurrent_p = false;
2488 uint32_t gc_heap::cm_in_progress = FALSE;
2490 BOOL gc_heap::dont_restart_ee_p = FALSE;
2492 BOOL gc_heap::keep_bgc_threads_p = FALSE;
2494 GCEvent gc_heap::bgc_threads_sync_event;
2496 BOOL gc_heap::do_ephemeral_gc_p = FALSE;
2498 BOOL gc_heap::do_concurrent_p = FALSE;
2500 size_t gc_heap::ephemeral_fgc_counts[max_generation];
2502 BOOL gc_heap::alloc_wait_event_p = FALSE;
2504 VOLATILE(c_gc_state) gc_heap::current_c_gc_state = c_gc_state_free;
2506 #endif //BACKGROUND_GC
2508 #ifndef MULTIPLE_HEAPS
2509 #ifdef SPINLOCK_HISTORY
2510 int gc_heap::spinlock_info_index = 0;
2511 spinlock_info gc_heap::last_spinlock_info[max_saved_spinlock_info + 8];
2512 #endif //SPINLOCK_HISTORY
2514 size_t gc_heap::fgn_last_alloc = 0;
2516 int gc_heap::generation_skip_ratio = 100;
2518 uint64_t gc_heap::loh_alloc_since_cg = 0;
2520 BOOL gc_heap::elevation_requested = FALSE;
2522 BOOL gc_heap::last_gc_before_oom = FALSE;
2524 #ifdef BACKGROUND_GC
2525 uint8_t* gc_heap::background_saved_lowest_address = 0;
2526 uint8_t* gc_heap::background_saved_highest_address = 0;
2527 uint8_t* gc_heap::next_sweep_obj = 0;
2528 uint8_t* gc_heap::current_sweep_pos = 0;
2529 exclusive_sync* gc_heap::bgc_alloc_lock;
2530 #endif //BACKGROUND_GC
2532 oom_history gc_heap::oom_info;
2534 fgm_history gc_heap::fgm_result;
2536 BOOL gc_heap::ro_segments_in_range;
2538 size_t gc_heap::gen0_big_free_spaces = 0;
2540 uint8_t* gc_heap::ephemeral_low;
2542 uint8_t* gc_heap::ephemeral_high;
2544 uint8_t* gc_heap::lowest_address;
2546 uint8_t* gc_heap::highest_address;
2548 BOOL gc_heap::ephemeral_promotion;
2550 uint8_t* gc_heap::saved_ephemeral_plan_start[NUMBERGENERATIONS-1];
2551 size_t gc_heap::saved_ephemeral_plan_start_size[NUMBERGENERATIONS-1];
2553 short* gc_heap::brick_table;
2555 uint32_t* gc_heap::card_table;
2558 uint32_t* gc_heap::card_bundle_table;
2559 #endif //CARD_BUNDLE
2561 uint8_t* gc_heap::gc_low;
2563 uint8_t* gc_heap::gc_high;
2565 uint8_t* gc_heap::demotion_low;
2567 uint8_t* gc_heap::demotion_high;
2569 BOOL gc_heap::demote_gen1_p = TRUE;
2571 uint8_t* gc_heap::last_gen1_pin_end;
2573 gen_to_condemn_tuning gc_heap::gen_to_condemn_reasons;
2575 size_t gc_heap::etw_allocation_running_amount[2];
2577 int gc_heap::gc_policy = 0;
2579 size_t gc_heap::allocation_running_time;
2581 size_t gc_heap::allocation_running_amount;
2583 heap_segment* gc_heap::ephemeral_heap_segment = 0;
2585 BOOL gc_heap::blocking_collection = FALSE;
2587 heap_segment* gc_heap::freeable_large_heap_segment = 0;
2589 size_t gc_heap::time_bgc_last = 0;
2591 size_t gc_heap::mark_stack_tos = 0;
2593 size_t gc_heap::mark_stack_bos = 0;
2595 size_t gc_heap::mark_stack_array_length = 0;
2597 mark* gc_heap::mark_stack_array = 0;
2599 BOOL gc_heap::verify_pinned_queue_p = FALSE;
2601 uint8_t* gc_heap::oldest_pinned_plug = 0;
2603 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
2604 size_t gc_heap::num_pinned_objects = 0;
2605 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
2607 #ifdef FEATURE_LOH_COMPACTION
2608 size_t gc_heap::loh_pinned_queue_tos = 0;
2610 size_t gc_heap::loh_pinned_queue_bos = 0;
2612 size_t gc_heap::loh_pinned_queue_length = 0;
2614 mark* gc_heap::loh_pinned_queue = 0;
2616 BOOL gc_heap::loh_compacted_p = FALSE;
2617 #endif //FEATURE_LOH_COMPACTION
2619 #ifdef BACKGROUND_GC
2621 EEThreadId gc_heap::bgc_thread_id;
2623 uint8_t* gc_heap::background_written_addresses [array_size+2];
2625 heap_segment* gc_heap::freeable_small_heap_segment = 0;
2627 size_t gc_heap::bgc_overflow_count = 0;
2629 size_t gc_heap::bgc_begin_loh_size = 0;
2630 size_t gc_heap::end_loh_size = 0;
2632 uint32_t gc_heap::bgc_alloc_spin_loh = 0;
2634 size_t gc_heap::bgc_loh_size_increased = 0;
2636 size_t gc_heap::bgc_loh_allocated_in_free = 0;
2638 size_t gc_heap::background_soh_alloc_count = 0;
2640 size_t gc_heap::background_loh_alloc_count = 0;
2642 uint8_t** gc_heap::background_mark_stack_tos = 0;
2644 uint8_t** gc_heap::background_mark_stack_array = 0;
2646 size_t gc_heap::background_mark_stack_array_length = 0;
2648 uint8_t* gc_heap::background_min_overflow_address =0;
2650 uint8_t* gc_heap::background_max_overflow_address =0;
2652 BOOL gc_heap::processed_soh_overflow_p = FALSE;
2654 uint8_t* gc_heap::background_min_soh_overflow_address =0;
2656 uint8_t* gc_heap::background_max_soh_overflow_address =0;
2658 heap_segment* gc_heap::saved_sweep_ephemeral_seg = 0;
2660 uint8_t* gc_heap::saved_sweep_ephemeral_start = 0;
2662 heap_segment* gc_heap::saved_overflow_ephemeral_seg = 0;
2664 Thread* gc_heap::bgc_thread = 0;
2666 BOOL gc_heap::expanded_in_fgc = FALSE;
2668 uint8_t** gc_heap::c_mark_list = 0;
2670 size_t gc_heap::c_mark_list_length = 0;
2672 size_t gc_heap::c_mark_list_index = 0;
2674 gc_history_per_heap gc_heap::bgc_data_per_heap;
2676 BOOL gc_heap::bgc_thread_running;
2678 CLRCriticalSection gc_heap::bgc_threads_timeout_cs;
2680 GCEvent gc_heap::gc_lh_block_event;
2682 #endif //BACKGROUND_GC
2685 uint8_t** gc_heap::mark_list;
2686 uint8_t** gc_heap::mark_list_index;
2687 uint8_t** gc_heap::mark_list_end;
2691 snoop_stats_data gc_heap::snoop_stat;
2692 #endif //SNOOP_STATS
2694 uint8_t* gc_heap::min_overflow_address = MAX_PTR;
2696 uint8_t* gc_heap::max_overflow_address = 0;
2698 uint8_t* gc_heap::shigh = 0;
2700 uint8_t* gc_heap::slow = MAX_PTR;
2702 size_t gc_heap::ordered_free_space_indices[MAX_NUM_BUCKETS];
2704 size_t gc_heap::saved_ordered_free_space_indices[MAX_NUM_BUCKETS];
2706 size_t gc_heap::ordered_plug_indices[MAX_NUM_BUCKETS];
2708 size_t gc_heap::saved_ordered_plug_indices[MAX_NUM_BUCKETS];
2710 BOOL gc_heap::ordered_plug_indices_init = FALSE;
2712 BOOL gc_heap::use_bestfit = FALSE;
2714 uint8_t* gc_heap::bestfit_first_pin = 0;
2716 BOOL gc_heap::commit_end_of_seg = FALSE;
2718 size_t gc_heap::max_free_space_items = 0;
2720 size_t gc_heap::free_space_buckets = 0;
2722 size_t gc_heap::free_space_items = 0;
2724 int gc_heap::trimmed_free_space_index = 0;
2726 size_t gc_heap::total_ephemeral_plugs = 0;
2728 seg_free_spaces* gc_heap::bestfit_seg = 0;
2730 size_t gc_heap::total_ephemeral_size = 0;
2734 size_t gc_heap::internal_root_array_length = initial_internal_roots;
2736 uint8_t** gc_heap::internal_root_array = 0;
2738 size_t gc_heap::internal_root_array_index = 0;
2740 BOOL gc_heap::heap_analyze_success = TRUE;
2742 uint8_t* gc_heap::current_obj = 0;
2743 size_t gc_heap::current_obj_size = 0;
2745 #endif //HEAP_ANALYZE
2747 #ifdef GC_CONFIG_DRIVEN
2748 size_t gc_heap::interesting_data_per_gc[max_idp_count];
2749 //size_t gc_heap::interesting_data_per_heap[max_idp_count];
2750 //size_t gc_heap::interesting_mechanisms_per_heap[max_im_count];
2751 #endif //GC_CONFIG_DRIVEN
2752 #endif //MULTIPLE_HEAPS
2754 no_gc_region_info gc_heap::current_no_gc_region_info;
2755 BOOL gc_heap::proceed_with_gc_p = FALSE;
2756 GCSpinLock gc_heap::gc_lock;
2758 size_t gc_heap::eph_gen_starts_size = 0;
2759 heap_segment* gc_heap::segment_standby_list;
2760 size_t gc_heap::last_gc_index = 0;
2761 #ifdef SEG_MAPPING_TABLE
2762 size_t gc_heap::min_segment_size = 0;
2763 size_t gc_heap::min_segment_size_shr = 0;
2764 #endif //SEG_MAPPING_TABLE
2765 size_t gc_heap::soh_segment_size = 0;
2766 size_t gc_heap::min_loh_segment_size = 0;
2767 size_t gc_heap::segment_info_size = 0;
2769 #ifdef GC_CONFIG_DRIVEN
2770 size_t gc_heap::time_init = 0;
2771 size_t gc_heap::time_since_init = 0;
2772 size_t gc_heap::compact_or_sweep_gcs[2];
2773 #endif //GC_CONFIG_DRIVEN
2775 #ifdef FEATURE_LOH_COMPACTION
2776 BOOL gc_heap::loh_compaction_always_p = FALSE;
2777 gc_loh_compaction_mode gc_heap::loh_compaction_mode = loh_compaction_default;
2778 int gc_heap::loh_pinned_queue_decay = LOH_PIN_DECAY;
2780 #endif //FEATURE_LOH_COMPACTION
2782 GCEvent gc_heap::full_gc_approach_event;
2784 GCEvent gc_heap::full_gc_end_event;
2786 uint32_t gc_heap::fgn_maxgen_percent = 0;
2788 uint32_t gc_heap::fgn_loh_percent = 0;
2790 #ifdef BACKGROUND_GC
2791 BOOL gc_heap::fgn_last_gc_was_concurrent = FALSE;
2792 #endif //BACKGROUND_GC
2794 VOLATILE(bool) gc_heap::full_gc_approach_event_set;
2796 size_t gc_heap::full_gc_counts[gc_type_max];
2798 BOOL gc_heap::should_expand_in_full_gc = FALSE;
2801 BOOL gc_heap::heap_analyze_enabled = FALSE;
2802 #endif //HEAP_ANALYZE
2804 #ifndef MULTIPLE_HEAPS
2806 alloc_list gc_heap::loh_alloc_list [NUM_LOH_ALIST-1];
2807 alloc_list gc_heap::gen2_alloc_list[NUM_GEN2_ALIST-1];
2809 dynamic_data gc_heap::dynamic_data_table [NUMBERGENERATIONS+1];
2810 gc_history_per_heap gc_heap::gc_data_per_heap;
2811 size_t gc_heap::maxgen_pinned_compact_before_advance = 0;
2813 uint8_t* gc_heap::alloc_allocated = 0;
2815 size_t gc_heap::allocation_quantum = CLR_SIZE;
2817 GCSpinLock gc_heap::more_space_lock;
2819 #ifdef SYNCHRONIZATION_STATS
2820 unsigned int gc_heap::good_suspension = 0;
2821 unsigned int gc_heap::bad_suspension = 0;
2822 uint64_t gc_heap::total_msl_acquire = 0;
2823 unsigned int gc_heap::num_msl_acquired = 0;
2824 unsigned int gc_heap::num_high_msl_acquire = 0;
2825 unsigned int gc_heap::num_low_msl_acquire = 0;
2826 #endif //SYNCHRONIZATION_STATS
2828 size_t gc_heap::alloc_contexts_used = 0;
2829 size_t gc_heap::soh_allocation_no_gc = 0;
2830 size_t gc_heap::loh_allocation_no_gc = 0;
2831 bool gc_heap::no_gc_oom_p = false;
2832 heap_segment* gc_heap::saved_loh_segment_no_gc = 0;
2834 #endif //MULTIPLE_HEAPS
2836 #ifndef MULTIPLE_HEAPS
2838 BOOL gc_heap::gen0_bricks_cleared = FALSE;
2841 int gc_heap::gen0_must_clear_bricks = 0;
2842 #endif //FFIND_OBJECT
2844 #ifdef FEATURE_PREMORTEM_FINALIZATION
2845 CFinalize* gc_heap::finalize_queue = 0;
2846 #endif // FEATURE_PREMORTEM_FINALIZATION
2848 generation gc_heap::generation_table [NUMBERGENERATIONS + 1];
2850 size_t gc_heap::interesting_data_per_heap[max_idp_count];
2852 size_t gc_heap::compact_reasons_per_heap[max_compact_reasons_count];
2854 size_t gc_heap::expand_mechanisms_per_heap[max_expand_mechanisms_count];
2856 size_t gc_heap::interesting_mechanism_bits_per_heap[max_gc_mechanism_bits_count];
2858 #endif // MULTIPLE_HEAPS
2860 /* end of per heap static initialization */
2862 /* end of static initialization */
2864 #ifndef DACCESS_COMPILE
2866 void gen_to_condemn_tuning::print (int heap_num)
2869 dprintf (DT_LOG_0, ("condemned reasons (%d %d)", condemn_reasons_gen, condemn_reasons_condition));
2870 dprintf (DT_LOG_0, ("%s", record_condemn_reasons_gen_header));
2871 gc_condemn_reason_gen r_gen;
2872 for (int i = 0; i < gcrg_max; i++)
2874 r_gen = (gc_condemn_reason_gen)(i);
2875 str_reasons_gen[i * 2] = get_gen_char (get_gen (r_gen));
2877 dprintf (DT_LOG_0, ("[%2d]%s", heap_num, str_reasons_gen));
2879 dprintf (DT_LOG_0, ("%s", record_condemn_reasons_condition_header));
2880 gc_condemn_reason_condition r_condition;
2881 for (int i = 0; i < gcrc_max; i++)
2883 r_condition = (gc_condemn_reason_condition)(i);
2884 str_reasons_condition[i * 2] = get_condition_char (get_condition (r_condition));
2887 dprintf (DT_LOG_0, ("[%2d]%s", heap_num, str_reasons_condition));
2889 UNREFERENCED_PARAMETER(heap_num);
2893 void gc_generation_data::print (int heap_num, int gen_num)
2895 #if defined(SIMPLE_DPRINTF) && defined(DT_LOG)
2896 dprintf (DT_LOG_0, ("[%2d]gen%d beg %Id fl %Id fo %Id end %Id fl %Id fo %Id in %Id p %Id np %Id alloc %Id",
2899 free_list_space_before, free_obj_space_before,
2901 free_list_space_after, free_obj_space_after,
2902 in, pinned_surv, npinned_surv,
2905 UNREFERENCED_PARAMETER(heap_num);
2906 UNREFERENCED_PARAMETER(gen_num);
2907 #endif //SIMPLE_DPRINTF && DT_LOG
2910 void gc_history_per_heap::set_mechanism (gc_mechanism_per_heap mechanism_per_heap, uint32_t value)
2912 uint32_t* mechanism = &mechanisms[mechanism_per_heap];
2914 *mechanism |= mechanism_mask;
2915 *mechanism |= (1 << value);
2918 gc_mechanism_descr* descr = &gc_mechanisms_descr[mechanism_per_heap];
2919 dprintf (DT_LOG_0, ("setting %s: %s",
2921 (descr->descr)[value]));
2925 void gc_history_per_heap::print()
2927 #if defined(SIMPLE_DPRINTF) && defined(DT_LOG)
2928 for (int i = 0; i < (sizeof (gen_data)/sizeof (gc_generation_data)); i++)
2930 gen_data[i].print (heap_index, i);
2933 dprintf (DT_LOG_0, ("fla %Id flr %Id esa %Id ca %Id pa %Id paa %Id, rfle %d, ec %Id",
2934 maxgen_size_info.free_list_allocated,
2935 maxgen_size_info.free_list_rejected,
2936 maxgen_size_info.end_seg_allocated,
2937 maxgen_size_info.condemned_allocated,
2938 maxgen_size_info.pinned_allocated,
2939 maxgen_size_info.pinned_allocated_advance,
2940 maxgen_size_info.running_free_list_efficiency,
2941 extra_gen0_committed));
2944 gc_mechanism_descr* descr = 0;
2946 for (int i = 0; i < max_mechanism_per_heap; i++)
2948 mechanism = get_mechanism ((gc_mechanism_per_heap)i);
2952 descr = &gc_mechanisms_descr[(gc_mechanism_per_heap)i];
2953 dprintf (DT_LOG_0, ("[%2d]%s%s",
2956 (descr->descr)[mechanism]));
2959 #endif //SIMPLE_DPRINTF && DT_LOG
2962 void gc_history_global::print()
2965 char str_settings[64];
2966 memset (str_settings, '|', sizeof (char) * 64);
2967 str_settings[max_global_mechanisms_count*2] = 0;
2969 for (int i = 0; i < max_global_mechanisms_count; i++)
2971 str_settings[i * 2] = (get_mechanism_p ((gc_global_mechanism_p)i) ? 'Y' : 'N');
2974 dprintf (DT_LOG_0, ("[hp]|c|p|o|d|b|e|"));
2976 dprintf (DT_LOG_0, ("%4d|%s", num_heaps, str_settings));
2977 dprintf (DT_LOG_0, ("Condemned gen%d(reason: %s; mode: %s), youngest budget %Id(%d), memload %d",
2978 condemned_generation,
2979 str_gc_reasons[reason],
2980 str_gc_pause_modes[pause_mode],
2981 final_youngest_desired,
2982 gen0_reduction_count,
2987 void gc_heap::fire_per_heap_hist_event (gc_history_per_heap* current_gc_data_per_heap, int heap_num)
2989 maxgen_size_increase* maxgen_size_info = &(current_gc_data_per_heap->maxgen_size_info);
2990 FIRE_EVENT(GCPerHeapHistory_V3,
2991 (void *)(maxgen_size_info->free_list_allocated),
2992 (void *)(maxgen_size_info->free_list_rejected),
2993 (void *)(maxgen_size_info->end_seg_allocated),
2994 (void *)(maxgen_size_info->condemned_allocated),
2995 (void *)(maxgen_size_info->pinned_allocated),
2996 (void *)(maxgen_size_info->pinned_allocated_advance),
2997 maxgen_size_info->running_free_list_efficiency,
2998 current_gc_data_per_heap->gen_to_condemn_reasons.get_reasons0(),
2999 current_gc_data_per_heap->gen_to_condemn_reasons.get_reasons1(),
3000 current_gc_data_per_heap->mechanisms[gc_heap_compact],
3001 current_gc_data_per_heap->mechanisms[gc_heap_expand],
3002 current_gc_data_per_heap->heap_index,
3003 (void *)(current_gc_data_per_heap->extra_gen0_committed),
3004 (max_generation + 2),
3005 (uint32_t)(sizeof (gc_generation_data)),
3006 (void *)&(current_gc_data_per_heap->gen_data[0]));
3008 current_gc_data_per_heap->print();
3009 current_gc_data_per_heap->gen_to_condemn_reasons.print (heap_num);
3012 void gc_heap::fire_pevents()
3015 settings.record (&gc_data_global);
3016 gc_data_global.print();
3018 FIRE_EVENT(GCGlobalHeapHistory_V2, gc_data_global.final_youngest_desired,
3019 gc_data_global.num_heaps,
3020 gc_data_global.condemned_generation,
3021 gc_data_global.gen0_reduction_count,
3022 gc_data_global.reason,
3023 gc_data_global.global_mechanims_p,
3024 gc_data_global.pause_mode,
3025 gc_data_global.mem_pressure);
3027 #ifdef MULTIPLE_HEAPS
3028 for (int i = 0; i < gc_heap::n_heaps; i++)
3030 gc_heap* hp = gc_heap::g_heaps[i];
3031 gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap();
3032 fire_per_heap_hist_event (current_gc_data_per_heap, hp->heap_number);
3035 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
3036 fire_per_heap_hist_event (current_gc_data_per_heap, heap_number);
3042 gc_heap::dt_low_ephemeral_space_p (gc_tuning_point tp)
3048 case tuning_deciding_condemned_gen:
3049 case tuning_deciding_compaction:
3050 case tuning_deciding_expansion:
3051 case tuning_deciding_full_gc:
3053 ret = (!ephemeral_gen_fit_p (tp));
3056 case tuning_deciding_promote_ephemeral:
3058 size_t new_gen0size = approximate_new_allocation();
3059 ptrdiff_t plan_ephemeral_size = total_ephemeral_size;
3061 dprintf (GTC_LOG, ("h%d: plan eph size is %Id, new gen0 is %Id",
3062 heap_number, plan_ephemeral_size, new_gen0size));
3064 // If we were in no_gc_region we could have allocated a larger than normal segment,
3065 // and the next seg we allocate will be a normal sized seg so if we can't fit the new
3066 // ephemeral generations there, do an ephemeral promotion.
3067 ret = ((soh_segment_size - segment_info_size) < (plan_ephemeral_size + new_gen0size));
3079 gc_heap::dt_high_frag_p (gc_tuning_point tp,
3087 case tuning_deciding_condemned_gen:
3089 dynamic_data* dd = dynamic_data_of (gen_number);
3090 float fragmentation_burden = 0;
3094 ret = (dd_fragmentation (dynamic_data_of (max_generation)) >= dd_max_size(dd));
3095 dprintf (GTC_LOG, ("h%d: frag is %Id, max size is %Id",
3096 heap_number, dd_fragmentation (dd), dd_max_size(dd)));
3100 #ifndef MULTIPLE_HEAPS
3101 if (gen_number == max_generation)
3103 float frag_ratio = (float)(dd_fragmentation (dynamic_data_of (max_generation))) / (float)generation_size (max_generation);
3104 if (frag_ratio > 0.65)
3106 dprintf (GTC_LOG, ("g2 FR: %d%%", (int)(frag_ratio*100)));
3110 #endif //!MULTIPLE_HEAPS
3111 size_t fr = generation_unusable_fragmentation (generation_of (gen_number));
3112 ret = (fr > dd_fragmentation_limit(dd));
3115 fragmentation_burden = (float)fr / generation_size (gen_number);
3116 ret = (fragmentation_burden > dd_v_fragmentation_burden_limit (dd));
3118 dprintf (GTC_LOG, ("h%d: gen%d, frag is %Id, alloc effi: %d%%, unusable frag is %Id, ratio is %d",
3119 heap_number, gen_number, dd_fragmentation (dd),
3120 (int)(100*generation_allocator_efficiency (generation_of (gen_number))),
3121 fr, (int)(fragmentation_burden*100)));
3133 gc_heap::dt_estimate_reclaim_space_p (gc_tuning_point tp, int gen_number)
3139 case tuning_deciding_condemned_gen:
3141 if (gen_number == max_generation)
3143 dynamic_data* dd = dynamic_data_of (gen_number);
3144 size_t maxgen_allocated = (dd_desired_allocation (dd) - dd_new_allocation (dd));
3145 size_t maxgen_total_size = maxgen_allocated + dd_current_size (dd);
3146 size_t est_maxgen_surv = (size_t)((float) (maxgen_total_size) * dd_surv (dd));
3147 size_t est_maxgen_free = maxgen_total_size - est_maxgen_surv + dd_fragmentation (dd);
3149 dprintf (GTC_LOG, ("h%d: Total gen2 size: %Id, est gen2 dead space: %Id (s: %d, allocated: %Id), frag: %Id",
3153 (int)(dd_surv (dd) * 100),
3155 dd_fragmentation (dd)));
3157 uint32_t num_heaps = 1;
3159 #ifdef MULTIPLE_HEAPS
3160 num_heaps = gc_heap::n_heaps;
3161 #endif //MULTIPLE_HEAPS
3163 size_t min_frag_th = min_reclaim_fragmentation_threshold (num_heaps);
3164 dprintf (GTC_LOG, ("h%d, min frag is %Id", heap_number, min_frag_th));
3165 ret = (est_maxgen_free >= min_frag_th);
3181 // DTREVIEW: Right now we only estimate gen2 fragmentation.
3182 // on 64-bit though we should consider gen1 or even gen0 fragmentatioin as
3185 gc_heap::dt_estimate_high_frag_p (gc_tuning_point tp, int gen_number, uint64_t available_mem)
3191 case tuning_deciding_condemned_gen:
3193 if (gen_number == max_generation)
3195 dynamic_data* dd = dynamic_data_of (gen_number);
3196 float est_frag_ratio = 0;
3197 if (dd_current_size (dd) == 0)
3201 else if ((dd_fragmentation (dd) == 0) || (dd_fragmentation (dd) + dd_current_size (dd) == 0))
3207 est_frag_ratio = (float)dd_fragmentation (dd) / (float)(dd_fragmentation (dd) + dd_current_size (dd));
3210 size_t est_frag = (dd_fragmentation (dd) + (size_t)((dd_desired_allocation (dd) - dd_new_allocation (dd)) * est_frag_ratio));
3211 dprintf (GTC_LOG, ("h%d: gen%d: current_size is %Id, frag is %Id, est_frag_ratio is %d%%, estimated frag is %Id",
3214 dd_current_size (dd),
3215 dd_fragmentation (dd),
3216 (int)(est_frag_ratio*100),
3219 uint32_t num_heaps = 1;
3221 #ifdef MULTIPLE_HEAPS
3222 num_heaps = gc_heap::n_heaps;
3223 #endif //MULTIPLE_HEAPS
3224 uint64_t min_frag_th = min_high_fragmentation_threshold(available_mem, num_heaps);
3225 //dprintf (GTC_LOG, ("h%d, min frag is %I64d", heap_number, min_frag_th));
3226 ret = (est_frag >= min_frag_th);
3243 gc_heap::dt_low_card_table_efficiency_p (gc_tuning_point tp)
3249 case tuning_deciding_condemned_gen:
3251 /* promote into max-generation if the card table has too many
3252 * generation faults besides the n -> 0
3254 ret = (generation_skip_ratio < 30);
3266 in_range_for_segment(uint8_t* add, heap_segment* seg)
3268 return ((add >= heap_segment_mem (seg)) && (add < heap_segment_reserved (seg)));
3271 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
3272 // The array we allocate is organized as follows:
3273 // 0th element is the address of the last array we allocated.
3274 // starting from the 1st element are the segment addresses, that's
3275 // what buckets() returns.
3288 bk* buckets() { return (slots + 1); }
3289 uint8_t*& last_slot (bk* arr) { return arr[0].add; }
3292 static sorted_table* make_sorted_table ();
3293 BOOL insert (uint8_t* add, size_t val);;
3294 size_t lookup (uint8_t*& add);
3295 void remove (uint8_t* add);
3297 void delete_sorted_table();
3298 void delete_old_slots();
3299 void enqueue_old_slot(bk* sl);
3300 BOOL ensure_space_for_insert();
3304 sorted_table::make_sorted_table ()
3308 // allocate one more bk to store the older slot address.
3309 sorted_table* res = (sorted_table*)new char [sizeof (sorted_table) + (size + 1) * sizeof (bk)];
3313 res->slots = (bk*)(res + 1);
3320 sorted_table::delete_sorted_table()
3322 if (slots != (bk*)(this+1))
3330 sorted_table::delete_old_slots()
3332 uint8_t* sl = (uint8_t*)old_slots;
3336 sl = last_slot ((bk*)sl);
3342 sorted_table::enqueue_old_slot(bk* sl)
3344 last_slot (sl) = (uint8_t*)old_slots;
3350 sorted_table::lookup (uint8_t*& add)
3352 ptrdiff_t high = (count-1);
3356 bk* buck = buckets();
3359 mid = ((low + high)/2);
3361 if (buck[ti].add > add)
3363 if ((ti > 0) && (buck[ti-1].add <= add))
3365 add = buck[ti-1].add;
3366 return buck[ti - 1].val;
3372 if (buck[ti+1].add > add)
3375 return buck[ti].val;
3385 sorted_table::ensure_space_for_insert()
3389 size = (size * 3)/2;
3390 assert((size * sizeof (bk)) > 0);
3391 bk* res = (bk*)new (nothrow) char [(size + 1) * sizeof (bk)];
3396 last_slot (res) = 0;
3397 memcpy (((bk*)res + 1), buckets(), count * sizeof (bk));
3398 bk* last_old_slots = slots;
3400 if (last_old_slots != (bk*)(this + 1))
3401 enqueue_old_slot (last_old_slots);
3407 sorted_table::insert (uint8_t* add, size_t val)
3409 //grow if no more room
3410 assert (count < size);
3413 ptrdiff_t high = (count-1);
3417 bk* buck = buckets();
3420 mid = ((low + high)/2);
3422 if (buck[ti].add > add)
3424 if ((ti == 0) || (buck[ti-1].add <= add))
3426 // found insertion point
3427 for (ptrdiff_t k = count; k > ti;k--)
3429 buck [k] = buck [k-1];
3440 if (buck[ti+1].add > add)
3442 //found the insertion point
3443 for (ptrdiff_t k = count; k > ti+1;k--)
3445 buck [k] = buck [k-1];
3447 buck[ti+1].add = add;
3448 buck[ti+1].val = val;
3460 sorted_table::remove (uint8_t* add)
3462 ptrdiff_t high = (count-1);
3466 bk* buck = buckets();
3469 mid = ((low + high)/2);
3471 if (buck[ti].add > add)
3473 if (buck[ti-1].add <= add)
3475 // found the guy to remove
3476 for (ptrdiff_t k = ti; k < count; k++)
3477 buck[k-1] = buck[k];
3485 if (buck[ti+1].add > add)
3487 // found the guy to remove
3488 for (ptrdiff_t k = ti+1; k < count; k++)
3489 buck[k-1] = buck[k];
3500 sorted_table::clear()
3503 buckets()[0].add = MAX_PTR;
3505 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
3507 #ifdef SEG_MAPPING_TABLE
3508 #ifdef GROWABLE_SEG_MAPPING_TABLE
3510 uint8_t* align_on_segment (uint8_t* add)
3512 return (uint8_t*)((size_t)(add + (gc_heap::min_segment_size - 1)) & ~(gc_heap::min_segment_size - 1));
3516 uint8_t* align_lower_segment (uint8_t* add)
3518 return (uint8_t*)((size_t)(add) & ~(gc_heap::min_segment_size - 1));
3521 size_t size_seg_mapping_table_of (uint8_t* from, uint8_t* end)
3523 from = align_lower_segment (from);
3524 end = align_on_segment (end);
3525 dprintf (1, ("from: %Ix, end: %Ix, size: %Ix", from, end, sizeof (seg_mapping)*(((size_t)(end - from) >> gc_heap::min_segment_size_shr))));
3526 return sizeof (seg_mapping)*((size_t)(end - from) >> gc_heap::min_segment_size_shr);
3529 // for seg_mapping_table we want it to start from a pointer sized address.
3531 size_t align_for_seg_mapping_table (size_t size)
3533 return ((size + (sizeof (uint8_t*) - 1)) &~ (sizeof (uint8_t*) - 1));
3537 size_t seg_mapping_word_of (uint8_t* add)
3539 return (size_t)add >> gc_heap::min_segment_size_shr;
3541 #else //GROWABLE_SEG_MAPPING_TABLE
3542 BOOL seg_mapping_table_init()
3545 uint64_t total_address_space = (uint64_t)8*1024*1024*1024*1024;
3547 uint64_t total_address_space = (uint64_t)4*1024*1024*1024;
3550 size_t num_entries = (size_t)(total_address_space >> gc_heap::min_segment_size_shr);
3551 seg_mapping_table = new seg_mapping[num_entries];
3553 if (seg_mapping_table)
3555 memset (seg_mapping_table, 0, num_entries * sizeof (seg_mapping));
3556 dprintf (1, ("created %d entries for heap mapping (%Id bytes)",
3557 num_entries, (num_entries * sizeof (seg_mapping))));
3562 dprintf (1, ("failed to create %d entries for heap mapping (%Id bytes)",
3563 num_entries, (num_entries * sizeof (seg_mapping))));
3567 #endif //GROWABLE_SEG_MAPPING_TABLE
3569 #ifdef FEATURE_BASICFREEZE
3571 size_t ro_seg_begin_index (heap_segment* seg)
3573 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3574 begin_index = max (begin_index, (size_t)g_gc_lowest_address >> gc_heap::min_segment_size_shr);
3579 size_t ro_seg_end_index (heap_segment* seg)
3581 size_t end_index = (size_t)(heap_segment_reserved (seg) - 1) >> gc_heap::min_segment_size_shr;
3582 end_index = min (end_index, (size_t)g_gc_highest_address >> gc_heap::min_segment_size_shr);
3586 void seg_mapping_table_add_ro_segment (heap_segment* seg)
3588 #ifdef GROWABLE_SEG_MAPPING_TABLE
3589 if ((heap_segment_reserved (seg) <= g_gc_lowest_address) || (heap_segment_mem (seg) >= g_gc_highest_address))
3591 #endif //GROWABLE_SEG_MAPPING_TABLE
3593 for (size_t entry_index = ro_seg_begin_index (seg); entry_index <= ro_seg_end_index (seg); entry_index++)
3594 seg_mapping_table[entry_index].seg1 = (heap_segment*)((size_t)seg_mapping_table[entry_index].seg1 | ro_in_entry);
3597 void seg_mapping_table_remove_ro_segment (heap_segment* seg)
3599 UNREFERENCED_PARAMETER(seg);
3601 // POSSIBLE PERF TODO: right now we are not doing anything because we can't simply remove the flag. If it proves
3602 // to be a perf problem, we can search in the current ro segs and see if any lands in this range and only
3603 // remove the flag if none lands in this range.
3607 heap_segment* ro_segment_lookup (uint8_t* o)
3609 uint8_t* ro_seg_start = o;
3610 heap_segment* seg = (heap_segment*)gc_heap::seg_table->lookup (ro_seg_start);
3612 if (ro_seg_start && in_range_for_segment (o, seg))
3618 #endif //FEATURE_BASICFREEZE
3620 void gc_heap::seg_mapping_table_add_segment (heap_segment* seg, gc_heap* hp)
3622 size_t seg_end = (size_t)(heap_segment_reserved (seg) - 1);
3623 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3624 seg_mapping* begin_entry = &seg_mapping_table[begin_index];
3625 size_t end_index = seg_end >> gc_heap::min_segment_size_shr;
3626 seg_mapping* end_entry = &seg_mapping_table[end_index];
3628 dprintf (1, ("adding seg %Ix(%d)-%Ix(%d)",
3629 seg, begin_index, heap_segment_reserved (seg), end_index));
3631 dprintf (1, ("before add: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3632 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3633 end_index, (seg_mapping_table[end_index].boundary + 1)));
3635 #ifdef MULTIPLE_HEAPS
3636 #ifdef SIMPLE_DPRINTF
3637 dprintf (1, ("begin %d: h0: %Ix(%d), h1: %Ix(%d); end %d: h0: %Ix(%d), h1: %Ix(%d)",
3638 begin_index, (uint8_t*)(begin_entry->h0), (begin_entry->h0 ? begin_entry->h0->heap_number : -1),
3639 (uint8_t*)(begin_entry->h1), (begin_entry->h1 ? begin_entry->h1->heap_number : -1),
3640 end_index, (uint8_t*)(end_entry->h0), (end_entry->h0 ? end_entry->h0->heap_number : -1),
3641 (uint8_t*)(end_entry->h1), (end_entry->h1 ? end_entry->h1->heap_number : -1)));
3642 #endif //SIMPLE_DPRINTF
3643 assert (end_entry->boundary == 0);
3644 assert (end_entry->h0 == 0);
3646 assert (begin_entry->h1 == 0);
3647 begin_entry->h1 = hp;
3649 UNREFERENCED_PARAMETER(hp);
3650 #endif //MULTIPLE_HEAPS
3652 end_entry->boundary = (uint8_t*)seg_end;
3654 dprintf (1, ("set entry %d seg1 and %d seg0 to %Ix", begin_index, end_index, seg));
3655 assert ((begin_entry->seg1 == 0) || ((size_t)(begin_entry->seg1) == ro_in_entry));
3656 begin_entry->seg1 = (heap_segment*)((size_t)(begin_entry->seg1) | (size_t)seg);
3657 end_entry->seg0 = seg;
3659 // for every entry inbetween we need to set its heap too.
3660 for (size_t entry_index = (begin_index + 1); entry_index <= (end_index - 1); entry_index++)
3662 assert (seg_mapping_table[entry_index].boundary == 0);
3663 #ifdef MULTIPLE_HEAPS
3664 assert (seg_mapping_table[entry_index].h0 == 0);
3665 seg_mapping_table[entry_index].h1 = hp;
3666 #endif //MULTIPLE_HEAPS
3667 seg_mapping_table[entry_index].seg1 = seg;
3670 dprintf (1, ("after add: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3671 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3672 end_index, (seg_mapping_table[end_index].boundary + 1)));
3673 #if defined(MULTIPLE_HEAPS) && defined(SIMPLE_DPRINTF)
3674 dprintf (1, ("begin %d: h0: %Ix(%d), h1: %Ix(%d); end: %d h0: %Ix(%d), h1: %Ix(%d)",
3675 begin_index, (uint8_t*)(begin_entry->h0), (begin_entry->h0 ? begin_entry->h0->heap_number : -1),
3676 (uint8_t*)(begin_entry->h1), (begin_entry->h1 ? begin_entry->h1->heap_number : -1),
3677 end_index, (uint8_t*)(end_entry->h0), (end_entry->h0 ? end_entry->h0->heap_number : -1),
3678 (uint8_t*)(end_entry->h1), (end_entry->h1 ? end_entry->h1->heap_number : -1)));
3679 #endif //MULTIPLE_HEAPS && SIMPLE_DPRINTF
3682 void gc_heap::seg_mapping_table_remove_segment (heap_segment* seg)
3684 size_t seg_end = (size_t)(heap_segment_reserved (seg) - 1);
3685 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3686 seg_mapping* begin_entry = &seg_mapping_table[begin_index];
3687 size_t end_index = seg_end >> gc_heap::min_segment_size_shr;
3688 seg_mapping* end_entry = &seg_mapping_table[end_index];
3689 dprintf (1, ("removing seg %Ix(%d)-%Ix(%d)",
3690 seg, begin_index, heap_segment_reserved (seg), end_index));
3692 assert (end_entry->boundary == (uint8_t*)seg_end);
3693 end_entry->boundary = 0;
3695 #ifdef MULTIPLE_HEAPS
3696 gc_heap* hp = heap_segment_heap (seg);
3697 assert (end_entry->h0 == hp);
3699 assert (begin_entry->h1 == hp);
3700 begin_entry->h1 = 0;
3701 #endif //MULTIPLE_HEAPS
3703 assert (begin_entry->seg1 != 0);
3704 begin_entry->seg1 = (heap_segment*)((size_t)(begin_entry->seg1) & ro_in_entry);
3705 end_entry->seg0 = 0;
3707 // for every entry inbetween we need to reset its heap too.
3708 for (size_t entry_index = (begin_index + 1); entry_index <= (end_index - 1); entry_index++)
3710 assert (seg_mapping_table[entry_index].boundary == 0);
3711 #ifdef MULTIPLE_HEAPS
3712 assert (seg_mapping_table[entry_index].h0 == 0);
3713 assert (seg_mapping_table[entry_index].h1 == hp);
3714 seg_mapping_table[entry_index].h1 = 0;
3715 #endif //MULTIPLE_HEAPS
3716 seg_mapping_table[entry_index].seg1 = 0;
3719 dprintf (1, ("after remove: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3720 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3721 end_index, (seg_mapping_table[end_index].boundary + 1)));
3722 #ifdef MULTIPLE_HEAPS
3723 dprintf (1, ("begin %d: h0: %Ix, h1: %Ix; end: %d h0: %Ix, h1: %Ix",
3724 begin_index, (uint8_t*)(begin_entry->h0), (uint8_t*)(begin_entry->h1),
3725 end_index, (uint8_t*)(end_entry->h0), (uint8_t*)(end_entry->h1)));
3726 #endif //MULTIPLE_HEAPS
3729 #ifdef MULTIPLE_HEAPS
3731 gc_heap* seg_mapping_table_heap_of_worker (uint8_t* o)
3733 size_t index = (size_t)o >> gc_heap::min_segment_size_shr;
3734 seg_mapping* entry = &seg_mapping_table[index];
3736 gc_heap* hp = ((o > entry->boundary) ? entry->h1 : entry->h0);
3738 dprintf (2, ("checking obj %Ix, index is %Id, entry: boundry: %Ix, h0: %Ix, seg0: %Ix, h1: %Ix, seg1: %Ix",
3739 o, index, (entry->boundary + 1),
3740 (uint8_t*)(entry->h0), (uint8_t*)(entry->seg0),
3741 (uint8_t*)(entry->h1), (uint8_t*)(entry->seg1)));
3744 heap_segment* seg = ((o > entry->boundary) ? entry->seg1 : entry->seg0);
3745 #ifdef FEATURE_BASICFREEZE
3746 if ((size_t)seg & ro_in_entry)
3747 seg = (heap_segment*)((size_t)seg & ~ro_in_entry);
3748 #endif //FEATURE_BASICFREEZE
3752 if (in_range_for_segment (o, seg))
3754 dprintf (2, ("obj %Ix belongs to segment %Ix(-%Ix)", o, seg, (uint8_t*)heap_segment_allocated (seg)));
3758 dprintf (2, ("found seg %Ix(-%Ix) for obj %Ix, but it's not on the seg",
3759 seg, (uint8_t*)heap_segment_allocated (seg), o));
3764 dprintf (2, ("could not find obj %Ix in any existing segments", o));
3771 gc_heap* seg_mapping_table_heap_of (uint8_t* o)
3773 #ifdef GROWABLE_SEG_MAPPING_TABLE
3774 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3776 #endif //GROWABLE_SEG_MAPPING_TABLE
3778 return seg_mapping_table_heap_of_worker (o);
3781 gc_heap* seg_mapping_table_heap_of_gc (uint8_t* o)
3783 #if defined(FEATURE_BASICFREEZE) && defined(GROWABLE_SEG_MAPPING_TABLE)
3784 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3786 #endif //FEATURE_BASICFREEZE || GROWABLE_SEG_MAPPING_TABLE
3788 return seg_mapping_table_heap_of_worker (o);
3790 #endif //MULTIPLE_HEAPS
3792 // Only returns a valid seg if we can actually find o on the seg.
3793 heap_segment* seg_mapping_table_segment_of (uint8_t* o)
3795 #if defined(FEATURE_BASICFREEZE) && defined(GROWABLE_SEG_MAPPING_TABLE)
3796 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3797 #ifdef FEATURE_BASICFREEZE
3798 return ro_segment_lookup (o);
3801 #endif //FEATURE_BASICFREEZE
3802 #endif //FEATURE_BASICFREEZE || GROWABLE_SEG_MAPPING_TABLE
3804 size_t index = (size_t)o >> gc_heap::min_segment_size_shr;
3805 seg_mapping* entry = &seg_mapping_table[index];
3807 dprintf (2, ("checking obj %Ix, index is %Id, entry: boundry: %Ix, seg0: %Ix, seg1: %Ix",
3808 o, index, (entry->boundary + 1),
3809 (uint8_t*)(entry->seg0), (uint8_t*)(entry->seg1)));
3811 heap_segment* seg = ((o > entry->boundary) ? entry->seg1 : entry->seg0);
3812 #ifdef FEATURE_BASICFREEZE
3813 if ((size_t)seg & ro_in_entry)
3814 seg = (heap_segment*)((size_t)seg & ~ro_in_entry);
3815 #endif //FEATURE_BASICFREEZE
3819 if (in_range_for_segment (o, seg))
3821 dprintf (2, ("obj %Ix belongs to segment %Ix(-%Ix)", o, (uint8_t*)heap_segment_mem(seg), (uint8_t*)heap_segment_reserved(seg)));
3825 dprintf (2, ("found seg %Ix(-%Ix) for obj %Ix, but it's not on the seg, setting it to 0",
3826 (uint8_t*)heap_segment_mem(seg), (uint8_t*)heap_segment_reserved(seg), o));
3832 dprintf (2, ("could not find obj %Ix in any existing segments", o));
3835 #ifdef FEATURE_BASICFREEZE
3836 // TODO: This was originally written assuming that the seg_mapping_table would always contain entries for ro
3837 // segments whenever the ro segment falls into the [g_gc_lowest_address,g_gc_highest_address) range. I.e., it had an
3838 // extra "&& (size_t)(entry->seg1) & ro_in_entry" expression. However, at the moment, grow_brick_card_table does
3839 // not correctly go through the ro segments and add them back to the seg_mapping_table when the [lowest,highest)
3840 // range changes. We should probably go ahead and modify grow_brick_card_table and put back the
3841 // "&& (size_t)(entry->seg1) & ro_in_entry" here.
3844 seg = ro_segment_lookup (o);
3845 if (seg && !in_range_for_segment (o, seg))
3848 #endif //FEATURE_BASICFREEZE
3852 #endif //SEG_MAPPING_TABLE
3854 size_t gcard_of ( uint8_t*);
3856 #define memref(i) *(uint8_t**)(i)
3859 #define GC_MARKED (size_t)0x1
3860 #define slot(i, j) ((uint8_t**)(i))[j+1]
3862 #define free_object_base_size (plug_skew + sizeof(ArrayBase))
3864 class CObjectHeader : public Object
3868 #if defined(FEATURE_REDHAWK) || defined(BUILD_AS_STANDALONE)
3869 // The GC expects the following methods that are provided by the Object class in the CLR but not provided
3870 // by Redhawk's version of Object.
3871 uint32_t GetNumComponents()
3873 return ((ArrayBase *)this)->GetNumComponents();
3876 void Validate(BOOL bDeep=TRUE, BOOL bVerifyNextHeader = TRUE)
3878 UNREFERENCED_PARAMETER(bVerifyNextHeader);
3883 MethodTable * pMT = GetMethodTable();
3885 _ASSERTE(pMT->SanityCheck());
3887 bool noRangeChecks =
3888 (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_NO_RANGE_CHECKS) == GCConfig::HEAPVERIFY_NO_RANGE_CHECKS;
3890 BOOL fSmallObjectHeapPtr = FALSE, fLargeObjectHeapPtr = FALSE;
3893 fSmallObjectHeapPtr = g_theGCHeap->IsHeapPointer(this, TRUE);
3894 if (!fSmallObjectHeapPtr)
3895 fLargeObjectHeapPtr = g_theGCHeap->IsHeapPointer(this);
3897 _ASSERTE(fSmallObjectHeapPtr || fLargeObjectHeapPtr);
3900 #ifdef FEATURE_STRUCTALIGN
3901 _ASSERTE(IsStructAligned((uint8_t *)this, GetMethodTable()->GetBaseAlignment()));
3902 #endif // FEATURE_STRUCTALIGN
3904 #ifdef FEATURE_64BIT_ALIGNMENT
3905 if (pMT->RequiresAlign8())
3907 _ASSERTE((((size_t)this) & 0x7) == (pMT->IsValueType() ? 4U : 0U));
3909 #endif // FEATURE_64BIT_ALIGNMENT
3912 if (bDeep && (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC))
3913 g_theGCHeap->ValidateObjectMember(this);
3915 if (fSmallObjectHeapPtr)
3917 #ifdef FEATURE_BASICFREEZE
3918 _ASSERTE(!g_theGCHeap->IsLargeObject(pMT) || g_theGCHeap->IsInFrozenSegment(this));
3920 _ASSERTE(!g_theGCHeap->IsLargeObject(pMT));
3925 void ValidatePromote(ScanContext *sc, uint32_t flags)
3927 UNREFERENCED_PARAMETER(sc);
3928 UNREFERENCED_PARAMETER(flags);
3933 void ValidateHeap(Object *from, BOOL bDeep)
3935 UNREFERENCED_PARAMETER(from);
3937 Validate(bDeep, FALSE);
3940 ADIndex GetAppDomainIndex()
3942 return (ADIndex)RH_DEFAULT_DOMAIN_ID;
3944 #endif //FEATURE_REDHAWK
3948 // Header Status Information
3951 MethodTable *GetMethodTable() const
3953 return( (MethodTable *) (((size_t) RawGetMethodTable()) & (~(GC_MARKED))));
3958 RawSetMethodTable((MethodTable *) (((size_t) RawGetMethodTable()) | GC_MARKED));
3961 BOOL IsMarked() const
3963 return !!(((size_t)RawGetMethodTable()) & GC_MARKED);
3968 assert (!(gc_heap::settings.concurrent));
3969 GetHeader()->SetGCBit();
3972 BOOL IsPinned() const
3974 return !!((((CObjectHeader*)this)->GetHeader()->GetBits()) & BIT_SBLK_GC_RESERVE);
3979 RawSetMethodTable( GetMethodTable() );
3982 CGCDesc *GetSlotMap ()
3984 assert (GetMethodTable()->ContainsPointers());
3985 return CGCDesc::GetCGCDescFromMT(GetMethodTable());
3988 void SetFree(size_t size)
3990 assert (size >= free_object_base_size);
3992 assert (g_gc_pFreeObjectMethodTable->GetBaseSize() == free_object_base_size);
3993 assert (g_gc_pFreeObjectMethodTable->RawGetComponentSize() == 1);
3995 RawSetMethodTable( g_gc_pFreeObjectMethodTable );
3997 size_t* numComponentsPtr = (size_t*) &((uint8_t*) this)[ArrayBase::GetOffsetOfNumComponents()];
3998 *numComponentsPtr = size - free_object_base_size;
4000 //This introduces a bug in the free list management.
4001 //((void**) this)[-1] = 0; // clear the sync block,
4002 assert (*numComponentsPtr >= 0);
4003 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
4004 memset (((uint8_t*)this)+sizeof(ArrayBase), 0xcc, *numComponentsPtr);
4005 #endif //VERIFY_HEAP
4010 size_t size = free_object_base_size - plug_skew;
4012 // since we only need to clear 2 ptr size, we do it manually
4013 PTR_PTR m = (PTR_PTR) this;
4014 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
4018 BOOL IsFree () const
4020 return (GetMethodTable() == g_gc_pFreeObjectMethodTable);
4023 #ifdef FEATURE_STRUCTALIGN
4024 int GetRequiredAlignment () const
4026 return GetMethodTable()->GetRequiredAlignment();
4028 #endif // FEATURE_STRUCTALIGN
4030 BOOL ContainsPointers() const
4032 return GetMethodTable()->ContainsPointers();
4035 #ifdef COLLECTIBLE_CLASS
4036 BOOL Collectible() const
4038 return GetMethodTable()->Collectible();
4041 FORCEINLINE BOOL ContainsPointersOrCollectible() const
4043 MethodTable *pMethodTable = GetMethodTable();
4044 return (pMethodTable->ContainsPointers() || pMethodTable->Collectible());
4046 #endif //COLLECTIBLE_CLASS
4048 Object* GetObjectBase() const
4050 return (Object*) this;
4054 #define header(i) ((CObjectHeader*)(i))
4056 #define free_list_slot(x) ((uint8_t**)(x))[2]
4057 #define free_list_undo(x) ((uint8_t**)(x))[-1]
4058 #define UNDO_EMPTY ((uint8_t*)1)
4062 void set_plug_padded (uint8_t* node)
4064 header(node)->SetMarked();
4067 void clear_plug_padded (uint8_t* node)
4069 header(node)->ClearMarked();
4072 BOOL is_plug_padded (uint8_t* node)
4074 return header(node)->IsMarked();
4077 inline void set_plug_padded (uint8_t* node){}
4078 inline void clear_plug_padded (uint8_t* node){}
4080 BOOL is_plug_padded (uint8_t* node){return FALSE;}
4081 #endif //SHORT_PLUGS
4084 inline size_t unused_array_size(uint8_t * p)
4086 assert(((CObjectHeader*)p)->IsFree());
4088 size_t* numComponentsPtr = (size_t*)(p + ArrayBase::GetOffsetOfNumComponents());
4089 return free_object_base_size + *numComponentsPtr;
4092 heap_segment* heap_segment_rw (heap_segment* ns)
4094 if ((ns == 0) || !heap_segment_read_only_p (ns))
4102 ns = heap_segment_next (ns);
4103 } while ((ns != 0) && heap_segment_read_only_p (ns));
4108 //returns the next non ro segment.
4109 heap_segment* heap_segment_next_rw (heap_segment* seg)
4111 heap_segment* ns = heap_segment_next (seg);
4112 return heap_segment_rw (ns);
4115 // returns the segment before seg.
4116 heap_segment* heap_segment_prev_rw (heap_segment* begin, heap_segment* seg)
4118 assert (begin != 0);
4119 heap_segment* prev = begin;
4120 heap_segment* current = heap_segment_next_rw (begin);
4122 while (current && current != seg)
4125 current = heap_segment_next_rw (current);
4138 // returns the segment before seg.
4139 heap_segment* heap_segment_prev (heap_segment* begin, heap_segment* seg)
4141 assert (begin != 0);
4142 heap_segment* prev = begin;
4143 heap_segment* current = heap_segment_next (begin);
4145 while (current && current != seg)
4148 current = heap_segment_next (current);
4161 heap_segment* heap_segment_in_range (heap_segment* ns)
4163 if ((ns == 0) || heap_segment_in_range_p (ns))
4171 ns = heap_segment_next (ns);
4172 } while ((ns != 0) && !heap_segment_in_range_p (ns));
4177 heap_segment* heap_segment_next_in_range (heap_segment* seg)
4179 heap_segment* ns = heap_segment_next (seg);
4180 return heap_segment_in_range (ns);
4185 uint8_t* memory_base;
4190 imemory_data *initial_memory;
4191 imemory_data *initial_normal_heap; // points into initial_memory_array
4192 imemory_data *initial_large_heap; // points into initial_memory_array
4194 size_t block_size_normal;
4195 size_t block_size_large;
4197 size_t block_count; // # of blocks in each
4198 size_t current_block_normal;
4199 size_t current_block_large;
4208 size_t allocation_pattern;
4209 } initial_memory_details;
4211 initial_memory_details memory_details;
4213 BOOL reserve_initial_memory (size_t normal_size, size_t large_size, size_t num_heaps)
4215 BOOL reserve_success = FALSE;
4217 // should only be called once
4218 assert (memory_details.initial_memory == 0);
4220 memory_details.initial_memory = new (nothrow) imemory_data[num_heaps*2];
4221 if (memory_details.initial_memory == 0)
4223 dprintf (2, ("failed to reserve %Id bytes for imemory_data", num_heaps*2*sizeof(imemory_data)));
4227 memory_details.initial_normal_heap = memory_details.initial_memory;
4228 memory_details.initial_large_heap = memory_details.initial_memory + num_heaps;
4229 memory_details.block_size_normal = normal_size;
4230 memory_details.block_size_large = large_size;
4231 memory_details.block_count = num_heaps;
4233 memory_details.current_block_normal = 0;
4234 memory_details.current_block_large = 0;
4236 g_gc_lowest_address = MAX_PTR;
4237 g_gc_highest_address = 0;
4239 if (((size_t)MAX_PTR - large_size) < normal_size)
4241 // we are already overflowing with just one heap.
4242 dprintf (2, ("0x%Ix + 0x%Ix already overflow", normal_size, large_size));
4246 if (((size_t)MAX_PTR / memory_details.block_count) < (normal_size + large_size))
4248 dprintf (2, ("(0x%Ix + 0x%Ix)*0x%Ix overflow", normal_size, large_size, memory_details.block_count));
4252 size_t requestedMemory = memory_details.block_count * (normal_size + large_size);
4254 uint8_t* allatonce_block = (uint8_t*)virtual_alloc (requestedMemory);
4255 if (allatonce_block)
4257 g_gc_lowest_address = allatonce_block;
4258 g_gc_highest_address = allatonce_block + (memory_details.block_count * (large_size + normal_size));
4259 memory_details.allocation_pattern = initial_memory_details::ALLATONCE;
4261 for(size_t i = 0; i < memory_details.block_count; i++)
4263 memory_details.initial_normal_heap[i].memory_base = allatonce_block + (i*normal_size);
4264 memory_details.initial_large_heap[i].memory_base = allatonce_block +
4265 (memory_details.block_count*normal_size) + (i*large_size);
4266 reserve_success = TRUE;
4271 // try to allocate 2 blocks
4274 b1 = (uint8_t*)virtual_alloc (memory_details.block_count * normal_size);
4277 b2 = (uint8_t*)virtual_alloc (memory_details.block_count * large_size);
4280 memory_details.allocation_pattern = initial_memory_details::TWO_STAGE;
4281 g_gc_lowest_address = min(b1,b2);
4282 g_gc_highest_address = max(b1 + memory_details.block_count*normal_size,
4283 b2 + memory_details.block_count*large_size);
4284 for(size_t i = 0; i < memory_details.block_count; i++)
4286 memory_details.initial_normal_heap[i].memory_base = b1 + (i*normal_size);
4287 memory_details.initial_large_heap[i].memory_base = b2 + (i*large_size);
4288 reserve_success = TRUE;
4293 // b2 allocation failed, we'll go on to try allocating each block.
4294 // We could preserve the b1 alloc, but code complexity increases
4295 virtual_free (b1, memory_details.block_count * normal_size);
4299 if ((b2==NULL) && ( memory_details.block_count > 1))
4301 memory_details.allocation_pattern = initial_memory_details::EACH_BLOCK;
4303 imemory_data *current_block = memory_details.initial_memory;
4304 for(size_t i = 0; i < (memory_details.block_count*2); i++, current_block++)
4306 size_t block_size = ((i < memory_details.block_count) ?
4307 memory_details.block_size_normal :
4308 memory_details.block_size_large);
4309 current_block->memory_base =
4310 (uint8_t*)virtual_alloc (block_size);
4311 if (current_block->memory_base == 0)
4313 // Free the blocks that we've allocated so far
4314 current_block = memory_details.initial_memory;
4315 for(size_t j = 0; j < i; j++, current_block++){
4316 if (current_block->memory_base != 0){
4317 block_size = ((j < memory_details.block_count) ?
4318 memory_details.block_size_normal :
4319 memory_details.block_size_large);
4320 virtual_free (current_block->memory_base , block_size);
4323 reserve_success = FALSE;
4328 if (current_block->memory_base < g_gc_lowest_address)
4329 g_gc_lowest_address = current_block->memory_base;
4330 if (((uint8_t *) current_block->memory_base + block_size) > g_gc_highest_address)
4331 g_gc_highest_address = (current_block->memory_base + block_size);
4333 reserve_success = TRUE;
4338 return reserve_success;
4341 void destroy_initial_memory()
4343 if (memory_details.initial_memory != NULL)
4345 if (memory_details.allocation_pattern == initial_memory_details::ALLATONCE)
4347 virtual_free(memory_details.initial_memory[0].memory_base,
4348 memory_details.block_count*(memory_details.block_size_normal +
4349 memory_details.block_size_large));
4351 else if (memory_details.allocation_pattern == initial_memory_details::TWO_STAGE)
4353 virtual_free (memory_details.initial_normal_heap[0].memory_base,
4354 memory_details.block_count*memory_details.block_size_normal);
4356 virtual_free (memory_details.initial_large_heap[0].memory_base,
4357 memory_details.block_count*memory_details.block_size_large);
4361 assert (memory_details.allocation_pattern == initial_memory_details::EACH_BLOCK);
4362 imemory_data *current_block = memory_details.initial_memory;
4363 for(size_t i = 0; i < (memory_details.block_count*2); i++, current_block++)
4365 size_t block_size = (i < memory_details.block_count) ? memory_details.block_size_normal :
4366 memory_details.block_size_large;
4367 if (current_block->memory_base != NULL)
4369 virtual_free (current_block->memory_base, block_size);
4374 delete [] memory_details.initial_memory;
4375 memory_details.initial_memory = NULL;
4376 memory_details.initial_normal_heap = NULL;
4377 memory_details.initial_large_heap = NULL;
4381 void* next_initial_memory (size_t size)
4383 assert ((size == memory_details.block_size_normal) || (size == memory_details.block_size_large));
4386 if ((size != memory_details.block_size_normal) ||
4387 ((memory_details.current_block_normal == memory_details.block_count) &&
4388 (memory_details.block_size_normal == memory_details.block_size_large)))
4390 // If the block sizes are the same, flow block requests from normal to large
4391 assert (memory_details.current_block_large < memory_details.block_count);
4392 assert (memory_details.initial_large_heap != 0);
4394 res = memory_details.initial_large_heap[memory_details.current_block_large].memory_base;
4395 memory_details.current_block_large++;
4399 assert (memory_details.current_block_normal < memory_details.block_count);
4400 assert (memory_details.initial_normal_heap != NULL);
4402 res = memory_details.initial_normal_heap[memory_details.current_block_normal].memory_base;
4403 memory_details.current_block_normal++;
4409 heap_segment* get_initial_segment (size_t size, int h_number)
4411 void* mem = next_initial_memory (size);
4412 heap_segment* res = gc_heap::make_heap_segment ((uint8_t*)mem, size , h_number);
4417 void* virtual_alloc (size_t size)
4419 size_t requested_size = size;
4421 if ((gc_heap::reserved_memory_limit - gc_heap::reserved_memory) < requested_size)
4423 gc_heap::reserved_memory_limit =
4424 GCScan::AskForMoreReservedMemory (gc_heap::reserved_memory_limit, requested_size);
4425 if ((gc_heap::reserved_memory_limit - gc_heap::reserved_memory) < requested_size)
4431 uint32_t flags = VirtualReserveFlags::None;
4432 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
4433 if (virtual_alloc_hardware_write_watch)
4435 flags = VirtualReserveFlags::WriteWatch;
4437 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
4438 void* prgmem = GCToOSInterface::VirtualReserve (requested_size, card_size * card_word_width, flags);
4439 void *aligned_mem = prgmem;
4441 // We don't want (prgmem + size) to be right at the end of the address space
4442 // because we'd have to worry about that everytime we do (address + size).
4443 // We also want to make sure that we leave LARGE_OBJECT_SIZE at the end
4444 // so we allocate a small object we don't need to worry about overflow there
4445 // when we do alloc_ptr+size.
4448 uint8_t* end_mem = (uint8_t*)prgmem + requested_size;
4450 if ((end_mem == 0) || ((size_t)(MAX_PTR - end_mem) <= END_SPACE_AFTER_GC))
4452 GCToOSInterface::VirtualRelease (prgmem, requested_size);
4453 dprintf (2, ("Virtual Alloc size %Id returned memory right against 4GB [%Ix, %Ix[ - discarding",
4454 requested_size, (size_t)prgmem, (size_t)((uint8_t*)prgmem+requested_size)));
4462 gc_heap::reserved_memory += requested_size;
4465 dprintf (2, ("Virtual Alloc size %Id: [%Ix, %Ix[",
4466 requested_size, (size_t)prgmem, (size_t)((uint8_t*)prgmem+requested_size)));
4471 void virtual_free (void* add, size_t size)
4473 GCToOSInterface::VirtualRelease (add, size);
4474 gc_heap::reserved_memory -= size;
4475 dprintf (2, ("Virtual Free size %Id: [%Ix, %Ix[",
4476 size, (size_t)add, (size_t)((uint8_t*)add+size)));
4479 static size_t get_valid_segment_size (BOOL large_seg=FALSE)
4481 size_t seg_size, initial_seg_size;
4485 initial_seg_size = INITIAL_ALLOC;
4486 seg_size = static_cast<size_t>(GCConfig::GetSegmentSize());
4490 initial_seg_size = LHEAP_ALLOC;
4491 seg_size = static_cast<size_t>(GCConfig::GetSegmentSize()) / 2;
4494 #ifdef MULTIPLE_HEAPS
4499 if (g_num_processors > 4)
4500 initial_seg_size /= 2;
4501 if (g_num_processors > 8)
4502 initial_seg_size /= 2;
4504 #endif //MULTIPLE_HEAPS
4506 // if seg_size is small but not 0 (0 is default if config not set)
4507 // then set the segment to the minimum size
4508 if (!g_theGCHeap->IsValidSegmentSize(seg_size))
4510 // if requested size is between 1 byte and 4MB, use min
4511 if ((seg_size >> 1) && !(seg_size >> 22))
4512 seg_size = 1024*1024*4;
4514 seg_size = initial_seg_size;
4517 #ifdef SEG_MAPPING_TABLE
4519 seg_size = round_up_power2 (seg_size);
4521 seg_size = round_down_power2 (seg_size);
4523 #endif //SEG_MAPPING_TABLE
4529 gc_heap::compute_new_ephemeral_size()
4531 int eph_gen_max = max_generation - 1 - (settings.promotion ? 1 : 0);
4532 size_t padding_size = 0;
4534 for (int i = 0; i <= eph_gen_max; i++)
4536 dynamic_data* dd = dynamic_data_of (i);
4537 total_ephemeral_size += (dd_survived_size (dd) - dd_pinned_survived_size (dd));
4538 #ifdef RESPECT_LARGE_ALIGNMENT
4539 total_ephemeral_size += dd_num_npinned_plugs (dd) * switch_alignment_size (FALSE);
4540 #endif //RESPECT_LARGE_ALIGNMENT
4541 #ifdef FEATURE_STRUCTALIGN
4542 total_ephemeral_size += dd_num_npinned_plugs (dd) * MAX_STRUCTALIGN;
4543 #endif //FEATURE_STRUCTALIGN
4546 padding_size += dd_padding_size (dd);
4547 #endif //SHORT_PLUGS
4550 total_ephemeral_size += eph_gen_starts_size;
4552 #ifdef RESPECT_LARGE_ALIGNMENT
4553 size_t planned_ephemeral_size = heap_segment_plan_allocated (ephemeral_heap_segment) -
4554 generation_plan_allocation_start (generation_of (max_generation-1));
4555 total_ephemeral_size = min (total_ephemeral_size, planned_ephemeral_size);
4556 #endif //RESPECT_LARGE_ALIGNMENT
4559 total_ephemeral_size = Align ((size_t)((double)total_ephemeral_size * short_plugs_pad_ratio) + 1);
4560 total_ephemeral_size += Align (DESIRED_PLUG_LENGTH);
4561 #endif //SHORT_PLUGS
4563 dprintf (3, ("total ephemeral size is %Ix, padding %Ix(%Ix)",
4564 total_ephemeral_size,
4565 padding_size, (total_ephemeral_size - padding_size)));
4569 #pragma warning(disable:4706) // "assignment within conditional expression" is intentional in this function.
4573 gc_heap::soh_get_segment_to_expand()
4575 size_t size = soh_segment_size;
4577 ordered_plug_indices_init = FALSE;
4578 use_bestfit = FALSE;
4580 //compute the size of the new ephemeral heap segment.
4581 compute_new_ephemeral_size();
4583 if ((settings.pause_mode != pause_low_latency) &&
4584 (settings.pause_mode != pause_no_gc)
4585 #ifdef BACKGROUND_GC
4586 && (!recursive_gc_sync::background_running_p())
4587 #endif //BACKGROUND_GC
4590 allocator* gen_alloc = ((settings.condemned_generation == max_generation) ? 0 :
4591 generation_allocator (generation_of (max_generation)));
4592 dprintf (2, ("(gen%d)soh_get_segment_to_expand", settings.condemned_generation));
4594 // try to find one in the gen 2 segment list, search backwards because the first segments
4595 // tend to be more compact than the later ones.
4596 heap_segment* fseg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
4598 PREFIX_ASSUME(fseg != NULL);
4600 #ifdef SEG_REUSE_STATS
4602 #endif //SEG_REUSE_STATS
4604 heap_segment* seg = ephemeral_heap_segment;
4605 while ((seg = heap_segment_prev_rw (fseg, seg)) && (seg != fseg))
4607 #ifdef SEG_REUSE_STATS
4609 #endif //SEG_REUSE_STATS
4611 if (can_expand_into_p (seg, size/3, total_ephemeral_size, gen_alloc))
4613 get_gc_data_per_heap()->set_mechanism (gc_heap_expand,
4614 (use_bestfit ? expand_reuse_bestfit : expand_reuse_normal));
4615 if (settings.condemned_generation == max_generation)
4619 build_ordered_free_spaces (seg);
4620 dprintf (GTC_LOG, ("can use best fit"));
4623 #ifdef SEG_REUSE_STATS
4624 dprintf (SEG_REUSE_LOG_0, ("(gen%d)soh_get_segment_to_expand: found seg #%d to reuse",
4625 settings.condemned_generation, try_reuse));
4626 #endif //SEG_REUSE_STATS
4627 dprintf (GTC_LOG, ("max_gen: Found existing segment to expand into %Ix", (size_t)seg));
4632 #ifdef SEG_REUSE_STATS
4633 dprintf (SEG_REUSE_LOG_0, ("(gen%d)soh_get_segment_to_expand: found seg #%d to reuse - returning",
4634 settings.condemned_generation, try_reuse));
4635 #endif //SEG_REUSE_STATS
4636 dprintf (GTC_LOG, ("max_gen-1: Found existing segment to expand into %Ix", (size_t)seg));
4638 // If we return 0 here, the allocator will think since we are short on end
4639 // of seg we neeed to trigger a full compacting GC. So if sustained low latency
4640 // is set we should acquire a new seg instead, that way we wouldn't be short.
4641 // The real solution, of course, is to actually implement seg reuse in gen1.
4642 if (settings.pause_mode != pause_sustained_low_latency)
4644 dprintf (GTC_LOG, ("max_gen-1: SustainedLowLatency is set, acquire a new seg"));
4645 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, expand_next_full_gc);
4653 heap_segment* result = get_segment (size, FALSE);
4657 #ifdef BACKGROUND_GC
4658 if (current_c_gc_state == c_gc_state_planning)
4660 // When we expand heap during bgc sweep, we set the seg to be swept so
4661 // we'll always look at cards for objects on the new segment.
4662 result->flags |= heap_segment_flags_swept;
4664 #endif //BACKGROUND_GC
4666 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(result),
4667 (size_t)(heap_segment_reserved (result) - heap_segment_mem(result)),
4668 gc_etw_segment_small_object_heap);
4671 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, (result ? expand_new_seg : expand_no_memory));
4675 dprintf (2, ("h%d: failed to allocate a new segment!", heap_number));
4679 #ifdef MULTIPLE_HEAPS
4680 heap_segment_heap (result) = this;
4681 #endif //MULTIPLE_HEAPS
4684 dprintf (GTC_LOG, ("(gen%d)creating new segment %Ix", settings.condemned_generation, result));
4689 #pragma warning(default:4706)
4692 //returns 0 in case of allocation failure
4694 gc_heap::get_segment (size_t size, BOOL loh_p)
4696 heap_segment* result = 0;
4698 if (segment_standby_list != 0)
4700 result = segment_standby_list;
4701 heap_segment* last = 0;
4704 size_t hs = (size_t)(heap_segment_reserved (result) - (uint8_t*)result);
4705 if ((hs >= size) && ((hs / 2) < size))
4707 dprintf (2, ("Hoarded segment %Ix found", (size_t) result));
4710 heap_segment_next (last) = heap_segment_next (result);
4714 segment_standby_list = heap_segment_next (result);
4721 result = heap_segment_next (result);
4728 init_heap_segment (result);
4729 #ifdef BACKGROUND_GC
4730 if (should_commit_mark_array())
4732 dprintf (GC_TABLE_LOG, ("hoarded seg %Ix, mark_array is %Ix", result, mark_array));
4733 if (!commit_mark_array_new_seg (__this, result))
4735 dprintf (GC_TABLE_LOG, ("failed to commit mark array for hoarded seg"));
4736 // If we can't use it we need to thread it back.
4737 if (segment_standby_list != 0)
4739 heap_segment_next (result) = segment_standby_list;
4740 segment_standby_list = result;
4744 segment_standby_list = result;
4750 #endif //BACKGROUND_GC
4752 #ifdef SEG_MAPPING_TABLE
4754 seg_mapping_table_add_segment (result, __this);
4755 #endif //SEG_MAPPING_TABLE
4760 #ifndef SEG_MAPPING_TABLE
4761 if (!seg_table->ensure_space_for_insert ())
4763 #endif //SEG_MAPPING_TABLE
4764 void* mem = virtual_alloc (size);
4767 fgm_result.set_fgm (fgm_reserve_segment, size, loh_p);
4771 result = gc_heap::make_heap_segment ((uint8_t*)mem, size, heap_number);
4777 if (mem < g_gc_lowest_address)
4779 start = (uint8_t*)mem;
4783 start = (uint8_t*)g_gc_lowest_address;
4786 if (((uint8_t*)mem + size) > g_gc_highest_address)
4788 end = (uint8_t*)mem + size;
4792 end = (uint8_t*)g_gc_highest_address;
4795 if (gc_heap::grow_brick_card_tables (start, end, size, result, __this, loh_p) != 0)
4797 virtual_free (mem, size);
4803 fgm_result.set_fgm (fgm_commit_segment_beg, SEGMENT_INITIAL_COMMIT, loh_p);
4804 virtual_free (mem, size);
4809 #ifdef SEG_MAPPING_TABLE
4810 seg_mapping_table_add_segment (result, __this);
4811 #else //SEG_MAPPING_TABLE
4812 gc_heap::seg_table->insert ((uint8_t*)result, delta);
4813 #endif //SEG_MAPPING_TABLE
4817 #ifdef BACKGROUND_GC
4820 ::record_changed_seg ((uint8_t*)result, heap_segment_reserved (result),
4821 settings.gc_index, current_bgc_state,
4823 bgc_verify_mark_array_cleared (result);
4825 #endif //BACKGROUND_GC
4827 dprintf (GC_TABLE_LOG, ("h%d: new seg: %Ix-%Ix (%Id)", heap_number, result, ((uint8_t*)result + size), size));
4831 void release_segment (heap_segment* sg)
4833 ptrdiff_t delta = 0;
4834 FIRE_EVENT(GCFreeSegment_V1, heap_segment_mem(sg));
4835 virtual_free (sg, (uint8_t*)heap_segment_reserved (sg)-(uint8_t*)sg);
4838 heap_segment* gc_heap::get_segment_for_loh (size_t size
4839 #ifdef MULTIPLE_HEAPS
4841 #endif //MULTIPLE_HEAPS
4844 #ifndef MULTIPLE_HEAPS
4846 #endif //MULTIPLE_HEAPS
4847 heap_segment* res = hp->get_segment (size, TRUE);
4850 #ifdef MULTIPLE_HEAPS
4851 heap_segment_heap (res) = hp;
4852 #endif //MULTIPLE_HEAPS
4853 res->flags |= heap_segment_flags_loh;
4855 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(res), (size_t)(heap_segment_reserved (res) - heap_segment_mem(res)), gc_etw_segment_large_object_heap);
4857 GCToEEInterface::DiagUpdateGenerationBounds();
4859 #ifdef MULTIPLE_HEAPS
4860 hp->thread_loh_segment (res);
4862 thread_loh_segment (res);
4863 #endif //MULTIPLE_HEAPS
4869 void gc_heap::thread_loh_segment (heap_segment* new_seg)
4871 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
4873 while (heap_segment_next_rw (seg))
4874 seg = heap_segment_next_rw (seg);
4875 heap_segment_next (seg) = new_seg;
4879 gc_heap::get_large_segment (size_t size, BOOL* did_full_compact_gc)
4881 *did_full_compact_gc = FALSE;
4882 size_t last_full_compact_gc_count = get_full_compact_gc_count();
4884 //access to get_segment needs to be serialized
4885 add_saved_spinlock_info (me_release, mt_get_large_seg);
4887 dprintf (SPINLOCK_LOG, ("[%d]Seg: Lmsl", heap_number));
4888 leave_spin_lock (&more_space_lock);
4889 enter_spin_lock (&gc_heap::gc_lock);
4890 dprintf (SPINLOCK_LOG, ("[%d]Seg: Egc", heap_number));
4891 // if a GC happened between here and before we ask for a segment in
4892 // get_large_segment, we need to count that GC.
4893 size_t current_full_compact_gc_count = get_full_compact_gc_count();
4895 if (current_full_compact_gc_count > last_full_compact_gc_count)
4897 *did_full_compact_gc = TRUE;
4900 #ifdef BACKGROUND_GC
4901 while (current_c_gc_state == c_gc_state_planning)
4903 dprintf (3, ("lh state planning, waiting to get a large seg"));
4905 dprintf (SPINLOCK_LOG, ("[%d]Seg: P, Lgc", heap_number));
4906 leave_spin_lock (&gc_lock);
4907 background_gc_wait_lh (awr_get_loh_seg);
4908 enter_spin_lock (&gc_lock);
4909 dprintf (SPINLOCK_LOG, ("[%d]Seg: P, Egc", heap_number));
4911 assert ((current_c_gc_state == c_gc_state_free) ||
4912 (current_c_gc_state == c_gc_state_marking));
4913 #endif //BACKGROUND_GC
4915 heap_segment* res = get_segment_for_loh (size
4916 #ifdef MULTIPLE_HEAPS
4918 #endif //MULTIPLE_HEAPS
4921 dprintf (SPINLOCK_LOG, ("[%d]Seg: A Lgc", heap_number));
4922 leave_spin_lock (&gc_heap::gc_lock);
4923 enter_spin_lock (&more_space_lock);
4924 dprintf (SPINLOCK_LOG, ("[%d]Seg: A Emsl", heap_number));
4925 add_saved_spinlock_info (me_acquire, mt_get_large_seg);
4927 #ifdef BACKGROUND_GC
4928 wait_for_background_planning (awr_get_loh_seg);
4929 #endif //BACKGROUND_GC
4935 BOOL gc_heap::unprotect_segment (heap_segment* seg)
4937 uint8_t* start = align_lower_page (heap_segment_mem (seg));
4938 ptrdiff_t region_size = heap_segment_allocated (seg) - start;
4940 if (region_size != 0 )
4942 dprintf (3, ("unprotecting segment %Ix:", (size_t)seg));
4944 BOOL status = GCToOSInterface::VirtualUnprotect (start, region_size);
4952 #ifdef MULTIPLE_HEAPS
4955 #pragma warning(disable:4035)
4956 static ptrdiff_t get_cycle_count()
4960 #pragma warning(default:4035)
4961 #elif defined(__GNUC__)
4962 static ptrdiff_t get_cycle_count()
4966 __asm__ __volatile__
4967 ("rdtsc":"=a" (cycles), "=d" (cyclesHi));
4971 #error Unknown compiler
4973 #elif defined(_TARGET_AMD64_)
4975 extern "C" uint64_t __rdtsc();
4976 #pragma intrinsic(__rdtsc)
4977 static ptrdiff_t get_cycle_count()
4979 return (ptrdiff_t)__rdtsc();
4981 #elif defined(__clang__)
4982 static ptrdiff_t get_cycle_count()
4986 __asm__ __volatile__
4987 ("rdtsc":"=a" (cycles), "=d" (cyclesHi));
4988 return (cyclesHi << 32) | cycles;
4991 extern "C" ptrdiff_t get_cycle_count(void);
4993 #elif defined(_TARGET_ARM_)
4994 static ptrdiff_t get_cycle_count()
4996 // @ARMTODO: cycle counter is not exposed to user mode by CoreARM. For now (until we can show this
4997 // makes a difference on the ARM configurations on which we'll run) just return 0. This will result in
4998 // all buffer access times being reported as equal in access_time().
5001 #elif defined(_TARGET_ARM64_)
5002 static ptrdiff_t get_cycle_count()
5004 // @ARM64TODO: cycle counter is not exposed to user mode by CoreARM. For now (until we can show this
5005 // makes a difference on the ARM configurations on which we'll run) just return 0. This will result in
5006 // all buffer access times being reported as equal in access_time().
5010 #error NYI platform: get_cycle_count
5011 #endif //_TARGET_X86_
5016 static uint8_t* sniff_buffer;
5017 static unsigned n_sniff_buffers;
5018 static unsigned cur_sniff_index;
5020 static uint16_t proc_no_to_heap_no[MAX_SUPPORTED_CPUS];
5021 static uint16_t heap_no_to_proc_no[MAX_SUPPORTED_CPUS];
5022 static uint16_t heap_no_to_numa_node[MAX_SUPPORTED_CPUS];
5023 static uint16_t heap_no_to_cpu_group[MAX_SUPPORTED_CPUS];
5024 static uint16_t heap_no_to_group_proc[MAX_SUPPORTED_CPUS];
5025 static uint16_t numa_node_to_heap_map[MAX_SUPPORTED_CPUS+4];
5027 static int access_time(uint8_t *sniff_buffer, int heap_number, unsigned sniff_index, unsigned n_sniff_buffers)
5029 ptrdiff_t start_cycles = get_cycle_count();
5030 uint8_t sniff = sniff_buffer[(1 + heap_number*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE];
5031 assert (sniff == 0);
5032 ptrdiff_t elapsed_cycles = get_cycle_count() - start_cycles;
5033 // add sniff here just to defeat the optimizer
5034 elapsed_cycles += sniff;
5035 return (int) elapsed_cycles;
5039 static BOOL init(int n_heaps)
5041 assert (sniff_buffer == NULL && n_sniff_buffers == 0);
5042 if (!GCToOSInterface::CanGetCurrentProcessorNumber())
5044 n_sniff_buffers = n_heaps*2+1;
5045 size_t n_cache_lines = 1 + n_heaps * n_sniff_buffers + 1;
5046 size_t sniff_buf_size = n_cache_lines * HS_CACHE_LINE_SIZE;
5047 if (sniff_buf_size / HS_CACHE_LINE_SIZE != n_cache_lines) // check for overlow
5052 sniff_buffer = new (nothrow) uint8_t[sniff_buf_size];
5053 if (sniff_buffer == 0)
5055 memset(sniff_buffer, 0, sniff_buf_size*sizeof(uint8_t));
5058 //can not enable gc numa aware, force all heaps to be in
5059 //one numa node by filling the array with all 0s
5060 if (!GCToOSInterface::CanEnableGCNumaAware())
5061 memset(heap_no_to_numa_node, 0, sizeof (heap_no_to_numa_node));
5066 static void init_cpu_mapping(gc_heap * /*heap*/, int heap_number)
5068 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5070 uint32_t proc_no = GCToOSInterface::GetCurrentProcessorNumber() % gc_heap::n_heaps;
5071 // We can safely cast heap_number to a uint16_t 'cause GetCurrentProcessCpuCount
5072 // only returns up to MAX_SUPPORTED_CPUS procs right now. We only ever create at most
5073 // MAX_SUPPORTED_CPUS GC threads.
5074 proc_no_to_heap_no[proc_no] = (uint16_t)heap_number;
5078 static void mark_heap(int heap_number)
5080 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5083 for (unsigned sniff_index = 0; sniff_index < n_sniff_buffers; sniff_index++)
5084 sniff_buffer[(1 + heap_number*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE] &= 1;
5087 static int select_heap(alloc_context* acontext, int /*hint*/)
5089 UNREFERENCED_PARAMETER(acontext); // only referenced by dprintf
5091 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5092 return proc_no_to_heap_no[GCToOSInterface::GetCurrentProcessorNumber() % gc_heap::n_heaps];
5094 unsigned sniff_index = Interlocked::Increment(&cur_sniff_index);
5095 sniff_index %= n_sniff_buffers;
5098 int best_access_time = 1000*1000*1000;
5099 int second_best_access_time = best_access_time;
5101 uint8_t *l_sniff_buffer = sniff_buffer;
5102 unsigned l_n_sniff_buffers = n_sniff_buffers;
5103 for (int heap_number = 0; heap_number < gc_heap::n_heaps; heap_number++)
5105 int this_access_time = access_time(l_sniff_buffer, heap_number, sniff_index, l_n_sniff_buffers);
5106 if (this_access_time < best_access_time)
5108 second_best_access_time = best_access_time;
5109 best_access_time = this_access_time;
5110 best_heap = heap_number;
5112 else if (this_access_time < second_best_access_time)
5114 second_best_access_time = this_access_time;
5118 if (best_access_time*2 < second_best_access_time)
5120 sniff_buffer[(1 + best_heap*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE] &= 1;
5122 dprintf (3, ("select_heap yields crisp %d for context %p\n", best_heap, (void *)acontext));
5126 dprintf (3, ("select_heap yields vague %d for context %p\n", best_heap, (void *)acontext ));
5132 static bool can_find_heap_fast()
5134 return GCToOSInterface::CanGetCurrentProcessorNumber();
5137 static uint16_t find_proc_no_from_heap_no(int heap_number)
5139 return heap_no_to_proc_no[heap_number];
5142 static void set_proc_no_for_heap(int heap_number, uint16_t proc_no)
5144 heap_no_to_proc_no[heap_number] = proc_no;
5147 static uint16_t find_numa_node_from_heap_no(int heap_number)
5149 return heap_no_to_numa_node[heap_number];
5152 static void set_numa_node_for_heap(int heap_number, uint16_t numa_node)
5154 heap_no_to_numa_node[heap_number] = numa_node;
5157 static uint16_t find_cpu_group_from_heap_no(int heap_number)
5159 return heap_no_to_cpu_group[heap_number];
5162 static void set_cpu_group_for_heap(int heap_number, uint16_t group_number)
5164 heap_no_to_cpu_group[heap_number] = group_number;
5167 static uint16_t find_group_proc_from_heap_no(int heap_number)
5169 return heap_no_to_group_proc[heap_number];
5172 static void set_group_proc_for_heap(int heap_number, uint16_t group_proc)
5174 heap_no_to_group_proc[heap_number] = group_proc;
5177 static void init_numa_node_to_heap_map(int nheaps)
5178 { // called right after GCHeap::Init() for each heap is finished
5179 // when numa is not enabled, heap_no_to_numa_node[] are all filled
5180 // with 0s during initialization, and will be treated as one node
5181 numa_node_to_heap_map[0] = 0;
5184 for (int i=1; i < nheaps; i++)
5186 if (heap_no_to_numa_node[i] != heap_no_to_numa_node[i-1])
5187 numa_node_to_heap_map[node_index++] = (uint16_t)i;
5189 numa_node_to_heap_map[node_index] = (uint16_t)nheaps; //mark the end with nheaps
5192 static void get_heap_range_for_heap(int hn, int* start, int* end)
5193 { // 1-tier/no numa case: heap_no_to_numa_node[] all zeros,
5194 // and treated as in one node. thus: start=0, end=n_heaps
5195 uint16_t numa_node = heap_no_to_numa_node[hn];
5196 *start = (int)numa_node_to_heap_map[numa_node];
5197 *end = (int)(numa_node_to_heap_map[numa_node+1]);
5200 uint8_t* heap_select::sniff_buffer;
5201 unsigned heap_select::n_sniff_buffers;
5202 unsigned heap_select::cur_sniff_index;
5203 uint16_t heap_select::proc_no_to_heap_no[MAX_SUPPORTED_CPUS];
5204 uint16_t heap_select::heap_no_to_proc_no[MAX_SUPPORTED_CPUS];
5205 uint16_t heap_select::heap_no_to_numa_node[MAX_SUPPORTED_CPUS];
5206 uint16_t heap_select::heap_no_to_cpu_group[MAX_SUPPORTED_CPUS];
5207 uint16_t heap_select::heap_no_to_group_proc[MAX_SUPPORTED_CPUS];
5208 uint16_t heap_select::numa_node_to_heap_map[MAX_SUPPORTED_CPUS+4];
5210 BOOL gc_heap::create_thread_support (unsigned number_of_heaps)
5213 if (!gc_start_event.CreateOSManualEventNoThrow (FALSE))
5217 if (!ee_suspend_event.CreateOSAutoEventNoThrow (FALSE))
5221 if (!gc_t_join.init (number_of_heaps, join_flavor_server_gc))
5232 destroy_thread_support();
5238 void gc_heap::destroy_thread_support ()
5240 if (ee_suspend_event.IsValid())
5242 ee_suspend_event.CloseEvent();
5244 if (gc_start_event.IsValid())
5246 gc_start_event.CloseEvent();
5250 void set_thread_group_affinity_for_heap(int heap_number, GCThreadAffinity* affinity)
5252 affinity->Group = GCThreadAffinity::None;
5253 affinity->Processor = GCThreadAffinity::None;
5256 GCToOSInterface::GetGroupForProcessor((uint16_t)heap_number, &gn, &gpn);
5259 for (uintptr_t mask = 1; mask !=0; mask <<=1)
5261 if (bit_number == gpn)
5263 dprintf(3, ("using processor group %d, mask %Ix for heap %d\n", gn, mask, heap_number));
5264 affinity->Processor = gpn;
5265 affinity->Group = gn;
5266 heap_select::set_cpu_group_for_heap(heap_number, gn);
5267 heap_select::set_group_proc_for_heap(heap_number, gpn);
5268 if (GCToOSInterface::CanEnableGCNumaAware())
5270 PROCESSOR_NUMBER proc_no;
5272 proc_no.Number = (uint8_t)gpn;
5273 proc_no.Reserved = 0;
5275 uint16_t node_no = 0;
5276 if (GCToOSInterface::GetNumaProcessorNode(&proc_no, &node_no))
5277 heap_select::set_numa_node_for_heap(heap_number, node_no);
5280 { // no numa setting, each cpu group is treated as a node
5281 heap_select::set_numa_node_for_heap(heap_number, gn);
5289 void set_thread_affinity_mask_for_heap(int heap_number, GCThreadAffinity* affinity)
5291 affinity->Group = GCThreadAffinity::None;
5292 affinity->Processor = GCThreadAffinity::None;
5294 uintptr_t pmask, smask;
5295 if (GCToOSInterface::GetCurrentProcessAffinityMask(&pmask, &smask))
5299 uint8_t proc_number = 0;
5300 for (uintptr_t mask = 1; mask != 0; mask <<= 1)
5302 if ((mask & pmask) != 0)
5304 if (bit_number == heap_number)
5306 dprintf (3, ("Using processor %d for heap %d", proc_number, heap_number));
5307 affinity->Processor = proc_number;
5308 heap_select::set_proc_no_for_heap(heap_number, proc_number);
5309 if (GCToOSInterface::CanEnableGCNumaAware())
5311 uint16_t node_no = 0;
5312 PROCESSOR_NUMBER proc_no;
5314 proc_no.Number = (uint8_t)proc_number;
5315 proc_no.Reserved = 0;
5316 if (GCToOSInterface::GetNumaProcessorNode(&proc_no, &node_no))
5318 heap_select::set_numa_node_for_heap(heap_number, node_no);
5330 bool gc_heap::create_gc_thread ()
5332 dprintf (3, ("Creating gc thread\n"));
5333 return GCToEEInterface::CreateThread(gc_thread_stub, this, false, ".NET Server GC");
5337 #pragma warning(disable:4715) //IA64 xcompiler recognizes that without the 'break;' the while(1) will never end and therefore not return a value for that code path
5339 void gc_heap::gc_thread_function ()
5341 assert (gc_done_event.IsValid());
5342 assert (gc_start_event.IsValid());
5343 dprintf (3, ("gc thread started"));
5345 heap_select::init_cpu_mapping(this, heap_number);
5349 assert (!gc_t_join.joined());
5351 if (heap_number == 0)
5353 gc_heap::ee_suspend_event.Wait(INFINITE, FALSE);
5355 BEGIN_TIMING(suspend_ee_during_log);
5356 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC);
5357 END_TIMING(suspend_ee_during_log);
5359 proceed_with_gc_p = TRUE;
5361 if (!should_proceed_with_gc())
5363 update_collection_counts_for_no_gc();
5364 proceed_with_gc_p = FALSE;
5368 settings.init_mechanisms();
5369 gc_start_event.Set();
5371 dprintf (3, ("%d gc thread waiting...", heap_number));
5375 gc_start_event.Wait(INFINITE, FALSE);
5376 dprintf (3, ("%d gc thread waiting... Done", heap_number));
5379 assert ((heap_number == 0) || proceed_with_gc_p);
5381 if (proceed_with_gc_p)
5382 garbage_collect (GCHeap::GcCondemnedGeneration);
5384 if (heap_number == 0)
5386 if (proceed_with_gc_p && (!settings.concurrent))
5391 #ifdef BACKGROUND_GC
5392 recover_bgc_settings();
5393 #endif //BACKGROUND_GC
5395 #ifdef MULTIPLE_HEAPS
5396 for (int i = 0; i < gc_heap::n_heaps; i++)
5398 gc_heap* hp = gc_heap::g_heaps[i];
5399 hp->add_saved_spinlock_info (me_release, mt_block_gc);
5400 dprintf (SPINLOCK_LOG, ("[%d]GC Lmsl", i));
5401 leave_spin_lock(&hp->more_space_lock);
5403 #endif //MULTIPLE_HEAPS
5405 gc_heap::gc_started = FALSE;
5407 BEGIN_TIMING(restart_ee_during_log);
5408 GCToEEInterface::RestartEE(TRUE);
5409 END_TIMING(restart_ee_during_log);
5410 process_sync_log_stats();
5412 dprintf (SPINLOCK_LOG, ("GC Lgc"));
5413 leave_spin_lock (&gc_heap::gc_lock);
5415 gc_heap::internal_gc_done = true;
5417 if (proceed_with_gc_p)
5421 // If we didn't actually do a GC, it means we didn't wait up the other threads,
5422 // we still need to set the gc_done_event for those threads.
5423 for (int i = 0; i < gc_heap::n_heaps; i++)
5425 gc_heap* hp = gc_heap::g_heaps[i];
5432 int spin_count = 32 * (gc_heap::n_heaps - 1);
5434 // wait until RestartEE has progressed to a stage where we can restart user threads
5435 while (!gc_heap::internal_gc_done && !GCHeap::SafeToRestartManagedThreads())
5437 spin_and_switch (spin_count, (gc_heap::internal_gc_done || GCHeap::SafeToRestartManagedThreads()));
5444 #pragma warning(default:4715) //IA64 xcompiler recognizes that without the 'break;' the while(1) will never end and therefore not return a value for that code path
5447 #endif //MULTIPLE_HEAPS
5449 bool virtual_alloc_commit_for_heap(void* addr, size_t size, int h_number)
5451 #if defined(MULTIPLE_HEAPS) && !defined(FEATURE_REDHAWK)
5452 // Currently there is no way for us to specific the numa node to allocate on via hosting interfaces to
5453 // a host. This will need to be added later.
5454 #if !defined(FEATURE_CORECLR)
5455 if (!CLRMemoryHosted())
5458 if (GCToOSInterface::CanEnableGCNumaAware())
5460 uint32_t numa_node = heap_select::find_numa_node_from_heap_no(h_number);
5461 if (GCToOSInterface::VirtualCommit(addr, size, numa_node))
5466 UNREFERENCED_PARAMETER(h_number);
5469 //numa aware not enabled, or call failed --> fallback to VirtualCommit()
5470 return GCToOSInterface::VirtualCommit(addr, size);
5473 #ifndef SEG_MAPPING_TABLE
5475 heap_segment* gc_heap::segment_of (uint8_t* add, ptrdiff_t& delta, BOOL verify_p)
5477 uint8_t* sadd = add;
5478 heap_segment* hs = 0;
5479 heap_segment* hs1 = 0;
5480 if (!((add >= g_gc_lowest_address) && (add < g_gc_highest_address)))
5485 //repeat in case there is a concurrent insertion in the table.
5490 seg_table->lookup (sadd);
5491 hs1 = (heap_segment*)sadd;
5492 } while (hs1 && !in_range_for_segment (add, hs1) && (hs != hs1));
5497 (verify_p && (add > heap_segment_reserved ((heap_segment*)(sadd + delta)))))
5501 #endif //SEG_MAPPING_TABLE
5509 // If we want to save space we can have a pool of plug_and_gap's instead of
5510 // always having 2 allocated for each pinned plug.
5511 gap_reloc_pair saved_pre_plug;
5512 // If we decide to not compact, we need to restore the original values.
5513 gap_reloc_pair saved_pre_plug_reloc;
5515 gap_reloc_pair saved_post_plug;
5517 // Supposedly Pinned objects cannot have references but we are seeing some from pinvoke
5518 // frames. Also if it's an artificially pinned plug created by us, it can certainly
5520 // We know these cases will be rare so we can optimize this to be only allocated on decommand.
5521 gap_reloc_pair saved_post_plug_reloc;
5523 // We need to calculate this after we are done with plan phase and before compact
5524 // phase because compact phase will change the bricks so relocate_address will no
5526 uint8_t* saved_pre_plug_info_reloc_start;
5528 // We need to save this because we will have no way to calculate it, unlike the
5529 // pre plug info start which is right before this plug.
5530 uint8_t* saved_post_plug_info_start;
5533 uint8_t* allocation_context_start_region;
5534 #endif //SHORT_PLUGS
5536 // How the bits in these bytes are organized:
5538 // bit to indicate whether it's a short obj | 3 bits for refs in this short obj | 2 unused bits | bit to indicate if it's collectible | last bit
5539 // last bit indicates if there's pre or post info associated with this plug. If it's not set all other bits will be 0.
5544 // We are seeing this is getting corrupted for a PP with a NP after.
5545 // Save it when we first set it and make sure it doesn't change.
5546 gap_reloc_pair saved_post_plug_debug;
5549 size_t get_max_short_bits()
5551 return (sizeof (gap_reloc_pair) / sizeof (uint8_t*));
5555 size_t get_pre_short_start_bit ()
5557 return (sizeof (saved_pre_p) * 8 - 1 - (sizeof (gap_reloc_pair) / sizeof (uint8_t*)));
5562 return (saved_pre_p & (1 << (sizeof (saved_pre_p) * 8 - 1)));
5565 void set_pre_short()
5567 saved_pre_p |= (1 << (sizeof (saved_pre_p) * 8 - 1));
5570 void set_pre_short_bit (size_t bit)
5572 saved_pre_p |= 1 << (get_pre_short_start_bit() + bit);
5575 BOOL pre_short_bit_p (size_t bit)
5577 return (saved_pre_p & (1 << (get_pre_short_start_bit() + bit)));
5580 #ifdef COLLECTIBLE_CLASS
5581 void set_pre_short_collectible()
5586 BOOL pre_short_collectible_p()
5588 return (saved_pre_p & 2);
5590 #endif //COLLECTIBLE_CLASS
5593 size_t get_post_short_start_bit ()
5595 return (sizeof (saved_post_p) * 8 - 1 - (sizeof (gap_reloc_pair) / sizeof (uint8_t*)));
5600 return (saved_post_p & (1 << (sizeof (saved_post_p) * 8 - 1)));
5603 void set_post_short()
5605 saved_post_p |= (1 << (sizeof (saved_post_p) * 8 - 1));
5608 void set_post_short_bit (size_t bit)
5610 saved_post_p |= 1 << (get_post_short_start_bit() + bit);
5613 BOOL post_short_bit_p (size_t bit)
5615 return (saved_post_p & (1 << (get_post_short_start_bit() + bit)));
5618 #ifdef COLLECTIBLE_CLASS
5619 void set_post_short_collectible()
5624 BOOL post_short_collectible_p()
5626 return (saved_post_p & 2);
5628 #endif //COLLECTIBLE_CLASS
5630 uint8_t* get_plug_address() { return first; }
5632 BOOL has_pre_plug_info() { return saved_pre_p; }
5633 BOOL has_post_plug_info() { return saved_post_p; }
5635 gap_reloc_pair* get_pre_plug_reloc_info() { return &saved_pre_plug_reloc; }
5636 gap_reloc_pair* get_post_plug_reloc_info() { return &saved_post_plug_reloc; }
5637 void set_pre_plug_info_reloc_start (uint8_t* reloc) { saved_pre_plug_info_reloc_start = reloc; }
5638 uint8_t* get_post_plug_info_start() { return saved_post_plug_info_start; }
5640 // We need to temporarily recover the shortened plugs for compact phase so we can
5641 // copy over the whole plug and their related info (mark bits/cards). But we will
5642 // need to set the artificial gap back so compact phase can keep reading the plug info.
5643 // We also need to recover the saved info because we'll need to recover it later.
5645 // So we would call swap_p*_plug_and_saved once to recover the object info; then call
5646 // it again to recover the artificial gap.
5647 void swap_pre_plug_and_saved()
5649 gap_reloc_pair temp;
5650 memcpy (&temp, (first - sizeof (plug_and_gap)), sizeof (temp));
5651 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug_reloc, sizeof (saved_pre_plug_reloc));
5652 saved_pre_plug_reloc = temp;
5655 void swap_post_plug_and_saved()
5657 gap_reloc_pair temp;
5658 memcpy (&temp, saved_post_plug_info_start, sizeof (temp));
5659 memcpy (saved_post_plug_info_start, &saved_post_plug_reloc, sizeof (saved_post_plug_reloc));
5660 saved_post_plug_reloc = temp;
5663 void swap_pre_plug_and_saved_for_profiler()
5665 gap_reloc_pair temp;
5666 memcpy (&temp, (first - sizeof (plug_and_gap)), sizeof (temp));
5667 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug, sizeof (saved_pre_plug));
5668 saved_pre_plug = temp;
5671 void swap_post_plug_and_saved_for_profiler()
5673 gap_reloc_pair temp;
5674 memcpy (&temp, saved_post_plug_info_start, sizeof (temp));
5675 memcpy (saved_post_plug_info_start, &saved_post_plug, sizeof (saved_post_plug));
5676 saved_post_plug = temp;
5679 // We should think about whether it's really necessary to have to copy back the pre plug
5680 // info since it was already copied during compacting plugs. But if a plug doesn't move
5681 // by >= 3 ptr size (the size of gap_reloc_pair), it means we'd have to recover pre plug info.
5682 void recover_plug_info()
5686 if (gc_heap::settings.compaction)
5688 dprintf (3, ("%Ix: REC Pre: %Ix-%Ix",
5690 &saved_pre_plug_reloc,
5691 saved_pre_plug_info_reloc_start));
5692 memcpy (saved_pre_plug_info_reloc_start, &saved_pre_plug_reloc, sizeof (saved_pre_plug_reloc));
5696 dprintf (3, ("%Ix: REC Pre: %Ix-%Ix",
5699 (first - sizeof (plug_and_gap))));
5700 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug, sizeof (saved_pre_plug));
5706 if (gc_heap::settings.compaction)
5708 dprintf (3, ("%Ix: REC Post: %Ix-%Ix",
5710 &saved_post_plug_reloc,
5711 saved_post_plug_info_start));
5712 memcpy (saved_post_plug_info_start, &saved_post_plug_reloc, sizeof (saved_post_plug_reloc));
5716 dprintf (3, ("%Ix: REC Post: %Ix-%Ix",
5719 saved_post_plug_info_start));
5720 memcpy (saved_post_plug_info_start, &saved_post_plug, sizeof (saved_post_plug));
5727 void gc_mechanisms::init_mechanisms()
5729 condemned_generation = 0;
5730 promotion = FALSE;//TRUE;
5732 #ifdef FEATURE_LOH_COMPACTION
5733 loh_compaction = gc_heap::should_compact_loh();
5735 loh_compaction = FALSE;
5736 #endif //FEATURE_LOH_COMPACTION
5737 heap_expansion = FALSE;
5740 elevation_reduced = FALSE;
5741 found_finalizers = FALSE;
5742 #ifdef BACKGROUND_GC
5743 background_p = recursive_gc_sync::background_running_p() != FALSE;
5744 allocations_allowed = TRUE;
5745 #endif //BACKGROUND_GC
5747 entry_memory_load = 0;
5748 exit_memory_load = 0;
5751 stress_induced = FALSE;
5752 #endif // STRESS_HEAP
5755 void gc_mechanisms::first_init()
5758 gen0_reduction_count = 0;
5759 should_lock_elevation = FALSE;
5760 elevation_locked_count = 0;
5761 reason = reason_empty;
5762 #ifdef BACKGROUND_GC
5763 pause_mode = gc_heap::gc_can_use_concurrent ? pause_interactive : pause_batch;
5765 int debug_pause_mode = static_cast<int>(GCConfig::GetLatencyMode());
5766 if (debug_pause_mode >= 0)
5768 assert (debug_pause_mode <= pause_sustained_low_latency);
5769 pause_mode = (gc_pause_mode)debug_pause_mode;
5772 #else //BACKGROUND_GC
5773 pause_mode = pause_batch;
5774 #endif //BACKGROUND_GC
5779 void gc_mechanisms::record (gc_history_global* history)
5781 #ifdef MULTIPLE_HEAPS
5782 history->num_heaps = gc_heap::n_heaps;
5784 history->num_heaps = 1;
5785 #endif //MULTIPLE_HEAPS
5787 history->condemned_generation = condemned_generation;
5788 history->gen0_reduction_count = gen0_reduction_count;
5789 history->reason = reason;
5790 history->pause_mode = (int)pause_mode;
5791 history->mem_pressure = entry_memory_load;
5792 history->global_mechanims_p = 0;
5794 // start setting the boolean values.
5796 history->set_mechanism_p (global_concurrent);
5799 history->set_mechanism_p (global_compaction);
5802 history->set_mechanism_p (global_promotion);
5805 history->set_mechanism_p (global_demotion);
5808 history->set_mechanism_p (global_card_bundles);
5810 if (elevation_reduced)
5811 history->set_mechanism_p (global_elevation);
5814 /**********************************
5815 called at the beginning of GC to fix the allocated size to
5816 what is really allocated, or to turn the free area into an unused object
5817 It needs to be called after all of the other allocation contexts have been
5818 fixed since it relies on alloc_allocated.
5819 ********************************/
5821 //for_gc_p indicates that the work is being done for GC,
5822 //as opposed to concurrent heap verification
5823 void gc_heap::fix_youngest_allocation_area (BOOL for_gc_p)
5825 UNREFERENCED_PARAMETER(for_gc_p);
5827 // The gen 0 alloc context is never used for allocation in the allocator path. It's
5828 // still used in the allocation path during GCs.
5829 assert (generation_allocation_pointer (youngest_generation) == nullptr);
5830 assert (generation_allocation_limit (youngest_generation) == nullptr);
5831 heap_segment_allocated (ephemeral_heap_segment) = alloc_allocated;
5834 void gc_heap::fix_large_allocation_area (BOOL for_gc_p)
5836 UNREFERENCED_PARAMETER(for_gc_p);
5839 alloc_context* acontext =
5841 generation_alloc_context (large_object_generation);
5842 assert (acontext->alloc_ptr == 0);
5843 assert (acontext->alloc_limit == 0);
5845 dprintf (3, ("Large object alloc context: ptr: %Ix, limit %Ix",
5846 (size_t)acontext->alloc_ptr, (size_t)acontext->alloc_limit));
5847 fix_allocation_context (acontext, FALSE, get_alignment_constant (FALSE));
5850 acontext->alloc_ptr = 0;
5851 acontext->alloc_limit = acontext->alloc_ptr;
5856 //for_gc_p indicates that the work is being done for GC,
5857 //as opposed to concurrent heap verification
5858 void gc_heap::fix_allocation_context (alloc_context* acontext, BOOL for_gc_p,
5861 dprintf (3, ("Fixing allocation context %Ix: ptr: %Ix, limit: %Ix",
5863 (size_t)acontext->alloc_ptr, (size_t)acontext->alloc_limit));
5865 if (((size_t)(alloc_allocated - acontext->alloc_limit) > Align (min_obj_size, align_const)) ||
5868 uint8_t* point = acontext->alloc_ptr;
5871 size_t size = (acontext->alloc_limit - acontext->alloc_ptr);
5872 // the allocation area was from the free list
5873 // it was shortened by Align (min_obj_size) to make room for
5874 // at least the shortest unused object
5875 size += Align (min_obj_size, align_const);
5876 assert ((size >= Align (min_obj_size)));
5878 dprintf(3,("Making unused area [%Ix, %Ix[", (size_t)point,
5879 (size_t)point + size ));
5880 make_unused_array (point, size);
5884 generation_free_obj_space (generation_of (0)) += size;
5885 alloc_contexts_used ++;
5891 alloc_allocated = acontext->alloc_ptr;
5892 assert (heap_segment_allocated (ephemeral_heap_segment) <=
5893 heap_segment_committed (ephemeral_heap_segment));
5894 alloc_contexts_used ++;
5899 // We need to update the alloc_bytes to reflect the portion that we have not used
5900 acontext->alloc_bytes -= (acontext->alloc_limit - acontext->alloc_ptr);
5901 acontext->alloc_ptr = 0;
5902 acontext->alloc_limit = acontext->alloc_ptr;
5906 //used by the heap verification for concurrent gc.
5907 //it nulls out the words set by fix_allocation_context for heap_verification
5908 void repair_allocation (gc_alloc_context* acontext, void*)
5910 uint8_t* point = acontext->alloc_ptr;
5914 dprintf (3, ("Clearing [%Ix, %Ix[", (size_t)acontext->alloc_ptr,
5915 (size_t)acontext->alloc_limit+Align(min_obj_size)));
5916 memclr (acontext->alloc_ptr - plug_skew,
5917 (acontext->alloc_limit - acontext->alloc_ptr)+Align (min_obj_size));
5921 void void_allocation (gc_alloc_context* acontext, void*)
5923 uint8_t* point = acontext->alloc_ptr;
5927 dprintf (3, ("Void [%Ix, %Ix[", (size_t)acontext->alloc_ptr,
5928 (size_t)acontext->alloc_limit+Align(min_obj_size)));
5929 acontext->alloc_ptr = 0;
5930 acontext->alloc_limit = acontext->alloc_ptr;
5934 void gc_heap::repair_allocation_contexts (BOOL repair_p)
5936 GCToEEInterface::GcEnumAllocContexts (repair_p ? repair_allocation : void_allocation, NULL);
5939 struct fix_alloc_context_args
5945 void fix_alloc_context(gc_alloc_context* acontext, void* param)
5947 fix_alloc_context_args* args = (fix_alloc_context_args*)param;
5948 g_theGCHeap->FixAllocContext(acontext, false, (void*)(size_t)(args->for_gc_p), args->heap);
5951 void gc_heap::fix_allocation_contexts(BOOL for_gc_p)
5953 fix_alloc_context_args args;
5954 args.for_gc_p = for_gc_p;
5957 GCToEEInterface::GcEnumAllocContexts(fix_alloc_context, &args);
5958 fix_youngest_allocation_area(for_gc_p);
5959 fix_large_allocation_area(for_gc_p);
5962 void gc_heap::fix_older_allocation_area (generation* older_gen)
5964 heap_segment* older_gen_seg = generation_allocation_segment (older_gen);
5965 if (generation_allocation_limit (older_gen) !=
5966 heap_segment_plan_allocated (older_gen_seg))
5968 uint8_t* point = generation_allocation_pointer (older_gen);
5970 size_t size = (generation_allocation_limit (older_gen) -
5971 generation_allocation_pointer (older_gen));
5974 assert ((size >= Align (min_obj_size)));
5975 dprintf(3,("Making unused area [%Ix, %Ix[", (size_t)point, (size_t)point+size));
5976 make_unused_array (point, size);
5981 assert (older_gen_seg != ephemeral_heap_segment);
5982 heap_segment_plan_allocated (older_gen_seg) =
5983 generation_allocation_pointer (older_gen);
5984 generation_allocation_limit (older_gen) =
5985 generation_allocation_pointer (older_gen);
5989 void gc_heap::set_allocation_heap_segment (generation* gen)
5991 uint8_t* p = generation_allocation_start (gen);
5993 heap_segment* seg = generation_allocation_segment (gen);
5994 if (in_range_for_segment (p, seg))
5997 // try ephemeral heap segment in case of heap expansion
5998 seg = ephemeral_heap_segment;
5999 if (!in_range_for_segment (p, seg))
6001 seg = heap_segment_rw (generation_start_segment (gen));
6003 PREFIX_ASSUME(seg != NULL);
6005 while (!in_range_for_segment (p, seg))
6007 seg = heap_segment_next_rw (seg);
6008 PREFIX_ASSUME(seg != NULL);
6012 generation_allocation_segment (gen) = seg;
6015 void gc_heap::reset_allocation_pointers (generation* gen, uint8_t* start)
6018 assert (Align ((size_t)start) == (size_t)start);
6019 generation_allocation_start (gen) = start;
6020 generation_allocation_pointer (gen) = 0;//start + Align (min_obj_size);
6021 generation_allocation_limit (gen) = 0;//generation_allocation_pointer (gen);
6022 set_allocation_heap_segment (gen);
6025 #ifdef BACKGROUND_GC
6026 //TODO BACKGROUND_GC this is for test only
6028 gc_heap::disallow_new_allocation (int gen_number)
6030 UNREFERENCED_PARAMETER(gen_number);
6031 settings.allocations_allowed = FALSE;
6034 gc_heap::allow_new_allocation (int gen_number)
6036 UNREFERENCED_PARAMETER(gen_number);
6037 settings.allocations_allowed = TRUE;
6040 #endif //BACKGROUND_GC
6042 bool gc_heap::new_allocation_allowed (int gen_number)
6044 #ifdef BACKGROUND_GC
6045 //TODO BACKGROUND_GC this is for test only
6046 if (!settings.allocations_allowed)
6048 dprintf (2, ("new allocation not allowed"));
6051 #endif //BACKGROUND_GC
6053 if (dd_new_allocation (dynamic_data_of (gen_number)) < 0)
6055 if (gen_number != 0)
6057 // For LOH we will give it more budget before we try a GC.
6058 if (settings.concurrent)
6060 dynamic_data* dd2 = dynamic_data_of (max_generation + 1 );
6062 if (dd_new_allocation (dd2) <= (ptrdiff_t)(-2 * dd_desired_allocation (dd2)))
6070 #ifndef MULTIPLE_HEAPS
6071 else if ((settings.pause_mode != pause_no_gc) && (gen_number == 0))
6073 dprintf (3, ("evaluating allocation rate"));
6074 dynamic_data* dd0 = dynamic_data_of (0);
6075 if ((allocation_running_amount - dd_new_allocation (dd0)) >
6078 uint32_t ctime = GCToOSInterface::GetLowPrecisionTimeStamp();
6079 if ((ctime - allocation_running_time) > 1000)
6081 dprintf (2, (">1s since last gen0 gc"));
6086 allocation_running_amount = dd_new_allocation (dd0);
6090 #endif //MULTIPLE_HEAPS
6095 ptrdiff_t gc_heap::get_desired_allocation (int gen_number)
6097 return dd_desired_allocation (dynamic_data_of (gen_number));
6101 ptrdiff_t gc_heap::get_new_allocation (int gen_number)
6103 return dd_new_allocation (dynamic_data_of (gen_number));
6106 //return the amount allocated so far in gen_number
6108 ptrdiff_t gc_heap::get_allocation (int gen_number)
6110 dynamic_data* dd = dynamic_data_of (gen_number);
6112 return dd_desired_allocation (dd) - dd_new_allocation (dd);
6116 BOOL grow_mark_stack (mark*& m, size_t& len, size_t init_len)
6118 size_t new_size = max (init_len, 2*len);
6119 mark* tmp = new (nothrow) mark [new_size];
6122 memcpy (tmp, m, len * sizeof (mark));
6130 dprintf (1, ("Failed to allocate %Id bytes for mark stack", (len * sizeof (mark))));
6136 uint8_t* pinned_plug (mark* m)
6142 size_t& pinned_len (mark* m)
6148 void set_new_pin_info (mark* m, uint8_t* pin_free_space_start)
6150 m->len = pinned_plug (m) - pin_free_space_start;
6152 m->allocation_context_start_region = pin_free_space_start;
6153 #endif //SHORT_PLUGS
6158 uint8_t*& pin_allocation_context_start_region (mark* m)
6160 return m->allocation_context_start_region;
6163 uint8_t* get_plug_start_in_saved (uint8_t* old_loc, mark* pinned_plug_entry)
6165 uint8_t* saved_pre_plug_info = (uint8_t*)(pinned_plug_entry->get_pre_plug_reloc_info());
6166 uint8_t* plug_start_in_saved = saved_pre_plug_info + (old_loc - (pinned_plug (pinned_plug_entry) - sizeof (plug_and_gap)));
6167 //dprintf (1, ("detected a very short plug: %Ix before PP %Ix, pad %Ix",
6168 // old_loc, pinned_plug (pinned_plug_entry), plug_start_in_saved));
6169 dprintf (1, ("EP: %Ix(%Ix), %Ix", old_loc, pinned_plug (pinned_plug_entry), plug_start_in_saved));
6170 return plug_start_in_saved;
6174 void set_padding_in_expand (uint8_t* old_loc,
6175 BOOL set_padding_on_saved_p,
6176 mark* pinned_plug_entry)
6178 if (set_padding_on_saved_p)
6180 set_plug_padded (get_plug_start_in_saved (old_loc, pinned_plug_entry));
6184 set_plug_padded (old_loc);
6189 void clear_padding_in_expand (uint8_t* old_loc,
6190 BOOL set_padding_on_saved_p,
6191 mark* pinned_plug_entry)
6193 if (set_padding_on_saved_p)
6195 clear_plug_padded (get_plug_start_in_saved (old_loc, pinned_plug_entry));
6199 clear_plug_padded (old_loc);
6202 #endif //SHORT_PLUGS
6204 void gc_heap::reset_pinned_queue()
6210 void gc_heap::reset_pinned_queue_bos()
6215 // last_pinned_plug is only for asserting purpose.
6216 void gc_heap::merge_with_last_pinned_plug (uint8_t* last_pinned_plug, size_t plug_size)
6218 if (last_pinned_plug)
6220 mark& last_m = mark_stack_array[mark_stack_tos - 1];
6221 assert (last_pinned_plug == last_m.first);
6222 if (last_m.saved_post_p)
6224 last_m.saved_post_p = FALSE;
6225 dprintf (3, ("setting last plug %Ix post to false", last_m.first));
6226 // We need to recover what the gap has overwritten.
6227 memcpy ((last_m.first + last_m.len - sizeof (plug_and_gap)), &(last_m.saved_post_plug), sizeof (gap_reloc_pair));
6229 last_m.len += plug_size;
6230 dprintf (3, ("recovered the last part of plug %Ix, setting its plug size to %Ix", last_m.first, last_m.len));
6234 void gc_heap::set_allocator_next_pin (uint8_t* alloc_pointer, uint8_t*& alloc_limit)
6236 dprintf (3, ("sanp: ptr: %Ix, limit: %Ix", alloc_pointer, alloc_limit));
6237 dprintf (3, ("oldest %Id: %Ix", mark_stack_bos, pinned_plug (oldest_pin())));
6238 if (!(pinned_plug_que_empty_p()))
6240 mark* oldest_entry = oldest_pin();
6241 uint8_t* plug = pinned_plug (oldest_entry);
6242 if ((plug >= alloc_pointer) && (plug < alloc_limit))
6244 alloc_limit = pinned_plug (oldest_entry);
6245 dprintf (3, ("now setting alloc context: %Ix->%Ix(%Id)",
6246 alloc_pointer, alloc_limit, (alloc_limit - alloc_pointer)));
6251 void gc_heap::set_allocator_next_pin (generation* gen)
6253 dprintf (3, ("SANP: gen%d, ptr; %Ix, limit: %Ix", gen->gen_num, generation_allocation_pointer (gen), generation_allocation_limit (gen)));
6254 if (!(pinned_plug_que_empty_p()))
6256 mark* oldest_entry = oldest_pin();
6257 uint8_t* plug = pinned_plug (oldest_entry);
6258 if ((plug >= generation_allocation_pointer (gen)) &&
6259 (plug < generation_allocation_limit (gen)))
6261 generation_allocation_limit (gen) = pinned_plug (oldest_entry);
6262 dprintf (3, ("SANP: get next pin free space in gen%d for alloc: %Ix->%Ix(%Id)",
6264 generation_allocation_pointer (gen), generation_allocation_limit (gen),
6265 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
6268 assert (!((plug < generation_allocation_pointer (gen)) &&
6269 (plug >= heap_segment_mem (generation_allocation_segment (gen)))));
6273 // After we set the info, we increase tos.
6274 void gc_heap::set_pinned_info (uint8_t* last_pinned_plug, size_t plug_len, uint8_t* alloc_pointer, uint8_t*& alloc_limit)
6276 UNREFERENCED_PARAMETER(last_pinned_plug);
6278 mark& m = mark_stack_array[mark_stack_tos];
6279 assert (m.first == last_pinned_plug);
6283 set_allocator_next_pin (alloc_pointer, alloc_limit);
6286 // After we set the info, we increase tos.
6287 void gc_heap::set_pinned_info (uint8_t* last_pinned_plug, size_t plug_len, generation* gen)
6289 UNREFERENCED_PARAMETER(last_pinned_plug);
6291 mark& m = mark_stack_array[mark_stack_tos];
6292 assert (m.first == last_pinned_plug);
6297 // Why are we checking here? gen is never 0.
6300 set_allocator_next_pin (gen);
6304 size_t gc_heap::deque_pinned_plug ()
6306 dprintf (3, ("dequed: %Id", mark_stack_bos));
6307 size_t m = mark_stack_bos;
6313 mark* gc_heap::pinned_plug_of (size_t bos)
6315 return &mark_stack_array [ bos ];
6319 mark* gc_heap::oldest_pin ()
6321 return pinned_plug_of (mark_stack_bos);
6325 BOOL gc_heap::pinned_plug_que_empty_p ()
6327 return (mark_stack_bos == mark_stack_tos);
6331 mark* gc_heap::before_oldest_pin()
6333 if (mark_stack_bos >= 1)
6334 return pinned_plug_of (mark_stack_bos-1);
6340 BOOL gc_heap::ephemeral_pointer_p (uint8_t* o)
6342 return ((o >= ephemeral_low) && (o < ephemeral_high));
6347 int& gc_heap::mark_stack_busy()
6349 return g_mark_stack_busy [(heap_number+2)*HS_CACHE_LINE_SIZE/sizeof(int)];
6353 void gc_heap::make_mark_stack (mark* arr)
6355 reset_pinned_queue();
6356 mark_stack_array = arr;
6357 mark_stack_array_length = MARK_STACK_INITIAL_LENGTH;
6359 mark_stack_busy() = 0;
6363 #ifdef BACKGROUND_GC
6365 size_t& gc_heap::bpromoted_bytes(int thread)
6367 #ifdef MULTIPLE_HEAPS
6368 return g_bpromoted [thread*16];
6369 #else //MULTIPLE_HEAPS
6370 UNREFERENCED_PARAMETER(thread);
6372 #endif //MULTIPLE_HEAPS
6375 void gc_heap::make_background_mark_stack (uint8_t** arr)
6377 background_mark_stack_array = arr;
6378 background_mark_stack_array_length = MARK_STACK_INITIAL_LENGTH;
6379 background_mark_stack_tos = arr;
6382 void gc_heap::make_c_mark_list (uint8_t** arr)
6385 c_mark_list_index = 0;
6386 c_mark_list_length = 1 + (OS_PAGE_SIZE / MIN_OBJECT_SIZE);
6388 #endif //BACKGROUND_GC
6393 // The card bundle keeps track of groups of card words.
6394 static const size_t card_bundle_word_width = 32;
6396 // How do we express the fact that 32 bits (card_word_width) is one uint32_t?
6397 static const size_t card_bundle_size = (size_t)(GC_PAGE_SIZE / (sizeof(uint32_t)*card_bundle_word_width));
6400 size_t card_bundle_word (size_t cardb)
6402 return cardb / card_bundle_word_width;
6406 uint32_t card_bundle_bit (size_t cardb)
6408 return (uint32_t)(cardb % card_bundle_word_width);
6411 size_t align_cardw_on_bundle (size_t cardw)
6413 return ((size_t)(cardw + card_bundle_size - 1) & ~(card_bundle_size - 1 ));
6416 // Get the card bundle representing a card word
6417 size_t cardw_card_bundle (size_t cardw)
6419 return cardw / card_bundle_size;
6422 // Get the first card word in a card bundle
6423 size_t card_bundle_cardw (size_t cardb)
6425 return cardb * card_bundle_size;
6428 // Clear the specified card bundle
6429 void gc_heap::card_bundle_clear (size_t cardb)
6431 card_bundle_table [card_bundle_word (cardb)] &= ~(1 << card_bundle_bit (cardb));
6432 dprintf (1,("Cleared card bundle %Ix [%Ix, %Ix[", cardb, (size_t)card_bundle_cardw (cardb),
6433 (size_t)card_bundle_cardw (cardb+1)));
6436 void gc_heap::card_bundle_set (size_t cardb)
6438 if (!card_bundle_set_p (cardb))
6440 card_bundle_table [card_bundle_word (cardb)] |= (1 << card_bundle_bit (cardb));
6444 // Set the card bundle bits between start_cardb and end_cardb
6445 void gc_heap::card_bundles_set (size_t start_cardb, size_t end_cardb)
6447 if (start_cardb == end_cardb)
6449 card_bundle_set(start_cardb);
6453 size_t start_word = card_bundle_word (start_cardb);
6454 size_t end_word = card_bundle_word (end_cardb);
6456 if (start_word < end_word)
6458 // Set the partial words
6459 card_bundle_table [start_word] |= highbits (~0u, card_bundle_bit (start_cardb));
6461 if (card_bundle_bit (end_cardb))
6462 card_bundle_table [end_word] |= lowbits (~0u, card_bundle_bit (end_cardb));
6464 // Set the full words
6465 for (size_t i = start_word + 1; i < end_word; i++)
6466 card_bundle_table [i] = ~0u;
6470 card_bundle_table [start_word] |= (highbits (~0u, card_bundle_bit (start_cardb)) &
6471 lowbits (~0u, card_bundle_bit (end_cardb)));
6475 // Indicates whether the specified bundle is set.
6476 BOOL gc_heap::card_bundle_set_p (size_t cardb)
6478 return (card_bundle_table[card_bundle_word(cardb)] & (1 << card_bundle_bit (cardb)));
6481 // Returns the size (in bytes) of a card bundle representing the region from 'from' to 'end'
6482 size_t size_card_bundle_of (uint8_t* from, uint8_t* end)
6484 // Number of heap bytes represented by a card bundle word
6485 size_t cbw_span = card_size * card_word_width * card_bundle_size * card_bundle_word_width;
6487 // Align the start of the region down
6488 from = (uint8_t*)((size_t)from & ~(cbw_span - 1));
6490 // Align the end of the region up
6491 end = (uint8_t*)((size_t)(end + (cbw_span - 1)) & ~(cbw_span - 1));
6493 // Make sure they're really aligned
6494 assert (((size_t)from & (cbw_span - 1)) == 0);
6495 assert (((size_t)end & (cbw_span - 1)) == 0);
6497 return ((end - from) / cbw_span) * sizeof (uint32_t);
6500 // Takes a pointer to a card bundle table and an address, and returns a pointer that represents
6501 // where a theoretical card bundle table that represents every address (starting from 0) would
6502 // start if the bundle word representing the address were to be located at the pointer passed in.
6503 // The returned 'translated' pointer makes it convenient/fast to calculate where the card bundle
6504 // for a given address is using a simple shift operation on the address.
6505 uint32_t* translate_card_bundle_table (uint32_t* cb, uint8_t* lowest_address)
6507 // The number of bytes of heap memory represented by a card bundle word
6508 const size_t heap_bytes_for_bundle_word = card_size * card_word_width * card_bundle_size * card_bundle_word_width;
6510 // Each card bundle word is 32 bits
6511 return (uint32_t*)((uint8_t*)cb - (((size_t)lowest_address / heap_bytes_for_bundle_word) * sizeof (uint32_t)));
6514 void gc_heap::enable_card_bundles ()
6516 if (can_use_write_watch_for_card_table() && (!card_bundles_enabled()))
6518 dprintf (1, ("Enabling card bundles"));
6520 // We initially set all of the card bundles
6521 card_bundles_set (cardw_card_bundle (card_word (card_of (lowest_address))),
6522 cardw_card_bundle (align_cardw_on_bundle (card_word (card_of (highest_address)))));
6523 settings.card_bundles = TRUE;
6527 BOOL gc_heap::card_bundles_enabled ()
6529 return settings.card_bundles;
6532 #endif // CARD_BUNDLE
6534 #if defined (_TARGET_AMD64_)
6535 #define brick_size ((size_t)4096)
6537 #define brick_size ((size_t)2048)
6538 #endif //_TARGET_AMD64_
6541 size_t gc_heap::brick_of (uint8_t* add)
6543 return (size_t)(add - lowest_address) / brick_size;
6547 uint8_t* gc_heap::brick_address (size_t brick)
6549 return lowest_address + (brick_size * brick);
6553 void gc_heap::clear_brick_table (uint8_t* from, uint8_t* end)
6555 for (size_t i = brick_of (from);i < brick_of (end); i++)
6559 //codes for the brick entries:
6560 //entry == 0 -> not assigned
6561 //entry >0 offset is entry-1
6562 //entry <0 jump back entry bricks
6566 void gc_heap::set_brick (size_t index, ptrdiff_t val)
6572 assert (val < 32767);
6574 brick_table [index] = (short)val+1;
6576 brick_table [index] = (short)val;
6580 int gc_heap::get_brick_entry (size_t index)
6582 #ifdef MULTIPLE_HEAPS
6583 return VolatileLoadWithoutBarrier(&brick_table [index]);
6585 return brick_table[index];
6591 uint8_t* align_on_brick (uint8_t* add)
6593 return (uint8_t*)((size_t)(add + brick_size - 1) & ~(brick_size - 1));
6597 uint8_t* align_lower_brick (uint8_t* add)
6599 return (uint8_t*)(((size_t)add) & ~(brick_size - 1));
6602 size_t size_brick_of (uint8_t* from, uint8_t* end)
6604 assert (((size_t)from & (brick_size-1)) == 0);
6605 assert (((size_t)end & (brick_size-1)) == 0);
6607 return ((end - from) / brick_size) * sizeof (short);
6611 uint8_t* gc_heap::card_address (size_t card)
6613 return (uint8_t*) (card_size * card);
6617 size_t gc_heap::card_of ( uint8_t* object)
6619 return (size_t)(object) / card_size;
6623 size_t gc_heap::card_to_brick (size_t card)
6625 return brick_of (card_address (card));
6629 uint8_t* align_on_card (uint8_t* add)
6631 return (uint8_t*)((size_t)(add + card_size - 1) & ~(card_size - 1 ));
6634 uint8_t* align_on_card_word (uint8_t* add)
6636 return (uint8_t*) ((size_t)(add + (card_size*card_word_width)-1) & ~(card_size*card_word_width - 1));
6640 uint8_t* align_lower_card (uint8_t* add)
6642 return (uint8_t*)((size_t)add & ~(card_size-1));
6646 void gc_heap::clear_card (size_t card)
6648 card_table [card_word (card)] =
6649 (card_table [card_word (card)] & ~(1 << card_bit (card)));
6650 dprintf (3,("Cleared card %Ix [%Ix, %Ix[", card, (size_t)card_address (card),
6651 (size_t)card_address (card+1)));
6655 void gc_heap::set_card (size_t card)
6657 size_t word = card_word (card);
6658 card_table[word] = (card_table [word] | (1 << card_bit (card)));
6660 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
6661 // Also set the card bundle that corresponds to the card
6662 size_t bundle_to_set = cardw_card_bundle(word);
6664 card_bundle_set(bundle_to_set);
6666 dprintf (3,("Set card %Ix [%Ix, %Ix[ and bundle %Ix", card, (size_t)card_address (card), (size_t)card_address (card+1), bundle_to_set));
6667 assert(card_bundle_set_p(bundle_to_set) != 0);
6672 BOOL gc_heap::card_set_p (size_t card)
6674 return ( card_table [ card_word (card) ] & (1 << card_bit (card)));
6677 // Returns the number of DWORDs in the card table that cover the
6678 // range of addresses [from, end[.
6679 size_t count_card_of (uint8_t* from, uint8_t* end)
6681 return card_word (gcard_of (end - 1)) - card_word (gcard_of (from)) + 1;
6684 // Returns the number of bytes to allocate for a card table
6685 // that covers the range of addresses [from, end[.
6686 size_t size_card_of (uint8_t* from, uint8_t* end)
6688 return count_card_of (from, end) * sizeof(uint32_t);
6691 // We don't store seg_mapping_table in card_table_info because there's only always one view.
6692 class card_table_info
6696 uint8_t* lowest_address;
6697 uint8_t* highest_address;
6701 uint32_t* card_bundle_table;
6702 #endif //CARD_BUNDLE
6704 // mark_array is always at the end of the data structure because we
6705 // want to be able to make one commit call for everything before it.
6707 uint32_t* mark_array;
6711 uint32_t* next_card_table;
6714 //These are accessors on untranslated cardtable
6716 unsigned& card_table_refcount (uint32_t* c_table)
6718 return *(unsigned*)((char*)c_table - sizeof (card_table_info));
6722 uint8_t*& card_table_lowest_address (uint32_t* c_table)
6724 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->lowest_address;
6727 uint32_t* translate_card_table (uint32_t* ct)
6729 return (uint32_t*)((uint8_t*)ct - card_word (gcard_of (card_table_lowest_address (ct))) * sizeof(uint32_t));
6733 uint8_t*& card_table_highest_address (uint32_t* c_table)
6735 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->highest_address;
6739 short*& card_table_brick_table (uint32_t* c_table)
6741 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->brick_table;
6745 // Get the card bundle table for the specified card table.
6747 uint32_t*& card_table_card_bundle_table (uint32_t* c_table)
6749 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->card_bundle_table;
6751 #endif //CARD_BUNDLE
6754 /* Support for mark_array */
6757 uint32_t*& card_table_mark_array (uint32_t* c_table)
6759 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->mark_array;
6763 #define mark_bit_pitch ((size_t)16)
6765 #define mark_bit_pitch ((size_t)8)
6767 #define mark_word_width ((size_t)32)
6768 #define mark_word_size (mark_word_width * mark_bit_pitch)
6771 uint8_t* align_on_mark_bit (uint8_t* add)
6773 return (uint8_t*)((size_t)(add + (mark_bit_pitch - 1)) & ~(mark_bit_pitch - 1));
6777 uint8_t* align_lower_mark_bit (uint8_t* add)
6779 return (uint8_t*)((size_t)(add) & ~(mark_bit_pitch - 1));
6783 BOOL is_aligned_on_mark_word (uint8_t* add)
6785 return ((size_t)add == ((size_t)(add) & ~(mark_word_size - 1)));
6789 uint8_t* align_on_mark_word (uint8_t* add)
6791 return (uint8_t*)((size_t)(add + mark_word_size - 1) & ~(mark_word_size - 1));
6795 uint8_t* align_lower_mark_word (uint8_t* add)
6797 return (uint8_t*)((size_t)(add) & ~(mark_word_size - 1));
6801 size_t mark_bit_of (uint8_t* add)
6803 return ((size_t)add / mark_bit_pitch);
6807 unsigned int mark_bit_bit (size_t mark_bit)
6809 return (unsigned int)(mark_bit % mark_word_width);
6813 size_t mark_bit_word (size_t mark_bit)
6815 return (mark_bit / mark_word_width);
6819 size_t mark_word_of (uint8_t* add)
6821 return ((size_t)add) / mark_word_size;
6824 uint8_t* mark_word_address (size_t wd)
6826 return (uint8_t*)(wd*mark_word_size);
6829 uint8_t* mark_bit_address (size_t mark_bit)
6831 return (uint8_t*)(mark_bit*mark_bit_pitch);
6835 size_t mark_bit_bit_of (uint8_t* add)
6837 return (((size_t)add / mark_bit_pitch) % mark_word_width);
6841 unsigned int gc_heap::mark_array_marked(uint8_t* add)
6843 return mark_array [mark_word_of (add)] & (1 << mark_bit_bit_of (add));
6847 BOOL gc_heap::is_mark_bit_set (uint8_t* add)
6849 return (mark_array [mark_word_of (add)] & (1 << mark_bit_bit_of (add)));
6853 void gc_heap::mark_array_set_marked (uint8_t* add)
6855 size_t index = mark_word_of (add);
6856 uint32_t val = (1 << mark_bit_bit_of (add));
6857 #ifdef MULTIPLE_HEAPS
6858 Interlocked::Or (&(mark_array [index]), val);
6860 mark_array [index] |= val;
6865 void gc_heap::mark_array_clear_marked (uint8_t* add)
6867 mark_array [mark_word_of (add)] &= ~(1 << mark_bit_bit_of (add));
6870 size_t size_mark_array_of (uint8_t* from, uint8_t* end)
6872 assert (((size_t)from & ((mark_word_size)-1)) == 0);
6873 assert (((size_t)end & ((mark_word_size)-1)) == 0);
6874 return sizeof (uint32_t)*(((end - from) / mark_word_size));
6877 //In order to eliminate the lowest_address in the mark array
6878 //computations (mark_word_of, etc) mark_array is offset
6879 // according to the lowest_address.
6880 uint32_t* translate_mark_array (uint32_t* ma)
6882 return (uint32_t*)((uint8_t*)ma - size_mark_array_of (0, g_gc_lowest_address));
6885 // from and end must be page aligned addresses.
6886 void gc_heap::clear_mark_array (uint8_t* from, uint8_t* end, BOOL check_only/*=TRUE*/
6887 #ifdef FEATURE_BASICFREEZE
6888 , BOOL read_only/*=FALSE*/
6889 #endif // FEATURE_BASICFREEZE
6892 if(!gc_can_use_concurrent)
6895 #ifdef FEATURE_BASICFREEZE
6897 #endif // FEATURE_BASICFREEZE
6899 assert (from == align_on_mark_word (from));
6901 assert (end == align_on_mark_word (end));
6903 #ifdef BACKGROUND_GC
6904 uint8_t* current_lowest_address = background_saved_lowest_address;
6905 uint8_t* current_highest_address = background_saved_highest_address;
6907 uint8_t* current_lowest_address = lowest_address;
6908 uint8_t* current_highest_address = highest_address;
6909 #endif //BACKGROUND_GC
6911 //there is a possibility of the addresses to be
6912 //outside of the covered range because of a newly allocated
6913 //large object segment
6914 if ((end <= current_highest_address) && (from >= current_lowest_address))
6916 size_t beg_word = mark_word_of (align_on_mark_word (from));
6917 MAYBE_UNUSED_VAR(beg_word);
6918 //align end word to make sure to cover the address
6919 size_t end_word = mark_word_of (align_on_mark_word (end));
6920 MAYBE_UNUSED_VAR(end_word);
6921 dprintf (3, ("Calling clearing mark array [%Ix, %Ix[ for addresses [%Ix, %Ix[(%s)",
6922 (size_t)mark_word_address (beg_word),
6923 (size_t)mark_word_address (end_word),
6924 (size_t)from, (size_t)end,
6925 (check_only ? "check_only" : "clear")));
6929 while (op < mark_word_address (beg_word))
6931 mark_array_clear_marked (op);
6932 op += mark_bit_pitch;
6935 memset (&mark_array[beg_word], 0, (end_word - beg_word)*sizeof (uint32_t));
6940 //Beware, it is assumed that the mark array word straddling
6941 //start has been cleared before
6942 //verify that the array is empty.
6943 size_t markw = mark_word_of (align_on_mark_word (from));
6944 size_t markw_end = mark_word_of (align_on_mark_word (end));
6945 while (markw < markw_end)
6947 assert (!(mark_array [markw]));
6950 uint8_t* p = mark_word_address (markw_end);
6953 assert (!(mark_array_marked (p)));
6962 //These work on untranslated card tables
6964 uint32_t*& card_table_next (uint32_t* c_table)
6966 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->next_card_table;
6970 size_t& card_table_size (uint32_t* c_table)
6972 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->size;
6975 void own_card_table (uint32_t* c_table)
6977 card_table_refcount (c_table) += 1;
6980 void destroy_card_table (uint32_t* c_table);
6982 void delete_next_card_table (uint32_t* c_table)
6984 uint32_t* n_table = card_table_next (c_table);
6987 if (card_table_next (n_table))
6989 delete_next_card_table (n_table);
6991 if (card_table_refcount (n_table) == 0)
6993 destroy_card_table (n_table);
6994 card_table_next (c_table) = 0;
6999 void release_card_table (uint32_t* c_table)
7001 assert (card_table_refcount (c_table) >0);
7002 card_table_refcount (c_table) -= 1;
7003 if (card_table_refcount (c_table) == 0)
7005 delete_next_card_table (c_table);
7006 if (card_table_next (c_table) == 0)
7008 destroy_card_table (c_table);
7009 // sever the link from the parent
7010 if (&g_gc_card_table[card_word (gcard_of(g_gc_lowest_address))] == c_table)
7012 g_gc_card_table = 0;
7014 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7015 g_gc_card_bundle_table = 0;
7017 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7018 SoftwareWriteWatch::StaticClose();
7019 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7023 uint32_t* p_table = &g_gc_card_table[card_word (gcard_of(g_gc_lowest_address))];
7026 while (p_table && (card_table_next (p_table) != c_table))
7027 p_table = card_table_next (p_table);
7028 card_table_next (p_table) = 0;
7035 void destroy_card_table (uint32_t* c_table)
7037 // delete (uint32_t*)&card_table_refcount(c_table);
7039 GCToOSInterface::VirtualRelease (&card_table_refcount(c_table), card_table_size(c_table));
7040 dprintf (2, ("Table Virtual Free : %Ix", (size_t)&card_table_refcount(c_table)));
7043 uint32_t* gc_heap::make_card_table (uint8_t* start, uint8_t* end)
7045 assert (g_gc_lowest_address == start);
7046 assert (g_gc_highest_address == end);
7048 uint32_t virtual_reserve_flags = VirtualReserveFlags::None;
7050 size_t bs = size_brick_of (start, end);
7051 size_t cs = size_card_of (start, end);
7053 size_t ms = (gc_can_use_concurrent ?
7054 size_mark_array_of (start, end) :
7063 if (can_use_write_watch_for_card_table())
7065 cb = size_card_bundle_of (g_gc_lowest_address, g_gc_highest_address);
7066 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7067 // If we're not manually managing the card bundles, we will need to use OS write
7068 // watch APIs over this region to track changes.
7069 virtual_reserve_flags |= VirtualReserveFlags::WriteWatch;
7072 #endif //CARD_BUNDLE
7075 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7076 size_t sw_ww_table_offset = 0;
7077 if (gc_can_use_concurrent)
7079 size_t sw_ww_size_before_table = sizeof(card_table_info) + cs + bs + cb;
7080 sw_ww_table_offset = SoftwareWriteWatch::GetTableStartByteOffset(sw_ww_size_before_table);
7081 wws = sw_ww_table_offset - sw_ww_size_before_table + SoftwareWriteWatch::GetTableByteSize(start, end);
7083 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7085 #ifdef GROWABLE_SEG_MAPPING_TABLE
7086 size_t st = size_seg_mapping_table_of (g_gc_lowest_address, g_gc_highest_address);
7087 size_t st_table_offset = sizeof(card_table_info) + cs + bs + cb + wws;
7088 size_t st_table_offset_aligned = align_for_seg_mapping_table (st_table_offset);
7090 st += (st_table_offset_aligned - st_table_offset);
7091 #else //GROWABLE_SEG_MAPPING_TABLE
7093 #endif //GROWABLE_SEG_MAPPING_TABLE
7095 // it is impossible for alloc_size to overflow due bounds on each of
7097 size_t alloc_size = sizeof (uint8_t)*(sizeof(card_table_info) + cs + bs + cb + wws + st + ms);
7098 uint8_t* mem = (uint8_t*)GCToOSInterface::VirtualReserve (alloc_size, 0, virtual_reserve_flags);
7103 dprintf (2, ("Init - Card table alloc for %Id bytes: [%Ix, %Ix[",
7104 alloc_size, (size_t)mem, (size_t)(mem+alloc_size)));
7106 // mark array will be committed separately (per segment).
7107 size_t commit_size = alloc_size - ms;
7109 if (!GCToOSInterface::VirtualCommit (mem, commit_size))
7111 dprintf (2, ("Card table commit failed"));
7112 GCToOSInterface::VirtualRelease (mem, alloc_size);
7116 // initialize the ref count
7117 uint32_t* ct = (uint32_t*)(mem+sizeof (card_table_info));
7118 card_table_refcount (ct) = 0;
7119 card_table_lowest_address (ct) = start;
7120 card_table_highest_address (ct) = end;
7121 card_table_brick_table (ct) = (short*)((uint8_t*)ct + cs);
7122 card_table_size (ct) = alloc_size;
7123 card_table_next (ct) = 0;
7126 card_table_card_bundle_table (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs);
7128 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7129 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), g_gc_lowest_address);
7132 #endif //CARD_BUNDLE
7134 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7135 if (gc_can_use_concurrent)
7137 SoftwareWriteWatch::InitializeUntranslatedTable(mem + sw_ww_table_offset, start);
7139 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7141 #ifdef GROWABLE_SEG_MAPPING_TABLE
7142 seg_mapping_table = (seg_mapping*)(mem + st_table_offset_aligned);
7143 seg_mapping_table = (seg_mapping*)((uint8_t*)seg_mapping_table -
7144 size_seg_mapping_table_of (0, (align_lower_segment (g_gc_lowest_address))));
7145 #endif //GROWABLE_SEG_MAPPING_TABLE
7148 if (gc_can_use_concurrent)
7149 card_table_mark_array (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs + cb + wws + st);
7151 card_table_mark_array (ct) = NULL;
7154 return translate_card_table(ct);
7157 void gc_heap::set_fgm_result (failure_get_memory f, size_t s, BOOL loh_p)
7159 #ifdef MULTIPLE_HEAPS
7160 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
7162 gc_heap* hp = gc_heap::g_heaps [hn];
7163 hp->fgm_result.set_fgm (f, s, loh_p);
7165 #else //MULTIPLE_HEAPS
7166 fgm_result.set_fgm (f, s, loh_p);
7167 #endif //MULTIPLE_HEAPS
7170 //returns 0 for success, -1 otherwise
7171 // We are doing all the decommitting here because we want to make sure we have
7172 // enough memory to do so - if we do this during copy_brick_card_table and
7173 // and fail to decommit it would make the failure case very complicated to
7174 // handle. This way we can waste some decommit if we call this multiple
7175 // times before the next FGC but it's easier to handle the failure case.
7176 int gc_heap::grow_brick_card_tables (uint8_t* start,
7179 heap_segment* new_seg,
7183 uint8_t* la = g_gc_lowest_address;
7184 uint8_t* ha = g_gc_highest_address;
7185 uint8_t* saved_g_lowest_address = min (start, g_gc_lowest_address);
7186 uint8_t* saved_g_highest_address = max (end, g_gc_highest_address);
7187 seg_mapping* new_seg_mapping_table = nullptr;
7188 #ifdef BACKGROUND_GC
7189 // This value is only for logging purpose - it's not necessarily exactly what we
7190 // would commit for mark array but close enough for diagnostics purpose.
7191 size_t logging_ma_commit_size = size_mark_array_of (0, (uint8_t*)size);
7192 #endif //BACKGROUND_GC
7194 // See if the address is already covered
7195 if ((la != saved_g_lowest_address ) || (ha != saved_g_highest_address))
7198 //modify the higest address so the span covered
7199 //is twice the previous one.
7200 uint8_t* top = (uint8_t*)0 + Align (GCToOSInterface::GetVirtualMemoryLimit());
7201 // On non-Windows systems, we get only an approximate value that can possibly be
7202 // slightly lower than the saved_g_highest_address.
7203 // In such case, we set the top to the saved_g_highest_address so that the
7204 // card and brick tables always cover the whole new range.
7205 if (top < saved_g_highest_address)
7207 top = saved_g_highest_address;
7211 if (ps > (uint64_t)200*1024*1024*1024)
7212 ps += (uint64_t)100*1024*1024*1024;
7217 if (saved_g_lowest_address < g_gc_lowest_address)
7219 if (ps > (size_t)g_gc_lowest_address)
7220 saved_g_lowest_address = (uint8_t*)(size_t)OS_PAGE_SIZE;
7223 assert (((size_t)g_gc_lowest_address - ps) >= OS_PAGE_SIZE);
7224 saved_g_lowest_address = min (saved_g_lowest_address, (g_gc_lowest_address - ps));
7228 if (saved_g_highest_address > g_gc_highest_address)
7230 saved_g_highest_address = max ((saved_g_lowest_address + ps), saved_g_highest_address);
7231 if (saved_g_highest_address > top)
7232 saved_g_highest_address = top;
7235 dprintf (GC_TABLE_LOG, ("Growing card table [%Ix, %Ix[",
7236 (size_t)saved_g_lowest_address,
7237 (size_t)saved_g_highest_address));
7239 bool write_barrier_updated = false;
7240 uint32_t virtual_reserve_flags = VirtualReserveFlags::None;
7241 uint32_t* saved_g_card_table = g_gc_card_table;
7243 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7244 uint32_t* saved_g_card_bundle_table = g_gc_card_bundle_table;
7248 uint32_t* translated_ct = 0;
7251 size_t cs = size_card_of (saved_g_lowest_address, saved_g_highest_address);
7252 size_t bs = size_brick_of (saved_g_lowest_address, saved_g_highest_address);
7255 size_t ms = (gc_heap::gc_can_use_concurrent ?
7256 size_mark_array_of (saved_g_lowest_address, saved_g_highest_address) :
7265 if (can_use_write_watch_for_card_table())
7267 cb = size_card_bundle_of (saved_g_lowest_address, saved_g_highest_address);
7269 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7270 // If we're not manually managing the card bundles, we will need to use OS write
7271 // watch APIs over this region to track changes.
7272 virtual_reserve_flags |= VirtualReserveFlags::WriteWatch;
7275 #endif //CARD_BUNDLE
7278 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7279 size_t sw_ww_table_offset = 0;
7280 if (gc_can_use_concurrent)
7282 size_t sw_ww_size_before_table = sizeof(card_table_info) + cs + bs + cb;
7283 sw_ww_table_offset = SoftwareWriteWatch::GetTableStartByteOffset(sw_ww_size_before_table);
7285 sw_ww_table_offset -
7286 sw_ww_size_before_table +
7287 SoftwareWriteWatch::GetTableByteSize(saved_g_lowest_address, saved_g_highest_address);
7289 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7291 #ifdef GROWABLE_SEG_MAPPING_TABLE
7292 size_t st = size_seg_mapping_table_of (saved_g_lowest_address, saved_g_highest_address);
7293 size_t st_table_offset = sizeof(card_table_info) + cs + bs + cb + wws;
7294 size_t st_table_offset_aligned = align_for_seg_mapping_table (st_table_offset);
7295 st += (st_table_offset_aligned - st_table_offset);
7296 #else //GROWABLE_SEG_MAPPING_TABLE
7298 #endif //GROWABLE_SEG_MAPPING_TABLE
7300 // it is impossible for alloc_size to overflow due bounds on each of
7302 size_t alloc_size = sizeof (uint8_t)*(sizeof(card_table_info) + cs + bs + cb + wws + st + ms);
7303 dprintf (GC_TABLE_LOG, ("card table: %Id; brick table: %Id; card bundle: %Id; sw ww table: %Id; seg table: %Id; mark array: %Id",
7304 cs, bs, cb, wws, st, ms));
7306 uint8_t* mem = (uint8_t*)GCToOSInterface::VirtualReserve (alloc_size, 0, virtual_reserve_flags);
7310 set_fgm_result (fgm_grow_table, alloc_size, loh_p);
7314 dprintf (GC_TABLE_LOG, ("Table alloc for %Id bytes: [%Ix, %Ix[",
7315 alloc_size, (size_t)mem, (size_t)((uint8_t*)mem+alloc_size)));
7318 // mark array will be committed separately (per segment).
7319 size_t commit_size = alloc_size - ms;
7321 if (!GCToOSInterface::VirtualCommit (mem, commit_size))
7323 dprintf (GC_TABLE_LOG, ("Table commit failed"));
7324 set_fgm_result (fgm_commit_table, commit_size, loh_p);
7329 ct = (uint32_t*)(mem + sizeof (card_table_info));
7330 card_table_refcount (ct) = 0;
7331 card_table_lowest_address (ct) = saved_g_lowest_address;
7332 card_table_highest_address (ct) = saved_g_highest_address;
7333 card_table_next (ct) = &g_gc_card_table[card_word (gcard_of (la))];
7335 //clear the card table
7337 memclr ((uint8_t*)ct,
7338 (((saved_g_highest_address - saved_g_lowest_address)*sizeof (uint32_t) /
7339 (card_size * card_word_width))
7340 + sizeof (uint32_t)));
7343 bt = (short*)((uint8_t*)ct + cs);
7345 // No initialization needed, will be done in copy_brick_card
7347 card_table_brick_table (ct) = bt;
7350 card_table_card_bundle_table (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs);
7351 //set all bundle to look at all of the cards
7352 memset(card_table_card_bundle_table (ct), 0xFF, cb);
7353 #endif //CARD_BUNDLE
7355 #ifdef GROWABLE_SEG_MAPPING_TABLE
7357 new_seg_mapping_table = (seg_mapping*)(mem + st_table_offset_aligned);
7358 new_seg_mapping_table = (seg_mapping*)((uint8_t*)new_seg_mapping_table -
7359 size_seg_mapping_table_of (0, (align_lower_segment (saved_g_lowest_address))));
7360 memcpy(&new_seg_mapping_table[seg_mapping_word_of(g_gc_lowest_address)],
7361 &seg_mapping_table[seg_mapping_word_of(g_gc_lowest_address)],
7362 size_seg_mapping_table_of(g_gc_lowest_address, g_gc_highest_address));
7364 // new_seg_mapping_table gets assigned to seg_mapping_table at the bottom of this function,
7365 // not here. The reason for this is that, if we fail at mark array committing (OOM) and we've
7366 // already switched seg_mapping_table to point to the new mapping table, we'll decommit it and
7367 // run into trouble. By not assigning here, we're making sure that we will not change seg_mapping_table
7368 // if an OOM occurs.
7370 #endif //GROWABLE_SEG_MAPPING_TABLE
7373 if(gc_can_use_concurrent)
7374 card_table_mark_array (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs + cb + wws + st);
7376 card_table_mark_array (ct) = NULL;
7379 translated_ct = translate_card_table (ct);
7381 dprintf (GC_TABLE_LOG, ("card table: %Ix(translated: %Ix), seg map: %Ix, mark array: %Ix",
7382 (size_t)ct, (size_t)translated_ct, (size_t)new_seg_mapping_table, (size_t)card_table_mark_array (ct)));
7384 #ifdef BACKGROUND_GC
7385 if (hp->should_commit_mark_array())
7387 dprintf (GC_TABLE_LOG, ("new low: %Ix, new high: %Ix, latest mark array is %Ix(translate: %Ix)",
7388 saved_g_lowest_address, saved_g_highest_address,
7389 card_table_mark_array (ct),
7390 translate_mark_array (card_table_mark_array (ct))));
7391 uint32_t* new_mark_array = (uint32_t*)((uint8_t*)card_table_mark_array (ct) - size_mark_array_of (0, saved_g_lowest_address));
7392 if (!commit_new_mark_array_global (new_mark_array))
7394 dprintf (GC_TABLE_LOG, ("failed to commit portions in the mark array for existing segments"));
7395 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7399 if (!commit_mark_array_new_seg (hp, new_seg, translated_ct, saved_g_lowest_address))
7401 dprintf (GC_TABLE_LOG, ("failed to commit mark array for the new seg"));
7402 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7408 clear_commit_flag_global();
7410 #endif //BACKGROUND_GC
7412 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7413 if (gc_can_use_concurrent)
7415 // The current design of software write watch requires that the runtime is suspended during resize. Suspending
7416 // on resize is preferred because it is a far less frequent operation than GetWriteWatch() / ResetWriteWatch().
7417 // Suspending here allows copying dirty state from the old table into the new table, and not have to merge old
7418 // table info lazily as done for card tables.
7420 // Either this thread was the thread that did the suspension which means we are suspended; or this is called
7421 // from a GC thread which means we are in a blocking GC and also suspended.
7422 bool is_runtime_suspended = GCToEEInterface::IsGCThread();
7423 if (!is_runtime_suspended)
7425 // Note on points where the runtime is suspended anywhere in this function. Upon an attempt to suspend the
7426 // runtime, a different thread may suspend first, causing this thread to block at the point of the suspend call.
7427 // So, at any suspend point, externally visible state needs to be consistent, as code that depends on that state
7428 // may run while this thread is blocked. This includes updates to g_gc_card_table, g_gc_lowest_address, and
7429 // g_gc_highest_address.
7433 g_gc_card_table = translated_ct;
7435 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7436 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), saved_g_lowest_address);
7439 SoftwareWriteWatch::SetResizedUntranslatedTable(
7440 mem + sw_ww_table_offset,
7441 saved_g_lowest_address,
7442 saved_g_highest_address);
7444 // Since the runtime is already suspended, update the write barrier here as well.
7445 // This passes a bool telling whether we need to switch to the post
7446 // grow version of the write barrier. This test tells us if the new
7447 // segment was allocated at a lower address than the old, requiring
7448 // that we start doing an upper bounds check in the write barrier.
7449 g_gc_lowest_address = saved_g_lowest_address;
7450 g_gc_highest_address = saved_g_highest_address;
7451 stomp_write_barrier_resize(true, la != saved_g_lowest_address);
7452 write_barrier_updated = true;
7454 if (!is_runtime_suspended)
7460 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7462 g_gc_card_table = translated_ct;
7464 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7465 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), saved_g_lowest_address);
7469 seg_mapping_table = new_seg_mapping_table;
7471 GCToOSInterface::FlushProcessWriteBuffers();
7472 g_gc_lowest_address = saved_g_lowest_address;
7473 g_gc_highest_address = saved_g_highest_address;
7475 if (!write_barrier_updated)
7477 // This passes a bool telling whether we need to switch to the post
7478 // grow version of the write barrier. This test tells us if the new
7479 // segment was allocated at a lower address than the old, requiring
7480 // that we start doing an upper bounds check in the write barrier.
7481 // This will also suspend the runtime if the write barrier type needs
7482 // to be changed, so we are doing this after all global state has
7483 // been updated. See the comment above suspend_EE() above for more
7485 stomp_write_barrier_resize(GCToEEInterface::IsGCThread(), la != saved_g_lowest_address);
7492 //cleanup mess and return -1;
7496 assert(g_gc_card_table == saved_g_card_table);
7498 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7499 assert(g_gc_card_bundle_table == saved_g_card_bundle_table);
7502 //delete (uint32_t*)((uint8_t*)ct - sizeof(card_table_info));
7503 if (!GCToOSInterface::VirtualRelease (mem, alloc_size))
7505 dprintf (GC_TABLE_LOG, ("GCToOSInterface::VirtualRelease failed"));
7506 assert (!"release failed");
7514 #ifdef BACKGROUND_GC
7515 if (hp->should_commit_mark_array())
7517 dprintf (GC_TABLE_LOG, ("in range new seg %Ix, mark_array is %Ix", new_seg, hp->mark_array));
7518 if (!commit_mark_array_new_seg (hp, new_seg))
7520 dprintf (GC_TABLE_LOG, ("failed to commit mark array for the new seg in range"));
7521 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7525 #endif //BACKGROUND_GC
7531 //copy all of the arrays managed by the card table for a page aligned range
7532 void gc_heap::copy_brick_card_range (uint8_t* la, uint32_t* old_card_table,
7533 short* old_brick_table,
7535 uint8_t* start, uint8_t* end)
7537 ptrdiff_t brick_offset = brick_of (start) - brick_of (la);
7540 dprintf (2, ("copying tables for range [%Ix %Ix[", (size_t)start, (size_t)end));
7543 short* brick_start = &brick_table [brick_of (start)];
7544 if (old_brick_table)
7546 // segments are always on page boundaries
7547 memcpy (brick_start, &old_brick_table[brick_offset],
7548 size_brick_of (start, end));
7553 // This is a large heap, just clear the brick table
7556 uint32_t* old_ct = &old_card_table[card_word (card_of (la))];
7558 #ifdef BACKGROUND_GC
7559 UNREFERENCED_PARAMETER(seg);
7560 if (recursive_gc_sync::background_running_p())
7562 uint32_t* old_mark_array = card_table_mark_array (old_ct);
7564 // We don't need to go through all the card tables here because
7565 // we only need to copy from the GC version of the mark array - when we
7566 // mark (even in allocate_large_object) we always use that mark array.
7567 if ((card_table_highest_address (old_ct) >= start) &&
7568 (card_table_lowest_address (old_ct) <= end))
7570 if ((background_saved_highest_address >= start) &&
7571 (background_saved_lowest_address <= end))
7573 //copy the mark bits
7574 // segments are always on page boundaries
7575 uint8_t* m_start = max (background_saved_lowest_address, start);
7576 uint8_t* m_end = min (background_saved_highest_address, end);
7577 memcpy (&mark_array[mark_word_of (m_start)],
7578 &old_mark_array[mark_word_of (m_start) - mark_word_of (la)],
7579 size_mark_array_of (m_start, m_end));
7584 //only large segments can be out of range
7585 assert (old_brick_table == 0);
7588 #else //BACKGROUND_GC
7590 clear_mark_array (start, heap_segment_committed(seg));
7591 #endif //BACKGROUND_GC
7594 // n way merge with all of the card table ever used in between
7595 uint32_t* ct = card_table_next (&card_table[card_word (card_of(lowest_address))]);
7598 while (card_table_next (old_ct) != ct)
7600 //copy if old card table contained [start, end[
7601 if ((card_table_highest_address (ct) >= end) &&
7602 (card_table_lowest_address (ct) <= start))
7604 // or the card_tables
7606 size_t start_word = card_word (card_of (start));
7608 uint32_t* dest = &card_table[start_word];
7609 uint32_t* src = &((translate_card_table (ct))[start_word]);
7610 ptrdiff_t count = count_card_of (start, end);
7611 for (int x = 0; x < count; x++)
7615 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7618 card_bundle_set(cardw_card_bundle(start_word+x));
7626 ct = card_table_next (ct);
7630 //initialize all of the arrays managed by the card table for a page aligned range when an existing ro segment becomes in range
7631 void gc_heap::init_brick_card_range (heap_segment* seg)
7633 dprintf (2, ("initialising tables for range [%Ix %Ix[",
7634 (size_t)heap_segment_mem (seg),
7635 (size_t)heap_segment_allocated (seg)));
7637 // initialize the brick table
7638 for (size_t b = brick_of (heap_segment_mem (seg));
7639 b < brick_of (align_on_brick (heap_segment_allocated (seg)));
7646 if (recursive_gc_sync::background_running_p() && (seg->flags & heap_segment_flags_ma_committed))
7649 clear_mark_array (heap_segment_mem (seg), heap_segment_committed(seg));
7653 clear_card_for_addresses (heap_segment_mem (seg),
7654 heap_segment_allocated (seg));
7657 void gc_heap::copy_brick_card_table()
7659 uint8_t* la = lowest_address;
7660 uint8_t* ha = highest_address;
7661 MAYBE_UNUSED_VAR(ha);
7662 uint32_t* old_card_table = card_table;
7663 short* old_brick_table = brick_table;
7665 assert (la == card_table_lowest_address (&old_card_table[card_word (card_of (la))]));
7666 assert (ha == card_table_highest_address (&old_card_table[card_word (card_of (la))]));
7668 /* todo: Need a global lock for this */
7669 uint32_t* ct = &g_gc_card_table[card_word (gcard_of (g_gc_lowest_address))];
7670 own_card_table (ct);
7671 card_table = translate_card_table (ct);
7672 /* End of global lock */
7673 highest_address = card_table_highest_address (ct);
7674 lowest_address = card_table_lowest_address (ct);
7676 brick_table = card_table_brick_table (ct);
7679 if (gc_can_use_concurrent)
7681 mark_array = translate_mark_array (card_table_mark_array (ct));
7682 assert (mark_word_of (g_gc_highest_address) ==
7683 mark_word_of (align_on_mark_word (g_gc_highest_address)));
7690 #if defined(MARK_ARRAY) && defined(_DEBUG)
7691 #ifdef GROWABLE_SEG_MAPPING_TABLE
7692 size_t st = size_seg_mapping_table_of (g_gc_lowest_address, g_gc_highest_address);
7693 #else //GROWABLE_SEG_MAPPING_TABLE
7695 #endif //GROWABLE_SEG_MAPPING_TABLE
7696 #endif //MARK_ARRAY && _DEBUG
7697 card_bundle_table = translate_card_bundle_table (card_table_card_bundle_table (ct), g_gc_lowest_address);
7699 // Ensure that the word that represents g_gc_lowest_address in the translated table is located at the
7700 // start of the untranslated table.
7701 assert (&card_bundle_table [card_bundle_word (cardw_card_bundle (card_word (card_of (g_gc_lowest_address))))] ==
7702 card_table_card_bundle_table (ct));
7704 //set the card table if we are in a heap growth scenario
7705 if (card_bundles_enabled())
7707 card_bundles_set (cardw_card_bundle (card_word (card_of (lowest_address))),
7708 cardw_card_bundle (align_cardw_on_bundle (card_word (card_of (highest_address)))));
7710 //check if we need to turn on card_bundles.
7711 #ifdef MULTIPLE_HEAPS
7712 // use INT64 arithmetic here because of possible overflow on 32p
7713 uint64_t th = (uint64_t)MH_TH_CARD_BUNDLE*gc_heap::n_heaps;
7715 // use INT64 arithmetic here because of possible overflow on 32p
7716 uint64_t th = (uint64_t)SH_TH_CARD_BUNDLE;
7717 #endif //MULTIPLE_HEAPS
7718 if (reserved_memory >= th)
7720 enable_card_bundles();
7723 #endif //CARD_BUNDLE
7725 // for each of the segments and heaps, copy the brick table and
7726 // or the card table
7727 heap_segment* seg = generation_start_segment (generation_of (max_generation));
7730 if (heap_segment_read_only_p (seg) && !heap_segment_in_range_p (seg))
7732 //check if it became in range
7733 if ((heap_segment_reserved (seg) > lowest_address) &&
7734 (heap_segment_mem (seg) < highest_address))
7736 set_ro_segment_in_range (seg);
7742 uint8_t* end = align_on_page (heap_segment_allocated (seg));
7743 copy_brick_card_range (la, old_card_table,
7746 align_lower_page (heap_segment_mem (seg)),
7749 seg = heap_segment_next (seg);
7752 seg = generation_start_segment (large_object_generation);
7755 if (heap_segment_read_only_p (seg) && !heap_segment_in_range_p (seg))
7757 //check if it became in range
7758 if ((heap_segment_reserved (seg) > lowest_address) &&
7759 (heap_segment_mem (seg) < highest_address))
7761 set_ro_segment_in_range (seg);
7766 uint8_t* end = align_on_page (heap_segment_allocated (seg));
7767 copy_brick_card_range (la, old_card_table,
7770 align_lower_page (heap_segment_mem (seg)),
7773 seg = heap_segment_next (seg);
7776 release_card_table (&old_card_table[card_word (card_of(la))]);
7779 #ifdef FEATURE_BASICFREEZE
7780 BOOL gc_heap::insert_ro_segment (heap_segment* seg)
7782 enter_spin_lock (&gc_heap::gc_lock);
7784 if (!gc_heap::seg_table->ensure_space_for_insert ()
7785 || (should_commit_mark_array() && !commit_mark_array_new_seg(__this, seg)))
7787 leave_spin_lock(&gc_heap::gc_lock);
7791 //insert at the head of the segment list
7792 generation* gen2 = generation_of (max_generation);
7793 heap_segment* oldhead = generation_start_segment (gen2);
7794 heap_segment_next (seg) = oldhead;
7795 generation_start_segment (gen2) = seg;
7797 seg_table->insert (heap_segment_mem(seg), (size_t)seg);
7799 #ifdef SEG_MAPPING_TABLE
7800 seg_mapping_table_add_ro_segment (seg);
7801 #endif //SEG_MAPPING_TABLE
7804 if ((heap_segment_reserved (seg) > lowest_address) &&
7805 (heap_segment_mem (seg) < highest_address))
7807 set_ro_segment_in_range (seg);
7810 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(seg), (size_t)(heap_segment_reserved (seg) - heap_segment_mem(seg)), gc_etw_segment_read_only_heap);
7812 leave_spin_lock (&gc_heap::gc_lock);
7816 // No one is calling this function right now. If this is getting called we need
7817 // to take care of decommitting the mark array for it - we will need to remember
7818 // which portion of the mark array was committed and only decommit that.
7819 void gc_heap::remove_ro_segment (heap_segment* seg)
7821 //clear the mark bits so a new segment allocated in its place will have a clear mark bits
7823 if (gc_can_use_concurrent)
7825 clear_mark_array (align_lower_mark_word (max (heap_segment_mem (seg), lowest_address)),
7826 align_on_card_word (min (heap_segment_allocated (seg), highest_address)),
7827 false); // read_only segments need the mark clear
7831 enter_spin_lock (&gc_heap::gc_lock);
7833 seg_table->remove ((uint8_t*)seg);
7835 #ifdef SEG_MAPPING_TABLE
7836 seg_mapping_table_remove_ro_segment (seg);
7837 #endif //SEG_MAPPING_TABLE
7839 // Locate segment (and previous segment) in the list.
7840 generation* gen2 = generation_of (max_generation);
7841 heap_segment* curr_seg = generation_start_segment (gen2);
7842 heap_segment* prev_seg = NULL;
7844 while (curr_seg && curr_seg != seg)
7846 prev_seg = curr_seg;
7847 curr_seg = heap_segment_next (curr_seg);
7849 assert (curr_seg == seg);
7851 // Patch previous segment (or list head if there is none) to skip the removed segment.
7853 heap_segment_next (prev_seg) = heap_segment_next (curr_seg);
7855 generation_start_segment (gen2) = heap_segment_next (curr_seg);
7857 leave_spin_lock (&gc_heap::gc_lock);
7859 #endif //FEATURE_BASICFREEZE
7861 BOOL gc_heap::set_ro_segment_in_range (heap_segment* seg)
7864 seg->flags |= heap_segment_flags_inrange;
7865 // init_brick_card_range (seg);
7866 ro_segments_in_range = TRUE;
7867 //right now, segments aren't protected
7868 //unprotect_segment (seg);
7874 uint8_t** make_mark_list (size_t size)
7876 uint8_t** mark_list = new (nothrow) uint8_t* [size];
7880 #define swap(a,b){uint8_t* t; t = a; a = b; b = t;}
7882 void verify_qsort_array (uint8_t* *low, uint8_t* *high)
7886 for (i = low+1; i <= high; i++)
7895 #ifndef USE_INTROSORT
7896 void qsort1( uint8_t* *low, uint8_t* *high, unsigned int depth)
7898 if (((low + 16) >= high) || (depth > 100))
7902 for (i = low+1; i <= high; i++)
7905 for (j=i;j >low && val<*(j-1);j--)
7914 uint8_t *pivot, **left, **right;
7916 //sort low middle and high
7917 if (*(low+((high-low)/2)) < *low)
7918 swap (*(low+((high-low)/2)), *low);
7921 if (*high < *(low+((high-low)/2)))
7922 swap (*(low+((high-low)/2)), *high);
7924 swap (*(low+((high-low)/2)), *(high-1));
7926 left = low; right = high-1;
7928 while (*(--right) > pivot);
7929 while (*(++left) < pivot);
7932 swap(*left, *right);
7937 swap (*left, *(high-1));
7938 qsort1(low, left-1, depth+1);
7939 qsort1(left+1, high, depth+1);
7942 #endif //USE_INTROSORT
7943 void rqsort1( uint8_t* *low, uint8_t* *high)
7945 if ((low + 16) >= high)
7949 for (i = low+1; i <= high; i++)
7952 for (j=i;j >low && val>*(j-1);j--)
7961 uint8_t *pivot, **left, **right;
7963 //sort low middle and high
7964 if (*(low+((high-low)/2)) > *low)
7965 swap (*(low+((high-low)/2)), *low);
7968 if (*high > *(low+((high-low)/2)))
7969 swap (*(low+((high-low)/2)), *high);
7971 swap (*(low+((high-low)/2)), *(high-1));
7973 left = low; right = high-1;
7975 while (*(--right) < pivot);
7976 while (*(++left) > pivot);
7979 swap(*left, *right);
7984 swap (*left, *(high-1));
7985 rqsort1(low, left-1);
7986 rqsort1(left+1, high);
7990 #ifdef USE_INTROSORT
7995 static const int size_threshold = 64;
7996 static const int max_depth = 100;
7999 inline static void swap_elements(uint8_t** i,uint8_t** j)
8007 static void sort (uint8_t** begin, uint8_t** end, int ignored)
8010 introsort_loop (begin, end, max_depth);
8011 insertionsort (begin, end);
8016 static void introsort_loop (uint8_t** lo, uint8_t** hi, int depth_limit)
8018 while (hi-lo >= size_threshold)
8020 if (depth_limit == 0)
8025 uint8_t** p=median_partition (lo, hi);
8026 depth_limit=depth_limit-1;
8027 introsort_loop (p, hi, depth_limit);
8032 static uint8_t** median_partition (uint8_t** low, uint8_t** high)
8034 uint8_t *pivot, **left, **right;
8036 //sort low middle and high
8037 if (*(low+((high-low)/2)) < *low)
8038 swap_elements ((low+((high-low)/2)), low);
8040 swap_elements (low, high);
8041 if (*high < *(low+((high-low)/2)))
8042 swap_elements ((low+((high-low)/2)), high);
8044 swap_elements ((low+((high-low)/2)), (high-1));
8046 left = low; right = high-1;
8048 while (*(--right) > pivot);
8049 while (*(++left) < pivot);
8052 swap_elements(left, right);
8057 swap_elements (left, (high-1));
8062 static void insertionsort (uint8_t** lo, uint8_t** hi)
8064 for (uint8_t** i=lo+1; i <= hi; i++)
8068 while((j > lo) && (t <*(j-1)))
8077 static void heapsort (uint8_t** lo, uint8_t** hi)
8079 size_t n = hi - lo + 1;
8080 for (size_t i=n / 2; i >= 1; i--)
8084 for (size_t i = n; i > 1; i--)
8086 swap_elements (lo, lo + i - 1);
8087 downheap(1, i - 1, lo);
8091 static void downheap (size_t i, size_t n, uint8_t** lo)
8093 uint8_t* d = *(lo + i - 1);
8098 if (child < n && *(lo + child - 1)<(*(lo + child)))
8102 if (!(d<*(lo + child - 1)))
8106 *(lo + i - 1) = *(lo + child - 1);
8114 #endif //USE_INTROSORT
8116 #ifdef MULTIPLE_HEAPS
8117 #ifdef PARALLEL_MARK_LIST_SORT
8118 void gc_heap::sort_mark_list()
8120 // if this heap had a mark list overflow, we don't do anything
8121 if (mark_list_index > mark_list_end)
8123 // printf("sort_mark_list: overflow on heap %d\n", heap_number);
8127 // if any other heap had a mark list overflow, we fake one too,
8128 // so we don't use an incomplete mark list by mistake
8129 for (int i = 0; i < n_heaps; i++)
8131 if (g_heaps[i]->mark_list_index > g_heaps[i]->mark_list_end)
8133 mark_list_index = mark_list_end + 1;
8134 // printf("sort_mark_list: overflow on heap %d\n", i);
8139 // unsigned long start = GetCycleCount32();
8141 dprintf (3, ("Sorting mark lists"));
8142 if (mark_list_index > mark_list)
8143 _sort (mark_list, mark_list_index - 1, 0);
8145 // printf("first phase of sort_mark_list for heap %d took %u cycles to sort %u entries\n", this->heap_number, GetCycleCount32() - start, mark_list_index - mark_list);
8146 // start = GetCycleCount32();
8148 // first set the pieces for all heaps to empty
8150 for (heap_num = 0; heap_num < n_heaps; heap_num++)
8152 mark_list_piece_start[heap_num] = NULL;
8153 mark_list_piece_end[heap_num] = NULL;
8156 uint8_t** x = mark_list;
8158 // predicate means: x is still within the mark list, and within the bounds of this heap
8159 #define predicate(x) (((x) < mark_list_index) && (*(x) < heap->ephemeral_high))
8162 while (x < mark_list_index)
8165 // find the heap x points into - searching cyclically from the last heap,
8166 // because in many cases the right heap is the next one or comes soon after
8167 int last_heap_num = heap_num;
8168 MAYBE_UNUSED_VAR(last_heap_num);
8172 if (heap_num >= n_heaps)
8174 assert(heap_num != last_heap_num); // we should always find the heap - infinite loop if not!
8175 heap = g_heaps[heap_num];
8177 while (!(*x >= heap->ephemeral_low && *x < heap->ephemeral_high));
8179 // x is the start of the mark list piece for this heap
8180 mark_list_piece_start[heap_num] = x;
8182 // to find the end of the mark list piece for this heap, find the first x
8183 // that has !predicate(x), i.e. that is either not in this heap, or beyond the end of the list
8186 // let's see if we get lucky and the whole rest belongs to this piece
8187 if (predicate(mark_list_index-1))
8189 x = mark_list_index;
8190 mark_list_piece_end[heap_num] = x;
8194 // we play a variant of binary search to find the point sooner.
8195 // the first loop advances by increasing steps until the predicate turns false.
8196 // then we retreat the last step, and the second loop advances by decreasing steps, keeping the predicate true.
8201 uint8_t** temp_x = x;
8208 while (predicate(x));
8209 // we know that only the last step was wrong, so we undo it
8213 // loop invariant - predicate holds at x, but not x + inc
8214 assert (predicate(x) && !(((x + inc) > x) && predicate(x + inc)));
8216 if (((x + inc) > x) && predicate(x + inc))
8222 // the termination condition and the loop invariant together imply this:
8223 assert(predicate(x) && !predicate(x + inc) && (inc == 1));
8224 // so the spot we're looking for is one further
8227 mark_list_piece_end[heap_num] = x;
8232 // printf("second phase of sort_mark_list for heap %d took %u cycles\n", this->heap_number, GetCycleCount32() - start);
8235 void gc_heap::append_to_mark_list(uint8_t **start, uint8_t **end)
8237 size_t slots_needed = end - start;
8238 size_t slots_available = mark_list_end + 1 - mark_list_index;
8239 size_t slots_to_copy = min(slots_needed, slots_available);
8240 memcpy(mark_list_index, start, slots_to_copy*sizeof(*start));
8241 mark_list_index += slots_to_copy;
8242 // printf("heap %d: appended %Id slots to mark_list\n", heap_number, slots_to_copy);
8245 void gc_heap::merge_mark_lists()
8247 uint8_t** source[MAX_SUPPORTED_CPUS];
8248 uint8_t** source_end[MAX_SUPPORTED_CPUS];
8249 int source_heap[MAX_SUPPORTED_CPUS];
8250 int source_count = 0;
8252 // in case of mark list overflow, don't bother
8253 if (mark_list_index > mark_list_end)
8255 // printf("merge_mark_lists: overflow\n");
8259 dprintf(3, ("merge_mark_lists: heap_number = %d starts out with %Id entries", heap_number, mark_list_index - mark_list));
8260 // unsigned long start = GetCycleCount32();
8261 for (int i = 0; i < n_heaps; i++)
8263 gc_heap* heap = g_heaps[i];
8264 if (heap->mark_list_piece_start[heap_number] < heap->mark_list_piece_end[heap_number])
8266 source[source_count] = heap->mark_list_piece_start[heap_number];
8267 source_end[source_count] = heap->mark_list_piece_end[heap_number];
8268 source_heap[source_count] = i;
8269 if (source_count < MAX_SUPPORTED_CPUS)
8273 // printf("first phase of merge_mark_lists for heap %d took %u cycles\n", heap_number, GetCycleCount32() - start);
8275 dprintf(3, ("heap_number = %d has %d sources\n", heap_number, source_count));
8276 #if defined(_DEBUG) || defined(TRACE_GC)
8277 for (int j = 0; j < source_count; j++)
8279 dprintf(3, ("heap_number = %d ", heap_number));
8280 dprintf(3, (" source from heap %d = %Ix .. %Ix (%Id entries)",
8281 (size_t)(source_heap[j]), (size_t)(source[j][0]), (size_t)(source_end[j][-1]), (size_t)(source_end[j] - source[j])));
8282 // the sources should all be sorted
8283 for (uint8_t **x = source[j]; x < source_end[j] - 1; x++)
8287 dprintf(3, ("oops, mark_list from source %d for heap %d isn't sorted\n", j, heap_number));
8292 #endif //_DEBUG || TRACE_GC
8294 // start = GetCycleCount32();
8296 mark_list = &g_mark_list_copy [heap_number*mark_list_size];
8297 mark_list_index = mark_list;
8298 mark_list_end = &mark_list [mark_list_size-1];
8299 int piece_count = 0;
8300 if (source_count == 0)
8304 else if (source_count == 1)
8306 mark_list = source[0];
8307 mark_list_index = source_end[0];
8308 mark_list_end = mark_list_index;
8313 while (source_count > 1)
8315 // find the lowest and second lowest value in the sources we're merging from
8316 int lowest_source = 0;
8317 uint8_t *lowest = *source[0];
8318 uint8_t *second_lowest = *source[1];
8319 for (int i = 1; i < source_count; i++)
8321 if (lowest > *source[i])
8323 second_lowest = lowest;
8324 lowest = *source[i];
8327 else if (second_lowest > *source[i])
8329 second_lowest = *source[i];
8333 // find the point in the lowest source where it either runs out or is not <= second_lowest anymore
8335 // let's first try to get lucky and see if the whole source is <= second_lowest -- this is actually quite common
8337 if (source_end[lowest_source][-1] <= second_lowest)
8338 x = source_end[lowest_source];
8341 // use linear search to find the end -- could also use binary search as in sort_mark_list,
8342 // but saw no improvement doing that
8343 for (x = source[lowest_source]; x < source_end[lowest_source] && *x <= second_lowest; x++)
8347 // blast this piece to the mark list
8348 append_to_mark_list(source[lowest_source], x);
8351 source[lowest_source] = x;
8353 // check whether this source is now exhausted
8354 if (x >= source_end[lowest_source])
8356 // if it's not the source with the highest index, copy the source with the highest index
8357 // over it so the non-empty sources are always at the beginning
8358 if (lowest_source < source_count-1)
8360 source[lowest_source] = source[source_count-1];
8361 source_end[lowest_source] = source_end[source_count-1];
8366 // we're left with just one source that we copy
8367 append_to_mark_list(source[0], source_end[0]);
8371 // printf("second phase of merge_mark_lists for heap %d took %u cycles to merge %d pieces\n", heap_number, GetCycleCount32() - start, piece_count);
8373 #if defined(_DEBUG) || defined(TRACE_GC)
8374 // the final mark list must be sorted
8375 for (uint8_t **x = mark_list; x < mark_list_index - 1; x++)
8379 dprintf(3, ("oops, mark_list for heap %d isn't sorted at the end of merge_mark_lists", heap_number));
8383 #endif //defined(_DEBUG) || defined(TRACE_GC)
8385 #else //PARALLEL_MARK_LIST_SORT
8386 void gc_heap::combine_mark_lists()
8388 dprintf (3, ("Combining mark lists"));
8389 //verify if a heap has overflowed its mark list
8390 BOOL use_mark_list = TRUE;
8391 for (int i = 0; i < n_heaps; i++)
8393 if (g_heaps [i]->mark_list_index > g_heaps [i]->mark_list_end)
8395 use_mark_list = FALSE;
8402 dprintf (3, ("Using mark list"));
8403 //compact the gaps out of the mark list
8405 uint8_t** current_gap = g_heaps [gn]->mark_list_index;
8406 uint8_t** current_gap_end = g_heaps[gn]->mark_list_end + 1;
8407 uint8_t** dst_last = current_gap-1;
8409 int srcn = n_heaps-1;
8410 gc_heap* srch = g_heaps [srcn];
8411 uint8_t** src = srch->mark_list_index - 1;
8412 uint8_t** src_beg = srch->mark_list;
8414 while (current_gap <= src)
8416 while ((gn < n_heaps-1) && (current_gap >= current_gap_end))
8418 //go to the next gap
8420 dprintf (3, ("Going to the next gap %d", gn));
8421 assert (gn < n_heaps);
8422 current_gap = g_heaps [gn]->mark_list_index;
8423 current_gap_end = g_heaps[gn]->mark_list_end + 1;
8424 assert ((gn == (n_heaps-1)) || (current_gap_end == g_heaps[gn+1]->mark_list));
8426 while ((srcn > 0) && (src < src_beg))
8428 //go to the previous source
8430 dprintf (3, ("going to the previous source %d", srcn));
8432 gc_heap* srch = g_heaps [srcn];
8433 src = srch->mark_list_index - 1;
8434 src_beg = srch->mark_list;
8436 if (current_gap < src)
8438 dst_last = current_gap;
8439 *current_gap++ = *src--;
8442 dprintf (3, ("src: %Ix dst_last: %Ix", (size_t)src, (size_t)dst_last));
8444 uint8_t** end_of_list = max (src, dst_last);
8446 //sort the resulting compacted list
8447 assert (end_of_list < &g_mark_list [n_heaps*mark_list_size]);
8448 if (end_of_list > &g_mark_list[0])
8449 _sort (&g_mark_list[0], end_of_list, 0);
8450 //adjust the mark_list to the begining of the resulting mark list.
8451 for (int i = 0; i < n_heaps; i++)
8453 g_heaps [i]->mark_list = g_mark_list;
8454 g_heaps [i]->mark_list_index = end_of_list + 1;
8455 g_heaps [i]->mark_list_end = end_of_list + 1;
8460 uint8_t** end_of_list = g_mark_list;
8461 //adjust the mark_list to the begining of the resulting mark list.
8462 //put the index beyond the end to turn off mark list processing
8463 for (int i = 0; i < n_heaps; i++)
8465 g_heaps [i]->mark_list = g_mark_list;
8466 g_heaps [i]->mark_list_index = end_of_list + 1;
8467 g_heaps [i]->mark_list_end = end_of_list;
8471 #endif // PARALLEL_MARK_LIST_SORT
8472 #endif //MULTIPLE_HEAPS
8475 class seg_free_spaces
8477 struct seg_free_space
8483 struct free_space_bucket
8485 seg_free_space* free_space;
8486 ptrdiff_t count_add; // Assigned when we first contruct the array.
8487 ptrdiff_t count_fit; // How many items left when we are fitting plugs.
8490 void move_bucket (int old_power2, int new_power2)
8492 // PREFAST warning 22015: old_power2 could be negative
8493 assert (old_power2 >= 0);
8494 assert (old_power2 >= new_power2);
8496 if (old_power2 == new_power2)
8501 seg_free_space* src_index = free_space_buckets[old_power2].free_space;
8502 for (int i = old_power2; i > new_power2; i--)
8504 seg_free_space** dest = &(free_space_buckets[i].free_space);
8507 seg_free_space* dest_index = free_space_buckets[i - 1].free_space;
8508 if (i > (new_power2 + 1))
8510 seg_free_space temp = *src_index;
8511 *src_index = *dest_index;
8514 src_index = dest_index;
8517 free_space_buckets[old_power2].count_fit--;
8518 free_space_buckets[new_power2].count_fit++;
8523 void dump_free_space (seg_free_space* item)
8530 mark* m = (mark*)(item->start);
8531 len = pinned_len (m);
8532 addr = pinned_plug (m) - len;
8536 heap_segment* seg = (heap_segment*)(item->start);
8537 addr = heap_segment_plan_allocated (seg);
8538 len = heap_segment_committed (seg) - addr;
8541 dprintf (SEG_REUSE_LOG_1, ("[%d]0x%Ix %Id", heap_num, addr, len));
8546 seg_free_space* item = NULL;
8549 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------\nnow the free spaces look like:", heap_num));
8550 for (i = 0; i < (free_space_bucket_count - 1); i++)
8552 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces for 2^%d bucket:", heap_num, (base_power2 + i)));
8553 dprintf (SEG_REUSE_LOG_1, ("[%d]%s %s", heap_num, "start", "len"));
8554 item = free_space_buckets[i].free_space;
8555 while (item < free_space_buckets[i + 1].free_space)
8557 dump_free_space (item);
8560 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------", heap_num));
8563 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces for 2^%d bucket:", heap_num, (base_power2 + i)));
8564 dprintf (SEG_REUSE_LOG_1, ("[%d]%s %s", heap_num, "start", "len"));
8565 item = free_space_buckets[i].free_space;
8567 while (item <= &seg_free_space_array[free_space_item_count - 1])
8569 dump_free_space (item);
8572 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------", heap_num));
8577 free_space_bucket* free_space_buckets;
8578 seg_free_space* seg_free_space_array;
8579 ptrdiff_t free_space_bucket_count;
8580 ptrdiff_t free_space_item_count;
8584 BOOL has_end_of_seg;
8589 seg_free_spaces (int h_number)
8591 heap_num = h_number;
8596 size_t total_prealloc_size =
8597 MAX_NUM_BUCKETS * sizeof (free_space_bucket) +
8598 MAX_NUM_FREE_SPACES * sizeof (seg_free_space);
8600 free_space_buckets = (free_space_bucket*) new (nothrow) uint8_t[total_prealloc_size];
8602 return (!!free_space_buckets);
8605 // We take the ordered free space array we got from the 1st pass,
8606 // and feed the portion that we decided to use to this method, ie,
8607 // the largest item_count free spaces.
8608 void add_buckets (int base, size_t* ordered_free_spaces, int bucket_count, size_t item_count)
8610 assert (free_space_buckets);
8611 assert (item_count <= (size_t)MAX_PTR);
8613 free_space_bucket_count = bucket_count;
8614 free_space_item_count = item_count;
8617 has_end_of_seg = FALSE;
8620 ptrdiff_t total_item_count = 0;
8623 seg_free_space_array = (seg_free_space*)(free_space_buckets + free_space_bucket_count);
8625 for (i = 0; i < (ptrdiff_t)item_count; i++)
8627 seg_free_space_array[i].start = 0;
8628 seg_free_space_array[i].is_plug = FALSE;
8631 for (i = 0; i < bucket_count; i++)
8633 free_space_buckets[i].count_add = ordered_free_spaces[i];
8634 free_space_buckets[i].count_fit = ordered_free_spaces[i];
8635 free_space_buckets[i].free_space = &seg_free_space_array[total_item_count];
8636 total_item_count += free_space_buckets[i].count_add;
8639 assert (total_item_count == (ptrdiff_t)item_count);
8642 // If we are adding a free space before a plug we pass the
8643 // mark stack position so we can update the length; we could
8644 // also be adding the free space after the last plug in which
8645 // case start is the segment which we'll need to update the
8646 // heap_segment_plan_allocated.
8647 void add (void* start, BOOL plug_p, BOOL first_p)
8649 size_t size = (plug_p ?
8650 pinned_len ((mark*)start) :
8651 (heap_segment_committed ((heap_segment*)start) -
8652 heap_segment_plan_allocated ((heap_segment*)start)));
8656 dprintf (SEG_REUSE_LOG_1, ("[%d]Adding a free space before plug: %Id", heap_num, size));
8660 dprintf (SEG_REUSE_LOG_1, ("[%d]Adding a free space at end of seg: %Id", heap_num, size));
8662 has_end_of_seg = TRUE;
8668 size_t eph_gen_starts = gc_heap::eph_gen_starts_size;
8669 size -= eph_gen_starts;
8672 mark* m = (mark*)(start);
8673 pinned_len (m) -= eph_gen_starts;
8677 heap_segment* seg = (heap_segment*)start;
8678 heap_segment_plan_allocated (seg) += eph_gen_starts;
8682 int bucket_power2 = index_of_highest_set_bit (size);
8683 if (bucket_power2 < base_power2)
8688 free_space_bucket* bucket = &free_space_buckets[bucket_power2 - base_power2];
8690 seg_free_space* bucket_free_space = bucket->free_space;
8691 assert (plug_p || (!plug_p && bucket->count_add));
8693 if (bucket->count_add == 0)
8695 dprintf (SEG_REUSE_LOG_1, ("[%d]Already have enough of 2^%d", heap_num, bucket_power2));
8699 ptrdiff_t index = bucket->count_add - 1;
8701 dprintf (SEG_REUSE_LOG_1, ("[%d]Building free spaces: adding %Ix; len: %Id (2^%d)",
8704 (pinned_plug ((mark*)start) - pinned_len ((mark*)start)) :
8705 heap_segment_plan_allocated ((heap_segment*)start)),
8711 bucket_free_space[index].is_plug = TRUE;
8714 bucket_free_space[index].start = start;
8715 bucket->count_add--;
8720 // Do a consistency check after all free spaces are added.
8724 int end_of_seg_count = 0;
8726 for (i = 0; i < free_space_item_count; i++)
8728 assert (seg_free_space_array[i].start);
8729 if (!(seg_free_space_array[i].is_plug))
8737 assert (end_of_seg_count == 1);
8741 assert (end_of_seg_count == 0);
8744 for (i = 0; i < free_space_bucket_count; i++)
8746 assert (free_space_buckets[i].count_add == 0);
8752 uint8_t* fit (uint8_t* old_loc,
8754 BOOL set_padding_on_saved_p,
8755 mark* pinned_plug_entry,
8756 #endif //SHORT_PLUGS
8758 REQD_ALIGN_AND_OFFSET_DCL)
8763 assert (!is_plug_padded (old_loc));
8764 #endif //SHORT_PLUGS
8765 assert (!node_realigned (old_loc));
8768 size_t saved_plug_size = plug_size;
8770 #ifdef FEATURE_STRUCTALIGN
8771 // BARTOKTODO (4841): this code path is disabled (see can_fit_all_blocks_p) until we take alignment requirements into account
8772 _ASSERTE(requiredAlignment == DATA_ALIGNMENT && false);
8773 #endif // FEATURE_STRUCTALIGN
8774 // TODO: this is also not large alignment ready. We would need to consider alignment when chosing the
8777 size_t plug_size_to_fit = plug_size;
8779 int pad_in_front = (old_loc != 0) ? USE_PADDING_FRONT : 0;
8782 plug_size_to_fit += (pad_in_front ? Align(min_obj_size) : 0);
8783 #endif //SHORT_PLUGS
8785 int plug_power2 = index_of_highest_set_bit (round_up_power2 (plug_size_to_fit + Align(min_obj_size)));
8787 uint8_t* new_address = 0;
8789 if (plug_power2 < base_power2)
8791 plug_power2 = base_power2;
8794 int chosen_power2 = plug_power2 - base_power2;
8796 for (i = chosen_power2; i < free_space_bucket_count; i++)
8798 if (free_space_buckets[i].count_fit != 0)
8805 dprintf (SEG_REUSE_LOG_1, ("[%d]Fitting plug len %Id (2^%d) using 2^%d free space",
8809 (chosen_power2 + base_power2)));
8811 assert (i < free_space_bucket_count);
8813 seg_free_space* bucket_free_space = free_space_buckets[chosen_power2].free_space;
8814 ptrdiff_t free_space_count = free_space_buckets[chosen_power2].count_fit;
8815 size_t new_free_space_size = 0;
8816 BOOL can_fit = FALSE;
8819 for (i = 0; i < free_space_count; i++)
8821 size_t free_space_size = 0;
8824 BOOL short_plugs_padding_p = FALSE;
8825 #endif //SHORT_PLUGS
8826 BOOL realign_padding_p = FALSE;
8828 if (bucket_free_space[i].is_plug)
8830 mark* m = (mark*)(bucket_free_space[i].start);
8831 uint8_t* plug_free_space_start = pinned_plug (m) - pinned_len (m);
8834 if ((pad_in_front & USE_PADDING_FRONT) &&
8835 (((plug_free_space_start - pin_allocation_context_start_region (m))==0) ||
8836 ((plug_free_space_start - pin_allocation_context_start_region (m))>=DESIRED_PLUG_LENGTH)))
8838 pad = Align (min_obj_size);
8839 short_plugs_padding_p = TRUE;
8841 #endif //SHORT_PLUGS
8843 if (!((old_loc == 0) || same_large_alignment_p (old_loc, plug_free_space_start+pad)))
8845 pad += switch_alignment_size (pad != 0);
8846 realign_padding_p = TRUE;
8849 plug_size = saved_plug_size + pad;
8851 free_space_size = pinned_len (m);
8852 new_address = pinned_plug (m) - pinned_len (m);
8854 if (free_space_size >= (plug_size + Align (min_obj_size)) ||
8855 free_space_size == plug_size)
8857 new_free_space_size = free_space_size - plug_size;
8858 pinned_len (m) = new_free_space_size;
8859 #ifdef SIMPLE_DPRINTF
8860 dprintf (SEG_REUSE_LOG_0, ("[%d]FP: 0x%Ix->0x%Ix(%Ix)(%Ix), [0x%Ix (2^%d) -> [0x%Ix (2^%d)",
8867 index_of_highest_set_bit (free_space_size),
8868 (pinned_plug (m) - pinned_len (m)),
8869 index_of_highest_set_bit (new_free_space_size)));
8870 #endif //SIMPLE_DPRINTF
8873 if (short_plugs_padding_p)
8875 pin_allocation_context_start_region (m) = plug_free_space_start;
8876 set_padding_in_expand (old_loc, set_padding_on_saved_p, pinned_plug_entry);
8878 #endif //SHORT_PLUGS
8880 if (realign_padding_p)
8882 set_node_realigned (old_loc);
8890 heap_segment* seg = (heap_segment*)(bucket_free_space[i].start);
8891 free_space_size = heap_segment_committed (seg) - heap_segment_plan_allocated (seg);
8893 if (!((old_loc == 0) || same_large_alignment_p (old_loc, heap_segment_plan_allocated (seg))))
8895 pad = switch_alignment_size (FALSE);
8896 realign_padding_p = TRUE;
8899 plug_size = saved_plug_size + pad;
8901 if (free_space_size >= (plug_size + Align (min_obj_size)) ||
8902 free_space_size == plug_size)
8904 new_address = heap_segment_plan_allocated (seg);
8905 new_free_space_size = free_space_size - plug_size;
8906 heap_segment_plan_allocated (seg) = new_address + plug_size;
8907 #ifdef SIMPLE_DPRINTF
8908 dprintf (SEG_REUSE_LOG_0, ("[%d]FS: 0x%Ix-> 0x%Ix(%Ix) (2^%d) -> 0x%Ix (2^%d)",
8913 index_of_highest_set_bit (free_space_size),
8914 heap_segment_plan_allocated (seg),
8915 index_of_highest_set_bit (new_free_space_size)));
8916 #endif //SIMPLE_DPRINTF
8918 if (realign_padding_p)
8919 set_node_realigned (old_loc);
8933 assert (chosen_power2 == 0);
8943 assert ((chosen_power2 && (i == 0)) ||
8944 (!chosen_power2) && (i < free_space_count));
8947 int new_bucket_power2 = index_of_highest_set_bit (new_free_space_size);
8949 if (new_bucket_power2 < base_power2)
8951 new_bucket_power2 = base_power2;
8954 move_bucket (chosen_power2, new_bucket_power2 - base_power2);
8963 if (free_space_buckets)
8965 delete [] free_space_buckets;
8967 if (seg_free_space_array)
8969 delete [] seg_free_space_array;
8975 #define marked(i) header(i)->IsMarked()
8976 #define set_marked(i) header(i)->SetMarked()
8977 #define clear_marked(i) header(i)->ClearMarked()
8978 #define pinned(i) header(i)->IsPinned()
8979 #define set_pinned(i) header(i)->SetPinned()
8980 #define clear_pinned(i) header(i)->GetHeader()->ClrGCBit();
8982 inline size_t my_get_size (Object* ob)
8984 MethodTable* mT = header(ob)->GetMethodTable();
8985 return (mT->GetBaseSize() +
8986 (mT->HasComponentSize() ?
8987 ((size_t)((CObjectHeader*)ob)->GetNumComponents() * mT->RawGetComponentSize()) : 0));
8990 //#define size(i) header(i)->GetSize()
8991 #define size(i) my_get_size (header(i))
8993 #define contain_pointers(i) header(i)->ContainsPointers()
8994 #ifdef COLLECTIBLE_CLASS
8995 #define contain_pointers_or_collectible(i) header(i)->ContainsPointersOrCollectible()
8997 #define get_class_object(i) GCToEEInterface::GetLoaderAllocatorObjectForGC((Object *)i)
8998 #define is_collectible(i) method_table(i)->Collectible()
8999 #else //COLLECTIBLE_CLASS
9000 #define contain_pointers_or_collectible(i) header(i)->ContainsPointers()
9001 #endif //COLLECTIBLE_CLASS
9003 #if defined (MARK_ARRAY) && defined (BACKGROUND_GC)
9005 void gc_heap::seg_clear_mark_array_bits_soh (heap_segment* seg)
9007 uint8_t* range_beg = 0;
9008 uint8_t* range_end = 0;
9009 if (bgc_mark_array_range (seg, FALSE, &range_beg, &range_end))
9011 clear_mark_array (range_beg, align_on_mark_word (range_end), FALSE
9012 #ifdef FEATURE_BASICFREEZE
9014 #endif // FEATURE_BASICFREEZE
9019 void gc_heap::clear_batch_mark_array_bits (uint8_t* start, uint8_t* end)
9021 if ((start < background_saved_highest_address) &&
9022 (end > background_saved_lowest_address))
9024 start = max (start, background_saved_lowest_address);
9025 end = min (end, background_saved_highest_address);
9027 size_t start_mark_bit = mark_bit_of (start);
9028 size_t end_mark_bit = mark_bit_of (end);
9029 unsigned int startbit = mark_bit_bit (start_mark_bit);
9030 unsigned int endbit = mark_bit_bit (end_mark_bit);
9031 size_t startwrd = mark_bit_word (start_mark_bit);
9032 size_t endwrd = mark_bit_word (end_mark_bit);
9034 dprintf (3, ("Clearing all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
9035 (size_t)start, (size_t)start_mark_bit,
9036 (size_t)end, (size_t)end_mark_bit));
9038 unsigned int firstwrd = lowbits (~0, startbit);
9039 unsigned int lastwrd = highbits (~0, endbit);
9041 if (startwrd == endwrd)
9043 unsigned int wrd = firstwrd | lastwrd;
9044 mark_array[startwrd] &= wrd;
9048 // clear the first mark word.
9051 mark_array[startwrd] &= firstwrd;
9055 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
9057 mark_array[wrdtmp] = 0;
9060 // clear the last mark word.
9063 mark_array[endwrd] &= lastwrd;
9068 void gc_heap::bgc_clear_batch_mark_array_bits (uint8_t* start, uint8_t* end)
9070 if ((start < background_saved_highest_address) &&
9071 (end > background_saved_lowest_address))
9073 start = max (start, background_saved_lowest_address);
9074 end = min (end, background_saved_highest_address);
9076 clear_batch_mark_array_bits (start, end);
9080 void gc_heap::clear_mark_array_by_objects (uint8_t* from, uint8_t* end, BOOL loh_p)
9082 dprintf (3, ("clearing mark array bits by objects for addr [%Ix,[%Ix",
9084 int align_const = get_alignment_constant (!loh_p);
9090 uint8_t* next_o = o + Align (size (o), align_const);
9092 if (background_object_marked (o, TRUE))
9094 dprintf (3, ("%Ix was marked by bgc, is now cleared", o));
9100 #endif //MARK_ARRAY && BACKGROUND_GC
9103 BOOL gc_heap::is_mark_set (uint8_t* o)
9108 #if defined (_MSC_VER) && defined (_TARGET_X86_)
9109 #pragma optimize("y", on) // Small critical routines, don't put in EBP frame
9110 #endif //_MSC_VER && _TARGET_X86_
9112 // return the generation number of an object.
9113 // It is assumed that the object is valid.
9114 //Note that this will return max_generation for a LOH object
9115 int gc_heap::object_gennum (uint8_t* o)
9117 if (in_range_for_segment (o, ephemeral_heap_segment) &&
9118 (o >= generation_allocation_start (generation_of (max_generation-1))))
9120 // in an ephemeral generation.
9121 for ( int i = 0; i < max_generation-1; i++)
9123 if ((o >= generation_allocation_start (generation_of (i))))
9126 return max_generation-1;
9130 return max_generation;
9134 int gc_heap::object_gennum_plan (uint8_t* o)
9136 if (in_range_for_segment (o, ephemeral_heap_segment))
9138 for (int i = 0; i <= max_generation-1; i++)
9140 uint8_t* plan_start = generation_plan_allocation_start (generation_of (i));
9141 if (plan_start && (o >= plan_start))
9147 return max_generation;
9150 #if defined(_MSC_VER) && defined(_TARGET_X86_)
9151 #pragma optimize("", on) // Go back to command line default optimizations
9152 #endif //_MSC_VER && _TARGET_X86_
9154 heap_segment* gc_heap::make_heap_segment (uint8_t* new_pages, size_t size, int h_number)
9156 size_t initial_commit = SEGMENT_INITIAL_COMMIT;
9158 //Commit the first page
9159 if (!virtual_alloc_commit_for_heap (new_pages, initial_commit, h_number))
9164 //overlay the heap_segment
9165 heap_segment* new_segment = (heap_segment*)new_pages;
9167 uint8_t* start = new_pages + segment_info_size;
9168 heap_segment_mem (new_segment) = start;
9169 heap_segment_used (new_segment) = start;
9170 heap_segment_reserved (new_segment) = new_pages + size;
9171 heap_segment_committed (new_segment) = new_pages + initial_commit;
9172 init_heap_segment (new_segment);
9173 dprintf (2, ("Creating heap segment %Ix", (size_t)new_segment));
9177 void gc_heap::init_heap_segment (heap_segment* seg)
9180 heap_segment_next (seg) = 0;
9181 heap_segment_plan_allocated (seg) = heap_segment_mem (seg);
9182 heap_segment_allocated (seg) = heap_segment_mem (seg);
9183 #ifdef BACKGROUND_GC
9184 heap_segment_background_allocated (seg) = 0;
9185 heap_segment_saved_bg_allocated (seg) = 0;
9186 #endif //BACKGROUND_GC
9189 //Releases the segment to the OS.
9190 // this is always called on one thread only so calling seg_table->remove is fine.
9191 void gc_heap::delete_heap_segment (heap_segment* seg, BOOL consider_hoarding)
9193 if (!heap_segment_loh_p (seg))
9195 //cleanup the brick table back to the empty value
9196 clear_brick_table (heap_segment_mem (seg), heap_segment_reserved (seg));
9199 if (consider_hoarding)
9201 assert ((heap_segment_mem (seg) - (uint8_t*)seg) <= ptrdiff_t(2*OS_PAGE_SIZE));
9202 size_t ss = (size_t) (heap_segment_reserved (seg) - (uint8_t*)seg);
9203 //Don't keep the big ones.
9204 if (ss <= INITIAL_ALLOC)
9206 dprintf (2, ("Hoarding segment %Ix", (size_t)seg));
9207 #ifdef BACKGROUND_GC
9208 // We don't need to clear the decommitted flag because when this segment is used
9209 // for a new segment the flags will be cleared.
9210 if (!heap_segment_decommitted_p (seg))
9211 #endif //BACKGROUND_GC
9213 decommit_heap_segment (seg);
9216 #ifdef SEG_MAPPING_TABLE
9217 seg_mapping_table_remove_segment (seg);
9218 #endif //SEG_MAPPING_TABLE
9220 heap_segment_next (seg) = segment_standby_list;
9221 segment_standby_list = seg;
9228 dprintf (2, ("h%d: del seg: [%Ix, %Ix[",
9229 heap_number, (size_t)seg,
9230 (size_t)(heap_segment_reserved (seg))));
9232 #ifdef BACKGROUND_GC
9233 ::record_changed_seg ((uint8_t*)seg, heap_segment_reserved (seg),
9234 settings.gc_index, current_bgc_state,
9236 decommit_mark_array_by_seg (seg);
9237 #endif //BACKGROUND_GC
9239 #ifdef SEG_MAPPING_TABLE
9240 seg_mapping_table_remove_segment (seg);
9241 #else //SEG_MAPPING_TABLE
9242 seg_table->remove ((uint8_t*)seg);
9243 #endif //SEG_MAPPING_TABLE
9245 release_segment (seg);
9249 //resets the pages beyond allocates size so they won't be swapped out and back in
9251 void gc_heap::reset_heap_segment_pages (heap_segment* seg)
9253 size_t page_start = align_on_page ((size_t)heap_segment_allocated (seg));
9254 size_t size = (size_t)heap_segment_committed (seg) - page_start;
9256 GCToOSInterface::VirtualReset((void*)page_start, size, false /* unlock */);
9259 void gc_heap::decommit_heap_segment_pages (heap_segment* seg,
9262 uint8_t* page_start = align_on_page (heap_segment_allocated(seg));
9263 size_t size = heap_segment_committed (seg) - page_start;
9264 extra_space = align_on_page (extra_space);
9265 if (size >= max ((extra_space + 2*OS_PAGE_SIZE), 100*OS_PAGE_SIZE))
9267 page_start += max(extra_space, 32*OS_PAGE_SIZE);
9268 size -= max (extra_space, 32*OS_PAGE_SIZE);
9270 GCToOSInterface::VirtualDecommit (page_start, size);
9271 dprintf (3, ("Decommitting heap segment [%Ix, %Ix[(%d)",
9273 (size_t)(page_start + size),
9275 heap_segment_committed (seg) = page_start;
9276 if (heap_segment_used (seg) > heap_segment_committed (seg))
9278 heap_segment_used (seg) = heap_segment_committed (seg);
9283 //decommit all pages except one or 2
9284 void gc_heap::decommit_heap_segment (heap_segment* seg)
9286 uint8_t* page_start = align_on_page (heap_segment_mem (seg));
9288 dprintf (3, ("Decommitting heap segment %Ix", (size_t)seg));
9290 #ifdef BACKGROUND_GC
9291 page_start += OS_PAGE_SIZE;
9292 #endif //BACKGROUND_GC
9294 size_t size = heap_segment_committed (seg) - page_start;
9295 GCToOSInterface::VirtualDecommit (page_start, size);
9297 //re-init the segment object
9298 heap_segment_committed (seg) = page_start;
9299 if (heap_segment_used (seg) > heap_segment_committed (seg))
9301 heap_segment_used (seg) = heap_segment_committed (seg);
9305 void gc_heap::clear_gen0_bricks()
9307 if (!gen0_bricks_cleared)
9309 gen0_bricks_cleared = TRUE;
9310 //initialize brick table for gen 0
9311 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
9312 b < brick_of (align_on_brick
9313 (heap_segment_allocated (ephemeral_heap_segment)));
9321 #ifdef BACKGROUND_GC
9322 void gc_heap::rearrange_small_heap_segments()
9324 heap_segment* seg = freeable_small_heap_segment;
9327 heap_segment* next_seg = heap_segment_next (seg);
9328 // TODO: we need to consider hoarding here.
9329 delete_heap_segment (seg, FALSE);
9332 freeable_small_heap_segment = 0;
9334 #endif //BACKGROUND_GC
9336 void gc_heap::rearrange_large_heap_segments()
9338 dprintf (2, ("deleting empty large segments"));
9339 heap_segment* seg = freeable_large_heap_segment;
9342 heap_segment* next_seg = heap_segment_next (seg);
9343 delete_heap_segment (seg, GCConfig::GetRetainVM());
9346 freeable_large_heap_segment = 0;
9349 void gc_heap::rearrange_heap_segments(BOOL compacting)
9352 generation_start_segment (generation_of (max_generation));
9354 heap_segment* prev_seg = 0;
9355 heap_segment* next_seg = 0;
9358 next_seg = heap_segment_next (seg);
9360 //link ephemeral segment when expanding
9361 if ((next_seg == 0) && (seg != ephemeral_heap_segment))
9363 seg->next = ephemeral_heap_segment;
9364 next_seg = heap_segment_next (seg);
9367 //re-used expanded heap segment
9368 if ((seg == ephemeral_heap_segment) && next_seg)
9370 heap_segment_next (prev_seg) = next_seg;
9371 heap_segment_next (seg) = 0;
9375 uint8_t* end_segment = (compacting ?
9376 heap_segment_plan_allocated (seg) :
9377 heap_segment_allocated (seg));
9378 // check if the segment was reached by allocation
9379 if ((end_segment == heap_segment_mem (seg))&&
9380 !heap_segment_read_only_p (seg))
9382 //if not, unthread and delete
9384 assert (seg != ephemeral_heap_segment);
9385 heap_segment_next (prev_seg) = next_seg;
9386 delete_heap_segment (seg, GCConfig::GetRetainVM());
9388 dprintf (2, ("Deleting heap segment %Ix", (size_t)seg));
9392 if (!heap_segment_read_only_p (seg))
9396 heap_segment_allocated (seg) =
9397 heap_segment_plan_allocated (seg);
9400 // reset the pages between allocated and committed.
9401 if (seg != ephemeral_heap_segment)
9403 decommit_heap_segment_pages (seg, 0);
9417 uint8_t* g_addresses [array_size+2]; // to get around the bug in GetWriteWatch
9419 #ifdef TIME_WRITE_WATCH
9420 static unsigned int tot_cycles = 0;
9421 #endif //TIME_WRITE_WATCH
9425 inline void gc_heap::verify_card_bundle_bits_set(size_t first_card_word, size_t last_card_word)
9428 for (size_t x = cardw_card_bundle (first_card_word); x < cardw_card_bundle (last_card_word); x++)
9430 if (!card_bundle_set_p (x))
9432 assert (!"Card bundle not set");
9433 dprintf (3, ("Card bundle %Ix not set", x));
9439 // Verifies that any bundles that are not set represent only cards that are not set.
9440 inline void gc_heap::verify_card_bundles()
9443 size_t lowest_card = card_word (card_of (lowest_address));
9444 size_t highest_card = card_word (card_of (highest_address));
9445 size_t cardb = cardw_card_bundle (lowest_card);
9446 size_t end_cardb = cardw_card_bundle (align_cardw_on_bundle (highest_card));
9448 while (cardb < end_cardb)
9450 uint32_t* card_word = &card_table[max(card_bundle_cardw (cardb), lowest_card)];
9451 uint32_t* card_word_end = &card_table[min(card_bundle_cardw (cardb+1), highest_card)];
9453 if (card_bundle_set_p (cardb) == 0)
9455 // Verify that no card is set
9456 while (card_word < card_word_end)
9458 if (*card_word != 0)
9460 dprintf (3, ("gc: %d, Card word %Ix for address %Ix set, card_bundle %Ix clear",
9461 dd_collection_count (dynamic_data_of (0)),
9462 (size_t)(card_word-&card_table[0]),
9463 (size_t)(card_address ((size_t)(card_word-&card_table[0]) * card_word_width)), cardb));
9466 assert((*card_word)==0);
9476 // If card bundles are enabled, use write watch to find pages in the card table that have
9477 // been dirtied, and set the corresponding card bundle bits.
9478 void gc_heap::update_card_table_bundle()
9480 if (card_bundles_enabled())
9482 // The address of the card word containing the card representing the lowest heap address
9483 uint8_t* base_address = (uint8_t*)(&card_table[card_word (card_of (lowest_address))]);
9485 // The address of the card word containing the card representing the highest heap address
9486 uint8_t* high_address = (uint8_t*)(&card_table[card_word (card_of (highest_address))]);
9488 uint8_t* saved_base_address = base_address;
9489 uintptr_t bcount = array_size;
9490 size_t saved_region_size = align_on_page (high_address) - saved_base_address;
9494 size_t region_size = align_on_page (high_address) - base_address;
9496 dprintf (3,("Probing card table pages [%Ix, %Ix[", (size_t)base_address, (size_t)base_address+region_size));
9497 bool success = GCToOSInterface::GetWriteWatch(false /* resetState */,
9500 (void**)g_addresses,
9502 assert (success && "GetWriteWatch failed!");
9504 dprintf (3,("Found %d pages written", bcount));
9505 for (unsigned i = 0; i < bcount; i++)
9507 // Offset of the dirty page from the start of the card table (clamped to base_address)
9508 size_t bcardw = (uint32_t*)(max(g_addresses[i],base_address)) - &card_table[0];
9510 // Offset of the end of the page from the start of the card table (clamped to high addr)
9511 size_t ecardw = (uint32_t*)(min(g_addresses[i]+OS_PAGE_SIZE, high_address)) - &card_table[0];
9512 assert (bcardw >= card_word (card_of (g_gc_lowest_address)));
9514 // Set the card bundle bits representing the dirty card table page
9515 card_bundles_set (cardw_card_bundle (bcardw), cardw_card_bundle (align_cardw_on_bundle (ecardw)));
9516 dprintf (3,("Set Card bundle [%Ix, %Ix[", cardw_card_bundle (bcardw), cardw_card_bundle (align_cardw_on_bundle (ecardw))));
9518 verify_card_bundle_bits_set(bcardw, ecardw);
9521 if (bcount >= array_size)
9523 base_address = g_addresses [array_size-1] + OS_PAGE_SIZE;
9524 bcount = array_size;
9527 } while ((bcount >= array_size) && (base_address < high_address));
9529 // Now that we've updated the card bundle bits, reset the write-tracking state.
9530 GCToOSInterface::ResetWriteWatch (saved_base_address, saved_region_size);
9533 #endif //CARD_BUNDLE
9536 void gc_heap::reset_write_watch_for_gc_heap(void* base_address, size_t region_size)
9538 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9539 SoftwareWriteWatch::ClearDirty(base_address, region_size);
9540 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9541 GCToOSInterface::ResetWriteWatch(base_address, region_size);
9542 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9546 void gc_heap::get_write_watch_for_gc_heap(bool reset, void *base_address, size_t region_size, void** dirty_pages, uintptr_t* dirty_page_count_ref, bool is_runtime_suspended)
9548 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9549 SoftwareWriteWatch::GetDirty(base_address, region_size, dirty_pages, dirty_page_count_ref, reset, is_runtime_suspended);
9550 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9551 UNREFERENCED_PARAMETER(is_runtime_suspended);
9552 bool success = GCToOSInterface::GetWriteWatch(reset, base_address, region_size, dirty_pages, dirty_page_count_ref);
9554 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9557 const size_t ww_reset_quantum = 128*1024*1024;
9560 void gc_heap::switch_one_quantum()
9562 enable_preemptive ();
9563 GCToOSInterface::Sleep (1);
9564 disable_preemptive (true);
9567 void gc_heap::reset_ww_by_chunk (uint8_t* start_address, size_t total_reset_size)
9569 size_t reset_size = 0;
9570 size_t remaining_reset_size = 0;
9571 size_t next_reset_size = 0;
9573 while (reset_size != total_reset_size)
9575 remaining_reset_size = total_reset_size - reset_size;
9576 next_reset_size = ((remaining_reset_size >= ww_reset_quantum) ? ww_reset_quantum : remaining_reset_size);
9577 if (next_reset_size)
9579 reset_write_watch_for_gc_heap(start_address, next_reset_size);
9580 reset_size += next_reset_size;
9582 switch_one_quantum();
9586 assert (reset_size == total_reset_size);
9589 // This does a Sleep(1) for every reset ww_reset_quantum bytes of reset
9590 // we do concurrently.
9591 void gc_heap::switch_on_reset (BOOL concurrent_p, size_t* current_total_reset_size, size_t last_reset_size)
9595 *current_total_reset_size += last_reset_size;
9597 dprintf (2, ("reset %Id bytes so far", *current_total_reset_size));
9599 if (*current_total_reset_size > ww_reset_quantum)
9601 switch_one_quantum();
9603 *current_total_reset_size = 0;
9608 void gc_heap::reset_write_watch (BOOL concurrent_p)
9610 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9611 // Software write watch currently requires the runtime to be suspended during reset. See SoftwareWriteWatch::ClearDirty().
9612 assert(!concurrent_p);
9613 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9615 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
9617 PREFIX_ASSUME(seg != NULL);
9619 size_t reset_size = 0;
9620 size_t region_size = 0;
9622 dprintf (2, ("bgc lowest: %Ix, bgc highest: %Ix", background_saved_lowest_address, background_saved_highest_address));
9626 uint8_t* base_address = align_lower_page (heap_segment_mem (seg));
9627 base_address = max (base_address, background_saved_lowest_address);
9629 uint8_t* high_address = 0;
9630 high_address = ((seg == ephemeral_heap_segment) ? alloc_allocated : heap_segment_allocated (seg));
9631 high_address = min (high_address, background_saved_highest_address);
9633 if (base_address < high_address)
9635 region_size = high_address - base_address;
9637 #ifdef TIME_WRITE_WATCH
9638 unsigned int time_start = GetCycleCount32();
9639 #endif //TIME_WRITE_WATCH
9640 dprintf (3, ("h%d: soh ww: [%Ix(%Id)", heap_number, (size_t)base_address, region_size));
9641 //reset_ww_by_chunk (base_address, region_size);
9642 reset_write_watch_for_gc_heap(base_address, region_size);
9644 #ifdef TIME_WRITE_WATCH
9645 unsigned int time_stop = GetCycleCount32();
9646 tot_cycles += time_stop - time_start;
9647 printf ("ResetWriteWatch Duration: %d, total: %d\n",
9648 time_stop - time_start, tot_cycles);
9649 #endif //TIME_WRITE_WATCH
9651 switch_on_reset (concurrent_p, &reset_size, region_size);
9654 seg = heap_segment_next_rw (seg);
9656 concurrent_print_time_delta ("CRWW soh");
9659 //concurrent_print_time_delta ("CRW soh");
9661 seg = heap_segment_rw (generation_start_segment (large_object_generation));
9663 PREFIX_ASSUME(seg != NULL);
9667 uint8_t* base_address = align_lower_page (heap_segment_mem (seg));
9668 uint8_t* high_address = heap_segment_allocated (seg);
9670 base_address = max (base_address, background_saved_lowest_address);
9671 high_address = min (high_address, background_saved_highest_address);
9673 if (base_address < high_address)
9675 region_size = high_address - base_address;
9677 #ifdef TIME_WRITE_WATCH
9678 unsigned int time_start = GetCycleCount32();
9679 #endif //TIME_WRITE_WATCH
9680 dprintf (3, ("h%d: loh ww: [%Ix(%Id)", heap_number, (size_t)base_address, region_size));
9681 //reset_ww_by_chunk (base_address, region_size);
9682 reset_write_watch_for_gc_heap(base_address, region_size);
9684 #ifdef TIME_WRITE_WATCH
9685 unsigned int time_stop = GetCycleCount32();
9686 tot_cycles += time_stop - time_start;
9687 printf ("ResetWriteWatch Duration: %d, total: %d\n",
9688 time_stop - time_start, tot_cycles);
9689 #endif //TIME_WRITE_WATCH
9691 switch_on_reset (concurrent_p, &reset_size, region_size);
9694 seg = heap_segment_next_rw (seg);
9696 concurrent_print_time_delta ("CRWW loh");
9699 #ifdef DEBUG_WRITE_WATCH
9700 debug_write_watch = (uint8_t**)~0;
9701 #endif //DEBUG_WRITE_WATCH
9704 #endif //WRITE_WATCH
9706 #ifdef BACKGROUND_GC
9707 void gc_heap::restart_vm()
9709 //assert (generation_allocation_pointer (youngest_generation) == 0);
9710 dprintf (3, ("Restarting EE"));
9711 STRESS_LOG0(LF_GC, LL_INFO10000, "Concurrent GC: Retarting EE\n");
9712 ee_proceed_event.Set();
9716 void fire_alloc_wait_event (alloc_wait_reason awr, BOOL begin_p)
9718 if (awr != awr_ignored)
9722 FIRE_EVENT(BGCAllocWaitBegin, awr);
9726 FIRE_EVENT(BGCAllocWaitEnd, awr);
9732 void gc_heap::fire_alloc_wait_event_begin (alloc_wait_reason awr)
9734 fire_alloc_wait_event (awr, TRUE);
9738 void gc_heap::fire_alloc_wait_event_end (alloc_wait_reason awr)
9740 fire_alloc_wait_event (awr, FALSE);
9742 #endif //BACKGROUND_GC
9743 void gc_heap::make_generation (generation& gen, heap_segment* seg, uint8_t* start, uint8_t* pointer)
9745 gen.allocation_start = start;
9746 gen.allocation_context.alloc_ptr = pointer;
9747 gen.allocation_context.alloc_limit = pointer;
9748 gen.allocation_context.alloc_bytes = 0;
9749 gen.allocation_context.alloc_bytes_loh = 0;
9750 gen.allocation_context_start_region = pointer;
9751 gen.start_segment = seg;
9752 gen.allocation_segment = seg;
9753 gen.plan_allocation_start = 0;
9754 gen.free_list_space = 0;
9755 gen.pinned_allocated = 0;
9756 gen.free_list_allocated = 0;
9757 gen.end_seg_allocated = 0;
9758 gen.condemned_allocated = 0;
9759 gen.free_obj_space = 0;
9760 gen.allocation_size = 0;
9761 gen.pinned_allocation_sweep_size = 0;
9762 gen.pinned_allocation_compact_size = 0;
9763 gen.allocate_end_seg_p = FALSE;
9764 gen.free_list_allocator.clear();
9766 #ifdef FREE_USAGE_STATS
9767 memset (gen.gen_free_spaces, 0, sizeof (gen.gen_free_spaces));
9768 memset (gen.gen_current_pinned_free_spaces, 0, sizeof (gen.gen_current_pinned_free_spaces));
9769 memset (gen.gen_plugs, 0, sizeof (gen.gen_plugs));
9770 #endif //FREE_USAGE_STATS
9773 void gc_heap::adjust_ephemeral_limits ()
9775 ephemeral_low = generation_allocation_start (generation_of (max_generation - 1));
9776 ephemeral_high = heap_segment_reserved (ephemeral_heap_segment);
9778 dprintf (3, ("new ephemeral low: %Ix new ephemeral high: %Ix",
9779 (size_t)ephemeral_low, (size_t)ephemeral_high))
9781 #ifndef MULTIPLE_HEAPS
9782 // This updates the write barrier helpers with the new info.
9783 stomp_write_barrier_ephemeral(ephemeral_low, ephemeral_high);
9784 #endif // MULTIPLE_HEAPS
9787 #if defined(TRACE_GC) || defined(GC_CONFIG_DRIVEN)
9788 FILE* CreateLogFile(const GCConfigStringHolder& temp_logfile_name, bool is_config)
9792 if (!temp_logfile_name.Get())
9797 char logfile_name[MAX_LONGPATH+1];
9798 uint32_t pid = GCToOSInterface::GetCurrentProcessId();
9799 const char* suffix = is_config ? ".config.log" : ".log";
9800 _snprintf_s(logfile_name, MAX_LONGPATH+1, _TRUNCATE, "%s.%d%s", temp_logfile_name.Get(), pid, suffix);
9801 logFile = fopen(logfile_name, "wb");
9804 #endif //TRACE_GC || GC_CONFIG_DRIVEN
9806 HRESULT gc_heap::initialize_gc (size_t segment_size,
9808 #ifdef MULTIPLE_HEAPS
9809 ,unsigned number_of_heaps
9810 #endif //MULTIPLE_HEAPS
9814 if (GCConfig::GetLogEnabled())
9816 gc_log = CreateLogFile(GCConfig::GetLogFile(), false);
9821 // GCLogFileSize in MBs.
9822 gc_log_file_size = static_cast<size_t>(GCConfig::GetLogFileSize());
9824 if (gc_log_file_size <= 0 || gc_log_file_size > 500)
9830 gc_log_lock.Initialize();
9831 gc_log_buffer = new (nothrow) uint8_t [gc_log_buffer_size];
9838 memset (gc_log_buffer, '*', gc_log_buffer_size);
9840 max_gc_buffers = gc_log_file_size * 1024 * 1024 / gc_log_buffer_size;
9844 #ifdef GC_CONFIG_DRIVEN
9845 if (GCConfig::GetConfigLogEnabled())
9847 gc_config_log = CreateLogFile(GCConfig::GetConfigLogFile(), true);
9849 if (gc_config_log == NULL)
9852 gc_config_log_buffer = new (nothrow) uint8_t [gc_config_log_buffer_size];
9853 if (!gc_config_log_buffer)
9855 fclose(gc_config_log);
9859 compact_ratio = static_cast<int>(GCConfig::GetCompactRatio());
9861 // h# | GC | gen | C | EX | NF | BF | ML | DM || PreS | PostS | Merge | Conv | Pre | Post | PrPo | PreP | PostP |
9862 cprintf (("%2s | %6s | %1s | %1s | %2s | %2s | %2s | %2s | %2s || %5s | %5s | %5s | %5s | %5s | %5s | %5s | %5s | %5s |",
9866 "C", // compaction (empty means sweeping), 'M' means it was mandatory, 'W' means it was not
9867 "EX", // heap expansion
9869 "BF", // best fit (if it indicates neither NF nor BF it means it had to acquire a new seg.
9872 "PreS", // short object before pinned plug
9873 "PostS", // short object after pinned plug
9874 "Merge", // merged pinned plugs
9875 "Conv", // converted to pinned plug
9876 "Pre", // plug before pinned plug but not after
9877 "Post", // plug after pinned plug but not before
9878 "PrPo", // plug both before and after pinned plug
9879 "PreP", // pre short object padded
9880 "PostP" // post short object padded
9883 #endif //GC_CONFIG_DRIVEN
9886 GCConfigStringHolder logFileName = GCConfig::GetMixLogFile();
9887 if (logFileName.Get() != nullptr)
9889 GCStatistics::logFileName = _strdup(logFileName.Get());
9890 GCStatistics::logFile = fopen(GCStatistics::logFileName, "a");
9891 if (!GCStatistics::logFile)
9898 HRESULT hres = S_OK;
9901 hardware_write_watch_api_supported();
9902 #ifdef BACKGROUND_GC
9903 if (can_use_write_watch_for_gc_heap() && GCConfig::GetConcurrentGC())
9905 gc_can_use_concurrent = true;
9906 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9907 virtual_alloc_hardware_write_watch = true;
9908 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9912 gc_can_use_concurrent = false;
9914 #endif //BACKGROUND_GC
9915 #endif //WRITE_WATCH
9917 #ifdef BACKGROUND_GC
9918 // leave the first page to contain only segment info
9919 // because otherwise we could need to revisit the first page frequently in
9921 segment_info_size = OS_PAGE_SIZE;
9923 segment_info_size = Align (sizeof (heap_segment), get_alignment_constant (FALSE));
9924 #endif //BACKGROUND_GC
9926 reserved_memory = 0;
9927 unsigned block_count;
9928 #ifdef MULTIPLE_HEAPS
9929 reserved_memory_limit = (segment_size + heap_size) * number_of_heaps;
9930 block_count = number_of_heaps;
9931 #else //MULTIPLE_HEAPS
9932 reserved_memory_limit = segment_size + heap_size;
9934 #endif //MULTIPLE_HEAPS
9936 if (!reserve_initial_memory(segment_size,heap_size,block_count))
9937 return E_OUTOFMEMORY;
9940 //check if we need to turn on card_bundles.
9941 #ifdef MULTIPLE_HEAPS
9942 // use INT64 arithmetic here because of possible overflow on 32p
9943 uint64_t th = (uint64_t)MH_TH_CARD_BUNDLE*number_of_heaps;
9945 // use INT64 arithmetic here because of possible overflow on 32p
9946 uint64_t th = (uint64_t)SH_TH_CARD_BUNDLE;
9947 #endif //MULTIPLE_HEAPS
9949 if (can_use_write_watch_for_card_table() && reserved_memory >= th)
9951 settings.card_bundles = TRUE;
9955 settings.card_bundles = FALSE;
9957 #endif //CARD_BUNDLE
9959 settings.first_init();
9961 int latency_level_from_config = static_cast<int>(GCConfig::GetLatencyLevel());
9962 if (latency_level_from_config >= latency_level_first && latency_level_from_config <= latency_level_last)
9964 gc_heap::latency_level = static_cast<gc_latency_level>(latency_level_from_config);
9969 g_gc_card_table = make_card_table (g_gc_lowest_address, g_gc_highest_address);
9971 if (!g_gc_card_table)
9972 return E_OUTOFMEMORY;
9976 #ifdef MULTIPLE_HEAPS
9977 n_heaps = number_of_heaps;
9979 g_heaps = new (nothrow) gc_heap* [number_of_heaps];
9981 return E_OUTOFMEMORY;
9984 #pragma warning(push)
9985 #pragma warning(disable:22011) // Suppress PREFast warning about integer underflow/overflow
9987 g_promoted = new (nothrow) size_t [number_of_heaps*16];
9988 g_bpromoted = new (nothrow) size_t [number_of_heaps*16];
9990 g_mark_stack_busy = new (nothrow) int[(number_of_heaps+2)*HS_CACHE_LINE_SIZE/sizeof(int)];
9993 #pragma warning(pop)
9995 if (!g_promoted || !g_bpromoted)
9996 return E_OUTOFMEMORY;
9999 if (!g_mark_stack_busy)
10000 return E_OUTOFMEMORY;
10001 #endif //MH_SC_MARK
10003 if (!create_thread_support (number_of_heaps))
10004 return E_OUTOFMEMORY;
10006 if (!heap_select::init (number_of_heaps))
10007 return E_OUTOFMEMORY;
10009 #endif //MULTIPLE_HEAPS
10011 if (!init_semi_shared())
10019 //Initializes PER_HEAP_ISOLATED data members.
10021 gc_heap::init_semi_shared()
10025 // This is used for heap expansion - it's to fix exactly the start for gen 0
10026 // through (max_generation-1). When we expand the heap we allocate all these
10027 // gen starts at the beginning of the new ephemeral seg.
10028 eph_gen_starts_size = (Align (min_obj_size)) * max_generation;
10031 #ifdef MULTIPLE_HEAPS
10032 mark_list_size = min (150*1024, max (8192, soh_segment_size/(2*10*32)));
10033 g_mark_list = make_mark_list (mark_list_size*n_heaps);
10035 min_balance_threshold = alloc_quantum_balance_units * CLR_SIZE * 2;
10036 #ifdef PARALLEL_MARK_LIST_SORT
10037 g_mark_list_copy = make_mark_list (mark_list_size*n_heaps);
10038 if (!g_mark_list_copy)
10042 #endif //PARALLEL_MARK_LIST_SORT
10044 #else //MULTIPLE_HEAPS
10046 mark_list_size = max (8192, soh_segment_size/(64*32));
10047 g_mark_list = make_mark_list (mark_list_size);
10049 #endif //MULTIPLE_HEAPS
10051 dprintf (3, ("mark_list_size: %d", mark_list_size));
10059 #if defined(SEG_MAPPING_TABLE) && !defined(GROWABLE_SEG_MAPPING_TABLE)
10060 if (!seg_mapping_table_init())
10062 #endif //SEG_MAPPING_TABLE && !GROWABLE_SEG_MAPPING_TABLE
10064 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
10065 seg_table = sorted_table::make_sorted_table();
10069 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
10071 segment_standby_list = 0;
10073 if (!full_gc_approach_event.CreateManualEventNoThrow(FALSE))
10077 if (!full_gc_end_event.CreateManualEventNoThrow(FALSE))
10082 fgn_maxgen_percent = 0;
10083 fgn_loh_percent = 0;
10084 full_gc_approach_event_set = false;
10086 memset (full_gc_counts, 0, sizeof (full_gc_counts));
10089 should_expand_in_full_gc = FALSE;
10091 #ifdef FEATURE_LOH_COMPACTION
10092 loh_compaction_always_p = GCConfig::GetLOHCompactionMode() != 0;
10093 loh_compaction_mode = loh_compaction_default;
10094 #endif //FEATURE_LOH_COMPACTION
10096 #ifdef BACKGROUND_GC
10097 memset (ephemeral_fgc_counts, 0, sizeof (ephemeral_fgc_counts));
10098 bgc_alloc_spin_count = static_cast<uint32_t>(GCConfig::GetBGCSpinCount());
10099 bgc_alloc_spin = static_cast<uint32_t>(GCConfig::GetBGCSpin());
10102 int number_bgc_threads = 1;
10103 #ifdef MULTIPLE_HEAPS
10104 number_bgc_threads = n_heaps;
10105 #endif //MULTIPLE_HEAPS
10106 if (!create_bgc_threads_support (number_bgc_threads))
10111 #endif //BACKGROUND_GC
10113 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
10115 #ifdef GC_CONFIG_DRIVEN
10116 compact_or_sweep_gcs[0] = 0;
10117 compact_or_sweep_gcs[1] = 0;
10118 #endif //GC_CONFIG_DRIVEN
10121 short_plugs_pad_ratio = (double)DESIRED_PLUG_LENGTH / (double)(DESIRED_PLUG_LENGTH - Align (min_obj_size));
10122 #endif //SHORT_PLUGS
10130 if (full_gc_approach_event.IsValid())
10132 full_gc_approach_event.CloseEvent();
10134 if (full_gc_end_event.IsValid())
10136 full_gc_end_event.CloseEvent();
10143 gc_heap* gc_heap::make_gc_heap (
10144 #ifdef MULTIPLE_HEAPS
10147 #endif //MULTIPLE_HEAPS
10152 #ifdef MULTIPLE_HEAPS
10153 res = new (nothrow) gc_heap;
10157 res->vm_heap = vm_hp;
10158 res->alloc_context_count = 0;
10161 #ifdef PARALLEL_MARK_LIST_SORT
10162 res->mark_list_piece_start = new (nothrow) uint8_t**[n_heaps];
10163 if (!res->mark_list_piece_start)
10167 #pragma warning(push)
10168 #pragma warning(disable:22011) // Suppress PREFast warning about integer underflow/overflow
10169 #endif // _PREFAST_
10170 res->mark_list_piece_end = new (nothrow) uint8_t**[n_heaps + 32]; // +32 is padding to reduce false sharing
10172 #pragma warning(pop)
10173 #endif // _PREFAST_
10175 if (!res->mark_list_piece_end)
10177 #endif //PARALLEL_MARK_LIST_SORT
10181 #endif //MULTIPLE_HEAPS
10183 if (res->init_gc_heap (
10184 #ifdef MULTIPLE_HEAPS
10186 #else //MULTIPLE_HEAPS
10188 #endif //MULTIPLE_HEAPS
10194 #ifdef MULTIPLE_HEAPS
10197 return (gc_heap*)1;
10198 #endif //MULTIPLE_HEAPS
10202 gc_heap::wait_for_gc_done(int32_t timeOut)
10204 bool cooperative_mode = enable_preemptive ();
10206 uint32_t dwWaitResult = NOERROR;
10208 gc_heap* wait_heap = NULL;
10209 while (gc_heap::gc_started)
10211 #ifdef MULTIPLE_HEAPS
10212 wait_heap = GCHeap::GetHeap(heap_select::select_heap(NULL, 0))->pGenGCHeap;
10213 dprintf(2, ("waiting for the gc_done_event on heap %d", wait_heap->heap_number));
10214 #endif // MULTIPLE_HEAPS
10217 PREFIX_ASSUME(wait_heap != NULL);
10218 #endif // _PREFAST_
10220 dwWaitResult = wait_heap->gc_done_event.Wait(timeOut, FALSE);
10222 disable_preemptive (cooperative_mode);
10224 return dwWaitResult;
10228 gc_heap::set_gc_done()
10230 enter_gc_done_event_lock();
10231 if (!gc_done_event_set)
10233 gc_done_event_set = true;
10234 dprintf (2, ("heap %d: setting gc_done_event", heap_number));
10235 gc_done_event.Set();
10237 exit_gc_done_event_lock();
10241 gc_heap::reset_gc_done()
10243 enter_gc_done_event_lock();
10244 if (gc_done_event_set)
10246 gc_done_event_set = false;
10247 dprintf (2, ("heap %d: resetting gc_done_event", heap_number));
10248 gc_done_event.Reset();
10250 exit_gc_done_event_lock();
10254 gc_heap::enter_gc_done_event_lock()
10256 uint32_t dwSwitchCount = 0;
10259 if (Interlocked::CompareExchange(&gc_done_event_lock, 0, -1) >= 0)
10261 while (gc_done_event_lock >= 0)
10263 if (g_num_processors > 1)
10265 int spin_count = 32 * g_num_processors;
10266 for (int j = 0; j < spin_count; j++)
10268 if (gc_done_event_lock < 0)
10270 YieldProcessor(); // indicate to the processor that we are spining
10272 if (gc_done_event_lock >= 0)
10273 GCToOSInterface::YieldThread(++dwSwitchCount);
10276 GCToOSInterface::YieldThread(++dwSwitchCount);
10283 gc_heap::exit_gc_done_event_lock()
10285 gc_done_event_lock = -1;
10288 #ifndef MULTIPLE_HEAPS
10290 #ifdef RECORD_LOH_STATE
10291 int gc_heap::loh_state_index = 0;
10292 gc_heap::loh_state_info gc_heap::last_loh_states[max_saved_loh_states];
10293 #endif //RECORD_LOH_STATE
10295 VOLATILE(int32_t) gc_heap::gc_done_event_lock;
10296 VOLATILE(bool) gc_heap::gc_done_event_set;
10297 GCEvent gc_heap::gc_done_event;
10298 #endif //!MULTIPLE_HEAPS
10299 VOLATILE(bool) gc_heap::internal_gc_done;
10301 void gc_heap::add_saved_spinlock_info (
10302 msl_enter_state enter_state,
10303 msl_take_state take_state)
10306 #ifdef SPINLOCK_HISTORY
10307 spinlock_info* current = &last_spinlock_info[spinlock_info_index];
10309 current->enter_state = enter_state;
10310 current->take_state = take_state;
10311 current->thread_id.SetToCurrentThread();
10313 spinlock_info_index++;
10315 assert (spinlock_info_index <= max_saved_spinlock_info);
10317 if (spinlock_info_index >= max_saved_spinlock_info)
10319 spinlock_info_index = 0;
10322 MAYBE_UNUSED_VAR(enter_state);
10323 MAYBE_UNUSED_VAR(take_state);
10324 #endif //SPINLOCK_HISTORY
10328 gc_heap::init_gc_heap (int h_number)
10330 #ifdef MULTIPLE_HEAPS
10334 #ifdef SPINLOCK_HISTORY
10335 spinlock_info_index = 0;
10336 memset (last_spinlock_info, 0, sizeof(last_spinlock_info));
10337 #endif //SPINLOCK_HISTORY
10339 // initialize per heap members.
10340 ephemeral_low = (uint8_t*)1;
10342 ephemeral_high = MAX_PTR;
10344 ephemeral_heap_segment = 0;
10346 freeable_large_heap_segment = 0;
10348 condemned_generation_num = 0;
10350 blocking_collection = FALSE;
10352 generation_skip_ratio = 100;
10354 mark_stack_tos = 0;
10356 mark_stack_bos = 0;
10358 mark_stack_array_length = 0;
10360 mark_stack_array = 0;
10362 verify_pinned_queue_p = FALSE;
10364 loh_pinned_queue_tos = 0;
10366 loh_pinned_queue_bos = 0;
10368 loh_pinned_queue_length = 0;
10370 loh_pinned_queue_decay = LOH_PIN_DECAY;
10372 loh_pinned_queue = 0;
10374 min_overflow_address = MAX_PTR;
10376 max_overflow_address = 0;
10378 gen0_bricks_cleared = FALSE;
10380 gen0_must_clear_bricks = 0;
10382 allocation_quantum = CLR_SIZE;
10384 more_space_lock = gc_lock;
10386 ro_segments_in_range = FALSE;
10388 loh_alloc_since_cg = 0;
10390 new_heap_segment = NULL;
10392 #ifdef RECORD_LOH_STATE
10393 loh_state_index = 0;
10394 #endif //RECORD_LOH_STATE
10395 #endif //MULTIPLE_HEAPS
10397 #ifdef MULTIPLE_HEAPS
10398 if (h_number > n_heaps)
10400 assert (!"Number of heaps exceeded");
10404 heap_number = h_number;
10405 #endif //MULTIPLE_HEAPS
10407 memset (&oom_info, 0, sizeof (oom_info));
10408 memset (&fgm_result, 0, sizeof (fgm_result));
10409 if (!gc_done_event.CreateManualEventNoThrow(FALSE))
10413 gc_done_event_lock = -1;
10414 gc_done_event_set = false;
10416 #ifndef SEG_MAPPING_TABLE
10417 if (!gc_heap::seg_table->ensure_space_for_insert ())
10421 #endif //!SEG_MAPPING_TABLE
10423 heap_segment* seg = get_initial_segment (soh_segment_size, h_number);
10427 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(seg),
10428 (size_t)(heap_segment_reserved (seg) - heap_segment_mem(seg)),
10429 gc_etw_segment_small_object_heap);
10431 #ifdef SEG_MAPPING_TABLE
10432 seg_mapping_table_add_segment (seg, __this);
10433 #else //SEG_MAPPING_TABLE
10434 seg_table->insert ((uint8_t*)seg, sdelta);
10435 #endif //SEG_MAPPING_TABLE
10437 #ifdef MULTIPLE_HEAPS
10438 heap_segment_heap (seg) = this;
10439 #endif //MULTIPLE_HEAPS
10441 /* todo: Need a global lock for this */
10442 uint32_t* ct = &g_gc_card_table [card_word (card_of (g_gc_lowest_address))];
10443 own_card_table (ct);
10444 card_table = translate_card_table (ct);
10445 /* End of global lock */
10447 brick_table = card_table_brick_table (ct);
10448 highest_address = card_table_highest_address (ct);
10449 lowest_address = card_table_lowest_address (ct);
10452 card_bundle_table = translate_card_bundle_table (card_table_card_bundle_table (ct), g_gc_lowest_address);
10453 assert (&card_bundle_table [card_bundle_word (cardw_card_bundle (card_word (card_of (g_gc_lowest_address))))] ==
10454 card_table_card_bundle_table (ct));
10455 #endif //CARD_BUNDLE
10458 if (gc_can_use_concurrent)
10459 mark_array = translate_mark_array (card_table_mark_array (&g_gc_card_table[card_word (card_of (g_gc_lowest_address))]));
10462 #endif //MARK_ARRAY
10464 uint8_t* start = heap_segment_mem (seg);
10466 for (int i = 0; i < 1 + max_generation; i++)
10468 make_generation (generation_table [ (max_generation - i) ],
10470 generation_table [(max_generation - i)].gen_num = max_generation - i;
10471 start += Align (min_obj_size);
10474 heap_segment_allocated (seg) = start;
10475 alloc_allocated = start;
10476 heap_segment_used (seg) = start - plug_skew;
10478 ephemeral_heap_segment = seg;
10480 #ifndef SEG_MAPPING_TABLE
10481 if (!gc_heap::seg_table->ensure_space_for_insert ())
10485 #endif //!SEG_MAPPING_TABLE
10486 //Create the large segment generation
10487 heap_segment* lseg = get_initial_segment(min_loh_segment_size, h_number);
10490 lseg->flags |= heap_segment_flags_loh;
10492 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(lseg),
10493 (size_t)(heap_segment_reserved (lseg) - heap_segment_mem(lseg)),
10494 gc_etw_segment_large_object_heap);
10496 #ifdef SEG_MAPPING_TABLE
10497 seg_mapping_table_add_segment (lseg, __this);
10498 #else //SEG_MAPPING_TABLE
10499 seg_table->insert ((uint8_t*)lseg, sdelta);
10500 #endif //SEG_MAPPING_TABLE
10502 generation_table [max_generation].free_list_allocator = allocator(NUM_GEN2_ALIST, BASE_GEN2_ALIST, gen2_alloc_list);
10503 //assign the alloc_list for the large generation
10504 generation_table [max_generation+1].free_list_allocator = allocator(NUM_LOH_ALIST, BASE_LOH_ALIST, loh_alloc_list);
10505 generation_table [max_generation+1].gen_num = max_generation+1;
10506 make_generation (generation_table [max_generation+1],lseg, heap_segment_mem (lseg), 0);
10507 heap_segment_allocated (lseg) = heap_segment_mem (lseg) + Align (min_obj_size, get_alignment_constant (FALSE));
10508 heap_segment_used (lseg) = heap_segment_allocated (lseg) - plug_skew;
10510 for (int gen_num = 0; gen_num <= 1 + max_generation; gen_num++)
10512 generation* gen = generation_of (gen_num);
10513 make_unused_array (generation_allocation_start (gen), Align (min_obj_size));
10516 #ifdef MULTIPLE_HEAPS
10517 heap_segment_heap (lseg) = this;
10519 //initialize the alloc context heap
10520 generation_alloc_context (generation_of (0))->set_alloc_heap(vm_heap);
10522 //initialize the alloc context heap
10523 generation_alloc_context (generation_of (max_generation+1))->set_alloc_heap(vm_heap);
10525 #endif //MULTIPLE_HEAPS
10527 //Do this only once
10528 #ifdef MULTIPLE_HEAPS
10530 #endif //MULTIPLE_HEAPS
10532 #ifndef INTERIOR_POINTERS
10533 //set the brick_table for large objects
10534 //but default value is clearded
10535 //clear_brick_table ((uint8_t*)heap_segment_mem (lseg),
10536 // (uint8_t*)heap_segment_reserved (lseg));
10538 #else //INTERIOR_POINTERS
10540 //Because of the interior pointer business, we have to clear
10541 //the whole brick table
10542 //but the default value is cleared
10543 // clear_brick_table (lowest_address, highest_address);
10544 #endif //INTERIOR_POINTERS
10547 if (!init_dynamic_data())
10552 etw_allocation_running_amount[0] = 0;
10553 etw_allocation_running_amount[1] = 0;
10555 //needs to be done after the dynamic data has been initialized
10556 #ifndef MULTIPLE_HEAPS
10557 allocation_running_amount = dd_min_size (dynamic_data_of (0));
10558 #endif //!MULTIPLE_HEAPS
10560 fgn_last_alloc = dd_min_size (dynamic_data_of (0));
10562 mark* arr = new (nothrow) (mark [MARK_STACK_INITIAL_LENGTH]);
10566 make_mark_stack(arr);
10568 #ifdef BACKGROUND_GC
10569 freeable_small_heap_segment = 0;
10570 gchist_index_per_heap = 0;
10571 uint8_t** b_arr = new (nothrow) (uint8_t* [MARK_STACK_INITIAL_LENGTH]);
10575 make_background_mark_stack (b_arr);
10576 #endif //BACKGROUND_GC
10578 ephemeral_low = generation_allocation_start(generation_of(max_generation - 1));
10579 ephemeral_high = heap_segment_reserved(ephemeral_heap_segment);
10580 if (heap_number == 0)
10582 stomp_write_barrier_initialize(
10583 #ifdef MULTIPLE_HEAPS
10584 reinterpret_cast<uint8_t*>(1), reinterpret_cast<uint8_t*>(~0)
10586 ephemeral_low, ephemeral_high
10587 #endif //!MULTIPLE_HEAPS
10592 // why would we clear the mark array for this page? it should be cleared..
10593 // clear the first committed page
10594 //if(gc_can_use_concurrent)
10596 // clear_mark_array (align_lower_page (heap_segment_mem (seg)), heap_segment_committed (seg));
10598 #endif //MARK_ARRAY
10600 #ifdef MULTIPLE_HEAPS
10601 //register the heap in the heaps array
10603 if (!create_gc_thread ())
10606 g_heaps [heap_number] = this;
10608 #endif //MULTIPLE_HEAPS
10610 #ifdef FEATURE_PREMORTEM_FINALIZATION
10611 HRESULT hr = AllocateCFinalize(&finalize_queue);
10614 #endif // FEATURE_PREMORTEM_FINALIZATION
10616 max_free_space_items = MAX_NUM_FREE_SPACES;
10618 bestfit_seg = new (nothrow) seg_free_spaces (heap_number);
10625 if (!bestfit_seg->alloc())
10630 last_gc_before_oom = FALSE;
10632 #ifdef MULTIPLE_HEAPS
10634 #ifdef HEAP_ANALYZE
10636 heap_analyze_success = TRUE;
10638 internal_root_array = 0;
10640 internal_root_array_index = 0;
10642 internal_root_array_length = initial_internal_roots;
10646 current_obj_size = 0;
10648 #endif //HEAP_ANALYZE
10650 #endif // MULTIPLE_HEAPS
10652 #ifdef BACKGROUND_GC
10653 bgc_thread_id.Clear();
10655 if (!create_bgc_thread_support())
10660 bgc_alloc_lock = new (nothrow) exclusive_sync;
10661 if (!bgc_alloc_lock)
10666 bgc_alloc_lock->init();
10670 if (!recursive_gc_sync::init())
10674 bgc_thread_running = 0;
10676 bgc_threads_timeout_cs.Initialize();
10677 expanded_in_fgc = 0;
10678 current_bgc_state = bgc_not_in_process;
10679 background_soh_alloc_count = 0;
10680 background_loh_alloc_count = 0;
10681 bgc_overflow_count = 0;
10682 end_loh_size = dd_min_size (dynamic_data_of (max_generation + 1));
10683 #endif //BACKGROUND_GC
10685 #ifdef GC_CONFIG_DRIVEN
10686 memset (interesting_data_per_heap, 0, sizeof (interesting_data_per_heap));
10687 memset(compact_reasons_per_heap, 0, sizeof (compact_reasons_per_heap));
10688 memset(expand_mechanisms_per_heap, 0, sizeof (expand_mechanisms_per_heap));
10689 memset(interesting_mechanism_bits_per_heap, 0, sizeof (interesting_mechanism_bits_per_heap));
10690 #endif //GC_CONFIG_DRIVEN
10696 gc_heap::destroy_semi_shared()
10698 //TODO: will need to move this to per heap
10699 //#ifdef BACKGROUND_GC
10700 // if (c_mark_list)
10701 // delete c_mark_list;
10702 //#endif //BACKGROUND_GC
10706 delete g_mark_list;
10709 #if defined(SEG_MAPPING_TABLE) && !defined(GROWABLE_SEG_MAPPING_TABLE)
10710 if (seg_mapping_table)
10711 delete seg_mapping_table;
10712 #endif //SEG_MAPPING_TABLE && !GROWABLE_SEG_MAPPING_TABLE
10714 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
10715 //destroy the segment map
10716 seg_table->delete_sorted_table();
10717 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
10721 gc_heap::self_destroy()
10723 #ifdef BACKGROUND_GC
10725 #endif //BACKGROUND_GC
10727 if (gc_done_event.IsValid())
10729 gc_done_event.CloseEvent();
10732 // destroy every segment.
10733 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
10735 PREFIX_ASSUME(seg != NULL);
10737 heap_segment* next_seg;
10740 next_seg = heap_segment_next_rw (seg);
10741 delete_heap_segment (seg);
10745 seg = heap_segment_rw (generation_start_segment (generation_of (max_generation+1)));
10747 PREFIX_ASSUME(seg != NULL);
10751 next_seg = heap_segment_next_rw (seg);
10752 delete_heap_segment (seg);
10756 // get rid of the card table
10757 release_card_table (card_table);
10759 // destroy the mark stack
10760 delete mark_stack_array;
10762 #ifdef FEATURE_PREMORTEM_FINALIZATION
10763 if (finalize_queue)
10764 delete finalize_queue;
10765 #endif // FEATURE_PREMORTEM_FINALIZATION
10769 gc_heap::destroy_gc_heap(gc_heap* heap)
10771 heap->self_destroy();
10775 // Destroys resources owned by gc. It is assumed that a last GC has been performed and that
10776 // the finalizer queue has been drained.
10777 void gc_heap::shutdown_gc()
10779 destroy_semi_shared();
10781 #ifdef MULTIPLE_HEAPS
10782 //delete the heaps array
10784 destroy_thread_support();
10786 #endif //MULTIPLE_HEAPS
10787 //destroy seg_manager
10789 destroy_initial_memory();
10791 GCToOSInterface::Shutdown();
10795 BOOL gc_heap::size_fit_p (size_t size REQD_ALIGN_AND_OFFSET_DCL, uint8_t* alloc_pointer, uint8_t* alloc_limit,
10796 uint8_t* old_loc, int use_padding)
10798 BOOL already_padded = FALSE;
10800 if ((old_loc != 0) && (use_padding & USE_PADDING_FRONT))
10802 alloc_pointer = alloc_pointer + Align (min_obj_size);
10803 already_padded = TRUE;
10805 #endif //SHORT_PLUGS
10807 if (!((old_loc == 0) || same_large_alignment_p (old_loc, alloc_pointer)))
10808 size = size + switch_alignment_size (already_padded);
10810 #ifdef FEATURE_STRUCTALIGN
10811 alloc_pointer = StructAlign(alloc_pointer, requiredAlignment, alignmentOffset);
10812 #endif // FEATURE_STRUCTALIGN
10814 // in allocate_in_condemned_generation we can have this when we
10815 // set the alloc_limit to plan_allocated which could be less than
10817 if (alloc_limit < alloc_pointer)
10824 return (((size_t)(alloc_limit - alloc_pointer) >= (size + ((use_padding & USE_PADDING_TAIL)? Align(min_obj_size) : 0)))
10826 ||((!(use_padding & USE_PADDING_FRONT)) && ((alloc_pointer + size) == alloc_limit))
10827 #else //SHORT_PLUGS
10828 ||((alloc_pointer + size) == alloc_limit)
10829 #endif //SHORT_PLUGS
10834 assert (size == Align (min_obj_size));
10835 return ((size_t)(alloc_limit - alloc_pointer) >= size);
10840 BOOL gc_heap::a_size_fit_p (size_t size, uint8_t* alloc_pointer, uint8_t* alloc_limit,
10843 // We could have run into cases where this is true when alloc_allocated is the
10844 // the same as the seg committed.
10845 if (alloc_limit < alloc_pointer)
10850 return ((size_t)(alloc_limit - alloc_pointer) >= (size + Align(min_obj_size, align_const)));
10853 // Grow by committing more pages
10854 BOOL gc_heap::grow_heap_segment (heap_segment* seg, uint8_t* high_address)
10856 assert (high_address <= heap_segment_reserved (seg));
10858 //return 0 if we are at the end of the segment.
10859 if (align_on_page (high_address) > heap_segment_reserved (seg))
10862 if (high_address <= heap_segment_committed (seg))
10865 size_t c_size = align_on_page ((size_t)(high_address - heap_segment_committed (seg)));
10866 c_size = max (c_size, 16*OS_PAGE_SIZE);
10867 c_size = min (c_size, (size_t)(heap_segment_reserved (seg) - heap_segment_committed (seg)));
10872 STRESS_LOG2(LF_GC, LL_INFO10000,
10873 "Growing heap_segment: %Ix high address: %Ix\n",
10874 (size_t)seg, (size_t)high_address);
10876 dprintf(3, ("Growing segment allocation %Ix %Ix", (size_t)heap_segment_committed(seg),c_size));
10878 if (!virtual_alloc_commit_for_heap(heap_segment_committed (seg), c_size, heap_number))
10880 dprintf(3, ("Cannot grow heap segment"));
10884 #ifndef BACKGROUND_GC
10885 clear_mark_array (heap_segment_committed (seg),
10886 heap_segment_committed (seg)+c_size, TRUE);
10887 #endif //BACKGROUND_GC
10888 #endif //MARK_ARRAY
10889 heap_segment_committed (seg) += c_size;
10890 STRESS_LOG1(LF_GC, LL_INFO10000, "New commit: %Ix",
10891 (size_t)heap_segment_committed (seg));
10893 assert (heap_segment_committed (seg) <= heap_segment_reserved (seg));
10895 assert (high_address <= heap_segment_committed (seg));
10901 int gc_heap::grow_heap_segment (heap_segment* seg, uint8_t* allocated, uint8_t* old_loc, size_t size, BOOL pad_front_p REQD_ALIGN_AND_OFFSET_DCL)
10904 if ((old_loc != 0) && pad_front_p)
10906 allocated = allocated + Align (min_obj_size);
10908 #endif //SHORT_PLUGS
10910 if (!((old_loc == 0) || same_large_alignment_p (old_loc, allocated)))
10911 size = size + switch_alignment_size (FALSE);
10912 #ifdef FEATURE_STRUCTALIGN
10913 size_t pad = ComputeStructAlignPad(allocated, requiredAlignment, alignmentOffset);
10914 return grow_heap_segment (seg, allocated + pad + size);
10915 #else // FEATURE_STRUCTALIGN
10916 return grow_heap_segment (seg, allocated + size);
10917 #endif // FEATURE_STRUCTALIGN
10920 //used only in older generation allocation (i.e during gc).
10921 void gc_heap::adjust_limit (uint8_t* start, size_t limit_size, generation* gen,
10924 UNREFERENCED_PARAMETER(gennum);
10925 dprintf (3, ("gc Expanding segment allocation"));
10926 heap_segment* seg = generation_allocation_segment (gen);
10927 if ((generation_allocation_limit (gen) != start) || (start != heap_segment_plan_allocated (seg)))
10929 if (generation_allocation_limit (gen) == heap_segment_plan_allocated (seg))
10931 assert (generation_allocation_pointer (gen) >= heap_segment_mem (seg));
10932 assert (generation_allocation_pointer (gen) <= heap_segment_committed (seg));
10933 heap_segment_plan_allocated (generation_allocation_segment (gen)) = generation_allocation_pointer (gen);
10937 uint8_t* hole = generation_allocation_pointer (gen);
10938 size_t size = (generation_allocation_limit (gen) - generation_allocation_pointer (gen));
10942 dprintf (3, ("filling up hole: %Ix, size %Ix", hole, size));
10943 size_t allocated_size = generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen);
10944 if (size >= Align (min_free_list))
10946 if (allocated_size < min_free_list)
10948 if (size >= (Align (min_free_list) + Align (min_obj_size)))
10950 //split hole into min obj + threadable free item
10951 make_unused_array (hole, min_obj_size);
10952 generation_free_obj_space (gen) += Align (min_obj_size);
10953 make_unused_array (hole + Align (min_obj_size), size - Align (min_obj_size));
10954 generation_free_list_space (gen) += size - Align (min_obj_size);
10955 generation_allocator(gen)->thread_item_front (hole + Align (min_obj_size),
10956 size - Align (min_obj_size));
10957 add_gen_free (gen->gen_num, (size - Align (min_obj_size)));
10961 dprintf (3, ("allocated size too small, can't put back rest on free list %Ix", allocated_size));
10962 make_unused_array (hole, size);
10963 generation_free_obj_space (gen) += size;
10968 dprintf (3, ("threading hole in front of free list"));
10969 make_unused_array (hole, size);
10970 generation_free_list_space (gen) += size;
10971 generation_allocator(gen)->thread_item_front (hole, size);
10972 add_gen_free (gen->gen_num, size);
10977 make_unused_array (hole, size);
10978 generation_free_obj_space (gen) += size;
10982 generation_allocation_pointer (gen) = start;
10983 generation_allocation_context_start_region (gen) = start;
10985 generation_allocation_limit (gen) = (start + limit_size);
10988 void verify_mem_cleared (uint8_t* start, size_t size)
10990 if (!Aligned (size))
10995 PTR_PTR curr_ptr = (PTR_PTR) start;
10996 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
10998 if (*(curr_ptr++) != 0)
11005 #if defined (VERIFY_HEAP) && defined (BACKGROUND_GC)
11006 void gc_heap::set_batch_mark_array_bits (uint8_t* start, uint8_t* end)
11008 size_t start_mark_bit = mark_bit_of (start);
11009 size_t end_mark_bit = mark_bit_of (end);
11010 unsigned int startbit = mark_bit_bit (start_mark_bit);
11011 unsigned int endbit = mark_bit_bit (end_mark_bit);
11012 size_t startwrd = mark_bit_word (start_mark_bit);
11013 size_t endwrd = mark_bit_word (end_mark_bit);
11015 dprintf (3, ("Setting all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
11016 (size_t)start, (size_t)start_mark_bit,
11017 (size_t)end, (size_t)end_mark_bit));
11019 unsigned int firstwrd = ~(lowbits (~0, startbit));
11020 unsigned int lastwrd = ~(highbits (~0, endbit));
11022 if (startwrd == endwrd)
11024 unsigned int wrd = firstwrd & lastwrd;
11025 mark_array[startwrd] |= wrd;
11029 // set the first mark word.
11032 mark_array[startwrd] |= firstwrd;
11036 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
11038 mark_array[wrdtmp] = ~(unsigned int)0;
11041 // set the last mark word.
11044 mark_array[endwrd] |= lastwrd;
11048 // makes sure that the mark array bits between start and end are 0.
11049 void gc_heap::check_batch_mark_array_bits (uint8_t* start, uint8_t* end)
11051 size_t start_mark_bit = mark_bit_of (start);
11052 size_t end_mark_bit = mark_bit_of (end);
11053 unsigned int startbit = mark_bit_bit (start_mark_bit);
11054 unsigned int endbit = mark_bit_bit (end_mark_bit);
11055 size_t startwrd = mark_bit_word (start_mark_bit);
11056 size_t endwrd = mark_bit_word (end_mark_bit);
11058 //dprintf (3, ("Setting all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
11059 // (size_t)start, (size_t)start_mark_bit,
11060 // (size_t)end, (size_t)end_mark_bit));
11062 unsigned int firstwrd = ~(lowbits (~0, startbit));
11063 unsigned int lastwrd = ~(highbits (~0, endbit));
11065 if (startwrd == endwrd)
11067 unsigned int wrd = firstwrd & lastwrd;
11068 if (mark_array[startwrd] & wrd)
11070 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11072 mark_array [startwrd], mark_word_address (startwrd)));
11078 // set the first mark word.
11081 if (mark_array[startwrd] & firstwrd)
11083 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11084 firstwrd, startwrd,
11085 mark_array [startwrd], mark_word_address (startwrd)));
11092 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
11094 if (mark_array[wrdtmp])
11096 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11098 mark_array [wrdtmp], mark_word_address (wrdtmp)));
11103 // set the last mark word.
11106 if (mark_array[endwrd] & lastwrd)
11108 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11110 mark_array [lastwrd], mark_word_address (lastwrd)));
11115 #endif //VERIFY_HEAP && BACKGROUND_GC
11117 allocator::allocator (unsigned int num_b, size_t fbs, alloc_list* b)
11119 assert (num_b < MAX_BUCKET_COUNT);
11120 num_buckets = num_b;
11121 frst_bucket_size = fbs;
11125 alloc_list& allocator::alloc_list_of (unsigned int bn)
11127 assert (bn < num_buckets);
11129 return first_bucket;
11131 return buckets [bn-1];
11134 size_t& allocator::alloc_list_damage_count_of (unsigned int bn)
11136 assert (bn < num_buckets);
11138 return first_bucket.alloc_list_damage_count();
11140 return buckets [bn-1].alloc_list_damage_count();
11143 void allocator::unlink_item (unsigned int bn, uint8_t* item, uint8_t* prev_item, BOOL use_undo_p)
11145 //unlink the free_item
11146 alloc_list* al = &alloc_list_of (bn);
11149 if (use_undo_p && (free_list_undo (prev_item) == UNDO_EMPTY))
11151 assert (item == free_list_slot (prev_item));
11152 free_list_undo (prev_item) = item;
11153 alloc_list_damage_count_of (bn)++;
11155 free_list_slot (prev_item) = free_list_slot(item);
11159 al->alloc_list_head() = (uint8_t*)free_list_slot(item);
11161 if (al->alloc_list_tail() == item)
11163 al->alloc_list_tail() = prev_item;
11167 void allocator::clear()
11169 for (unsigned int i = 0; i < num_buckets; i++)
11171 alloc_list_head_of (i) = 0;
11172 alloc_list_tail_of (i) = 0;
11176 //always thread to the end.
11177 void allocator::thread_free_item (uint8_t* item, uint8_t*& head, uint8_t*& tail)
11179 free_list_slot (item) = 0;
11180 free_list_undo (item) = UNDO_EMPTY;
11181 assert (item != head);
11187 //TODO: This shouldn't happen anymore - verify that's the case.
11188 //the following is necessary because the last free element
11189 //may have been truncated, and tail isn't updated.
11190 else if (free_list_slot (head) == 0)
11192 free_list_slot (head) = item;
11196 assert (item != tail);
11197 assert (free_list_slot(tail) == 0);
11198 free_list_slot (tail) = item;
11203 void allocator::thread_item (uint8_t* item, size_t size)
11205 size_t sz = frst_bucket_size;
11206 unsigned int a_l_number = 0;
11208 for (; a_l_number < (num_buckets-1); a_l_number++)
11216 alloc_list* al = &alloc_list_of (a_l_number);
11217 thread_free_item (item,
11218 al->alloc_list_head(),
11219 al->alloc_list_tail());
11222 void allocator::thread_item_front (uint8_t* item, size_t size)
11224 //find right free list
11225 size_t sz = frst_bucket_size;
11226 unsigned int a_l_number = 0;
11227 for (; a_l_number < (num_buckets-1); a_l_number++)
11235 alloc_list* al = &alloc_list_of (a_l_number);
11236 free_list_slot (item) = al->alloc_list_head();
11237 free_list_undo (item) = UNDO_EMPTY;
11239 if (al->alloc_list_tail() == 0)
11241 al->alloc_list_tail() = al->alloc_list_head();
11243 al->alloc_list_head() = item;
11244 if (al->alloc_list_tail() == 0)
11246 al->alloc_list_tail() = item;
11250 void allocator::copy_to_alloc_list (alloc_list* toalist)
11252 for (unsigned int i = 0; i < num_buckets; i++)
11254 toalist [i] = alloc_list_of (i);
11255 #ifdef FL_VERIFICATION
11256 uint8_t* free_item = alloc_list_head_of (i);
11261 free_item = free_list_slot (free_item);
11264 toalist[i].item_count = count;
11265 #endif //FL_VERIFICATION
11269 void allocator::copy_from_alloc_list (alloc_list* fromalist)
11271 BOOL repair_list = !discard_if_no_fit_p ();
11272 for (unsigned int i = 0; i < num_buckets; i++)
11274 size_t count = alloc_list_damage_count_of (i);
11275 alloc_list_of (i) = fromalist [i];
11276 assert (alloc_list_damage_count_of (i) == 0);
11280 //repair the the list
11281 //new items may have been added during the plan phase
11282 //items may have been unlinked.
11283 uint8_t* free_item = alloc_list_head_of (i);
11284 while (free_item && count)
11286 assert (((CObjectHeader*)free_item)->IsFree());
11287 if ((free_list_undo (free_item) != UNDO_EMPTY))
11290 free_list_slot (free_item) = free_list_undo (free_item);
11291 free_list_undo (free_item) = UNDO_EMPTY;
11294 free_item = free_list_slot (free_item);
11297 #ifdef FL_VERIFICATION
11298 free_item = alloc_list_head_of (i);
11299 size_t item_count = 0;
11303 free_item = free_list_slot (free_item);
11306 assert (item_count == alloc_list_of (i).item_count);
11307 #endif //FL_VERIFICATION
11310 uint8_t* tail_item = alloc_list_tail_of (i);
11311 assert ((tail_item == 0) || (free_list_slot (tail_item) == 0));
11316 void allocator::commit_alloc_list_changes()
11318 BOOL repair_list = !discard_if_no_fit_p ();
11321 for (unsigned int i = 0; i < num_buckets; i++)
11323 //remove the undo info from list.
11324 uint8_t* free_item = alloc_list_head_of (i);
11325 size_t count = alloc_list_damage_count_of (i);
11326 while (free_item && count)
11328 assert (((CObjectHeader*)free_item)->IsFree());
11330 if (free_list_undo (free_item) != UNDO_EMPTY)
11332 free_list_undo (free_item) = UNDO_EMPTY;
11336 free_item = free_list_slot (free_item);
11339 alloc_list_damage_count_of (i) = 0;
11344 void gc_heap::adjust_limit_clr (uint8_t* start, size_t limit_size,
11345 alloc_context* acontext, heap_segment* seg,
11346 int align_const, int gen_number)
11348 size_t aligned_min_obj_size = Align(min_obj_size, align_const);
11350 //probably should pass seg==0 for free lists.
11353 assert (heap_segment_used (seg) <= heap_segment_committed (seg));
11356 dprintf (3, ("Expanding segment allocation [%Ix, %Ix[", (size_t)start,
11357 (size_t)start + limit_size - aligned_min_obj_size));
11359 if ((acontext->alloc_limit != start) &&
11360 (acontext->alloc_limit + aligned_min_obj_size)!= start)
11362 uint8_t* hole = acontext->alloc_ptr;
11365 size_t size = (acontext->alloc_limit - acontext->alloc_ptr);
11366 dprintf (3, ("filling up hole [%Ix, %Ix[", (size_t)hole, (size_t)hole + size + Align (min_obj_size, align_const)));
11367 // when we are finishing an allocation from a free list
11368 // we know that the free area was Align(min_obj_size) larger
11369 acontext->alloc_bytes -= size;
11370 size_t free_obj_size = size + aligned_min_obj_size;
11371 make_unused_array (hole, free_obj_size);
11372 generation_free_obj_space (generation_of (gen_number)) += free_obj_size;
11374 acontext->alloc_ptr = start;
11378 // If the next alloc context is right up against the current one it means we are absorbing the min
11379 // object, so need to account for that.
11380 acontext->alloc_bytes += (start - acontext->alloc_limit);
11383 acontext->alloc_limit = (start + limit_size - aligned_min_obj_size);
11384 acontext->alloc_bytes += limit_size - ((gen_number < max_generation + 1) ? aligned_min_obj_size : 0);
11386 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
11389 AppDomain* alloc_appdomain = GetAppDomain();
11390 alloc_appdomain->RecordAllocBytes (limit_size, heap_number);
11392 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
11394 uint8_t* saved_used = 0;
11398 saved_used = heap_segment_used (seg);
11401 if (seg == ephemeral_heap_segment)
11403 //Sometimes the allocated size is advanced without clearing the
11404 //memory. Let's catch up here
11405 if (heap_segment_used (seg) < (alloc_allocated - plug_skew))
11408 #ifndef BACKGROUND_GC
11409 clear_mark_array (heap_segment_used (seg) + plug_skew, alloc_allocated);
11410 #endif //BACKGROUND_GC
11411 #endif //MARK_ARRAY
11412 heap_segment_used (seg) = alloc_allocated - plug_skew;
11415 #ifdef BACKGROUND_GC
11418 uint8_t* old_allocated = heap_segment_allocated (seg) - plug_skew - limit_size;
11419 #ifdef FEATURE_LOH_COMPACTION
11420 old_allocated -= Align (loh_padding_obj_size, align_const);
11421 #endif //FEATURE_LOH_COMPACTION
11423 assert (heap_segment_used (seg) >= old_allocated);
11425 #endif //BACKGROUND_GC
11427 (start - plug_skew + limit_size) <= heap_segment_used (seg))
11429 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to clear memory(1)", heap_number));
11430 add_saved_spinlock_info (me_release, mt_clr_mem);
11431 leave_spin_lock (&more_space_lock);
11432 dprintf (3, ("clearing memory at %Ix for %d bytes", (start - plug_skew), limit_size));
11433 memclr (start - plug_skew, limit_size);
11437 uint8_t* used = heap_segment_used (seg);
11438 heap_segment_used (seg) = start + limit_size - plug_skew;
11440 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to clear memory", heap_number));
11441 add_saved_spinlock_info (me_release, mt_clr_mem);
11442 leave_spin_lock (&more_space_lock);
11443 if ((start - plug_skew) < used)
11445 if (used != saved_used)
11450 dprintf (2, ("clearing memory before used at %Ix for %Id bytes",
11451 (start - plug_skew), (plug_skew + used - start)));
11452 memclr (start - plug_skew, used - (start - plug_skew));
11456 //this portion can be done after we release the lock
11457 if (seg == ephemeral_heap_segment)
11459 #ifdef FFIND_OBJECT
11460 if (gen0_must_clear_bricks > 0)
11462 //set the brick table to speed up find_object
11463 size_t b = brick_of (acontext->alloc_ptr);
11464 set_brick (b, acontext->alloc_ptr - brick_address (b));
11466 dprintf (3, ("Allocation Clearing bricks [%Ix, %Ix[",
11467 b, brick_of (align_on_brick (start + limit_size))));
11468 volatile short* x = &brick_table [b];
11469 short* end_x = &brick_table [brick_of (align_on_brick (start + limit_size))];
11471 for (;x < end_x;x++)
11475 #endif //FFIND_OBJECT
11477 gen0_bricks_cleared = FALSE;
11481 // verifying the memory is completely cleared.
11482 //verify_mem_cleared (start - plug_skew, limit_size);
11485 /* in order to make the allocator faster, allocate returns a
11486 * 0 filled object. Care must be taken to set the allocation limit to the
11487 * allocation pointer after gc
11490 size_t gc_heap::limit_from_size (size_t size, size_t room, int gen_number,
11493 size_t new_limit = new_allocation_limit ((size + Align (min_obj_size, align_const)),
11494 min (room,max (size + Align (min_obj_size, align_const),
11495 ((gen_number < max_generation+1) ?
11496 allocation_quantum :
11499 assert (new_limit >= (size + Align (min_obj_size, align_const)));
11500 dprintf (100, ("requested to allocate %Id bytes, actual size is %Id", size, new_limit));
11504 void gc_heap::handle_oom (int heap_num, oom_reason reason, size_t alloc_size,
11505 uint8_t* allocated, uint8_t* reserved)
11507 dprintf (1, ("total committed on the heap is %Id", get_total_committed_size()));
11509 UNREFERENCED_PARAMETER(heap_num);
11511 if (reason == oom_budget)
11513 alloc_size = dd_min_size (dynamic_data_of (0)) / 2;
11516 if ((reason == oom_budget) && ((!fgm_result.loh_p) && (fgm_result.fgm != fgm_no_failure)))
11518 // This means during the last GC we needed to reserve and/or commit more memory
11519 // but we couldn't. We proceeded with the GC and ended up not having enough
11520 // memory at the end. This is a legitimate OOM situtation. Otherwise we
11521 // probably made a mistake and didn't expand the heap when we should have.
11522 reason = oom_low_mem;
11525 oom_info.reason = reason;
11526 oom_info.allocated = allocated;
11527 oom_info.reserved = reserved;
11528 oom_info.alloc_size = alloc_size;
11529 oom_info.gc_index = settings.gc_index;
11530 oom_info.fgm = fgm_result.fgm;
11531 oom_info.size = fgm_result.size;
11532 oom_info.available_pagefile_mb = fgm_result.available_pagefile_mb;
11533 oom_info.loh_p = fgm_result.loh_p;
11535 fgm_result.fgm = fgm_no_failure;
11537 // Break early - before the more_space_lock is release so no other threads
11538 // could have allocated on the same heap when OOM happened.
11539 if (GCConfig::GetBreakOnOOM())
11541 GCToOSInterface::DebugBreak();
11545 #ifdef BACKGROUND_GC
11546 BOOL gc_heap::background_allowed_p()
11548 return ( gc_can_use_concurrent && ((settings.pause_mode == pause_interactive) || (settings.pause_mode == pause_sustained_low_latency)) );
11550 #endif //BACKGROUND_GC
11552 void gc_heap::check_for_full_gc (int gen_num, size_t size)
11554 BOOL should_notify = FALSE;
11555 // if we detect full gc because of the allocation budget specified this is TRUE;
11556 // it's FALSE if it's due to other factors.
11557 BOOL alloc_factor = TRUE;
11560 int n_initial = gen_num;
11561 BOOL local_blocking_collection = FALSE;
11562 BOOL local_elevation_requested = FALSE;
11563 int new_alloc_remain_percent = 0;
11565 if (full_gc_approach_event_set)
11570 if (gen_num != (max_generation + 1))
11572 gen_num = max_generation;
11575 dynamic_data* dd_full = dynamic_data_of (gen_num);
11576 ptrdiff_t new_alloc_remain = 0;
11577 uint32_t pct = ((gen_num == (max_generation + 1)) ? fgn_loh_percent : fgn_maxgen_percent);
11579 for (int gen_index = 0; gen_index <= (max_generation + 1); gen_index++)
11581 dprintf (2, ("FGN: h#%d: gen%d: %Id(%Id)",
11582 heap_number, gen_index,
11583 dd_new_allocation (dynamic_data_of (gen_index)),
11584 dd_desired_allocation (dynamic_data_of (gen_index))));
11587 // For small object allocations we only check every fgn_check_quantum bytes.
11588 if (n_initial == 0)
11590 dprintf (2, ("FGN: gen0 last recorded alloc: %Id", fgn_last_alloc));
11591 dynamic_data* dd_0 = dynamic_data_of (n_initial);
11592 if (((fgn_last_alloc - dd_new_allocation (dd_0)) < fgn_check_quantum) &&
11593 (dd_new_allocation (dd_0) >= 0))
11599 fgn_last_alloc = dd_new_allocation (dd_0);
11600 dprintf (2, ("FGN: gen0 last recorded alloc is now: %Id", fgn_last_alloc));
11603 // We don't consider the size that came from soh 'cause it doesn't contribute to the
11608 for (i = n+1; i <= max_generation; i++)
11610 if (get_new_allocation (i) <= 0)
11612 n = min (i, max_generation);
11618 dprintf (2, ("FGN: h#%d: gen%d budget exceeded", heap_number, n));
11619 if (gen_num == max_generation)
11621 // If it's small object heap we should first see if we will even be looking at gen2 budget
11622 // in the next GC or not. If not we should go directly to checking other factors.
11623 if (n < (max_generation - 1))
11625 goto check_other_factors;
11629 new_alloc_remain = dd_new_allocation (dd_full) - size;
11631 new_alloc_remain_percent = (int)(((float)(new_alloc_remain) / (float)dd_desired_allocation (dd_full)) * 100);
11633 dprintf (2, ("FGN: alloc threshold for gen%d is %d%%, current threshold is %d%%",
11634 gen_num, pct, new_alloc_remain_percent));
11636 if (new_alloc_remain_percent <= (int)pct)
11638 #ifdef BACKGROUND_GC
11639 // If background GC is enabled, we still want to check whether this will
11640 // be a blocking GC or not because we only want to notify when it's a
11641 // blocking full GC.
11642 if (background_allowed_p())
11644 goto check_other_factors;
11646 #endif //BACKGROUND_GC
11648 should_notify = TRUE;
11652 check_other_factors:
11654 dprintf (2, ("FGC: checking other factors"));
11655 n = generation_to_condemn (n,
11656 &local_blocking_collection,
11657 &local_elevation_requested,
11660 if (local_elevation_requested && (n == max_generation))
11662 if (settings.should_lock_elevation)
11664 int local_elevation_locked_count = settings.elevation_locked_count + 1;
11665 if (local_elevation_locked_count != 6)
11667 dprintf (2, ("FGN: lock count is %d - Condemning max_generation-1",
11668 local_elevation_locked_count));
11669 n = max_generation - 1;
11674 dprintf (2, ("FGN: we estimate gen%d will be collected", n));
11676 #ifdef BACKGROUND_GC
11677 // When background GC is enabled it decreases the accuracy of our predictability -
11678 // by the time the GC happens, we may not be under BGC anymore. If we try to
11679 // predict often enough it should be ok.
11680 if ((n == max_generation) &&
11681 (recursive_gc_sync::background_running_p()))
11683 n = max_generation - 1;
11684 dprintf (2, ("FGN: bgc - 1 instead of 2"));
11687 if ((n == max_generation) && !local_blocking_collection)
11689 if (!background_allowed_p())
11691 local_blocking_collection = TRUE;
11694 #endif //BACKGROUND_GC
11696 dprintf (2, ("FGN: we estimate gen%d will be collected: %s",
11698 (local_blocking_collection ? "blocking" : "background")));
11700 if ((n == max_generation) && local_blocking_collection)
11702 alloc_factor = FALSE;
11703 should_notify = TRUE;
11711 dprintf (2, ("FGN: gen%d detecting full GC approaching(%s) (GC#%d) (%Id%% left in gen%d)",
11713 (alloc_factor ? "alloc" : "other"),
11714 dd_collection_count (dynamic_data_of (0)),
11715 new_alloc_remain_percent,
11718 send_full_gc_notification (n_initial, alloc_factor);
11722 void gc_heap::send_full_gc_notification (int gen_num, BOOL due_to_alloc_p)
11724 if (!full_gc_approach_event_set)
11726 assert (full_gc_approach_event.IsValid());
11727 FIRE_EVENT(GCFullNotify_V1, gen_num, due_to_alloc_p);
11729 full_gc_end_event.Reset();
11730 full_gc_approach_event.Set();
11731 full_gc_approach_event_set = true;
11735 wait_full_gc_status gc_heap::full_gc_wait (GCEvent *event, int time_out_ms)
11737 if (fgn_maxgen_percent == 0)
11739 return wait_full_gc_na;
11742 uint32_t wait_result = user_thread_wait(event, FALSE, time_out_ms);
11744 if ((wait_result == WAIT_OBJECT_0) || (wait_result == WAIT_TIMEOUT))
11746 if (fgn_maxgen_percent == 0)
11748 return wait_full_gc_cancelled;
11751 if (wait_result == WAIT_OBJECT_0)
11753 #ifdef BACKGROUND_GC
11754 if (fgn_last_gc_was_concurrent)
11756 fgn_last_gc_was_concurrent = FALSE;
11757 return wait_full_gc_na;
11760 #endif //BACKGROUND_GC
11762 return wait_full_gc_success;
11767 return wait_full_gc_timeout;
11772 return wait_full_gc_failed;
11776 size_t gc_heap::get_full_compact_gc_count()
11778 return full_gc_counts[gc_type_compacting];
11781 // DTREVIEW - we should check this in dt_low_ephemeral_space_p
11784 BOOL gc_heap::short_on_end_of_seg (int gen_number,
11788 UNREFERENCED_PARAMETER(gen_number);
11789 uint8_t* allocated = heap_segment_allocated(seg);
11791 return (!a_size_fit_p (end_space_after_gc(),
11793 heap_segment_reserved (seg),
11798 #pragma warning(disable:4706) // "assignment within conditional expression" is intentional in this function.
11802 BOOL gc_heap::a_fit_free_list_p (int gen_number,
11804 alloc_context* acontext,
11807 BOOL can_fit = FALSE;
11808 generation* gen = generation_of (gen_number);
11809 allocator* gen_allocator = generation_allocator (gen);
11810 size_t sz_list = gen_allocator->first_bucket_size();
11811 for (unsigned int a_l_idx = 0; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
11813 if ((size < sz_list) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
11815 uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
11816 uint8_t* prev_free_item = 0;
11818 while (free_list != 0)
11820 dprintf (3, ("considering free list %Ix", (size_t)free_list));
11821 size_t free_list_size = unused_array_size (free_list);
11822 if ((size + Align (min_obj_size, align_const)) <= free_list_size)
11824 dprintf (3, ("Found adequate unused area: [%Ix, size: %Id",
11825 (size_t)free_list, free_list_size));
11827 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
11828 // We ask for more Align (min_obj_size)
11829 // to make sure that we can insert a free object
11830 // in adjust_limit will set the limit lower
11831 size_t limit = limit_from_size (size, free_list_size, gen_number, align_const);
11833 uint8_t* remain = (free_list + limit);
11834 size_t remain_size = (free_list_size - limit);
11835 if (remain_size >= Align(min_free_list, align_const))
11837 make_unused_array (remain, remain_size);
11838 gen_allocator->thread_item_front (remain, remain_size);
11839 assert (remain_size >= Align (min_obj_size, align_const));
11843 //absorb the entire free list
11844 limit += remain_size;
11846 generation_free_list_space (gen) -= limit;
11848 adjust_limit_clr (free_list, limit, acontext, 0, align_const, gen_number);
11853 else if (gen_allocator->discard_if_no_fit_p())
11855 assert (prev_free_item == 0);
11856 dprintf (3, ("couldn't use this free area, discarding"));
11857 generation_free_obj_space (gen) += free_list_size;
11859 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
11860 generation_free_list_space (gen) -= free_list_size;
11864 prev_free_item = free_list;
11866 free_list = free_list_slot (free_list);
11869 sz_list = sz_list * 2;
11876 #ifdef BACKGROUND_GC
11877 void gc_heap::bgc_loh_alloc_clr (uint8_t* alloc_start,
11879 alloc_context* acontext,
11885 make_unused_array (alloc_start, size);
11887 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
11890 AppDomain* alloc_appdomain = GetAppDomain();
11891 alloc_appdomain->RecordAllocBytes (size, heap_number);
11893 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
11895 size_t size_of_array_base = sizeof(ArrayBase);
11897 bgc_alloc_lock->loh_alloc_done_with_index (lock_index);
11899 // clear memory while not holding the lock.
11900 size_t size_to_skip = size_of_array_base;
11901 size_t size_to_clear = size - size_to_skip - plug_skew;
11902 size_t saved_size_to_clear = size_to_clear;
11905 uint8_t* end = alloc_start + size - plug_skew;
11906 uint8_t* used = heap_segment_used (seg);
11909 if ((alloc_start + size_to_skip) < used)
11911 size_to_clear = used - (alloc_start + size_to_skip);
11917 dprintf (2, ("bgc loh: setting used to %Ix", end));
11918 heap_segment_used (seg) = end;
11921 dprintf (2, ("bgc loh: used: %Ix, alloc: %Ix, end of alloc: %Ix, clear %Id bytes",
11922 used, alloc_start, end, size_to_clear));
11926 dprintf (2, ("bgc loh: [%Ix-[%Ix(%Id)", alloc_start, alloc_start+size, size));
11930 // since we filled in 0xcc for free object when we verify heap,
11931 // we need to make sure we clear those bytes.
11932 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
11934 if (size_to_clear < saved_size_to_clear)
11936 size_to_clear = saved_size_to_clear;
11939 #endif //VERIFY_HEAP
11941 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to clear large obj", heap_number));
11942 add_saved_spinlock_info (me_release, mt_clr_large_mem);
11943 leave_spin_lock (&more_space_lock);
11944 memclr (alloc_start + size_to_skip, size_to_clear);
11946 bgc_alloc_lock->loh_alloc_set (alloc_start);
11948 acontext->alloc_ptr = alloc_start;
11949 acontext->alloc_limit = (alloc_start + size - Align (min_obj_size, align_const));
11951 // need to clear the rest of the object before we hand it out.
11952 clear_unused_array(alloc_start, size);
11954 #endif //BACKGROUND_GC
11956 BOOL gc_heap::a_fit_free_list_large_p (size_t size,
11957 alloc_context* acontext,
11960 #ifdef BACKGROUND_GC
11961 wait_for_background_planning (awr_loh_alloc_during_plan);
11962 #endif //BACKGROUND_GC
11964 BOOL can_fit = FALSE;
11965 int gen_number = max_generation + 1;
11966 generation* gen = generation_of (gen_number);
11967 allocator* loh_allocator = generation_allocator (gen);
11969 #ifdef FEATURE_LOH_COMPACTION
11970 size_t loh_pad = Align (loh_padding_obj_size, align_const);
11971 #endif //FEATURE_LOH_COMPACTION
11973 #ifdef BACKGROUND_GC
11975 #endif //BACKGROUND_GC
11976 size_t sz_list = loh_allocator->first_bucket_size();
11977 for (unsigned int a_l_idx = 0; a_l_idx < loh_allocator->number_of_buckets(); a_l_idx++)
11979 if ((size < sz_list) || (a_l_idx == (loh_allocator->number_of_buckets()-1)))
11981 uint8_t* free_list = loh_allocator->alloc_list_head_of (a_l_idx);
11982 uint8_t* prev_free_item = 0;
11983 while (free_list != 0)
11985 dprintf (3, ("considering free list %Ix", (size_t)free_list));
11987 size_t free_list_size = unused_array_size(free_list);
11989 #ifdef FEATURE_LOH_COMPACTION
11990 if ((size + loh_pad) <= free_list_size)
11992 if (((size + Align (min_obj_size, align_const)) <= free_list_size)||
11993 (size == free_list_size))
11994 #endif //FEATURE_LOH_COMPACTION
11996 #ifdef BACKGROUND_GC
11997 cookie = bgc_alloc_lock->loh_alloc_set (free_list);
11998 #endif //BACKGROUND_GC
12000 //unlink the free_item
12001 loh_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
12003 // Substract min obj size because limit_from_size adds it. Not needed for LOH
12004 size_t limit = limit_from_size (size - Align(min_obj_size, align_const), free_list_size,
12005 gen_number, align_const);
12007 #ifdef FEATURE_LOH_COMPACTION
12008 make_unused_array (free_list, loh_pad);
12010 free_list += loh_pad;
12011 free_list_size -= loh_pad;
12012 #endif //FEATURE_LOH_COMPACTION
12014 uint8_t* remain = (free_list + limit);
12015 size_t remain_size = (free_list_size - limit);
12016 if (remain_size != 0)
12018 assert (remain_size >= Align (min_obj_size, align_const));
12019 make_unused_array (remain, remain_size);
12021 if (remain_size >= Align(min_free_list, align_const))
12023 loh_thread_gap_front (remain, remain_size, gen);
12024 assert (remain_size >= Align (min_obj_size, align_const));
12028 generation_free_obj_space (gen) += remain_size;
12030 generation_free_list_space (gen) -= free_list_size;
12031 dprintf (3, ("found fit on loh at %Ix", free_list));
12032 #ifdef BACKGROUND_GC
12035 bgc_loh_alloc_clr (free_list, limit, acontext, align_const, cookie, FALSE, 0);
12038 #endif //BACKGROUND_GC
12040 adjust_limit_clr (free_list, limit, acontext, 0, align_const, gen_number);
12043 //fix the limit to compensate for adjust_limit_clr making it too short
12044 acontext->alloc_limit += Align (min_obj_size, align_const);
12048 prev_free_item = free_list;
12049 free_list = free_list_slot (free_list);
12052 sz_list = sz_list * 2;
12059 #pragma warning(default:4706)
12062 BOOL gc_heap::a_fit_segment_end_p (int gen_number,
12065 alloc_context* acontext,
12067 BOOL* commit_failed_p)
12069 *commit_failed_p = FALSE;
12071 #ifdef BACKGROUND_GC
12073 #endif //BACKGROUND_GC
12075 uint8_t*& allocated = ((gen_number == 0) ?
12077 heap_segment_allocated(seg));
12079 size_t pad = Align (min_obj_size, align_const);
12081 #ifdef FEATURE_LOH_COMPACTION
12082 if (gen_number == (max_generation + 1))
12084 pad += Align (loh_padding_obj_size, align_const);
12086 #endif //FEATURE_LOH_COMPACTION
12088 uint8_t* end = heap_segment_committed (seg) - pad;
12090 if (a_size_fit_p (size, allocated, end, align_const))
12092 limit = limit_from_size (size,
12094 gen_number, align_const);
12098 end = heap_segment_reserved (seg) - pad;
12100 if (a_size_fit_p (size, allocated, end, align_const))
12102 limit = limit_from_size (size,
12104 gen_number, align_const);
12105 if (grow_heap_segment (seg, allocated + limit))
12111 dprintf (2, ("can't grow segment, doing a full gc"));
12112 *commit_failed_p = TRUE;
12119 #ifdef BACKGROUND_GC
12120 if (gen_number != 0)
12122 cookie = bgc_alloc_lock->loh_alloc_set (allocated);
12124 #endif //BACKGROUND_GC
12126 uint8_t* old_alloc;
12127 old_alloc = allocated;
12128 #ifdef FEATURE_LOH_COMPACTION
12129 if (gen_number == (max_generation + 1))
12131 size_t loh_pad = Align (loh_padding_obj_size, align_const);
12132 make_unused_array (old_alloc, loh_pad);
12133 old_alloc += loh_pad;
12134 allocated += loh_pad;
12137 #endif //FEATURE_LOH_COMPACTION
12139 #if defined (VERIFY_HEAP) && defined (_DEBUG)
12140 ((void**) allocated)[-1] = 0; //clear the sync block
12141 #endif //VERIFY_HEAP && _DEBUG
12142 allocated += limit;
12144 dprintf (3, ("found fit at end of seg: %Ix", old_alloc));
12146 #ifdef BACKGROUND_GC
12149 bgc_loh_alloc_clr (old_alloc, limit, acontext, align_const, cookie, TRUE, seg);
12152 #endif //BACKGROUND_GC
12154 adjust_limit_clr (old_alloc, limit, acontext, seg, align_const, gen_number);
12164 BOOL gc_heap::loh_a_fit_segment_end_p (int gen_number,
12166 alloc_context* acontext,
12168 BOOL* commit_failed_p,
12171 *commit_failed_p = FALSE;
12172 heap_segment* seg = generation_allocation_segment (generation_of (gen_number));
12173 BOOL can_allocate_p = FALSE;
12177 if (a_fit_segment_end_p (gen_number, seg, (size - Align (min_obj_size, align_const)),
12178 acontext, align_const, commit_failed_p))
12180 acontext->alloc_limit += Align (min_obj_size, align_const);
12181 can_allocate_p = TRUE;
12186 if (*commit_failed_p)
12188 *oom_r = oom_cant_commit;
12193 seg = heap_segment_next_rw (seg);
12198 return can_allocate_p;
12201 #ifdef BACKGROUND_GC
12203 void gc_heap::wait_for_background (alloc_wait_reason awr)
12205 dprintf (2, ("BGC is already in progress, waiting for it to finish"));
12206 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to wait for bgc done", heap_number));
12207 add_saved_spinlock_info (me_release, mt_wait_bgc);
12208 leave_spin_lock (&more_space_lock);
12209 background_gc_wait (awr);
12210 enter_spin_lock (&more_space_lock);
12211 add_saved_spinlock_info (me_acquire, mt_wait_bgc);
12212 dprintf (SPINLOCK_LOG, ("[%d]Emsl after waiting for bgc done", heap_number));
12215 void gc_heap::wait_for_bgc_high_memory (alloc_wait_reason awr)
12217 if (recursive_gc_sync::background_running_p())
12219 uint32_t memory_load;
12220 get_memory_info (&memory_load);
12221 if (memory_load >= 95)
12223 dprintf (GTC_LOG, ("high mem - wait for BGC to finish, wait reason: %d", awr));
12224 wait_for_background (awr);
12229 #endif //BACKGROUND_GC
12231 // We request to trigger an ephemeral GC but we may get a full compacting GC.
12232 // return TRUE if that's the case.
12233 BOOL gc_heap::trigger_ephemeral_gc (gc_reason gr)
12235 #ifdef BACKGROUND_GC
12236 wait_for_bgc_high_memory (awr_loh_oos_bgc);
12237 #endif //BACKGROUND_GC
12239 BOOL did_full_compact_gc = FALSE;
12241 dprintf (2, ("triggering a gen1 GC"));
12242 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12243 vm_heap->GarbageCollectGeneration(max_generation - 1, gr);
12245 #ifdef MULTIPLE_HEAPS
12246 enter_spin_lock (&more_space_lock);
12247 add_saved_spinlock_info (me_acquire, mt_t_eph_gc);
12248 dprintf (SPINLOCK_LOG, ("[%d]Emsl after a GC", heap_number));
12249 #endif //MULTIPLE_HEAPS
12251 size_t current_full_compact_gc_count = get_full_compact_gc_count();
12253 if (current_full_compact_gc_count > last_full_compact_gc_count)
12255 dprintf (2, ("attempted to trigger an ephemeral GC and got a full compacting GC"));
12256 did_full_compact_gc = TRUE;
12259 return did_full_compact_gc;
12262 BOOL gc_heap::soh_try_fit (int gen_number,
12264 alloc_context* acontext,
12266 BOOL* commit_failed_p,
12267 BOOL* short_seg_end_p)
12269 BOOL can_allocate = TRUE;
12270 if (short_seg_end_p)
12272 *short_seg_end_p = FALSE;
12275 can_allocate = a_fit_free_list_p (gen_number, size, acontext, align_const);
12278 if (short_seg_end_p)
12280 *short_seg_end_p = short_on_end_of_seg (gen_number, ephemeral_heap_segment, align_const);
12282 // If the caller doesn't care, we always try to fit at the end of seg;
12283 // otherwise we would only try if we are actually not short at end of seg.
12284 if (!short_seg_end_p || !(*short_seg_end_p))
12286 can_allocate = a_fit_segment_end_p (gen_number, ephemeral_heap_segment, size,
12287 acontext, align_const, commit_failed_p);
12291 return can_allocate;
12294 BOOL gc_heap::allocate_small (int gen_number,
12296 alloc_context* acontext,
12299 #if defined (BACKGROUND_GC) && !defined (MULTIPLE_HEAPS)
12300 if (recursive_gc_sync::background_running_p())
12302 background_soh_alloc_count++;
12303 if ((background_soh_alloc_count % bgc_alloc_spin_count) == 0)
12305 add_saved_spinlock_info (me_release, mt_alloc_small);
12306 dprintf (SPINLOCK_LOG, ("[%d]spin Lmsl", heap_number));
12307 leave_spin_lock (&more_space_lock);
12308 bool cooperative_mode = enable_preemptive ();
12309 GCToOSInterface::Sleep (bgc_alloc_spin);
12310 disable_preemptive (cooperative_mode);
12311 enter_spin_lock (&more_space_lock);
12312 add_saved_spinlock_info (me_acquire, mt_alloc_small);
12313 dprintf (SPINLOCK_LOG, ("[%d]spin Emsl", heap_number));
12317 //GCToOSInterface::YieldThread (0);
12320 #endif //BACKGROUND_GC && !MULTIPLE_HEAPS
12322 gc_reason gr = reason_oos_soh;
12323 oom_reason oom_r = oom_no_failure;
12325 // No variable values should be "carried over" from one state to the other.
12326 // That's why there are local variable for each state
12328 allocation_state soh_alloc_state = a_state_start;
12330 // If we can get a new seg it means allocation will succeed.
12333 dprintf (3, ("[h%d]soh state is %s", heap_number, allocation_state_str[soh_alloc_state]));
12334 switch (soh_alloc_state)
12336 case a_state_can_allocate:
12337 case a_state_cant_allocate:
12341 case a_state_start:
12343 soh_alloc_state = a_state_try_fit;
12346 case a_state_try_fit:
12348 BOOL commit_failed_p = FALSE;
12349 BOOL can_use_existing_p = FALSE;
12351 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12352 align_const, &commit_failed_p,
12354 soh_alloc_state = (can_use_existing_p ?
12355 a_state_can_allocate :
12357 a_state_trigger_full_compact_gc :
12358 a_state_trigger_ephemeral_gc));
12361 case a_state_try_fit_after_bgc:
12363 BOOL commit_failed_p = FALSE;
12364 BOOL can_use_existing_p = FALSE;
12365 BOOL short_seg_end_p = FALSE;
12367 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12368 align_const, &commit_failed_p,
12370 soh_alloc_state = (can_use_existing_p ?
12371 a_state_can_allocate :
12373 a_state_trigger_2nd_ephemeral_gc :
12374 a_state_trigger_full_compact_gc));
12377 case a_state_try_fit_after_cg:
12379 BOOL commit_failed_p = FALSE;
12380 BOOL can_use_existing_p = FALSE;
12381 BOOL short_seg_end_p = FALSE;
12383 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12384 align_const, &commit_failed_p,
12386 if (short_seg_end_p)
12388 soh_alloc_state = a_state_cant_allocate;
12389 oom_r = oom_budget;
12393 if (can_use_existing_p)
12395 soh_alloc_state = a_state_can_allocate;
12399 #ifdef MULTIPLE_HEAPS
12400 if (!commit_failed_p)
12402 // some other threads already grabbed the more space lock and allocated
12403 // so we should attempt an ephemeral GC again.
12404 assert (heap_segment_allocated (ephemeral_heap_segment) < alloc_allocated);
12405 soh_alloc_state = a_state_trigger_ephemeral_gc;
12408 #endif //MULTIPLE_HEAPS
12410 assert (commit_failed_p);
12411 soh_alloc_state = a_state_cant_allocate;
12412 oom_r = oom_cant_commit;
12418 case a_state_check_and_wait_for_bgc:
12420 BOOL bgc_in_progress_p = FALSE;
12421 BOOL did_full_compacting_gc = FALSE;
12423 bgc_in_progress_p = check_and_wait_for_bgc (awr_gen0_oos_bgc, &did_full_compacting_gc);
12424 soh_alloc_state = (did_full_compacting_gc ?
12425 a_state_try_fit_after_cg :
12426 a_state_try_fit_after_bgc);
12429 case a_state_trigger_ephemeral_gc:
12431 BOOL commit_failed_p = FALSE;
12432 BOOL can_use_existing_p = FALSE;
12433 BOOL short_seg_end_p = FALSE;
12434 BOOL bgc_in_progress_p = FALSE;
12435 BOOL did_full_compacting_gc = FALSE;
12437 did_full_compacting_gc = trigger_ephemeral_gc (gr);
12438 if (did_full_compacting_gc)
12440 soh_alloc_state = a_state_try_fit_after_cg;
12444 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12445 align_const, &commit_failed_p,
12447 #ifdef BACKGROUND_GC
12448 bgc_in_progress_p = recursive_gc_sync::background_running_p();
12449 #endif //BACKGROUND_GC
12451 if (short_seg_end_p)
12453 soh_alloc_state = (bgc_in_progress_p ?
12454 a_state_check_and_wait_for_bgc :
12455 a_state_trigger_full_compact_gc);
12457 if (fgn_maxgen_percent)
12459 dprintf (2, ("FGN: doing last GC before we throw OOM"));
12460 send_full_gc_notification (max_generation, FALSE);
12465 if (can_use_existing_p)
12467 soh_alloc_state = a_state_can_allocate;
12471 #ifdef MULTIPLE_HEAPS
12472 if (!commit_failed_p)
12474 // some other threads already grabbed the more space lock and allocated
12475 // so we should attempt an ephemeral GC again.
12476 assert (heap_segment_allocated (ephemeral_heap_segment) < alloc_allocated);
12477 soh_alloc_state = a_state_trigger_ephemeral_gc;
12480 #endif //MULTIPLE_HEAPS
12482 soh_alloc_state = a_state_trigger_full_compact_gc;
12483 if (fgn_maxgen_percent)
12485 dprintf (2, ("FGN: failed to commit, doing full compacting GC"));
12486 send_full_gc_notification (max_generation, FALSE);
12494 case a_state_trigger_2nd_ephemeral_gc:
12496 BOOL commit_failed_p = FALSE;
12497 BOOL can_use_existing_p = FALSE;
12498 BOOL short_seg_end_p = FALSE;
12499 BOOL did_full_compacting_gc = FALSE;
12502 did_full_compacting_gc = trigger_ephemeral_gc (gr);
12504 if (did_full_compacting_gc)
12506 soh_alloc_state = a_state_try_fit_after_cg;
12510 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12511 align_const, &commit_failed_p,
12513 if (short_seg_end_p || commit_failed_p)
12515 soh_alloc_state = a_state_trigger_full_compact_gc;
12519 assert (can_use_existing_p);
12520 soh_alloc_state = a_state_can_allocate;
12525 case a_state_trigger_full_compact_gc:
12527 BOOL got_full_compacting_gc = FALSE;
12529 got_full_compacting_gc = trigger_full_compact_gc (gr, &oom_r);
12530 soh_alloc_state = (got_full_compacting_gc ? a_state_try_fit_after_cg : a_state_cant_allocate);
12535 assert (!"Invalid state!");
12542 if (soh_alloc_state == a_state_cant_allocate)
12544 assert (oom_r != oom_no_failure);
12545 handle_oom (heap_number,
12548 heap_segment_allocated (ephemeral_heap_segment),
12549 heap_segment_reserved (ephemeral_heap_segment));
12551 dprintf (SPINLOCK_LOG, ("[%d]Lmsl for oom", heap_number));
12552 add_saved_spinlock_info (me_release, mt_alloc_small_cant);
12553 leave_spin_lock (&more_space_lock);
12556 return (soh_alloc_state == a_state_can_allocate);
12559 #ifdef BACKGROUND_GC
12561 void gc_heap::wait_for_background_planning (alloc_wait_reason awr)
12563 while (current_c_gc_state == c_gc_state_planning)
12565 dprintf (3, ("lh state planning, cannot allocate"));
12567 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to wait for bgc plan", heap_number));
12568 add_saved_spinlock_info (me_release, mt_wait_bgc_plan);
12569 leave_spin_lock (&more_space_lock);
12570 background_gc_wait_lh (awr);
12571 enter_spin_lock (&more_space_lock);
12572 add_saved_spinlock_info (me_acquire, mt_wait_bgc_plan);
12573 dprintf (SPINLOCK_LOG, ("[%d]Emsl after waiting for bgc plan", heap_number));
12575 assert ((current_c_gc_state == c_gc_state_free) ||
12576 (current_c_gc_state == c_gc_state_marking));
12579 BOOL gc_heap::bgc_loh_should_allocate()
12581 size_t min_gc_size = dd_min_size(dynamic_data_of (max_generation + 1));
12583 if ((bgc_begin_loh_size + bgc_loh_size_increased) < (min_gc_size * 10))
12588 if (((bgc_begin_loh_size / end_loh_size) >= 2) || (bgc_loh_size_increased >= bgc_begin_loh_size))
12590 if ((bgc_begin_loh_size / end_loh_size) > 2)
12592 dprintf (3, ("alloc-ed too much before bgc started"));
12596 dprintf (3, ("alloc-ed too much after bgc started"));
12602 bgc_alloc_spin_loh = (uint32_t)(((float)bgc_loh_size_increased / (float)bgc_begin_loh_size) * 10);
12606 #endif //BACKGROUND_GC
12608 size_t gc_heap::get_large_seg_size (size_t size)
12610 size_t default_seg_size = min_loh_segment_size;
12611 #ifdef SEG_MAPPING_TABLE
12612 size_t align_size = default_seg_size;
12613 #else //SEG_MAPPING_TABLE
12614 size_t align_size = default_seg_size / 2;
12615 #endif //SEG_MAPPING_TABLE
12616 int align_const = get_alignment_constant (FALSE);
12617 size_t large_seg_size = align_on_page (
12618 max (default_seg_size,
12619 ((size + 2 * Align(min_obj_size, align_const) + OS_PAGE_SIZE +
12620 align_size) / align_size * align_size)));
12621 return large_seg_size;
12624 BOOL gc_heap::loh_get_new_seg (generation* gen,
12627 BOOL* did_full_compact_gc,
12630 UNREFERENCED_PARAMETER(gen);
12631 UNREFERENCED_PARAMETER(align_const);
12633 *did_full_compact_gc = FALSE;
12635 size_t seg_size = get_large_seg_size (size);
12637 heap_segment* new_seg = get_large_segment (seg_size, did_full_compact_gc);
12641 loh_alloc_since_cg += seg_size;
12648 return (new_seg != 0);
12651 BOOL gc_heap::retry_full_compact_gc (size_t size)
12653 size_t seg_size = get_large_seg_size (size);
12655 if (loh_alloc_since_cg >= (2 * (uint64_t)seg_size))
12660 #ifdef MULTIPLE_HEAPS
12661 uint64_t total_alloc_size = 0;
12662 for (int i = 0; i < n_heaps; i++)
12664 total_alloc_size += g_heaps[i]->loh_alloc_since_cg;
12667 if (total_alloc_size >= (2 * (uint64_t)seg_size))
12671 #endif //MULTIPLE_HEAPS
12676 BOOL gc_heap::check_and_wait_for_bgc (alloc_wait_reason awr,
12677 BOOL* did_full_compact_gc)
12679 BOOL bgc_in_progress = FALSE;
12680 *did_full_compact_gc = FALSE;
12681 #ifdef BACKGROUND_GC
12682 if (recursive_gc_sync::background_running_p())
12684 bgc_in_progress = TRUE;
12685 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12686 wait_for_background (awr);
12687 size_t current_full_compact_gc_count = get_full_compact_gc_count();
12688 if (current_full_compact_gc_count > last_full_compact_gc_count)
12690 *did_full_compact_gc = TRUE;
12693 #endif //BACKGROUND_GC
12695 return bgc_in_progress;
12698 BOOL gc_heap::loh_try_fit (int gen_number,
12700 alloc_context* acontext,
12702 BOOL* commit_failed_p,
12705 BOOL can_allocate = TRUE;
12707 if (!a_fit_free_list_large_p (size, acontext, align_const))
12709 can_allocate = loh_a_fit_segment_end_p (gen_number, size,
12710 acontext, align_const,
12711 commit_failed_p, oom_r);
12713 #ifdef BACKGROUND_GC
12714 if (can_allocate && recursive_gc_sync::background_running_p())
12716 bgc_loh_size_increased += size;
12718 #endif //BACKGROUND_GC
12720 #ifdef BACKGROUND_GC
12723 if (recursive_gc_sync::background_running_p())
12725 bgc_loh_allocated_in_free += size;
12728 #endif //BACKGROUND_GC
12730 return can_allocate;
12733 BOOL gc_heap::trigger_full_compact_gc (gc_reason gr,
12736 BOOL did_full_compact_gc = FALSE;
12738 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12740 // Set this so the next GC will be a full compacting GC.
12741 if (!last_gc_before_oom)
12743 last_gc_before_oom = TRUE;
12746 #ifdef BACKGROUND_GC
12747 if (recursive_gc_sync::background_running_p())
12749 wait_for_background ((gr == reason_oos_soh) ? awr_gen0_oos_bgc : awr_loh_oos_bgc);
12750 dprintf (2, ("waited for BGC - done"));
12752 #endif //BACKGROUND_GC
12754 size_t current_full_compact_gc_count = get_full_compact_gc_count();
12755 if (current_full_compact_gc_count > last_full_compact_gc_count)
12757 dprintf (3, ("a full compacting GC triggered while waiting for BGC (%d->%d)", last_full_compact_gc_count, current_full_compact_gc_count));
12758 assert (current_full_compact_gc_count > last_full_compact_gc_count);
12759 did_full_compact_gc = TRUE;
12763 dprintf (3, ("h%d full GC", heap_number));
12764 vm_heap->GarbageCollectGeneration(max_generation, gr);
12766 #ifdef MULTIPLE_HEAPS
12767 enter_spin_lock (&more_space_lock);
12768 dprintf (SPINLOCK_LOG, ("[%d]Emsl after full gc", heap_number));
12769 add_saved_spinlock_info (me_acquire, mt_t_full_gc);
12770 #endif //MULTIPLE_HEAPS
12772 current_full_compact_gc_count = get_full_compact_gc_count();
12774 if (current_full_compact_gc_count == last_full_compact_gc_count)
12776 dprintf (2, ("attempted to trigger a full compacting GC but didn't get it"));
12777 // We requested a full GC but didn't get because of the elevation logic
12778 // which means we should fail.
12779 *oom_r = oom_unproductive_full_gc;
12783 dprintf (3, ("h%d: T full compacting GC (%d->%d)",
12785 last_full_compact_gc_count,
12786 current_full_compact_gc_count));
12788 assert (current_full_compact_gc_count > last_full_compact_gc_count);
12789 did_full_compact_gc = TRUE;
12793 return did_full_compact_gc;
12796 #ifdef RECORD_LOH_STATE
12797 void gc_heap::add_saved_loh_state (allocation_state loh_state_to_save, EEThreadId thread_id)
12799 // When the state is can_allocate we already have released the more
12800 // space lock. So we are not logging states here since this code
12801 // is not thread safe.
12802 if (loh_state_to_save != a_state_can_allocate)
12804 last_loh_states[loh_state_index].alloc_state = loh_state_to_save;
12805 last_loh_states[loh_state_index].thread_id = thread_id;
12808 if (loh_state_index == max_saved_loh_states)
12810 loh_state_index = 0;
12813 assert (loh_state_index < max_saved_loh_states);
12816 #endif //RECORD_LOH_STATE
12818 BOOL gc_heap::allocate_large (int gen_number,
12820 alloc_context* acontext,
12823 #ifdef BACKGROUND_GC
12824 if (recursive_gc_sync::background_running_p() && (current_c_gc_state != c_gc_state_planning))
12826 background_loh_alloc_count++;
12827 //if ((background_loh_alloc_count % bgc_alloc_spin_count_loh) == 0)
12829 if (bgc_loh_should_allocate())
12831 if (!bgc_alloc_spin_loh)
12833 add_saved_spinlock_info (me_release, mt_alloc_large);
12834 dprintf (SPINLOCK_LOG, ("[%d]spin Lmsl loh", heap_number));
12835 leave_spin_lock (&more_space_lock);
12836 bool cooperative_mode = enable_preemptive ();
12837 GCToOSInterface::YieldThread (bgc_alloc_spin_loh);
12838 disable_preemptive (cooperative_mode);
12839 enter_spin_lock (&more_space_lock);
12840 add_saved_spinlock_info (me_acquire, mt_alloc_large);
12841 dprintf (SPINLOCK_LOG, ("[%d]spin Emsl loh", heap_number));
12846 wait_for_background (awr_loh_alloc_during_bgc);
12850 #endif //BACKGROUND_GC
12852 gc_reason gr = reason_oos_loh;
12853 generation* gen = generation_of (gen_number);
12854 oom_reason oom_r = oom_no_failure;
12855 size_t current_full_compact_gc_count = 0;
12857 // No variable values should be "carried over" from one state to the other.
12858 // That's why there are local variable for each state
12859 allocation_state loh_alloc_state = a_state_start;
12860 #ifdef RECORD_LOH_STATE
12861 EEThreadId current_thread_id;
12862 current_thread_id.SetToCurrentThread();
12863 #endif //RECORD_LOH_STATE
12865 // If we can get a new seg it means allocation will succeed.
12868 dprintf (3, ("[h%d]loh state is %s", heap_number, allocation_state_str[loh_alloc_state]));
12870 #ifdef RECORD_LOH_STATE
12871 add_saved_loh_state (loh_alloc_state, current_thread_id);
12872 #endif //RECORD_LOH_STATE
12873 switch (loh_alloc_state)
12875 case a_state_can_allocate:
12876 case a_state_cant_allocate:
12880 case a_state_start:
12882 loh_alloc_state = a_state_try_fit;
12885 case a_state_try_fit:
12887 BOOL commit_failed_p = FALSE;
12888 BOOL can_use_existing_p = FALSE;
12890 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12891 align_const, &commit_failed_p, &oom_r);
12892 loh_alloc_state = (can_use_existing_p ?
12893 a_state_can_allocate :
12895 a_state_trigger_full_compact_gc :
12896 a_state_acquire_seg));
12897 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12900 case a_state_try_fit_new_seg:
12902 BOOL commit_failed_p = FALSE;
12903 BOOL can_use_existing_p = FALSE;
12905 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12906 align_const, &commit_failed_p, &oom_r);
12907 // Even after we got a new seg it doesn't necessarily mean we can allocate,
12908 // another LOH allocating thread could have beat us to acquire the msl so
12909 // we need to try again.
12910 loh_alloc_state = (can_use_existing_p ? a_state_can_allocate : a_state_try_fit);
12911 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12914 case a_state_try_fit_new_seg_after_cg:
12916 BOOL commit_failed_p = FALSE;
12917 BOOL can_use_existing_p = FALSE;
12919 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12920 align_const, &commit_failed_p, &oom_r);
12921 // Even after we got a new seg it doesn't necessarily mean we can allocate,
12922 // another LOH allocating thread could have beat us to acquire the msl so
12923 // we need to try again. However, if we failed to commit, which means we
12924 // did have space on the seg, we bail right away 'cause we already did a
12925 // full compacting GC.
12926 loh_alloc_state = (can_use_existing_p ?
12927 a_state_can_allocate :
12929 a_state_cant_allocate :
12930 a_state_acquire_seg_after_cg));
12931 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12934 case a_state_try_fit_no_seg:
12936 BOOL commit_failed_p = FALSE;
12937 BOOL can_use_existing_p = FALSE;
12939 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12940 align_const, &commit_failed_p, &oom_r);
12941 loh_alloc_state = (can_use_existing_p ? a_state_can_allocate : a_state_cant_allocate);
12942 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12943 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
12946 case a_state_try_fit_after_cg:
12948 BOOL commit_failed_p = FALSE;
12949 BOOL can_use_existing_p = FALSE;
12951 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12952 align_const, &commit_failed_p, &oom_r);
12953 loh_alloc_state = (can_use_existing_p ?
12954 a_state_can_allocate :
12956 a_state_cant_allocate :
12957 a_state_acquire_seg_after_cg));
12958 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12961 case a_state_try_fit_after_bgc:
12963 BOOL commit_failed_p = FALSE;
12964 BOOL can_use_existing_p = FALSE;
12966 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12967 align_const, &commit_failed_p, &oom_r);
12968 loh_alloc_state = (can_use_existing_p ?
12969 a_state_can_allocate :
12971 a_state_trigger_full_compact_gc :
12972 a_state_acquire_seg_after_bgc));
12973 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12976 case a_state_acquire_seg:
12978 BOOL can_get_new_seg_p = FALSE;
12979 BOOL did_full_compacting_gc = FALSE;
12981 current_full_compact_gc_count = get_full_compact_gc_count();
12983 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
12984 loh_alloc_state = (can_get_new_seg_p ?
12985 a_state_try_fit_new_seg :
12986 (did_full_compacting_gc ?
12987 a_state_check_retry_seg :
12988 a_state_check_and_wait_for_bgc));
12991 case a_state_acquire_seg_after_cg:
12993 BOOL can_get_new_seg_p = FALSE;
12994 BOOL did_full_compacting_gc = FALSE;
12996 current_full_compact_gc_count = get_full_compact_gc_count();
12998 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
12999 // Since we release the msl before we try to allocate a seg, other
13000 // threads could have allocated a bunch of segments before us so
13001 // we might need to retry.
13002 loh_alloc_state = (can_get_new_seg_p ?
13003 a_state_try_fit_new_seg_after_cg :
13004 a_state_check_retry_seg);
13007 case a_state_acquire_seg_after_bgc:
13009 BOOL can_get_new_seg_p = FALSE;
13010 BOOL did_full_compacting_gc = FALSE;
13012 current_full_compact_gc_count = get_full_compact_gc_count();
13014 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
13015 loh_alloc_state = (can_get_new_seg_p ?
13016 a_state_try_fit_new_seg :
13017 (did_full_compacting_gc ?
13018 a_state_check_retry_seg :
13019 a_state_trigger_full_compact_gc));
13020 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13023 case a_state_check_and_wait_for_bgc:
13025 BOOL bgc_in_progress_p = FALSE;
13026 BOOL did_full_compacting_gc = FALSE;
13028 if (fgn_maxgen_percent)
13030 dprintf (2, ("FGN: failed to acquire seg, may need to do a full blocking GC"));
13031 send_full_gc_notification (max_generation, FALSE);
13034 bgc_in_progress_p = check_and_wait_for_bgc (awr_loh_oos_bgc, &did_full_compacting_gc);
13035 loh_alloc_state = (!bgc_in_progress_p ?
13036 a_state_trigger_full_compact_gc :
13037 (did_full_compacting_gc ?
13038 a_state_try_fit_after_cg :
13039 a_state_try_fit_after_bgc));
13042 case a_state_trigger_full_compact_gc:
13044 BOOL got_full_compacting_gc = FALSE;
13046 got_full_compacting_gc = trigger_full_compact_gc (gr, &oom_r);
13047 loh_alloc_state = (got_full_compacting_gc ? a_state_try_fit_after_cg : a_state_cant_allocate);
13048 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13051 case a_state_check_retry_seg:
13053 BOOL should_retry_gc = retry_full_compact_gc (size);
13054 BOOL should_retry_get_seg = FALSE;
13055 if (!should_retry_gc)
13057 size_t last_full_compact_gc_count = current_full_compact_gc_count;
13058 current_full_compact_gc_count = get_full_compact_gc_count();
13060 if (current_full_compact_gc_count > (last_full_compact_gc_count + 1))
13062 should_retry_get_seg = TRUE;
13066 loh_alloc_state = (should_retry_gc ?
13067 a_state_trigger_full_compact_gc :
13068 (should_retry_get_seg ?
13069 a_state_acquire_seg_after_cg :
13070 a_state_cant_allocate));
13071 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13076 assert (!"Invalid state!");
13083 if (loh_alloc_state == a_state_cant_allocate)
13085 assert (oom_r != oom_no_failure);
13086 handle_oom (heap_number,
13092 add_saved_spinlock_info (me_release, mt_alloc_large_cant);
13093 dprintf (SPINLOCK_LOG, ("[%d]Lmsl for loh oom", heap_number));
13094 leave_spin_lock (&more_space_lock);
13097 return (loh_alloc_state == a_state_can_allocate);
13100 int gc_heap::try_allocate_more_space (alloc_context* acontext, size_t size,
13103 if (gc_heap::gc_started)
13105 wait_for_gc_done();
13109 #ifdef SYNCHRONIZATION_STATS
13110 int64_t msl_acquire_start = GCToOSInterface::QueryPerformanceCounter();
13111 #endif //SYNCHRONIZATION_STATS
13112 enter_spin_lock (&more_space_lock);
13113 add_saved_spinlock_info (me_acquire, mt_try_alloc);
13114 dprintf (SPINLOCK_LOG, ("[%d]Emsl for alloc", heap_number));
13115 #ifdef SYNCHRONIZATION_STATS
13116 int64_t msl_acquire = GCToOSInterface::QueryPerformanceCounter() - msl_acquire_start;
13117 total_msl_acquire += msl_acquire;
13118 num_msl_acquired++;
13119 if (msl_acquire > 200)
13121 num_high_msl_acquire++;
13125 num_low_msl_acquire++;
13127 #endif //SYNCHRONIZATION_STATS
13130 // We are commenting this out 'cause we don't see the point - we already
13131 // have checked gc_started when we were acquiring the msl - no need to check
13132 // again. This complicates the logic in bgc_suspend_EE 'cause that one would
13133 // need to release msl which causes all sorts of trouble.
13134 if (gc_heap::gc_started)
13136 #ifdef SYNCHRONIZATION_STATS
13138 #endif //SYNCHRONIZATION_STATS
13139 BOOL fStress = (g_pConfig->GetGCStressLevel() & GCConfig::GCSTRESS_TRANSITION) != 0;
13142 //Rendez vous early (MP scaling issue)
13143 //dprintf (1, ("[%d]waiting for gc", heap_number));
13144 wait_for_gc_done();
13145 #ifdef MULTIPLE_HEAPS
13147 #endif //MULTIPLE_HEAPS
13152 dprintf (3, ("requested to allocate %d bytes on gen%d", size, gen_number));
13154 int align_const = get_alignment_constant (gen_number != (max_generation+1));
13156 if (fgn_maxgen_percent)
13158 check_for_full_gc (gen_number, size);
13161 if (!(new_allocation_allowed (gen_number)))
13163 if (fgn_maxgen_percent && (gen_number == 0))
13165 // We only check gen0 every so often, so take this opportunity to check again.
13166 check_for_full_gc (gen_number, size);
13169 #ifdef BACKGROUND_GC
13170 wait_for_bgc_high_memory (awr_gen0_alloc);
13171 #endif //BACKGROUND_GC
13173 #ifdef SYNCHRONIZATION_STATS
13175 #endif //SYNCHRONIZATION_STATS
13176 dprintf (/*100*/ 2, ("running out of budget on gen%d, gc", gen_number));
13178 if (!settings.concurrent || (gen_number == 0))
13180 vm_heap->GarbageCollectGeneration (0, ((gen_number == 0) ? reason_alloc_soh : reason_alloc_loh));
13181 #ifdef MULTIPLE_HEAPS
13182 enter_spin_lock (&more_space_lock);
13183 add_saved_spinlock_info (me_acquire, mt_try_budget);
13184 dprintf (SPINLOCK_LOG, ("[%d]Emsl out budget", heap_number));
13185 #endif //MULTIPLE_HEAPS
13189 BOOL can_allocate = ((gen_number == 0) ?
13190 allocate_small (gen_number, size, acontext, align_const) :
13191 allocate_large (gen_number, size, acontext, align_const));
13195 size_t alloc_context_bytes = acontext->alloc_limit + Align (min_obj_size, align_const) - acontext->alloc_ptr;
13196 int etw_allocation_index = ((gen_number == 0) ? 0 : 1);
13198 etw_allocation_running_amount[etw_allocation_index] += alloc_context_bytes;
13201 if (etw_allocation_running_amount[etw_allocation_index] > etw_allocation_tick)
13203 #ifdef FEATURE_REDHAWK
13204 FIRE_EVENT(GCAllocationTick_V1, (uint32_t)etw_allocation_running_amount[etw_allocation_index],
13205 (gen_number == 0) ? gc_etw_alloc_soh : gc_etw_alloc_loh);
13207 // Unfortunately some of the ETW macros do not check whether the ETW feature is enabled.
13208 // The ones that do are much less efficient.
13209 #if defined(FEATURE_EVENT_TRACE)
13210 if (EVENT_ENABLED(GCAllocationTick_V3))
13212 fire_etw_allocation_event (etw_allocation_running_amount[etw_allocation_index], gen_number, acontext->alloc_ptr);
13214 #endif //FEATURE_EVENT_TRACE
13215 #endif //FEATURE_REDHAWK
13216 etw_allocation_running_amount[etw_allocation_index] = 0;
13220 return (int)can_allocate;
13223 #ifdef MULTIPLE_HEAPS
13224 void gc_heap::balance_heaps (alloc_context* acontext)
13227 if (acontext->alloc_count < 4)
13229 if (acontext->alloc_count == 0)
13231 acontext->set_home_heap(GCHeap::GetHeap( heap_select::select_heap(acontext, 0) ));
13232 gc_heap* hp = acontext->get_home_heap()->pGenGCHeap;
13233 dprintf (3, ("First allocation for context %Ix on heap %d\n", (size_t)acontext, (size_t)hp->heap_number));
13234 acontext->set_alloc_heap(acontext->get_home_heap());
13235 hp->alloc_context_count++;
13240 BOOL set_home_heap = FALSE;
13243 if (heap_select::can_find_heap_fast())
13245 if (acontext->get_home_heap() != NULL)
13246 hint = acontext->get_home_heap()->pGenGCHeap->heap_number;
13247 if (acontext->get_home_heap() != GCHeap::GetHeap(hint = heap_select::select_heap(acontext, hint)) || ((acontext->alloc_count & 15) == 0))
13249 set_home_heap = TRUE;
13255 if ((acontext->alloc_count & 3) == 0)
13256 set_home_heap = TRUE;
13262 // Since we are balancing up to MAX_SUPPORTED_CPUS, no need for this.
13263 if (n_heaps > MAX_SUPPORTED_CPUS)
13265 // on machines with many processors cache affinity is really king, so don't even try
13266 // to balance on these.
13267 acontext->home_heap = GCHeap::GetHeap( heap_select::select_heap(acontext, hint) );
13268 acontext->alloc_heap = acontext->home_heap;
13273 gc_heap* org_hp = acontext->get_alloc_heap()->pGenGCHeap;
13275 dynamic_data* dd = org_hp->dynamic_data_of (0);
13276 ptrdiff_t org_size = dd_new_allocation (dd);
13277 int org_alloc_context_count;
13278 int max_alloc_context_count;
13280 ptrdiff_t max_size;
13281 size_t delta = dd_min_size (dd)/4;
13283 int start, end, finish;
13284 heap_select::get_heap_range_for_heap(org_hp->heap_number, &start, &end);
13285 finish = start + n_heaps;
13291 max_size = org_size + delta;
13292 acontext->set_home_heap(GCHeap::GetHeap( heap_select::select_heap(acontext, hint) ));
13294 if (org_hp == acontext->get_home_heap()->pGenGCHeap)
13295 max_size = max_size + delta;
13297 org_alloc_context_count = org_hp->alloc_context_count;
13298 max_alloc_context_count = org_alloc_context_count;
13299 if (max_alloc_context_count > 1)
13300 max_size /= max_alloc_context_count;
13302 for (int i = start; i < end; i++)
13304 gc_heap* hp = GCHeap::GetHeap(i%n_heaps)->pGenGCHeap;
13305 dd = hp->dynamic_data_of (0);
13306 ptrdiff_t size = dd_new_allocation (dd);
13307 if (hp == acontext->get_home_heap()->pGenGCHeap)
13308 size = size + delta;
13309 int hp_alloc_context_count = hp->alloc_context_count;
13310 if (hp_alloc_context_count > 0)
13311 size /= (hp_alloc_context_count + 1);
13312 if (size > max_size)
13316 max_alloc_context_count = hp_alloc_context_count;
13320 while (org_alloc_context_count != org_hp->alloc_context_count ||
13321 max_alloc_context_count != max_hp->alloc_context_count);
13323 if ((max_hp == org_hp) && (end < finish))
13325 start = end; end = finish;
13326 delta = dd_min_size(dd)/2; // Make it twice as hard to balance to remote nodes on NUMA.
13330 if (max_hp != org_hp)
13332 org_hp->alloc_context_count--;
13333 max_hp->alloc_context_count++;
13334 acontext->set_alloc_heap(GCHeap::GetHeap(max_hp->heap_number));
13335 if (GCToOSInterface::CanEnableGCCPUGroups())
13336 { //only set ideal processor when max_hp and org_hp are in the same cpu
13337 //group. DO NOT MOVE THREADS ACROSS CPU GROUPS
13338 uint16_t org_gn = heap_select::find_cpu_group_from_heap_no(org_hp->heap_number);
13339 uint16_t max_gn = heap_select::find_cpu_group_from_heap_no(max_hp->heap_number);
13340 if (org_gn == max_gn) //only set within CPU group, so SetThreadIdealProcessor is enough
13342 uint16_t group_proc_no = heap_select::find_group_proc_from_heap_no(max_hp->heap_number);
13344 GCThreadAffinity affinity;
13345 affinity.Processor = group_proc_no;
13346 affinity.Group = org_gn;
13347 if (!GCToOSInterface::SetCurrentThreadIdealAffinity(&affinity))
13349 dprintf (3, ("Failed to set the ideal processor and group for heap %d.",
13350 org_hp->heap_number));
13356 uint16_t proc_no = heap_select::find_proc_no_from_heap_no(max_hp->heap_number);
13358 GCThreadAffinity affinity;
13359 affinity.Processor = proc_no;
13360 affinity.Group = GCThreadAffinity::None;
13362 if (!GCToOSInterface::SetCurrentThreadIdealAffinity(&affinity))
13364 dprintf (3, ("Failed to set the ideal processor for heap %d.",
13365 org_hp->heap_number));
13368 dprintf (3, ("Switching context %p (home heap %d) ",
13370 acontext->get_home_heap()->pGenGCHeap->heap_number));
13371 dprintf (3, (" from heap %d (%Id free bytes, %d contexts) ",
13372 org_hp->heap_number,
13374 org_alloc_context_count));
13375 dprintf (3, (" to heap %d (%Id free bytes, %d contexts)\n",
13376 max_hp->heap_number,
13377 dd_new_allocation(max_hp->dynamic_data_of(0)),
13378 max_alloc_context_count));
13383 acontext->alloc_count++;
13386 gc_heap* gc_heap::balance_heaps_loh (alloc_context* acontext, size_t /*size*/)
13388 gc_heap* org_hp = acontext->get_alloc_heap()->pGenGCHeap;
13389 //dprintf (1, ("LA: %Id", size));
13391 //if (size > 128*1024)
13394 dynamic_data* dd = org_hp->dynamic_data_of (max_generation + 1);
13396 ptrdiff_t org_size = dd_new_allocation (dd);
13398 ptrdiff_t max_size;
13399 size_t delta = dd_min_size (dd) * 4;
13401 int start, end, finish;
13402 heap_select::get_heap_range_for_heap(org_hp->heap_number, &start, &end);
13403 finish = start + n_heaps;
13408 max_size = org_size + delta;
13409 dprintf (3, ("orig hp: %d, max size: %d",
13410 org_hp->heap_number,
13413 for (int i = start; i < end; i++)
13415 gc_heap* hp = GCHeap::GetHeap(i%n_heaps)->pGenGCHeap;
13416 dd = hp->dynamic_data_of (max_generation + 1);
13417 ptrdiff_t size = dd_new_allocation (dd);
13418 dprintf (3, ("hp: %d, size: %d",
13421 if (size > max_size)
13425 dprintf (3, ("max hp: %d, max size: %d",
13426 max_hp->heap_number,
13432 if ((max_hp == org_hp) && (end < finish))
13434 start = end; end = finish;
13435 delta = dd_min_size(dd) * 4; // Need to tuning delta
13439 if (max_hp != org_hp)
13441 dprintf (3, ("loh: %d(%Id)->%d(%Id)",
13442 org_hp->heap_number, dd_new_allocation (org_hp->dynamic_data_of (max_generation + 1)),
13443 max_hp->heap_number, dd_new_allocation (max_hp->dynamic_data_of (max_generation + 1))));
13453 #endif //MULTIPLE_HEAPS
13455 BOOL gc_heap::allocate_more_space(alloc_context* acontext, size_t size,
13456 int alloc_generation_number)
13461 #ifdef MULTIPLE_HEAPS
13462 if (alloc_generation_number == 0)
13464 balance_heaps (acontext);
13465 status = acontext->get_alloc_heap()->pGenGCHeap->try_allocate_more_space (acontext, size, alloc_generation_number);
13469 gc_heap* alloc_heap = balance_heaps_loh (acontext, size);
13470 status = alloc_heap->try_allocate_more_space (acontext, size, alloc_generation_number);
13473 status = try_allocate_more_space (acontext, size, alloc_generation_number);
13474 #endif //MULTIPLE_HEAPS
13476 while (status == -1);
13478 return (status != 0);
13482 CObjectHeader* gc_heap::allocate (size_t jsize, alloc_context* acontext)
13484 size_t size = Align (jsize);
13485 assert (size >= Align (min_obj_size));
13488 uint8_t* result = acontext->alloc_ptr;
13489 acontext->alloc_ptr+=size;
13490 if (acontext->alloc_ptr <= acontext->alloc_limit)
13492 CObjectHeader* obj = (CObjectHeader*)result;
13498 acontext->alloc_ptr -= size;
13501 #pragma inline_depth(0)
13504 if (! allocate_more_space (acontext, size, 0))
13508 #pragma inline_depth(20)
13517 CObjectHeader* gc_heap::try_fast_alloc (size_t jsize)
13519 size_t size = Align (jsize);
13520 assert (size >= Align (min_obj_size));
13521 generation* gen = generation_of (0);
13522 uint8_t* result = generation_allocation_pointer (gen);
13523 generation_allocation_pointer (gen) += size;
13524 if (generation_allocation_pointer (gen) <=
13525 generation_allocation_limit (gen))
13527 return (CObjectHeader*)result;
13531 generation_allocation_pointer (gen) -= size;
13535 void gc_heap::leave_allocation_segment (generation* gen)
13537 adjust_limit (0, 0, gen, max_generation);
13540 void gc_heap::init_free_and_plug()
13542 #ifdef FREE_USAGE_STATS
13543 for (int i = 0; i <= settings.condemned_generation; i++)
13545 generation* gen = generation_of (i);
13546 memset (gen->gen_free_spaces, 0, sizeof (gen->gen_free_spaces));
13547 memset (gen->gen_plugs, 0, sizeof (gen->gen_plugs));
13548 memset (gen->gen_current_pinned_free_spaces, 0, sizeof (gen->gen_current_pinned_free_spaces));
13551 if (settings.condemned_generation != max_generation)
13553 for (int i = (settings.condemned_generation + 1); i <= max_generation; i++)
13555 generation* gen = generation_of (i);
13556 memset (gen->gen_plugs, 0, sizeof (gen->gen_plugs));
13559 #endif //FREE_USAGE_STATS
13562 void gc_heap::print_free_and_plug (const char* msg)
13564 #if defined(FREE_USAGE_STATS) && defined(SIMPLE_DPRINTF)
13565 int older_gen = ((settings.condemned_generation == max_generation) ? max_generation : (settings.condemned_generation + 1));
13566 for (int i = 0; i <= older_gen; i++)
13568 generation* gen = generation_of (i);
13569 for (int j = 0; j < NUM_GEN_POWER2; j++)
13571 if ((gen->gen_free_spaces[j] != 0) || (gen->gen_plugs[j] != 0))
13573 dprintf (2, ("[%s][h%d][%s#%d]gen%d: 2^%d: F: %Id, P: %Id",
13576 (settings.concurrent ? "BGC" : "GC"),
13579 (j + 9), gen->gen_free_spaces[j], gen->gen_plugs[j]));
13584 UNREFERENCED_PARAMETER(msg);
13585 #endif //FREE_USAGE_STATS && SIMPLE_DPRINTF
13588 void gc_heap::add_gen_plug (int gen_number, size_t plug_size)
13590 #ifdef FREE_USAGE_STATS
13591 dprintf (3, ("adding plug size %Id to gen%d", plug_size, gen_number));
13592 generation* gen = generation_of (gen_number);
13593 size_t sz = BASE_GEN_SIZE;
13596 for (; i < NUM_GEN_POWER2; i++)
13598 if (plug_size < sz)
13605 (gen->gen_plugs[i])++;
13607 UNREFERENCED_PARAMETER(gen_number);
13608 UNREFERENCED_PARAMETER(plug_size);
13609 #endif //FREE_USAGE_STATS
13612 void gc_heap::add_item_to_current_pinned_free (int gen_number, size_t free_size)
13614 #ifdef FREE_USAGE_STATS
13615 generation* gen = generation_of (gen_number);
13616 size_t sz = BASE_GEN_SIZE;
13619 for (; i < NUM_GEN_POWER2; i++)
13621 if (free_size < sz)
13628 (gen->gen_current_pinned_free_spaces[i])++;
13629 generation_pinned_free_obj_space (gen) += free_size;
13630 dprintf (3, ("left pin free %Id(2^%d) to gen%d, total %Id bytes (%Id)",
13631 free_size, (i + 10), gen_number,
13632 generation_pinned_free_obj_space (gen),
13633 gen->gen_current_pinned_free_spaces[i]));
13635 UNREFERENCED_PARAMETER(gen_number);
13636 UNREFERENCED_PARAMETER(free_size);
13637 #endif //FREE_USAGE_STATS
13640 void gc_heap::add_gen_free (int gen_number, size_t free_size)
13642 #ifdef FREE_USAGE_STATS
13643 dprintf (3, ("adding free size %Id to gen%d", free_size, gen_number));
13644 generation* gen = generation_of (gen_number);
13645 size_t sz = BASE_GEN_SIZE;
13648 for (; i < NUM_GEN_POWER2; i++)
13650 if (free_size < sz)
13657 (gen->gen_free_spaces[i])++;
13659 UNREFERENCED_PARAMETER(gen_number);
13660 UNREFERENCED_PARAMETER(free_size);
13661 #endif //FREE_USAGE_STATS
13664 void gc_heap::remove_gen_free (int gen_number, size_t free_size)
13666 #ifdef FREE_USAGE_STATS
13667 dprintf (3, ("removing free %Id from gen%d", free_size, gen_number));
13668 generation* gen = generation_of (gen_number);
13669 size_t sz = BASE_GEN_SIZE;
13672 for (; i < NUM_GEN_POWER2; i++)
13674 if (free_size < sz)
13681 (gen->gen_free_spaces[i])--;
13683 UNREFERENCED_PARAMETER(gen_number);
13684 UNREFERENCED_PARAMETER(free_size);
13685 #endif //FREE_USAGE_STATS
13688 uint8_t* gc_heap::allocate_in_older_generation (generation* gen, size_t size,
13689 int from_gen_number,
13690 uint8_t* old_loc REQD_ALIGN_AND_OFFSET_DCL)
13692 size = Align (size);
13693 assert (size >= Align (min_obj_size));
13694 assert (from_gen_number < max_generation);
13695 assert (from_gen_number >= 0);
13696 assert (generation_of (from_gen_number + 1) == gen);
13698 allocator* gen_allocator = generation_allocator (gen);
13699 BOOL discard_p = gen_allocator->discard_if_no_fit_p ();
13700 int pad_in_front = (old_loc != 0)? USE_PADDING_FRONT : 0;
13702 size_t real_size = size + Align (min_obj_size);
13704 real_size += Align (min_obj_size);
13706 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
13707 generation_allocation_limit (gen), old_loc, USE_PADDING_TAIL | pad_in_front)))
13709 size_t sz_list = gen_allocator->first_bucket_size();
13710 for (unsigned int a_l_idx = 0; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
13712 if ((real_size < (sz_list / 2)) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
13714 uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
13715 uint8_t* prev_free_item = 0;
13716 while (free_list != 0)
13718 dprintf (3, ("considering free list %Ix", (size_t)free_list));
13720 size_t free_list_size = unused_array_size (free_list);
13722 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, free_list, (free_list + free_list_size),
13723 old_loc, USE_PADDING_TAIL | pad_in_front))
13725 dprintf (4, ("F:%Ix-%Id",
13726 (size_t)free_list, free_list_size));
13728 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, !discard_p);
13729 generation_free_list_space (gen) -= free_list_size;
13730 remove_gen_free (gen->gen_num, free_list_size);
13732 adjust_limit (free_list, free_list_size, gen, from_gen_number+1);
13735 // We do first fit on bucket 0 because we are not guaranteed to find a fit there.
13736 else if (discard_p || (a_l_idx == 0))
13738 dprintf (3, ("couldn't use this free area, discarding"));
13739 generation_free_obj_space (gen) += free_list_size;
13741 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
13742 generation_free_list_space (gen) -= free_list_size;
13743 remove_gen_free (gen->gen_num, free_list_size);
13747 prev_free_item = free_list;
13749 free_list = free_list_slot (free_list);
13752 sz_list = sz_list * 2;
13754 //go back to the beginning of the segment list
13755 generation_allocate_end_seg_p (gen) = TRUE;
13756 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
13757 if (seg != generation_allocation_segment (gen))
13759 leave_allocation_segment (gen);
13760 generation_allocation_segment (gen) = seg;
13762 while (seg != ephemeral_heap_segment)
13764 if (size_fit_p(size REQD_ALIGN_AND_OFFSET_ARG, heap_segment_plan_allocated (seg),
13765 heap_segment_committed (seg), old_loc, USE_PADDING_TAIL | pad_in_front))
13767 dprintf (3, ("using what's left in committed"));
13768 adjust_limit (heap_segment_plan_allocated (seg),
13769 heap_segment_committed (seg) -
13770 heap_segment_plan_allocated (seg),
13771 gen, from_gen_number+1);
13772 // dformat (t, 3, "Expanding segment allocation");
13773 heap_segment_plan_allocated (seg) =
13774 heap_segment_committed (seg);
13779 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, heap_segment_plan_allocated (seg),
13780 heap_segment_reserved (seg), old_loc, USE_PADDING_TAIL | pad_in_front) &&
13781 grow_heap_segment (seg, heap_segment_plan_allocated (seg), old_loc, size, pad_in_front REQD_ALIGN_AND_OFFSET_ARG))
13783 dprintf (3, ("using what's left in reserved"));
13784 adjust_limit (heap_segment_plan_allocated (seg),
13785 heap_segment_committed (seg) -
13786 heap_segment_plan_allocated (seg),
13787 gen, from_gen_number+1);
13788 heap_segment_plan_allocated (seg) =
13789 heap_segment_committed (seg);
13795 leave_allocation_segment (gen);
13796 heap_segment* next_seg = heap_segment_next_rw (seg);
13799 dprintf (3, ("getting next segment"));
13800 generation_allocation_segment (gen) = next_seg;
13801 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
13802 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
13811 seg = generation_allocation_segment (gen);
13813 //No need to fix the last region. Will be done later
13824 uint8_t* result = generation_allocation_pointer (gen);
13828 if ((pad_in_front & USE_PADDING_FRONT) &&
13829 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
13830 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
13832 pad = Align (min_obj_size);
13833 set_plug_padded (old_loc);
13835 #endif //SHORT_PLUGS
13837 #ifdef FEATURE_STRUCTALIGN
13838 _ASSERTE(!old_loc || alignmentOffset != 0);
13839 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
13842 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
13843 set_node_aligninfo (old_loc, requiredAlignment, pad1);
13846 #else // FEATURE_STRUCTALIGN
13847 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
13849 pad += switch_alignment_size (is_plug_padded (old_loc));
13850 set_node_realigned (old_loc);
13851 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
13852 (size_t)old_loc, (size_t)(result+pad)));
13853 assert (same_large_alignment_p (result + pad, old_loc));
13855 #endif // FEATURE_STRUCTALIGN
13856 dprintf (3, ("Allocate %Id bytes", size));
13858 if ((old_loc == 0) || (pad != 0))
13860 //allocating a non plug or a gap, so reset the start region
13861 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
13864 generation_allocation_pointer (gen) += size + pad;
13865 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
13866 if (generation_allocate_end_seg_p (gen))
13868 generation_end_seg_allocated (gen) += size;
13872 generation_free_list_allocated (gen) += size;
13874 generation_allocation_size (gen) += size;
13876 dprintf (3, ("aio: ptr: %Ix, limit: %Ix, sr: %Ix",
13877 generation_allocation_pointer (gen), generation_allocation_limit (gen),
13878 generation_allocation_context_start_region (gen)));
13880 return result + pad;;
13884 void gc_heap::repair_allocation_in_expanded_heap (generation* consing_gen)
13886 //make sure that every generation has a planned allocation start
13887 int gen_number = max_generation - 1;
13888 while (gen_number>= 0)
13890 generation* gen = generation_of (gen_number);
13891 if (0 == generation_plan_allocation_start (gen))
13893 realloc_plan_generation_start (gen, consing_gen);
13895 assert (generation_plan_allocation_start (gen));
13900 // now we know the planned allocation size
13901 size_t size = (generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
13902 heap_segment* seg = generation_allocation_segment (consing_gen);
13903 if (generation_allocation_limit (consing_gen) == heap_segment_plan_allocated (seg))
13907 heap_segment_plan_allocated (seg) = generation_allocation_pointer (consing_gen);
13912 assert (settings.condemned_generation == max_generation);
13913 uint8_t* first_address = generation_allocation_limit (consing_gen);
13914 //look through the pinned plugs for relevant ones.
13915 //Look for the right pinned plug to start from.
13918 while (mi != mark_stack_tos)
13920 m = pinned_plug_of (mi);
13921 if ((pinned_plug (m) == first_address))
13926 assert (mi != mark_stack_tos);
13927 pinned_len (m) = size;
13931 //tododefrag optimize for new segment (plan_allocated == mem)
13932 uint8_t* gc_heap::allocate_in_expanded_heap (generation* gen,
13937 BOOL set_padding_on_saved_p,
13938 mark* pinned_plug_entry,
13939 #endif //SHORT_PLUGS
13940 BOOL consider_bestfit,
13941 int active_new_gen_number
13942 REQD_ALIGN_AND_OFFSET_DCL)
13944 UNREFERENCED_PARAMETER(active_new_gen_number);
13945 dprintf (3, ("aie: P: %Ix, size: %Ix", old_loc, size));
13947 size = Align (size);
13948 assert (size >= Align (min_obj_size));
13949 int pad_in_front = (old_loc != 0) ? USE_PADDING_FRONT : 0;
13951 if (consider_bestfit && use_bestfit)
13953 assert (bestfit_seg);
13954 dprintf (SEG_REUSE_LOG_1, ("reallocating 0x%Ix in expanded heap, size: %Id",
13956 return bestfit_seg->fit (old_loc,
13958 set_padding_on_saved_p,
13960 #endif //SHORT_PLUGS
13961 size REQD_ALIGN_AND_OFFSET_ARG);
13964 heap_segment* seg = generation_allocation_segment (gen);
13966 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
13967 generation_allocation_limit (gen), old_loc,
13968 ((generation_allocation_limit (gen) !=
13969 heap_segment_plan_allocated (seg))? USE_PADDING_TAIL : 0) | pad_in_front)))
13971 dprintf (3, ("aie: can't fit: ptr: %Ix, limit: %Ix", generation_allocation_pointer (gen),
13972 generation_allocation_limit (gen)));
13975 uint8_t* first_address = (generation_allocation_limit (gen) ?
13976 generation_allocation_limit (gen) :
13977 heap_segment_mem (seg));
13978 assert (in_range_for_segment (first_address, seg));
13980 uint8_t* end_address = heap_segment_reserved (seg);
13982 dprintf (3, ("aie: first_addr: %Ix, gen alloc limit: %Ix, end_address: %Ix",
13983 first_address, generation_allocation_limit (gen), end_address));
13988 if (heap_segment_allocated (seg) != heap_segment_mem (seg))
13990 assert (settings.condemned_generation == max_generation);
13991 //look through the pinned plugs for relevant ones.
13992 //Look for the right pinned plug to start from.
13993 while (mi != mark_stack_tos)
13995 m = pinned_plug_of (mi);
13996 if ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address))
13998 dprintf (3, ("aie: found pin: %Ix", pinned_plug (m)));
14004 if (mi != mark_stack_tos)
14006 //fix old free list.
14007 size_t hsize = (generation_allocation_limit (gen) - generation_allocation_pointer (gen));
14009 dprintf(3,("gc filling up hole"));
14010 ptrdiff_t mi1 = (ptrdiff_t)mi;
14011 while ((mi1 >= 0) &&
14012 (pinned_plug (pinned_plug_of(mi1)) != generation_allocation_limit (gen)))
14014 dprintf (3, ("aie: checking pin %Ix", pinned_plug (pinned_plug_of(mi1))));
14019 size_t saved_pinned_len = pinned_len (pinned_plug_of(mi1));
14020 pinned_len (pinned_plug_of(mi1)) = hsize;
14021 dprintf (3, ("changing %Ix len %Ix->%Ix",
14022 pinned_plug (pinned_plug_of(mi1)),
14023 saved_pinned_len, pinned_len (pinned_plug_of(mi1))));
14030 assert (generation_allocation_limit (gen) ==
14031 generation_allocation_pointer (gen));
14032 mi = mark_stack_tos;
14035 while ((mi != mark_stack_tos) && in_range_for_segment (pinned_plug (m), seg))
14037 size_t len = pinned_len (m);
14038 uint8_t* free_list = (pinned_plug (m) - len);
14039 dprintf (3, ("aie: testing free item: %Ix->%Ix(%Ix)",
14040 free_list, (free_list + len), len));
14041 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, free_list, (free_list + len), old_loc, USE_PADDING_TAIL | pad_in_front))
14043 dprintf (3, ("aie: Found adequate unused area: %Ix, size: %Id",
14044 (size_t)free_list, len));
14046 generation_allocation_pointer (gen) = free_list;
14047 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14048 generation_allocation_limit (gen) = (free_list + len);
14050 goto allocate_in_free;
14053 m = pinned_plug_of (mi);
14056 //switch to the end of the segment.
14057 generation_allocation_pointer (gen) = heap_segment_plan_allocated (seg);
14058 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14059 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14060 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14061 dprintf (3, ("aie: switching to end of seg: %Ix->%Ix(%Ix)",
14062 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14063 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
14065 if (!size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14066 generation_allocation_limit (gen), old_loc, USE_PADDING_TAIL | pad_in_front))
14068 dprintf (3, ("aie: ptr: %Ix, limit: %Ix, can't alloc", generation_allocation_pointer (gen),
14069 generation_allocation_limit (gen)));
14070 assert (!"Can't allocate if no free space");
14081 uint8_t* result = generation_allocation_pointer (gen);
14085 if ((pad_in_front & USE_PADDING_FRONT) &&
14086 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
14087 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
14090 pad = Align (min_obj_size);
14091 set_padding_in_expand (old_loc, set_padding_on_saved_p, pinned_plug_entry);
14093 #endif //SHORT_PLUGS
14095 #ifdef FEATURE_STRUCTALIGN
14096 _ASSERTE(!old_loc || alignmentOffset != 0);
14097 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
14100 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
14101 set_node_aligninfo (old_loc, requiredAlignment, pad1);
14105 #else // FEATURE_STRUCTALIGN
14106 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
14108 pad += switch_alignment_size (is_plug_padded (old_loc));
14109 set_node_realigned (old_loc);
14110 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
14111 (size_t)old_loc, (size_t)(result+pad)));
14112 assert (same_large_alignment_p (result + pad, old_loc));
14115 #endif // FEATURE_STRUCTALIGN
14117 if ((old_loc == 0) || (pad != 0))
14119 //allocating a non plug or a gap, so reset the start region
14120 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14123 generation_allocation_pointer (gen) += size + pad;
14124 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
14125 dprintf (3, ("Allocated in expanded heap %Ix:%Id", (size_t)(result+pad), size));
14127 dprintf (3, ("aie: ptr: %Ix, limit: %Ix, sr: %Ix",
14128 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14129 generation_allocation_context_start_region (gen)));
14131 return result + pad;
14135 generation* gc_heap::ensure_ephemeral_heap_segment (generation* consing_gen)
14137 heap_segment* seg = generation_allocation_segment (consing_gen);
14138 if (seg != ephemeral_heap_segment)
14140 assert (generation_allocation_pointer (consing_gen)>= heap_segment_mem (seg));
14141 assert (generation_allocation_pointer (consing_gen)<= heap_segment_committed (seg));
14143 //fix the allocated size of the segment.
14144 heap_segment_plan_allocated (seg) = generation_allocation_pointer (consing_gen);
14146 generation* new_consing_gen = generation_of (max_generation - 1);
14147 generation_allocation_pointer (new_consing_gen) =
14148 heap_segment_mem (ephemeral_heap_segment);
14149 generation_allocation_limit (new_consing_gen) =
14150 generation_allocation_pointer (new_consing_gen);
14151 generation_allocation_context_start_region (new_consing_gen) =
14152 generation_allocation_pointer (new_consing_gen);
14153 generation_allocation_segment (new_consing_gen) = ephemeral_heap_segment;
14155 return new_consing_gen;
14158 return consing_gen;
14161 uint8_t* gc_heap::allocate_in_condemned_generations (generation* gen,
14163 int from_gen_number,
14165 BOOL* convert_to_pinned_p,
14166 uint8_t* next_pinned_plug,
14167 heap_segment* current_seg,
14168 #endif //SHORT_PLUGS
14170 REQD_ALIGN_AND_OFFSET_DCL)
14172 // Make sure that the youngest generation gap hasn't been allocated
14173 if (settings.promotion)
14175 assert (generation_plan_allocation_start (youngest_generation) == 0);
14178 size = Align (size);
14179 assert (size >= Align (min_obj_size));
14180 int to_gen_number = from_gen_number;
14181 if (from_gen_number != (int)max_generation)
14183 to_gen_number = from_gen_number + (settings.promotion ? 1 : 0);
14186 dprintf (3, ("aic gen%d: s: %Id, %d->%d, %Ix->%Ix", gen->gen_num, size, from_gen_number,
14187 to_gen_number, generation_allocation_pointer(gen), generation_allocation_limit(gen)));
14189 int pad_in_front = (old_loc != 0) ? USE_PADDING_FRONT : 0;
14191 if ((from_gen_number != -1) && (from_gen_number != (int)max_generation) && settings.promotion)
14193 generation_condemned_allocated (generation_of (from_gen_number + (settings.promotion ? 1 : 0))) += size;
14194 generation_allocation_size (generation_of (from_gen_number + (settings.promotion ? 1 : 0))) += size;
14198 heap_segment* seg = generation_allocation_segment (gen);
14199 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14200 generation_allocation_limit (gen), old_loc,
14201 ((generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))?USE_PADDING_TAIL:0)|pad_in_front)))
14203 if ((! (pinned_plug_que_empty_p()) &&
14204 (generation_allocation_limit (gen) ==
14205 pinned_plug (oldest_pin()))))
14207 size_t entry = deque_pinned_plug();
14208 mark* pinned_plug_entry = pinned_plug_of (entry);
14209 size_t len = pinned_len (pinned_plug_entry);
14210 uint8_t* plug = pinned_plug (pinned_plug_entry);
14211 set_new_pin_info (pinned_plug_entry, generation_allocation_pointer (gen));
14213 #ifdef FREE_USAGE_STATS
14214 generation_allocated_in_pinned_free (gen) += generation_allocated_since_last_pin (gen);
14215 dprintf (3, ("allocated %Id so far within pin %Ix, total->%Id",
14216 generation_allocated_since_last_pin (gen),
14218 generation_allocated_in_pinned_free (gen)));
14219 generation_allocated_since_last_pin (gen) = 0;
14221 add_item_to_current_pinned_free (gen->gen_num, pinned_len (pinned_plug_of (entry)));
14222 #endif //FREE_USAGE_STATS
14224 dprintf (3, ("mark stack bos: %Id, tos: %Id, aic: p %Ix len: %Ix->%Ix",
14225 mark_stack_bos, mark_stack_tos, plug, len, pinned_len (pinned_plug_of (entry))));
14227 assert(mark_stack_array[entry].len == 0 ||
14228 mark_stack_array[entry].len >= Align(min_obj_size));
14229 generation_allocation_pointer (gen) = plug + len;
14230 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14231 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14232 set_allocator_next_pin (gen);
14234 //Add the size of the pinned plug to the right pinned allocations
14235 //find out which gen this pinned plug came from
14236 int frgn = object_gennum (plug);
14237 if ((frgn != (int)max_generation) && settings.promotion)
14239 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
14240 int togn = object_gennum_plan (plug);
14243 generation_pinned_allocation_compact_size (generation_of (togn)) += len;
14249 if (generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))
14251 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14252 dprintf (3, ("changed limit to plan alloc: %Ix", generation_allocation_limit (gen)));
14256 if (heap_segment_plan_allocated (seg) != heap_segment_committed (seg))
14258 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14259 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14260 dprintf (3, ("changed limit to commit: %Ix", generation_allocation_limit (gen)));
14264 #ifndef RESPECT_LARGE_ALIGNMENT
14265 assert (gen != youngest_generation);
14266 #endif //RESPECT_LARGE_ALIGNMENT
14268 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14269 heap_segment_reserved (seg), old_loc, USE_PADDING_TAIL | pad_in_front) &&
14270 (grow_heap_segment (seg, generation_allocation_pointer (gen), old_loc,
14271 size, pad_in_front REQD_ALIGN_AND_OFFSET_ARG)))
14273 dprintf (3, ("Expanded segment allocation by committing more memory"));
14274 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14275 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14279 heap_segment* next_seg = heap_segment_next (seg);
14280 assert (generation_allocation_pointer (gen)>=
14281 heap_segment_mem (seg));
14282 // Verify that all pinned plugs for this segment are consumed
14283 if (!pinned_plug_que_empty_p() &&
14284 ((pinned_plug (oldest_pin()) <
14285 heap_segment_allocated (seg)) &&
14286 (pinned_plug (oldest_pin()) >=
14287 generation_allocation_pointer (gen))))
14289 LOG((LF_GC, LL_INFO10, "remaining pinned plug %Ix while leaving segment on allocation",
14290 pinned_plug (oldest_pin())));
14293 assert (generation_allocation_pointer (gen)>=
14294 heap_segment_mem (seg));
14295 assert (generation_allocation_pointer (gen)<=
14296 heap_segment_committed (seg));
14297 heap_segment_plan_allocated (seg) = generation_allocation_pointer (gen);
14301 generation_allocation_segment (gen) = next_seg;
14302 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
14303 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
14304 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14308 return 0; //should only happen during allocation of generation 0 gap
14309 // in that case we are going to grow the heap anyway
14314 set_allocator_next_pin (gen);
14321 assert (generation_allocation_pointer (gen)>=
14322 heap_segment_mem (generation_allocation_segment (gen)));
14323 uint8_t* result = generation_allocation_pointer (gen);
14326 if ((pad_in_front & USE_PADDING_FRONT) &&
14327 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
14328 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
14330 ptrdiff_t dist = old_loc - result;
14333 dprintf (3, ("old alloc: %Ix, same as new alloc, not padding", old_loc));
14338 if ((dist > 0) && (dist < (ptrdiff_t)Align (min_obj_size)))
14340 dprintf (3, ("old alloc: %Ix, only %d bytes > new alloc! Shouldn't happen", old_loc, dist));
14344 pad = Align (min_obj_size);
14345 set_plug_padded (old_loc);
14348 #endif //SHORT_PLUGS
14349 #ifdef FEATURE_STRUCTALIGN
14350 _ASSERTE(!old_loc || alignmentOffset != 0);
14351 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
14352 if ((old_loc != 0))
14354 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
14355 set_node_aligninfo (old_loc, requiredAlignment, pad1);
14358 #else // FEATURE_STRUCTALIGN
14359 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
14361 pad += switch_alignment_size (is_plug_padded (old_loc));
14362 set_node_realigned(old_loc);
14363 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
14364 (size_t)old_loc, (size_t)(result+pad)));
14365 assert (same_large_alignment_p (result + pad, old_loc));
14367 #endif // FEATURE_STRUCTALIGN
14370 if ((next_pinned_plug != 0) && (pad != 0) && (generation_allocation_segment (gen) == current_seg))
14372 assert (old_loc != 0);
14373 ptrdiff_t dist_to_next_pin = (ptrdiff_t)(next_pinned_plug - (generation_allocation_pointer (gen) + size + pad));
14374 assert (dist_to_next_pin >= 0);
14376 if ((dist_to_next_pin >= 0) && (dist_to_next_pin < (ptrdiff_t)Align (min_obj_size)))
14378 dprintf (3, ("%Ix->(%Ix,%Ix),%Ix(%Ix)(%Ix),NP->PP",
14380 generation_allocation_pointer (gen),
14381 generation_allocation_limit (gen),
14384 dist_to_next_pin));
14385 clear_plug_padded (old_loc);
14387 *convert_to_pinned_p = TRUE;
14388 record_interesting_data_point (idp_converted_pin);
14393 #endif //SHORT_PLUGS
14395 if ((old_loc == 0) || (pad != 0))
14397 //allocating a non plug or a gap, so reset the start region
14398 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14401 generation_allocation_pointer (gen) += size + pad;
14402 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
14404 #ifdef FREE_USAGE_STATS
14405 generation_allocated_since_last_pin (gen) += size;
14406 #endif //FREE_USAGE_STATS
14408 dprintf (3, ("aic: ptr: %Ix, limit: %Ix, sr: %Ix",
14409 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14410 generation_allocation_context_start_region (gen)));
14412 assert (result + pad);
14413 return result + pad;
14417 inline int power (int x, int y)
14420 for (int i = 0; i < y; i++)
14427 int gc_heap::joined_generation_to_condemn (BOOL should_evaluate_elevation,
14429 BOOL* blocking_collection_p
14430 STRESS_HEAP_ARG(int n_original))
14433 #ifdef MULTIPLE_HEAPS
14434 BOOL blocking_p = *blocking_collection_p;
14437 for (int i = 0; i < n_heaps; i++)
14439 if (g_heaps[i]->last_gc_before_oom)
14441 dprintf (GTC_LOG, ("h%d is setting blocking to TRUE", i));
14442 *blocking_collection_p = TRUE;
14447 #endif //MULTIPLE_HEAPS
14449 if (should_evaluate_elevation && (n == max_generation))
14451 dprintf (GTC_LOG, ("lock: %d(%d)",
14452 (settings.should_lock_elevation ? 1 : 0),
14453 settings.elevation_locked_count));
14455 if (settings.should_lock_elevation)
14457 settings.elevation_locked_count++;
14458 if (settings.elevation_locked_count == 6)
14460 settings.elevation_locked_count = 0;
14464 n = max_generation - 1;
14465 settings.elevation_reduced = TRUE;
14470 settings.elevation_locked_count = 0;
14475 settings.should_lock_elevation = FALSE;
14476 settings.elevation_locked_count = 0;
14480 #ifdef BACKGROUND_GC
14481 // We can only do Concurrent GC Stress if the caller did not explicitly ask for all
14482 // generations to be collected,
14484 // [LOCALGC TODO] STRESS_HEAP is not defined for a standalone GC so there are multiple
14485 // things that need to be fixed in this code block.
14486 if (n_original != max_generation &&
14487 g_pConfig->GetGCStressLevel() && gc_can_use_concurrent)
14489 #ifndef FEATURE_REDHAWK
14490 // for the GC stress mix mode throttle down gen2 collections
14491 if (g_pConfig->IsGCStressMix())
14493 size_t current_gc_count = 0;
14495 #ifdef MULTIPLE_HEAPS
14496 current_gc_count = (size_t)dd_collection_count (g_heaps[0]->dynamic_data_of (0));
14498 current_gc_count = (size_t)dd_collection_count (dynamic_data_of (0));
14499 #endif //MULTIPLE_HEAPS
14500 // in gc stress, only escalate every 10th non-gen2 collection to a gen2...
14501 if ((current_gc_count % 10) == 0)
14503 n = max_generation;
14506 // for traditional GC stress
14508 #endif // !FEATURE_REDHAWK
14509 if (*blocking_collection_p)
14511 // We call StressHeap() a lot for Concurrent GC Stress. However,
14512 // if we can not do a concurrent collection, no need to stress anymore.
14513 // @TODO: Enable stress when the memory pressure goes down again
14514 GCStressPolicy::GlobalDisable();
14518 n = max_generation;
14521 #endif //BACKGROUND_GC
14522 #endif //STRESS_HEAP
14528 size_t get_survived_size (gc_history_per_heap* hist)
14530 size_t surv_size = 0;
14531 gc_generation_data* gen_data;
14533 for (int gen_number = 0; gen_number <= (max_generation + 1); gen_number++)
14535 gen_data = &(hist->gen_data[gen_number]);
14536 surv_size += (gen_data->size_after -
14537 gen_data->free_list_space_after -
14538 gen_data->free_obj_space_after);
14544 size_t gc_heap::get_total_survived_size()
14546 size_t total_surv_size = 0;
14547 #ifdef MULTIPLE_HEAPS
14548 for (int i = 0; i < gc_heap::n_heaps; i++)
14550 gc_heap* hp = gc_heap::g_heaps[i];
14551 gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap();
14552 total_surv_size += get_survived_size (current_gc_data_per_heap);
14555 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
14556 total_surv_size = get_survived_size (current_gc_data_per_heap);
14557 #endif //MULTIPLE_HEAPS
14558 return total_surv_size;
14561 // Gets what's allocated on both SOH and LOH that hasn't been collected.
14562 size_t gc_heap::get_current_allocated()
14564 dynamic_data* dd = dynamic_data_of (0);
14565 size_t current_alloc = dd_desired_allocation (dd) - dd_new_allocation (dd);
14566 dd = dynamic_data_of (max_generation + 1);
14567 current_alloc += dd_desired_allocation (dd) - dd_new_allocation (dd);
14569 return current_alloc;
14572 size_t gc_heap::get_total_allocated()
14574 size_t total_current_allocated = 0;
14575 #ifdef MULTIPLE_HEAPS
14576 for (int i = 0; i < gc_heap::n_heaps; i++)
14578 gc_heap* hp = gc_heap::g_heaps[i];
14579 total_current_allocated += hp->get_current_allocated();
14582 total_current_allocated = get_current_allocated();
14583 #endif //MULTIPLE_HEAPS
14584 return total_current_allocated;
14587 size_t gc_heap::current_generation_size (int gen_number)
14589 dynamic_data* dd = dynamic_data_of (gen_number);
14590 size_t gen_size = (dd_current_size (dd) + dd_desired_allocation (dd)
14591 - dd_new_allocation (dd));
14597 #pragma warning(push)
14598 #pragma warning(disable:6326) // "Potential comparison of a constant with another constant" is intentional in this function.
14602 This is called by when we are actually doing a GC, or when we are just checking whether
14603 we would do a full blocking GC, in which case check_only_p is TRUE.
14605 The difference between calling this with check_only_p TRUE and FALSE is that when it's
14607 settings.reason is ignored
14608 budgets are not checked (since they are checked before this is called)
14609 it doesn't change anything non local like generation_skip_ratio
14611 int gc_heap::generation_to_condemn (int n_initial,
14612 BOOL* blocking_collection_p,
14613 BOOL* elevation_requested_p,
14616 gc_mechanisms temp_settings = settings;
14617 gen_to_condemn_tuning temp_condemn_reasons;
14618 gc_mechanisms* local_settings = (check_only_p ? &temp_settings : &settings);
14619 gen_to_condemn_tuning* local_condemn_reasons = (check_only_p ? &temp_condemn_reasons : &gen_to_condemn_reasons);
14622 if ((local_settings->reason == reason_oos_soh) || (local_settings->reason == reason_oos_loh))
14624 assert (n_initial >= 1);
14627 assert (settings.reason != reason_empty);
14630 local_condemn_reasons->init();
14634 if (heap_number == 0)
14636 dprintf (GTC_LOG, ("init: %d(%d)", n_initial, settings.reason));
14640 BOOL low_memory_detected = g_low_memory_status;
14641 uint32_t memory_load = 0;
14642 uint64_t available_physical = 0;
14643 uint64_t available_page_file = 0;
14644 BOOL check_memory = FALSE;
14645 BOOL high_fragmentation = FALSE;
14646 BOOL v_high_memory_load = FALSE;
14647 BOOL high_memory_load = FALSE;
14648 BOOL low_ephemeral_space = FALSE;
14649 BOOL evaluate_elevation = TRUE;
14650 *elevation_requested_p = FALSE;
14651 *blocking_collection_p = FALSE;
14653 BOOL check_max_gen_alloc = TRUE;
14657 #endif //STRESS_HEAP
14661 dd_fragmentation (dynamic_data_of (0)) =
14662 generation_free_list_space (youngest_generation) +
14663 generation_free_obj_space (youngest_generation);
14665 dd_fragmentation (dynamic_data_of (max_generation + 1)) =
14666 generation_free_list_space (large_object_generation) +
14667 generation_free_obj_space (large_object_generation);
14669 //save new_allocation
14670 for (i = 0; i <= max_generation+1; i++)
14672 dynamic_data* dd = dynamic_data_of (i);
14673 dprintf (GTC_LOG, ("h%d: g%d: l: %Id (%Id)",
14675 dd_new_allocation (dd),
14676 dd_desired_allocation (dd)));
14677 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
14680 local_condemn_reasons->set_gen (gen_initial, n);
14683 #ifdef BACKGROUND_GC
14684 if (recursive_gc_sync::background_running_p())
14686 dprintf (GTC_LOG, ("bgc in prog, 1"));
14687 check_max_gen_alloc = FALSE;
14689 #endif //BACKGROUND_GC
14691 if (check_max_gen_alloc)
14693 //figure out if large objects need to be collected.
14694 if (get_new_allocation (max_generation+1) <= 0)
14696 n = max_generation;
14697 local_condemn_reasons->set_gen (gen_alloc_budget, n);
14701 //figure out which generation ran out of allocation
14702 for (i = n+1; i <= (check_max_gen_alloc ? max_generation : (max_generation - 1)); i++)
14704 if (get_new_allocation (i) <= 0)
14715 local_condemn_reasons->set_gen (gen_alloc_budget, n);
14718 dprintf (GTC_LOG, ("h%d: g%d budget", heap_number, ((get_new_allocation (max_generation+1) <= 0) ? 3 : n)));
14722 #if defined(BACKGROUND_GC) && !defined(MULTIPLE_HEAPS)
14723 //time based tuning
14724 // if enough time has elapsed since the last gc
14725 // and the number of gc is too low (1/10 of lower gen) then collect
14726 // This should also be enabled if we have memory concerns
14727 int n_time_max = max_generation;
14731 if (recursive_gc_sync::background_running_p())
14733 n_time_max = max_generation - 1;
14737 if ((local_settings->pause_mode == pause_interactive) ||
14738 (local_settings->pause_mode == pause_sustained_low_latency))
14740 dynamic_data* dd0 = dynamic_data_of (0);
14741 size_t now = GetHighPrecisionTimeStamp();
14743 for (i = (temp_gen+1); i <= n_time_max; i++)
14745 dynamic_data* dd = dynamic_data_of (i);
14746 if ((now > dd_time_clock(dd) + dd_time_clock_interval(dd)) &&
14747 (dd_gc_clock (dd0) > (dd_gc_clock (dd) + dd_gc_clock_interval(dd))) &&
14748 ((n < max_generation) || ((dd_current_size (dd) < dd_max_size (dd0)))))
14750 n = min (i, n_time_max);
14751 dprintf (GTC_LOG, ("time %d", n));
14756 local_condemn_reasons->set_gen (gen_time_tuning, n);
14762 dprintf (GTC_LOG, ("Condemning %d based on time tuning and fragmentation", n));
14764 #endif //BACKGROUND_GC && !MULTIPLE_HEAPS
14766 if (n < (max_generation - 1))
14768 if (dt_low_card_table_efficiency_p (tuning_deciding_condemned_gen))
14770 n = max (n, max_generation - 1);
14771 local_settings->promotion = TRUE;
14772 dprintf (GTC_LOG, ("h%d: skip %d, c %d",
14773 heap_number, generation_skip_ratio, n));
14774 local_condemn_reasons->set_condition (gen_low_card_p);
14780 generation_skip_ratio = 100;
14783 if (dt_low_ephemeral_space_p (check_only_p ?
14784 tuning_deciding_full_gc :
14785 tuning_deciding_condemned_gen))
14787 low_ephemeral_space = TRUE;
14789 n = max (n, max_generation - 1);
14790 local_condemn_reasons->set_condition (gen_low_ephemeral_p);
14791 dprintf (GTC_LOG, ("h%d: low eph", heap_number));
14793 #ifdef BACKGROUND_GC
14794 if (!gc_can_use_concurrent || (generation_free_list_space (generation_of (max_generation)) == 0))
14795 #endif //BACKGROUND_GC
14797 //It is better to defragment first if we are running out of space for
14798 //the ephemeral generation but we have enough fragmentation to make up for it
14799 //in the non ephemeral generation. Essentially we are trading a gen2 for
14800 // having to expand heap in ephemeral collections.
14801 if (dt_high_frag_p (tuning_deciding_condemned_gen,
14802 max_generation - 1,
14805 high_fragmentation = TRUE;
14806 local_condemn_reasons->set_condition (gen_max_high_frag_e_p);
14807 dprintf (GTC_LOG, ("heap%d: gen1 frag", heap_number));
14812 //figure out which ephemeral generation is too fragramented
14814 for (i = n+1; i < max_generation; i++)
14816 if (dt_high_frag_p (tuning_deciding_condemned_gen, i))
14818 dprintf (GTC_LOG, ("h%d g%d too frag", heap_number, i));
14825 if (low_ephemeral_space)
14828 local_settings->promotion = TRUE;
14833 local_condemn_reasons->set_condition (gen_eph_high_frag_p);
14838 if (settings.pause_mode == pause_low_latency)
14840 if (!is_induced (settings.reason))
14842 n = min (n, max_generation - 1);
14843 dprintf (GTC_LOG, ("low latency mode is enabled, condemning %d", n));
14844 evaluate_elevation = FALSE;
14850 // It's hard to catch when we get to the point that the memory load is so high
14851 // we get an induced GC from the finalizer thread so we are checking the memory load
14852 // for every gen0 GC.
14853 check_memory = (check_only_p ?
14855 ((n >= 1) || low_memory_detected));
14859 //find out if we are short on memory
14860 get_memory_info (&memory_load, &available_physical, &available_page_file);
14861 if (heap_number == 0)
14863 dprintf (GTC_LOG, ("ml: %d", memory_load));
14866 // Need to get it early enough for all heaps to use.
14867 entry_available_physical_mem = available_physical;
14868 local_settings->entry_memory_load = memory_load;
14870 // @TODO: Force compaction more often under GCSTRESS
14871 if (memory_load >= high_memory_load_th || low_memory_detected)
14873 #ifdef SIMPLE_DPRINTF
14874 // stress log can't handle any parameter that's bigger than a void*.
14875 if (heap_number == 0)
14877 dprintf (GTC_LOG, ("tp: %I64d, ap: %I64d", total_physical_mem, available_physical));
14879 #endif //SIMPLE_DPRINTF
14881 high_memory_load = TRUE;
14883 if (memory_load >= v_high_memory_load_th || low_memory_detected)
14885 // TODO: Perhaps in 64-bit we should be estimating gen1's fragmentation as well since
14886 // gen1/gen0 may take a lot more memory than gen2.
14887 if (!high_fragmentation)
14889 high_fragmentation = dt_estimate_reclaim_space_p (tuning_deciding_condemned_gen, max_generation);
14891 v_high_memory_load = TRUE;
14895 if (!high_fragmentation)
14897 high_fragmentation = dt_estimate_high_frag_p (tuning_deciding_condemned_gen, max_generation, available_physical);
14901 if (high_fragmentation)
14903 if (high_memory_load)
14905 local_condemn_reasons->set_condition (gen_max_high_frag_m_p);
14907 else if (v_high_memory_load)
14909 local_condemn_reasons->set_condition (gen_max_high_frag_vm_p);
14915 dprintf (GTC_LOG, ("h%d: le: %d, hm: %d, vm: %d, f: %d",
14916 heap_number, low_ephemeral_space, high_memory_load, v_high_memory_load,
14917 high_fragmentation));
14919 if (should_expand_in_full_gc)
14921 dprintf (GTC_LOG, ("h%d: expand_in_full - BLOCK", heap_number));
14922 *blocking_collection_p = TRUE;
14925 should_expand_in_full_gc = FALSE;
14927 evaluate_elevation = FALSE;
14928 n = max_generation;
14929 local_condemn_reasons->set_condition (gen_expand_fullgc_p);
14932 if (last_gc_before_oom)
14934 dprintf (GTC_LOG, ("h%d: alloc full - BLOCK", heap_number));
14935 n = max_generation;
14936 *blocking_collection_p = TRUE;
14937 if ((local_settings->reason == reason_oos_loh) ||
14938 (local_settings->reason == reason_alloc_loh))
14939 evaluate_elevation = FALSE;
14941 local_condemn_reasons->set_condition (gen_before_oom);
14946 if (is_induced_blocking (settings.reason) &&
14947 n_initial == max_generation
14948 IN_STRESS_HEAP( && !settings.stress_induced ))
14950 if (heap_number == 0)
14952 dprintf (GTC_LOG, ("induced - BLOCK"));
14955 *blocking_collection_p = TRUE;
14956 local_condemn_reasons->set_condition (gen_induced_fullgc_p);
14957 evaluate_elevation = FALSE;
14960 if (settings.reason == reason_induced_noforce)
14962 local_condemn_reasons->set_condition (gen_induced_noforce_p);
14963 evaluate_elevation = FALSE;
14967 if (evaluate_elevation && (low_ephemeral_space || high_memory_load || v_high_memory_load))
14969 *elevation_requested_p = TRUE;
14971 // if we are in high memory load and have consumed 10% of the gen2 budget, do a gen2 now.
14972 if (high_memory_load || v_high_memory_load)
14974 dynamic_data* dd_max = dynamic_data_of (max_generation);
14975 if (((float)dd_new_allocation (dd_max) / (float)dd_desired_allocation (dd_max)) < 0.9)
14977 dprintf (GTC_LOG, ("%Id left in gen2 alloc (%Id)",
14978 dd_new_allocation (dd_max), dd_desired_allocation (dd_max)));
14979 n = max_generation;
14980 local_condemn_reasons->set_condition (gen_almost_max_alloc);
14984 if (n <= max_generation)
14987 if (high_fragmentation)
14989 //elevate to max_generation
14990 n = max_generation;
14991 dprintf (GTC_LOG, ("h%d: f full", heap_number));
14993 #ifdef BACKGROUND_GC
14994 if (high_memory_load || v_high_memory_load)
14996 // For background GC we want to do blocking collections more eagerly because we don't
14997 // want to get into the situation where the memory load becomes high while we are in
14998 // a background GC and we'd have to wait for the background GC to finish to start
14999 // a blocking collection (right now the implemenation doesn't handle converting
15000 // a background GC to a blocking collection midway.
15001 dprintf (GTC_LOG, ("h%d: bgc - BLOCK", heap_number));
15002 *blocking_collection_p = TRUE;
15005 if (v_high_memory_load)
15007 dprintf (GTC_LOG, ("h%d: - BLOCK", heap_number));
15008 *blocking_collection_p = TRUE;
15010 #endif //BACKGROUND_GC
15014 n = max (n, max_generation - 1);
15015 dprintf (GTC_LOG, ("h%d: nf c %d", heap_number, n));
15022 if ((n == (max_generation - 1)) && (n_alloc < (max_generation -1)))
15024 dprintf (GTC_LOG, ("h%d: budget %d, check 2",
15025 heap_number, n_alloc));
15026 if (get_new_allocation (max_generation) <= 0)
15028 dprintf (GTC_LOG, ("h%d: budget alloc", heap_number));
15029 n = max_generation;
15030 local_condemn_reasons->set_condition (gen_max_gen1);
15034 //figure out if max_generation is too fragmented -> blocking collection
15035 if (n == max_generation)
15037 if (dt_high_frag_p (tuning_deciding_condemned_gen, n))
15039 dprintf (GTC_LOG, ("h%d: g%d too frag", heap_number, n));
15040 local_condemn_reasons->set_condition (gen_max_high_frag_p);
15041 if (local_settings->pause_mode != pause_sustained_low_latency)
15043 *blocking_collection_p = TRUE;
15048 #ifdef BACKGROUND_GC
15049 if (n == max_generation)
15051 if (heap_number == 0)
15053 BOOL bgc_heap_too_small = TRUE;
15054 size_t gen2size = 0;
15055 size_t gen3size = 0;
15056 #ifdef MULTIPLE_HEAPS
15057 for (int i = 0; i < n_heaps; i++)
15059 if (((g_heaps[i]->current_generation_size (max_generation)) > bgc_min_per_heap) ||
15060 ((g_heaps[i]->current_generation_size (max_generation + 1)) > bgc_min_per_heap))
15062 bgc_heap_too_small = FALSE;
15066 #else //MULTIPLE_HEAPS
15067 if ((current_generation_size (max_generation) > bgc_min_per_heap) ||
15068 (current_generation_size (max_generation + 1) > bgc_min_per_heap))
15070 bgc_heap_too_small = FALSE;
15072 #endif //MULTIPLE_HEAPS
15074 if (bgc_heap_too_small)
15076 dprintf (GTC_LOG, ("gen2 and gen3 too small"));
15079 // do not turn stress-induced collections into blocking GCs
15080 if (!settings.stress_induced)
15081 #endif //STRESS_HEAP
15083 *blocking_collection_p = TRUE;
15086 local_condemn_reasons->set_condition (gen_gen2_too_small);
15090 #endif //BACKGROUND_GC
15096 #ifdef BACKGROUND_GC
15097 // We can only do Concurrent GC Stress if the caller did not explicitly ask for all
15098 // generations to be collected,
15100 if (orig_gen != max_generation &&
15101 g_pConfig->GetGCStressLevel() && gc_can_use_concurrent)
15103 *elevation_requested_p = FALSE;
15105 #endif //BACKGROUND_GC
15106 #endif //STRESS_HEAP
15110 fgm_result.available_pagefile_mb = (size_t)(available_page_file / (1024 * 1024));
15113 local_condemn_reasons->set_gen (gen_final_per_heap, n);
15114 get_gc_data_per_heap()->gen_to_condemn_reasons.init (local_condemn_reasons);
15117 local_condemn_reasons->print (heap_number);
15120 if ((local_settings->reason == reason_oos_soh) ||
15121 (local_settings->reason == reason_oos_loh))
15127 if (n == max_generation && GCToEEInterface::ForceFullGCToBeBlocking())
15129 #ifdef BACKGROUND_GC
15130 // do not turn stress-induced collections into blocking GCs, unless there
15131 // have already been more full BGCs than full NGCs
15133 // This exposes DevDiv 94129, so we'll leave this out for now
15134 if (!settings.stress_induced ||
15135 full_gc_counts[gc_type_blocking] <= full_gc_counts[gc_type_background])
15137 #endif // BACKGROUND_GC
15139 *blocking_collection_p = TRUE;
15147 #pragma warning(pop)
15151 size_t gc_heap::min_reclaim_fragmentation_threshold (uint32_t num_heaps)
15153 // if the memory load is higher, the threshold we'd want to collect gets lower.
15154 size_t min_mem_based_on_available =
15155 (500 - (settings.entry_memory_load - high_memory_load_th) * 40) * 1024 * 1024 / num_heaps;
15156 size_t ten_percent_size = (size_t)((float)generation_size (max_generation) * 0.10);
15157 uint64_t three_percent_mem = mem_one_percent * 3 / num_heaps;
15159 #ifdef SIMPLE_DPRINTF
15160 dprintf (GTC_LOG, ("min av: %Id, 10%% gen2: %Id, 3%% mem: %I64d",
15161 min_mem_based_on_available, ten_percent_size, three_percent_mem));
15162 #endif //SIMPLE_DPRINTF
15163 return (size_t)(min (min_mem_based_on_available, min (ten_percent_size, three_percent_mem)));
15167 uint64_t gc_heap::min_high_fragmentation_threshold(uint64_t available_mem, uint32_t num_heaps)
15169 return min (available_mem, (256*1024*1024)) / num_heaps;
15173 CORINFO_EXCEPTION_GC = 0xE0004743 // 'GC'
15177 #ifdef BACKGROUND_GC
15178 void gc_heap::init_background_gc ()
15180 //reset the allocation so foreground gc can allocate into older (max_generation) generation
15181 generation* gen = generation_of (max_generation);
15182 generation_allocation_pointer (gen)= 0;
15183 generation_allocation_limit (gen) = 0;
15184 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
15186 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
15188 //reset the plan allocation for each segment
15189 for (heap_segment* seg = generation_allocation_segment (gen); seg != ephemeral_heap_segment;
15190 seg = heap_segment_next_rw (seg))
15192 heap_segment_plan_allocated (seg) = heap_segment_allocated (seg);
15195 if (heap_number == 0)
15197 dprintf (2, ("heap%d: bgc lowest: %Ix, highest: %Ix",
15199 background_saved_lowest_address,
15200 background_saved_highest_address));
15203 gc_lh_block_event.Reset();
15206 #endif //BACKGROUND_GC
15209 void fire_drain_mark_list_event (size_t mark_list_objects)
15211 FIRE_EVENT(BGCDrainMark, mark_list_objects);
15215 void fire_revisit_event (size_t dirtied_pages,
15216 size_t marked_objects,
15217 BOOL large_objects_p)
15219 FIRE_EVENT(BGCRevisit, dirtied_pages, marked_objects, large_objects_p);
15223 void fire_overflow_event (uint8_t* overflow_min,
15224 uint8_t* overflow_max,
15225 size_t marked_objects,
15226 int large_objects_p)
15228 FIRE_EVENT(BGCOverflow, (uint64_t)overflow_min, (uint64_t)overflow_max, marked_objects, large_objects_p);
15231 void gc_heap::concurrent_print_time_delta (const char* msg)
15234 size_t current_time = GetHighPrecisionTimeStamp();
15235 size_t elapsed_time = current_time - time_bgc_last;
15236 time_bgc_last = current_time;
15238 dprintf (2, ("h%d: %s T %Id ms", heap_number, msg, elapsed_time));
15240 UNREFERENCED_PARAMETER(msg);
15244 void gc_heap::free_list_info (int gen_num, const char* msg)
15246 UNREFERENCED_PARAMETER(gen_num);
15247 #if defined (BACKGROUND_GC) && defined (TRACE_GC)
15248 dprintf (3, ("h%d: %s", heap_number, msg));
15249 for (int i = 0; i <= (max_generation + 1); i++)
15251 generation* gen = generation_of (i);
15252 if ((generation_allocation_size (gen) == 0) &&
15253 (generation_free_list_space (gen) == 0) &&
15254 (generation_free_obj_space (gen) == 0))
15256 // don't print if everything is 0.
15260 dprintf (3, ("h%d: g%d: a-%Id, fl-%Id, fo-%Id",
15262 generation_allocation_size (gen),
15263 generation_free_list_space (gen),
15264 generation_free_obj_space (gen)));
15268 UNREFERENCED_PARAMETER(msg);
15269 #endif // BACKGROUND_GC && TRACE_GC
15272 void gc_heap::update_collection_counts_for_no_gc()
15274 assert (settings.pause_mode == pause_no_gc);
15276 settings.condemned_generation = max_generation;
15277 #ifdef MULTIPLE_HEAPS
15278 for (int i = 0; i < n_heaps; i++)
15279 g_heaps[i]->update_collection_counts();
15280 #else //MULTIPLE_HEAPS
15281 update_collection_counts();
15282 #endif //MULTIPLE_HEAPS
15284 full_gc_counts[gc_type_blocking]++;
15287 BOOL gc_heap::should_proceed_with_gc()
15289 if (gc_heap::settings.pause_mode == pause_no_gc)
15291 if (current_no_gc_region_info.started)
15293 // The no_gc mode was already in progress yet we triggered another GC,
15294 // this effectively exits the no_gc mode.
15295 restore_data_for_no_gc();
15298 return should_proceed_for_no_gc();
15304 //internal part of gc used by the serial and concurrent version
15305 void gc_heap::gc1()
15307 #ifdef BACKGROUND_GC
15308 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15309 #endif //BACKGROUND_GC
15312 mark_time = plan_time = reloc_time = compact_time = sweep_time = 0;
15315 verify_soh_segment_list();
15317 int n = settings.condemned_generation;
15319 update_collection_counts ();
15321 #ifdef BACKGROUND_GC
15322 bgc_alloc_lock->check();
15323 #endif //BACKGROUND_GC
15325 free_list_info (max_generation, "beginning");
15327 vm_heap->GcCondemnedGeneration = settings.condemned_generation;
15329 assert (g_gc_card_table == card_table);
15331 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
15332 assert (g_gc_card_bundle_table == card_bundle_table);
15336 if (n == max_generation)
15338 gc_low = lowest_address;
15339 gc_high = highest_address;
15343 gc_low = generation_allocation_start (generation_of (n));
15344 gc_high = heap_segment_reserved (ephemeral_heap_segment);
15346 #ifdef BACKGROUND_GC
15347 if (settings.concurrent)
15350 time_bgc_last = GetHighPrecisionTimeStamp();
15353 FIRE_EVENT(BGCBegin);
15355 concurrent_print_time_delta ("BGC");
15357 //#ifdef WRITE_WATCH
15358 //reset_write_watch (FALSE);
15359 //#endif //WRITE_WATCH
15361 concurrent_print_time_delta ("RW");
15362 background_mark_phase();
15363 free_list_info (max_generation, "after mark phase");
15365 background_sweep();
15366 free_list_info (max_generation, "after sweep phase");
15369 #endif //BACKGROUND_GC
15371 mark_phase (n, FALSE);
15373 GCScan::GcRuntimeStructuresValid (FALSE);
15375 GCScan::GcRuntimeStructuresValid (TRUE);
15379 size_t end_gc_time = GetHighPrecisionTimeStamp();
15380 // printf ("generation: %d, elapsed time: %Id\n", n, end_gc_time - dd_time_clock (dynamic_data_of (0)));
15382 //adjust the allocation size from the pinned quantities.
15383 for (int gen_number = 0; gen_number <= min (max_generation,n+1); gen_number++)
15385 generation* gn = generation_of (gen_number);
15386 if (settings.compaction)
15388 generation_pinned_allocated (gn) += generation_pinned_allocation_compact_size (gn);
15389 generation_allocation_size (generation_of (gen_number)) += generation_pinned_allocation_compact_size (gn);
15393 generation_pinned_allocated (gn) += generation_pinned_allocation_sweep_size (gn);
15394 generation_allocation_size (generation_of (gen_number)) += generation_pinned_allocation_sweep_size (gn);
15396 generation_pinned_allocation_sweep_size (gn) = 0;
15397 generation_pinned_allocation_compact_size (gn) = 0;
15400 #ifdef BACKGROUND_GC
15401 if (settings.concurrent)
15403 dynamic_data* dd = dynamic_data_of (n);
15404 dd_gc_elapsed_time (dd) = end_gc_time - dd_time_clock (dd);
15406 free_list_info (max_generation, "after computing new dynamic data");
15408 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
15410 for (int gen_number = 0; gen_number < max_generation; gen_number++)
15412 dprintf (2, ("end of BGC: gen%d new_alloc: %Id",
15413 gen_number, dd_desired_allocation (dynamic_data_of (gen_number))));
15414 current_gc_data_per_heap->gen_data[gen_number].size_after = generation_size (gen_number);
15415 current_gc_data_per_heap->gen_data[gen_number].free_list_space_after = generation_free_list_space (generation_of (gen_number));
15416 current_gc_data_per_heap->gen_data[gen_number].free_obj_space_after = generation_free_obj_space (generation_of (gen_number));
15420 #endif //BACKGROUND_GC
15422 free_list_info (max_generation, "end");
15423 for (int gen_number = 0; gen_number <= n; gen_number++)
15425 dynamic_data* dd = dynamic_data_of (gen_number);
15426 dd_gc_elapsed_time (dd) = end_gc_time - dd_time_clock (dd);
15427 compute_new_dynamic_data (gen_number);
15430 if (n != max_generation)
15432 int gen_num_for_data = ((n < (max_generation - 1)) ? (n + 1) : (max_generation + 1));
15433 for (int gen_number = (n + 1); gen_number <= gen_num_for_data; gen_number++)
15435 get_gc_data_per_heap()->gen_data[gen_number].size_after = generation_size (gen_number);
15436 get_gc_data_per_heap()->gen_data[gen_number].free_list_space_after = generation_free_list_space (generation_of (gen_number));
15437 get_gc_data_per_heap()->gen_data[gen_number].free_obj_space_after = generation_free_obj_space (generation_of (gen_number));
15441 get_gc_data_per_heap()->maxgen_size_info.running_free_list_efficiency = (uint32_t)(generation_allocator_efficiency (generation_of (max_generation)) * 100);
15443 free_list_info (max_generation, "after computing new dynamic data");
15445 if (heap_number == 0)
15447 dprintf (GTC_LOG, ("GC#%d(gen%d) took %Idms",
15448 dd_collection_count (dynamic_data_of (0)),
15449 settings.condemned_generation,
15450 dd_gc_elapsed_time (dynamic_data_of (0))));
15453 for (int gen_number = 0; gen_number <= (max_generation + 1); gen_number++)
15455 dprintf (2, ("end of FGC/NGC: gen%d new_alloc: %Id",
15456 gen_number, dd_desired_allocation (dynamic_data_of (gen_number))));
15460 if (n < max_generation)
15462 compute_promoted_allocation (1 + n);
15464 dynamic_data* dd = dynamic_data_of (1 + n);
15465 size_t new_fragmentation = generation_free_list_space (generation_of (1 + n)) +
15466 generation_free_obj_space (generation_of (1 + n));
15468 #ifdef BACKGROUND_GC
15469 if (current_c_gc_state != c_gc_state_planning)
15470 #endif //BACKGROUND_GC
15472 if (settings.promotion)
15474 dd_fragmentation (dd) = new_fragmentation;
15478 //assert (dd_fragmentation (dd) == new_fragmentation);
15483 #ifdef BACKGROUND_GC
15484 if (!settings.concurrent)
15485 #endif //BACKGROUND_GC
15487 #ifndef FEATURE_REDHAWK
15488 // GCToEEInterface::IsGCThread() always returns false on CoreRT, but this assert is useful in CoreCLR.
15489 assert(GCToEEInterface::IsGCThread());
15490 #endif // FEATURE_REDHAWK
15491 adjust_ephemeral_limits();
15494 #ifdef BACKGROUND_GC
15495 assert (ephemeral_low == generation_allocation_start (generation_of ( max_generation -1)));
15496 assert (ephemeral_high == heap_segment_reserved (ephemeral_heap_segment));
15497 #endif //BACKGROUND_GC
15499 if (fgn_maxgen_percent)
15501 if (settings.condemned_generation == (max_generation - 1))
15503 check_for_full_gc (max_generation - 1, 0);
15505 else if (settings.condemned_generation == max_generation)
15507 if (full_gc_approach_event_set
15508 #ifdef MULTIPLE_HEAPS
15509 && (heap_number == 0)
15510 #endif //MULTIPLE_HEAPS
15513 dprintf (2, ("FGN-GC: setting gen2 end event"));
15515 full_gc_approach_event.Reset();
15516 #ifdef BACKGROUND_GC
15517 // By definition WaitForFullGCComplete only succeeds if it's full, *blocking* GC, otherwise need to return N/A
15518 fgn_last_gc_was_concurrent = settings.concurrent ? TRUE : FALSE;
15519 #endif //BACKGROUND_GC
15520 full_gc_end_event.Set();
15521 full_gc_approach_event_set = false;
15526 #ifdef BACKGROUND_GC
15527 if (!settings.concurrent)
15528 #endif //BACKGROUND_GC
15530 //decide on the next allocation quantum
15531 if (alloc_contexts_used >= 1)
15533 allocation_quantum = Align (min ((size_t)CLR_SIZE,
15534 (size_t)max (1024, get_new_allocation (0) / (2 * alloc_contexts_used))),
15535 get_alignment_constant(FALSE));
15536 dprintf (3, ("New allocation quantum: %d(0x%Ix)", allocation_quantum, allocation_quantum));
15540 descr_generations (FALSE);
15542 verify_soh_segment_list();
15544 #ifdef BACKGROUND_GC
15545 add_to_history_per_heap();
15546 if (heap_number == 0)
15550 #endif // BACKGROUND_GC
15553 if (GCStatistics::Enabled() && heap_number == 0)
15554 g_GCStatistics.AddGCStats(settings,
15555 dd_gc_elapsed_time(dynamic_data_of(settings.condemned_generation)));
15559 fprintf (stdout, "%d,%d,%d,%d,%d,%d\n",
15560 n, mark_time, plan_time, reloc_time, compact_time, sweep_time);
15563 #ifdef BACKGROUND_GC
15564 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15565 #endif //BACKGROUND_GC
15567 #if defined(VERIFY_HEAP) || (defined (FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC))
15570 // Note that right now g_pConfig->GetHeapVerifyLevel always returns the same
15571 // value. If we ever allow randomly adjusting this as the process runs,
15572 // we cannot call it this way as joins need to match - we must have the same
15573 // value for all heaps like we do with bgc_heap_walk_for_etw_p.
15574 || (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
15576 #if defined(FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC)
15577 || (bgc_heap_walk_for_etw_p && settings.concurrent)
15581 #ifdef BACKGROUND_GC
15582 bool cooperative_mode = true;
15584 if (settings.concurrent)
15586 cooperative_mode = enable_preemptive ();
15588 #ifdef MULTIPLE_HEAPS
15589 bgc_t_join.join(this, gc_join_suspend_ee_verify);
15590 if (bgc_t_join.joined())
15592 bgc_threads_sync_event.Reset();
15594 dprintf(2, ("Joining BGC threads to suspend EE for verify heap"));
15595 bgc_t_join.restart();
15597 if (heap_number == 0)
15600 bgc_threads_sync_event.Set();
15604 bgc_threads_sync_event.Wait(INFINITE, FALSE);
15605 dprintf (2, ("bgc_threads_sync_event is signalled"));
15609 #endif //MULTIPLE_HEAPS
15611 //fix the allocation area so verify_heap can proceed.
15612 fix_allocation_contexts (FALSE);
15614 #endif //BACKGROUND_GC
15616 #ifdef BACKGROUND_GC
15617 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15618 #ifdef FEATURE_EVENT_TRACE
15619 if (bgc_heap_walk_for_etw_p && settings.concurrent)
15621 GCToEEInterface::DiagWalkBGCSurvivors(__this);
15623 #ifdef MULTIPLE_HEAPS
15624 bgc_t_join.join(this, gc_join_after_profiler_heap_walk);
15625 if (bgc_t_join.joined())
15627 bgc_t_join.restart();
15629 #endif // MULTIPLE_HEAPS
15631 #endif // FEATURE_EVENT_TRACE
15632 #endif //BACKGROUND_GC
15635 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
15636 verify_heap (FALSE);
15637 #endif // VERIFY_HEAP
15639 #ifdef BACKGROUND_GC
15640 if (settings.concurrent)
15642 repair_allocation_contexts (TRUE);
15644 #ifdef MULTIPLE_HEAPS
15645 bgc_t_join.join(this, gc_join_restart_ee_verify);
15646 if (bgc_t_join.joined())
15648 bgc_threads_sync_event.Reset();
15650 dprintf(2, ("Joining BGC threads to restart EE after verify heap"));
15651 bgc_t_join.restart();
15653 if (heap_number == 0)
15656 bgc_threads_sync_event.Set();
15660 bgc_threads_sync_event.Wait(INFINITE, FALSE);
15661 dprintf (2, ("bgc_threads_sync_event is signalled"));
15665 #endif //MULTIPLE_HEAPS
15667 disable_preemptive (cooperative_mode);
15669 #endif //BACKGROUND_GC
15671 #endif // defined(VERIFY_HEAP) || (defined(FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC))
15673 #ifdef MULTIPLE_HEAPS
15674 if (!settings.concurrent)
15676 gc_t_join.join(this, gc_join_done);
15677 if (gc_t_join.joined ())
15679 gc_heap::internal_gc_done = false;
15681 //equalize the new desired size of the generations
15682 int limit = settings.condemned_generation;
15683 if (limit == max_generation)
15685 limit = max_generation+1;
15687 for (int gen = 0; gen <= limit; gen++)
15689 size_t total_desired = 0;
15691 for (int i = 0; i < gc_heap::n_heaps; i++)
15693 gc_heap* hp = gc_heap::g_heaps[i];
15694 dynamic_data* dd = hp->dynamic_data_of (gen);
15695 size_t temp_total_desired = total_desired + dd_desired_allocation (dd);
15696 if (temp_total_desired < total_desired)
15699 total_desired = (size_t)MAX_PTR;
15702 total_desired = temp_total_desired;
15705 size_t desired_per_heap = Align (total_desired/gc_heap::n_heaps,
15706 get_alignment_constant ((gen != (max_generation+1))));
15710 #if 1 //subsumed by the linear allocation model
15711 // to avoid spikes in mem usage due to short terms fluctuations in survivorship,
15712 // apply some smoothing.
15713 static size_t smoothed_desired_per_heap = 0;
15714 size_t smoothing = 3; // exponential smoothing factor
15715 if (smoothing > VolatileLoad(&settings.gc_index))
15716 smoothing = VolatileLoad(&settings.gc_index);
15717 smoothed_desired_per_heap = desired_per_heap / smoothing + ((smoothed_desired_per_heap / smoothing) * (smoothing-1));
15718 dprintf (1, ("sn = %Id n = %Id", smoothed_desired_per_heap, desired_per_heap));
15719 desired_per_heap = Align(smoothed_desired_per_heap, get_alignment_constant (true));
15722 // if desired_per_heap is close to min_gc_size, trim it
15723 // down to min_gc_size to stay in the cache
15724 gc_heap* hp = gc_heap::g_heaps[0];
15725 dynamic_data* dd = hp->dynamic_data_of (gen);
15726 size_t min_gc_size = dd_min_size(dd);
15727 // if min GC size larger than true on die cache, then don't bother
15728 // limiting the desired size
15729 if ((min_gc_size <= GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE)) &&
15730 desired_per_heap <= 2*min_gc_size)
15732 desired_per_heap = min_gc_size;
15735 desired_per_heap = joined_youngest_desired (desired_per_heap);
15736 dprintf (2, ("final gen0 new_alloc: %Id", desired_per_heap));
15739 gc_data_global.final_youngest_desired = desired_per_heap;
15741 #if 1 //subsumed by the linear allocation model
15742 if (gen == (max_generation + 1))
15744 // to avoid spikes in mem usage due to short terms fluctuations in survivorship,
15745 // apply some smoothing.
15746 static size_t smoothed_desired_per_heap_loh = 0;
15747 size_t smoothing = 3; // exponential smoothing factor
15748 size_t loh_count = dd_collection_count (dynamic_data_of (max_generation));
15749 if (smoothing > loh_count)
15750 smoothing = loh_count;
15751 smoothed_desired_per_heap_loh = desired_per_heap / smoothing + ((smoothed_desired_per_heap_loh / smoothing) * (smoothing-1));
15752 dprintf( 2, ("smoothed_desired_per_heap_loh = %Id desired_per_heap = %Id", smoothed_desired_per_heap_loh, desired_per_heap));
15753 desired_per_heap = Align(smoothed_desired_per_heap_loh, get_alignment_constant (false));
15756 for (int i = 0; i < gc_heap::n_heaps; i++)
15758 gc_heap* hp = gc_heap::g_heaps[i];
15759 dynamic_data* dd = hp->dynamic_data_of (gen);
15760 dd_desired_allocation (dd) = desired_per_heap;
15761 dd_gc_new_allocation (dd) = desired_per_heap;
15762 dd_new_allocation (dd) = desired_per_heap;
15766 hp->fgn_last_alloc = desired_per_heap;
15771 #ifdef FEATURE_LOH_COMPACTION
15772 BOOL all_heaps_compacted_p = TRUE;
15773 #endif //FEATURE_LOH_COMPACTION
15774 for (int i = 0; i < gc_heap::n_heaps; i++)
15776 gc_heap* hp = gc_heap::g_heaps[i];
15777 hp->decommit_ephemeral_segment_pages();
15778 hp->rearrange_large_heap_segments();
15779 #ifdef FEATURE_LOH_COMPACTION
15780 all_heaps_compacted_p &= hp->loh_compacted_p;
15781 #endif //FEATURE_LOH_COMPACTION
15784 #ifdef FEATURE_LOH_COMPACTION
15785 check_loh_compact_mode (all_heaps_compacted_p);
15786 #endif //FEATURE_LOH_COMPACTION
15790 gc_t_join.restart();
15792 alloc_context_count = 0;
15793 heap_select::mark_heap (heap_number);
15797 gc_data_global.final_youngest_desired =
15798 dd_desired_allocation (dynamic_data_of (0));
15800 check_loh_compact_mode (loh_compacted_p);
15802 decommit_ephemeral_segment_pages();
15805 if (!(settings.concurrent))
15807 rearrange_large_heap_segments();
15811 #ifdef BACKGROUND_GC
15812 recover_bgc_settings();
15813 #endif //BACKGROUND_GC
15814 #endif //MULTIPLE_HEAPS
15817 void gc_heap::save_data_for_no_gc()
15819 current_no_gc_region_info.saved_pause_mode = settings.pause_mode;
15820 #ifdef MULTIPLE_HEAPS
15821 // This is to affect heap balancing.
15822 for (int i = 0; i < n_heaps; i++)
15824 current_no_gc_region_info.saved_gen0_min_size = dd_min_size (g_heaps[i]->dynamic_data_of (0));
15825 dd_min_size (g_heaps[i]->dynamic_data_of (0)) = min_balance_threshold;
15826 current_no_gc_region_info.saved_gen3_min_size = dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1));
15827 dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1)) = 0;
15829 #endif //MULTIPLE_HEAPS
15832 void gc_heap::restore_data_for_no_gc()
15834 gc_heap::settings.pause_mode = current_no_gc_region_info.saved_pause_mode;
15835 #ifdef MULTIPLE_HEAPS
15836 for (int i = 0; i < n_heaps; i++)
15838 dd_min_size (g_heaps[i]->dynamic_data_of (0)) = current_no_gc_region_info.saved_gen0_min_size;
15839 dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1)) = current_no_gc_region_info.saved_gen3_min_size;
15841 #endif //MULTIPLE_HEAPS
15844 start_no_gc_region_status gc_heap::prepare_for_no_gc_region (uint64_t total_size,
15845 BOOL loh_size_known,
15847 BOOL disallow_full_blocking)
15849 if (current_no_gc_region_info.started)
15851 return start_no_gc_in_progress;
15854 start_no_gc_region_status status = start_no_gc_success;
15856 save_data_for_no_gc();
15857 settings.pause_mode = pause_no_gc;
15858 current_no_gc_region_info.start_status = start_no_gc_success;
15860 uint64_t allocation_no_gc_loh = 0;
15861 uint64_t allocation_no_gc_soh = 0;
15862 assert(total_size != 0);
15863 if (loh_size_known)
15865 assert(loh_size != 0);
15866 assert(loh_size <= total_size);
15867 allocation_no_gc_loh = loh_size;
15868 allocation_no_gc_soh = total_size - loh_size;
15872 allocation_no_gc_soh = total_size;
15873 allocation_no_gc_loh = total_size;
15876 int soh_align_const = get_alignment_constant (TRUE);
15877 size_t max_soh_allocated = soh_segment_size - segment_info_size - eph_gen_starts_size;
15878 size_t size_per_heap = 0;
15879 const double scale_factor = 1.05;
15882 #ifdef MULTIPLE_HEAPS
15883 num_heaps = n_heaps;
15884 #endif // MULTIPLE_HEAPS
15886 uint64_t total_allowed_soh_allocation = max_soh_allocated * num_heaps;
15888 // In theory, the upper limit here is the physical memory of the machine, not
15889 // SIZE_T_MAX. This is not true today because total_physical_mem can be
15890 // larger than SIZE_T_MAX if running in wow64 on a machine with more than
15891 // 4GB of RAM. Once Local GC code divergence is resolved and code is flowing
15892 // more freely between branches, it would be good to clean this up to use
15893 // total_physical_mem instead of SIZE_T_MAX.
15894 assert(total_allowed_soh_allocation <= SIZE_T_MAX);
15895 uint64_t total_allowed_loh_allocation = SIZE_T_MAX;
15896 uint64_t total_allowed_soh_alloc_scaled = allocation_no_gc_soh > 0 ? static_cast<uint64_t>(total_allowed_soh_allocation / scale_factor) : 0;
15897 uint64_t total_allowed_loh_alloc_scaled = allocation_no_gc_loh > 0 ? static_cast<uint64_t>(total_allowed_loh_allocation / scale_factor) : 0;
15899 if (allocation_no_gc_soh > total_allowed_soh_alloc_scaled ||
15900 allocation_no_gc_loh > total_allowed_loh_alloc_scaled)
15902 status = start_no_gc_too_large;
15906 if (allocation_no_gc_soh > 0)
15908 allocation_no_gc_soh = static_cast<uint64_t>(allocation_no_gc_soh * scale_factor);
15909 allocation_no_gc_soh = min (allocation_no_gc_soh, total_allowed_soh_alloc_scaled);
15912 if (allocation_no_gc_loh > 0)
15914 allocation_no_gc_loh = static_cast<uint64_t>(allocation_no_gc_loh * scale_factor);
15915 allocation_no_gc_loh = min (allocation_no_gc_loh, total_allowed_loh_alloc_scaled);
15918 if (disallow_full_blocking)
15919 current_no_gc_region_info.minimal_gc_p = TRUE;
15921 if (allocation_no_gc_soh != 0)
15923 current_no_gc_region_info.soh_allocation_size = static_cast<size_t>(allocation_no_gc_soh);
15924 size_per_heap = current_no_gc_region_info.soh_allocation_size;
15925 #ifdef MULTIPLE_HEAPS
15926 size_per_heap /= n_heaps;
15927 for (int i = 0; i < n_heaps; i++)
15929 // due to heap balancing we need to allow some room before we even look to balance to another heap.
15930 g_heaps[i]->soh_allocation_no_gc = min (Align ((size_per_heap + min_balance_threshold), soh_align_const), max_soh_allocated);
15932 #else //MULTIPLE_HEAPS
15933 soh_allocation_no_gc = min (Align (size_per_heap, soh_align_const), max_soh_allocated);
15934 #endif //MULTIPLE_HEAPS
15937 if (allocation_no_gc_loh != 0)
15939 current_no_gc_region_info.loh_allocation_size = static_cast<size_t>(allocation_no_gc_loh);
15940 size_per_heap = current_no_gc_region_info.loh_allocation_size;
15941 #ifdef MULTIPLE_HEAPS
15942 size_per_heap /= n_heaps;
15943 for (int i = 0; i < n_heaps; i++)
15944 g_heaps[i]->loh_allocation_no_gc = Align (size_per_heap, get_alignment_constant (FALSE));
15945 #else //MULTIPLE_HEAPS
15946 loh_allocation_no_gc = Align (size_per_heap, get_alignment_constant (FALSE));
15947 #endif //MULTIPLE_HEAPS
15951 if (status != start_no_gc_success)
15952 restore_data_for_no_gc();
15956 void gc_heap::handle_failure_for_no_gc()
15958 gc_heap::restore_data_for_no_gc();
15959 // sets current_no_gc_region_info.started to FALSE here.
15960 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
15963 start_no_gc_region_status gc_heap::get_start_no_gc_region_status()
15965 return current_no_gc_region_info.start_status;
15968 void gc_heap::record_gcs_during_no_gc()
15970 if (current_no_gc_region_info.started)
15972 current_no_gc_region_info.num_gcs++;
15973 if (is_induced (settings.reason))
15974 current_no_gc_region_info.num_gcs_induced++;
15978 BOOL gc_heap::find_loh_free_for_no_gc()
15980 allocator* loh_allocator = generation_allocator (generation_of (max_generation + 1));
15981 size_t sz_list = loh_allocator->first_bucket_size();
15982 size_t size = loh_allocation_no_gc;
15983 for (unsigned int a_l_idx = 0; a_l_idx < loh_allocator->number_of_buckets(); a_l_idx++)
15985 if ((size < sz_list) || (a_l_idx == (loh_allocator->number_of_buckets()-1)))
15987 uint8_t* free_list = loh_allocator->alloc_list_head_of (a_l_idx);
15990 size_t free_list_size = unused_array_size(free_list);
15992 if (free_list_size > loh_allocation_no_gc)
15994 dprintf (3, ("free item %Ix(%Id) for no gc", (size_t)free_list, free_list_size));
15998 free_list = free_list_slot (free_list);
16001 sz_list = sz_list * 2;
16007 BOOL gc_heap::find_loh_space_for_no_gc()
16009 saved_loh_segment_no_gc = 0;
16011 if (find_loh_free_for_no_gc())
16014 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16018 size_t remaining = heap_segment_reserved (seg) - heap_segment_allocated (seg);
16019 if (remaining >= loh_allocation_no_gc)
16021 saved_loh_segment_no_gc = seg;
16024 seg = heap_segment_next (seg);
16027 if (!saved_loh_segment_no_gc && current_no_gc_region_info.minimal_gc_p)
16029 // If no full GC is allowed, we try to get a new seg right away.
16030 saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc)
16031 #ifdef MULTIPLE_HEAPS
16033 #endif //MULTIPLE_HEAPS
16037 return (saved_loh_segment_no_gc != 0);
16040 BOOL gc_heap::loh_allocated_for_no_gc()
16042 if (!saved_loh_segment_no_gc)
16045 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16048 if (seg == saved_loh_segment_no_gc)
16052 seg = heap_segment_next (seg);
16058 BOOL gc_heap::commit_loh_for_no_gc (heap_segment* seg)
16060 uint8_t* end_committed = heap_segment_allocated (seg) + loh_allocation_no_gc;
16061 assert (end_committed <= heap_segment_reserved (seg));
16062 return (grow_heap_segment (seg, end_committed));
16065 void gc_heap::thread_no_gc_loh_segments()
16067 #ifdef MULTIPLE_HEAPS
16068 for (int i = 0; i < n_heaps; i++)
16070 gc_heap* hp = g_heaps[i];
16071 if (hp->loh_allocated_for_no_gc())
16073 hp->thread_loh_segment (hp->saved_loh_segment_no_gc);
16074 hp->saved_loh_segment_no_gc = 0;
16077 #else //MULTIPLE_HEAPS
16078 if (loh_allocated_for_no_gc())
16080 thread_loh_segment (saved_loh_segment_no_gc);
16081 saved_loh_segment_no_gc = 0;
16083 #endif //MULTIPLE_HEAPS
16086 void gc_heap::set_loh_allocations_for_no_gc()
16088 if (current_no_gc_region_info.loh_allocation_size != 0)
16090 dynamic_data* dd = dynamic_data_of (max_generation + 1);
16091 dd_new_allocation (dd) = loh_allocation_no_gc;
16092 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
16096 void gc_heap::set_soh_allocations_for_no_gc()
16098 if (current_no_gc_region_info.soh_allocation_size != 0)
16100 dynamic_data* dd = dynamic_data_of (0);
16101 dd_new_allocation (dd) = soh_allocation_no_gc;
16102 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
16103 #ifdef MULTIPLE_HEAPS
16104 alloc_context_count = 0;
16105 #endif //MULTIPLE_HEAPS
16109 void gc_heap::set_allocations_for_no_gc()
16111 #ifdef MULTIPLE_HEAPS
16112 for (int i = 0; i < n_heaps; i++)
16114 gc_heap* hp = g_heaps[i];
16115 hp->set_loh_allocations_for_no_gc();
16116 hp->set_soh_allocations_for_no_gc();
16118 #else //MULTIPLE_HEAPS
16119 set_loh_allocations_for_no_gc();
16120 set_soh_allocations_for_no_gc();
16121 #endif //MULTIPLE_HEAPS
16124 BOOL gc_heap::should_proceed_for_no_gc()
16126 BOOL gc_requested = FALSE;
16127 BOOL loh_full_gc_requested = FALSE;
16128 BOOL soh_full_gc_requested = FALSE;
16129 BOOL no_gc_requested = FALSE;
16130 BOOL get_new_loh_segments = FALSE;
16132 if (current_no_gc_region_info.soh_allocation_size)
16134 #ifdef MULTIPLE_HEAPS
16135 for (int i = 0; i < n_heaps; i++)
16137 gc_heap* hp = g_heaps[i];
16138 if ((size_t)(heap_segment_reserved (hp->ephemeral_heap_segment) - hp->alloc_allocated) < hp->soh_allocation_no_gc)
16140 gc_requested = TRUE;
16144 #else //MULTIPLE_HEAPS
16145 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated) < soh_allocation_no_gc)
16146 gc_requested = TRUE;
16147 #endif //MULTIPLE_HEAPS
16151 #ifdef MULTIPLE_HEAPS
16152 for (int i = 0; i < n_heaps; i++)
16154 gc_heap* hp = g_heaps[i];
16155 if (!(hp->grow_heap_segment (hp->ephemeral_heap_segment, (hp->alloc_allocated + hp->soh_allocation_no_gc))))
16157 soh_full_gc_requested = TRUE;
16161 #else //MULTIPLE_HEAPS
16162 if (!grow_heap_segment (ephemeral_heap_segment, (alloc_allocated + soh_allocation_no_gc)))
16163 soh_full_gc_requested = TRUE;
16164 #endif //MULTIPLE_HEAPS
16168 if (!current_no_gc_region_info.minimal_gc_p && gc_requested)
16170 soh_full_gc_requested = TRUE;
16173 no_gc_requested = !(soh_full_gc_requested || gc_requested);
16175 if (soh_full_gc_requested && current_no_gc_region_info.minimal_gc_p)
16177 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16181 if (!soh_full_gc_requested && current_no_gc_region_info.loh_allocation_size)
16183 // Check to see if we have enough reserved space.
16184 #ifdef MULTIPLE_HEAPS
16185 for (int i = 0; i < n_heaps; i++)
16187 gc_heap* hp = g_heaps[i];
16188 if (!hp->find_loh_space_for_no_gc())
16190 loh_full_gc_requested = TRUE;
16194 #else //MULTIPLE_HEAPS
16195 if (!find_loh_space_for_no_gc())
16196 loh_full_gc_requested = TRUE;
16197 #endif //MULTIPLE_HEAPS
16199 // Check to see if we have committed space.
16200 if (!loh_full_gc_requested)
16202 #ifdef MULTIPLE_HEAPS
16203 for (int i = 0; i < n_heaps; i++)
16205 gc_heap* hp = g_heaps[i];
16206 if (hp->saved_loh_segment_no_gc &&!hp->commit_loh_for_no_gc (hp->saved_loh_segment_no_gc))
16208 loh_full_gc_requested = TRUE;
16212 #else //MULTIPLE_HEAPS
16213 if (saved_loh_segment_no_gc && !commit_loh_for_no_gc (saved_loh_segment_no_gc))
16214 loh_full_gc_requested = TRUE;
16215 #endif //MULTIPLE_HEAPS
16219 if (loh_full_gc_requested || soh_full_gc_requested)
16221 if (current_no_gc_region_info.minimal_gc_p)
16222 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16225 no_gc_requested = !(loh_full_gc_requested || soh_full_gc_requested || gc_requested);
16227 if (current_no_gc_region_info.start_status == start_no_gc_success)
16229 if (no_gc_requested)
16230 set_allocations_for_no_gc();
16235 if ((current_no_gc_region_info.start_status == start_no_gc_success) && !no_gc_requested)
16239 // We are done with starting the no_gc_region.
16240 current_no_gc_region_info.started = TRUE;
16245 end_no_gc_region_status gc_heap::end_no_gc_region()
16247 dprintf (1, ("end no gc called"));
16249 end_no_gc_region_status status = end_no_gc_success;
16251 if (!(current_no_gc_region_info.started))
16252 status = end_no_gc_not_in_progress;
16253 if (current_no_gc_region_info.num_gcs_induced)
16254 status = end_no_gc_induced;
16255 else if (current_no_gc_region_info.num_gcs)
16256 status = end_no_gc_alloc_exceeded;
16258 if (settings.pause_mode == pause_no_gc)
16259 restore_data_for_no_gc();
16261 // sets current_no_gc_region_info.started to FALSE here.
16262 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
16268 void gc_heap::update_collection_counts ()
16270 dynamic_data* dd0 = dynamic_data_of (0);
16271 dd_gc_clock (dd0) += 1;
16273 size_t now = GetHighPrecisionTimeStamp();
16275 for (int i = 0; i <= settings.condemned_generation;i++)
16277 dynamic_data* dd = dynamic_data_of (i);
16278 dd_collection_count (dd)++;
16279 //this is needed by the linear allocation model
16280 if (i == max_generation)
16281 dd_collection_count (dynamic_data_of (max_generation+1))++;
16282 dd_gc_clock (dd) = dd_gc_clock (dd0);
16283 dd_time_clock (dd) = now;
16287 BOOL gc_heap::expand_soh_with_minimal_gc()
16289 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment)) >= soh_allocation_no_gc)
16292 heap_segment* new_seg = soh_get_segment_to_expand();
16295 if (g_gc_card_table != card_table)
16296 copy_brick_card_table();
16298 settings.promotion = TRUE;
16299 settings.demotion = FALSE;
16300 ephemeral_promotion = TRUE;
16301 int condemned_gen_number = max_generation - 1;
16303 generation* gen = 0;
16304 int align_const = get_alignment_constant (TRUE);
16306 for (int i = 0; i <= condemned_gen_number; i++)
16308 gen = generation_of (i);
16309 saved_ephemeral_plan_start[i] = generation_allocation_start (gen);
16310 saved_ephemeral_plan_start_size[i] = Align (size (generation_allocation_start (gen)), align_const);
16313 // We do need to clear the bricks here as we are converting a bunch of ephemeral objects to gen2
16314 // and need to make sure that there are no left over bricks from the previous GCs for the space
16315 // we just used for gen0 allocation. We will need to go through the bricks for these objects for
16316 // ephemeral GCs later.
16317 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
16318 b < brick_of (align_on_brick (heap_segment_allocated (ephemeral_heap_segment)));
16324 size_t ephemeral_size = (heap_segment_allocated (ephemeral_heap_segment) -
16325 generation_allocation_start (generation_of (max_generation - 1)));
16326 heap_segment_next (ephemeral_heap_segment) = new_seg;
16327 ephemeral_heap_segment = new_seg;
16328 uint8_t* start = heap_segment_mem (ephemeral_heap_segment);
16330 for (int i = condemned_gen_number; i >= 0; i--)
16332 gen = generation_of (i);
16333 size_t gen_start_size = Align (min_obj_size);
16334 make_generation (generation_table[i], ephemeral_heap_segment, start, 0);
16335 generation_plan_allocation_start (gen) = start;
16336 generation_plan_allocation_start_size (gen) = gen_start_size;
16337 start += gen_start_size;
16339 heap_segment_used (ephemeral_heap_segment) = start - plug_skew;
16340 heap_segment_plan_allocated (ephemeral_heap_segment) = start;
16342 fix_generation_bounds (condemned_gen_number, generation_of (0));
16344 dd_gc_new_allocation (dynamic_data_of (max_generation)) -= ephemeral_size;
16345 dd_new_allocation (dynamic_data_of (max_generation)) = dd_gc_new_allocation (dynamic_data_of (max_generation));
16347 adjust_ephemeral_limits();
16354 // Only to be done on the thread that calls restart in a join for server GC
16355 // and reset the oom status per heap.
16356 void gc_heap::check_and_set_no_gc_oom()
16358 #ifdef MULTIPLE_HEAPS
16359 for (int i = 0; i < n_heaps; i++)
16361 gc_heap* hp = g_heaps[i];
16362 if (hp->no_gc_oom_p)
16364 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16365 hp->no_gc_oom_p = false;
16371 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16372 no_gc_oom_p = false;
16374 #endif //MULTIPLE_HEAPS
16377 void gc_heap::allocate_for_no_gc_after_gc()
16379 if (current_no_gc_region_info.minimal_gc_p)
16380 repair_allocation_contexts (TRUE);
16382 no_gc_oom_p = false;
16384 if (current_no_gc_region_info.start_status != start_no_gc_no_memory)
16386 if (current_no_gc_region_info.soh_allocation_size != 0)
16388 if (((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment)) < soh_allocation_no_gc) ||
16389 (!grow_heap_segment (ephemeral_heap_segment, (heap_segment_allocated (ephemeral_heap_segment) + soh_allocation_no_gc))))
16391 no_gc_oom_p = true;
16394 #ifdef MULTIPLE_HEAPS
16395 gc_t_join.join(this, gc_join_after_commit_soh_no_gc);
16396 if (gc_t_join.joined())
16398 #endif //MULTIPLE_HEAPS
16400 check_and_set_no_gc_oom();
16402 #ifdef MULTIPLE_HEAPS
16403 gc_t_join.restart();
16405 #endif //MULTIPLE_HEAPS
16408 if ((current_no_gc_region_info.start_status == start_no_gc_success) &&
16409 !(current_no_gc_region_info.minimal_gc_p) &&
16410 (current_no_gc_region_info.loh_allocation_size != 0))
16412 gc_policy = policy_compact;
16413 saved_loh_segment_no_gc = 0;
16415 if (!find_loh_free_for_no_gc())
16417 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16418 BOOL found_seg_p = FALSE;
16421 if ((size_t)(heap_segment_reserved (seg) - heap_segment_allocated (seg)) >= loh_allocation_no_gc)
16423 found_seg_p = TRUE;
16424 if (!commit_loh_for_no_gc (seg))
16426 no_gc_oom_p = true;
16430 seg = heap_segment_next (seg);
16434 gc_policy = policy_expand;
16437 #ifdef MULTIPLE_HEAPS
16438 gc_t_join.join(this, gc_join_expand_loh_no_gc);
16439 if (gc_t_join.joined())
16441 check_and_set_no_gc_oom();
16443 if (current_no_gc_region_info.start_status == start_no_gc_success)
16445 for (int i = 0; i < n_heaps; i++)
16447 gc_heap* hp = g_heaps[i];
16448 if (hp->gc_policy == policy_expand)
16450 hp->saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc), hp);
16451 if (!(hp->saved_loh_segment_no_gc))
16453 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16460 gc_t_join.restart();
16462 #else //MULTIPLE_HEAPS
16463 check_and_set_no_gc_oom();
16465 if ((current_no_gc_region_info.start_status == start_no_gc_success) && (gc_policy == policy_expand))
16467 saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc));
16468 if (!saved_loh_segment_no_gc)
16469 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16471 #endif //MULTIPLE_HEAPS
16473 if ((current_no_gc_region_info.start_status == start_no_gc_success) && saved_loh_segment_no_gc)
16475 if (!commit_loh_for_no_gc (saved_loh_segment_no_gc))
16477 no_gc_oom_p = true;
16483 #ifdef MULTIPLE_HEAPS
16484 gc_t_join.join(this, gc_join_final_no_gc);
16485 if (gc_t_join.joined())
16487 #endif //MULTIPLE_HEAPS
16489 check_and_set_no_gc_oom();
16491 if (current_no_gc_region_info.start_status == start_no_gc_success)
16493 set_allocations_for_no_gc();
16494 current_no_gc_region_info.started = TRUE;
16497 #ifdef MULTIPLE_HEAPS
16498 gc_t_join.restart();
16500 #endif //MULTIPLE_HEAPS
16503 void gc_heap::init_records()
16505 memset (&gc_data_per_heap, 0, sizeof (gc_data_per_heap));
16506 gc_data_per_heap.heap_index = heap_number;
16507 if (heap_number == 0)
16508 memset (&gc_data_global, 0, sizeof (gc_data_global));
16510 #ifdef GC_CONFIG_DRIVEN
16511 memset (interesting_data_per_gc, 0, sizeof (interesting_data_per_gc));
16512 #endif //GC_CONFIG_DRIVEN
16515 int gc_heap::garbage_collect (int n)
16517 //reset the number of alloc contexts
16518 alloc_contexts_used = 0;
16520 fix_allocation_contexts (TRUE);
16521 #ifdef MULTIPLE_HEAPS
16523 gc_t_join.start_ts(this);
16524 #endif //JOIN_STATS
16525 clear_gen0_bricks();
16526 #endif //MULTIPLE_HEAPS
16528 if ((settings.pause_mode == pause_no_gc) && current_no_gc_region_info.minimal_gc_p)
16530 #ifdef MULTIPLE_HEAPS
16531 gc_t_join.join(this, gc_join_minimal_gc);
16532 if (gc_t_join.joined())
16534 #endif //MULTIPLE_HEAPS
16536 #ifdef MULTIPLE_HEAPS
16537 // this is serialized because we need to get a segment
16538 for (int i = 0; i < n_heaps; i++)
16540 if (!(g_heaps[i]->expand_soh_with_minimal_gc()))
16541 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16544 if (!expand_soh_with_minimal_gc())
16545 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16546 #endif //MULTIPLE_HEAPS
16548 update_collection_counts_for_no_gc();
16550 #ifdef MULTIPLE_HEAPS
16551 gc_t_join.restart();
16553 #endif //MULTIPLE_HEAPS
16559 memset (&fgm_result, 0, sizeof (fgm_result));
16561 settings.reason = gc_trigger_reason;
16562 verify_pinned_queue_p = FALSE;
16564 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
16565 num_pinned_objects = 0;
16566 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
16569 if (settings.reason == reason_gcstress)
16571 settings.reason = reason_induced;
16572 settings.stress_induced = TRUE;
16574 #endif // STRESS_HEAP
16576 #ifdef MULTIPLE_HEAPS
16577 //align all heaps on the max generation to condemn
16578 dprintf (3, ("Joining for max generation to condemn"));
16579 condemned_generation_num = generation_to_condemn (n,
16580 &blocking_collection,
16581 &elevation_requested,
16583 gc_t_join.join(this, gc_join_generation_determined);
16584 if (gc_t_join.joined())
16585 #endif //MULTIPLE_HEAPS
16587 #ifdef MULTIPLE_HEAPS
16588 #if !defined(SEG_MAPPING_TABLE) && !defined(FEATURE_BASICFREEZE)
16589 //delete old slots from the segment table
16590 seg_table->delete_old_slots();
16591 #endif //!SEG_MAPPING_TABLE && !FEATURE_BASICFREEZE
16592 for (int i = 0; i < n_heaps; i++)
16594 //copy the card and brick tables
16595 if (g_gc_card_table != g_heaps[i]->card_table)
16597 g_heaps[i]->copy_brick_card_table();
16600 g_heaps[i]->rearrange_large_heap_segments();
16601 if (!recursive_gc_sync::background_running_p())
16603 g_heaps[i]->rearrange_small_heap_segments();
16606 #else //MULTIPLE_HEAPS
16607 #ifdef BACKGROUND_GC
16608 //delete old slots from the segment table
16609 #if !defined(SEG_MAPPING_TABLE) && !defined(FEATURE_BASICFREEZE)
16610 seg_table->delete_old_slots();
16611 #endif //!SEG_MAPPING_TABLE && !FEATURE_BASICFREEZE
16612 rearrange_large_heap_segments();
16613 if (!recursive_gc_sync::background_running_p())
16615 rearrange_small_heap_segments();
16617 #endif //BACKGROUND_GC
16618 // check for card table growth
16619 if (g_gc_card_table != card_table)
16620 copy_brick_card_table();
16622 #endif //MULTIPLE_HEAPS
16624 BOOL should_evaluate_elevation = FALSE;
16625 BOOL should_do_blocking_collection = FALSE;
16627 #ifdef MULTIPLE_HEAPS
16628 int gen_max = condemned_generation_num;
16629 for (int i = 0; i < n_heaps; i++)
16631 if (gen_max < g_heaps[i]->condemned_generation_num)
16632 gen_max = g_heaps[i]->condemned_generation_num;
16633 if ((!should_evaluate_elevation) && (g_heaps[i]->elevation_requested))
16634 should_evaluate_elevation = TRUE;
16635 if ((!should_do_blocking_collection) && (g_heaps[i]->blocking_collection))
16636 should_do_blocking_collection = TRUE;
16639 settings.condemned_generation = gen_max;
16640 #else //MULTIPLE_HEAPS
16641 settings.condemned_generation = generation_to_condemn (n,
16642 &blocking_collection,
16643 &elevation_requested,
16645 should_evaluate_elevation = elevation_requested;
16646 should_do_blocking_collection = blocking_collection;
16647 #endif //MULTIPLE_HEAPS
16649 settings.condemned_generation = joined_generation_to_condemn (
16650 should_evaluate_elevation,
16651 settings.condemned_generation,
16652 &should_do_blocking_collection
16656 STRESS_LOG1(LF_GCROOTS|LF_GC|LF_GCALLOC, LL_INFO10,
16657 "condemned generation num: %d\n", settings.condemned_generation);
16659 record_gcs_during_no_gc();
16661 if (settings.condemned_generation > 1)
16662 settings.promotion = TRUE;
16664 #ifdef HEAP_ANALYZE
16665 // At this point we've decided what generation is condemned
16666 // See if we've been requested to analyze survivors after the mark phase
16667 if (GCToEEInterface::AnalyzeSurvivorsRequested(settings.condemned_generation))
16669 heap_analyze_enabled = TRUE;
16671 #endif // HEAP_ANALYZE
16673 GCToEEInterface::DiagGCStart(settings.condemned_generation, settings.reason == reason_induced);
16675 #ifdef BACKGROUND_GC
16676 if ((settings.condemned_generation == max_generation) &&
16677 (recursive_gc_sync::background_running_p()))
16679 //TODO BACKGROUND_GC If we just wait for the end of gc, it won't work
16680 // because we have to collect 0 and 1 properly
16681 // in particular, the allocation contexts are gone.
16682 // For now, it is simpler to collect max_generation-1
16683 settings.condemned_generation = max_generation - 1;
16684 dprintf (GTC_LOG, ("bgc - 1 instead of 2"));
16687 if ((settings.condemned_generation == max_generation) &&
16688 (should_do_blocking_collection == FALSE) &&
16689 gc_can_use_concurrent &&
16690 !temp_disable_concurrent_p &&
16691 ((settings.pause_mode == pause_interactive) || (settings.pause_mode == pause_sustained_low_latency)))
16693 keep_bgc_threads_p = TRUE;
16694 c_write (settings.concurrent, TRUE);
16696 #endif //BACKGROUND_GC
16698 settings.gc_index = (uint32_t)dd_collection_count (dynamic_data_of (0)) + 1;
16700 // Call the EE for start of GC work
16701 // just one thread for MP GC
16702 GCToEEInterface::GcStartWork (settings.condemned_generation,
16705 // TODO: we could fire an ETW event to say this GC as a concurrent GC but later on due to not being able to
16706 // create threads or whatever, this could be a non concurrent GC. Maybe for concurrent GC we should fire
16707 // it in do_background_gc and if it failed to be a CGC we fire it in gc1... in other words, this should be
16711 #ifdef MULTIPLE_HEAPS
16712 gc_start_event.Reset();
16713 //start all threads on the roots.
16714 dprintf(3, ("Starting all gc threads for gc"));
16715 gc_t_join.restart();
16716 #endif //MULTIPLE_HEAPS
16720 int gen_num_for_data = max_generation + 1;
16721 for (int i = 0; i <= gen_num_for_data; i++)
16723 gc_data_per_heap.gen_data[i].size_before = generation_size (i);
16724 generation* gen = generation_of (i);
16725 gc_data_per_heap.gen_data[i].free_list_space_before = generation_free_list_space (gen);
16726 gc_data_per_heap.gen_data[i].free_obj_space_before = generation_free_obj_space (gen);
16729 descr_generations (TRUE);
16730 // descr_card_table();
16733 if ((GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC) &&
16734 !(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_POST_GC_ONLY))
16736 verify_heap (TRUE);
16738 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_BARRIERCHECK)
16739 checkGCWriteBarrier();
16741 #endif // VERIFY_HEAP
16743 #ifdef BACKGROUND_GC
16744 if (settings.concurrent)
16746 // We need to save the settings because we'll need to restore it after each FGC.
16747 assert (settings.condemned_generation == max_generation);
16748 settings.compaction = FALSE;
16749 saved_bgc_settings = settings;
16751 #ifdef MULTIPLE_HEAPS
16752 if (heap_number == 0)
16754 for (int i = 0; i < n_heaps; i++)
16756 prepare_bgc_thread (g_heaps[i]);
16758 dprintf (2, ("setting bgc_threads_sync_event"));
16759 bgc_threads_sync_event.Set();
16763 bgc_threads_sync_event.Wait(INFINITE, FALSE);
16764 dprintf (2, ("bgc_threads_sync_event is signalled"));
16767 prepare_bgc_thread(0);
16768 #endif //MULTIPLE_HEAPS
16770 #ifdef MULTIPLE_HEAPS
16771 gc_t_join.join(this, gc_join_start_bgc);
16772 if (gc_t_join.joined())
16773 #endif //MULTIPLE_HEAPS
16775 do_concurrent_p = TRUE;
16776 do_ephemeral_gc_p = FALSE;
16777 #ifdef MULTIPLE_HEAPS
16778 dprintf(2, ("Joined to perform a background GC"));
16780 for (int i = 0; i < n_heaps; i++)
16782 gc_heap* hp = g_heaps[i];
16783 if (!(hp->bgc_thread) || !hp->commit_mark_array_bgc_init (hp->mark_array))
16785 do_concurrent_p = FALSE;
16790 hp->background_saved_lowest_address = hp->lowest_address;
16791 hp->background_saved_highest_address = hp->highest_address;
16795 do_concurrent_p = (!!bgc_thread && commit_mark_array_bgc_init (mark_array));
16796 if (do_concurrent_p)
16798 background_saved_lowest_address = lowest_address;
16799 background_saved_highest_address = highest_address;
16801 #endif //MULTIPLE_HEAPS
16803 if (do_concurrent_p)
16805 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
16806 SoftwareWriteWatch::EnableForGCHeap();
16807 #endif //FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
16809 #ifdef MULTIPLE_HEAPS
16810 for (int i = 0; i < n_heaps; i++)
16811 g_heaps[i]->current_bgc_state = bgc_initialized;
16813 current_bgc_state = bgc_initialized;
16814 #endif //MULTIPLE_HEAPS
16816 int gen = check_for_ephemeral_alloc();
16817 // always do a gen1 GC before we start BGC.
16818 // This is temporary for testing purpose.
16819 //int gen = max_generation - 1;
16820 dont_restart_ee_p = TRUE;
16823 // If we decide to not do a GC before the BGC we need to
16824 // restore the gen0 alloc context.
16825 #ifdef MULTIPLE_HEAPS
16826 for (int i = 0; i < n_heaps; i++)
16828 generation_allocation_pointer (g_heaps[i]->generation_of (0)) = 0;
16829 generation_allocation_limit (g_heaps[i]->generation_of (0)) = 0;
16832 generation_allocation_pointer (youngest_generation) = 0;
16833 generation_allocation_limit (youngest_generation) = 0;
16834 #endif //MULTIPLE_HEAPS
16838 do_ephemeral_gc_p = TRUE;
16840 settings.init_mechanisms();
16841 settings.condemned_generation = gen;
16842 settings.gc_index = (size_t)dd_collection_count (dynamic_data_of (0)) + 2;
16845 // TODO BACKGROUND_GC need to add the profiling stuff here.
16846 dprintf (GTC_LOG, ("doing gen%d before doing a bgc", gen));
16849 //clear the cards so they don't bleed in gen 1 during collection
16850 // shouldn't this always be done at the beginning of any GC?
16851 //clear_card_for_addresses (
16852 // generation_allocation_start (generation_of (0)),
16853 // heap_segment_allocated (ephemeral_heap_segment));
16855 if (!do_ephemeral_gc_p)
16857 do_background_gc();
16862 settings.compaction = TRUE;
16863 c_write (settings.concurrent, FALSE);
16866 #ifdef MULTIPLE_HEAPS
16867 gc_t_join.restart();
16868 #endif //MULTIPLE_HEAPS
16871 if (do_concurrent_p)
16873 // At this point we are sure we'll be starting a BGC, so save its per heap data here.
16874 // global data is only calculated at the end of the GC so we don't need to worry about
16875 // FGCs overwriting it.
16876 memset (&bgc_data_per_heap, 0, sizeof (bgc_data_per_heap));
16877 memcpy (&bgc_data_per_heap, &gc_data_per_heap, sizeof(gc_data_per_heap));
16879 if (do_ephemeral_gc_p)
16881 dprintf (2, ("GC threads running, doing gen%d GC", settings.condemned_generation));
16883 gen_to_condemn_reasons.init();
16884 gen_to_condemn_reasons.set_condition (gen_before_bgc);
16885 gc_data_per_heap.gen_to_condemn_reasons.init (&gen_to_condemn_reasons);
16887 #ifdef MULTIPLE_HEAPS
16888 gc_t_join.join(this, gc_join_bgc_after_ephemeral);
16889 if (gc_t_join.joined())
16890 #endif //MULTIPLE_HEAPS
16892 #ifdef MULTIPLE_HEAPS
16894 #endif //MULTIPLE_HEAPS
16895 settings = saved_bgc_settings;
16896 assert (settings.concurrent);
16898 do_background_gc();
16900 #ifdef MULTIPLE_HEAPS
16901 gc_t_join.restart();
16902 #endif //MULTIPLE_HEAPS
16908 dprintf (2, ("couldn't create BGC threads, reverting to doing a blocking GC"));
16913 #endif //BACKGROUND_GC
16917 #ifndef MULTIPLE_HEAPS
16918 allocation_running_time = (size_t)GCToOSInterface::GetLowPrecisionTimeStamp();
16919 allocation_running_amount = dd_new_allocation (dynamic_data_of (0));
16920 fgn_last_alloc = dd_new_allocation (dynamic_data_of (0));
16921 #endif //MULTIPLE_HEAPS
16924 if (settings.pause_mode == pause_no_gc)
16925 allocate_for_no_gc_after_gc();
16927 int gn = settings.condemned_generation;
16931 #define mark_stack_empty_p() (mark_stack_base == mark_stack_tos)
16934 size_t& gc_heap::promoted_bytes(int thread)
16936 #ifdef MULTIPLE_HEAPS
16937 return g_promoted [thread*16];
16938 #else //MULTIPLE_HEAPS
16939 UNREFERENCED_PARAMETER(thread);
16941 #endif //MULTIPLE_HEAPS
16944 #ifdef INTERIOR_POINTERS
16945 heap_segment* gc_heap::find_segment (uint8_t* interior, BOOL small_segment_only_p)
16947 #ifdef SEG_MAPPING_TABLE
16948 heap_segment* seg = seg_mapping_table_segment_of (interior);
16951 if (small_segment_only_p && heap_segment_loh_p (seg))
16955 #else //SEG_MAPPING_TABLE
16956 #ifdef MULTIPLE_HEAPS
16957 for (int i = 0; i < gc_heap::n_heaps; i++)
16959 gc_heap* h = gc_heap::g_heaps [i];
16960 hs = h->find_segment_per_heap (o, small_segment_only_p);
16968 gc_heap* h = pGenGCHeap;
16969 hs = h->find_segment_per_heap (o, small_segment_only_p);
16971 #endif //MULTIPLE_HEAPS
16972 #endif //SEG_MAPPING_TABLE
16975 heap_segment* gc_heap::find_segment_per_heap (uint8_t* interior, BOOL small_segment_only_p)
16977 #ifdef SEG_MAPPING_TABLE
16978 return find_segment (interior, small_segment_only_p);
16979 #else //SEG_MAPPING_TABLE
16980 if (in_range_for_segment (interior, ephemeral_heap_segment))
16982 return ephemeral_heap_segment;
16986 heap_segment* found_seg = 0;
16989 heap_segment* seg = generation_start_segment (generation_of (max_generation));
16992 if (in_range_for_segment (interior, seg))
16995 goto end_find_segment;
16998 } while ((seg = heap_segment_next (seg)) != 0);
17000 if (!small_segment_only_p)
17002 #ifdef BACKGROUND_GC
17004 ptrdiff_t delta = 0;
17005 heap_segment* seg = segment_of (interior, delta);
17006 if (seg && in_range_for_segment (interior, seg))
17010 goto end_find_segment;
17012 #else //BACKGROUND_GC
17013 heap_segment* seg = generation_start_segment (generation_of (max_generation+1));
17016 if (in_range_for_segment(interior, seg))
17019 goto end_find_segment;
17022 } while ((seg = heap_segment_next (seg)) != 0);
17023 #endif //BACKGROUND_GC
17029 #endif //SEG_MAPPING_TABLE
17031 #endif //INTERIOR_POINTERS
17033 #if !defined(_DEBUG) && !defined(__GNUC__)
17034 inline // This causes link errors if global optimization is off
17035 #endif //!_DEBUG && !__GNUC__
17036 gc_heap* gc_heap::heap_of (uint8_t* o)
17038 #ifdef MULTIPLE_HEAPS
17040 return g_heaps [0];
17041 #ifdef SEG_MAPPING_TABLE
17042 gc_heap* hp = seg_mapping_table_heap_of (o);
17043 return (hp ? hp : g_heaps[0]);
17044 #else //SEG_MAPPING_TABLE
17045 ptrdiff_t delta = 0;
17046 heap_segment* seg = segment_of (o, delta);
17047 return (seg ? heap_segment_heap (seg) : g_heaps [0]);
17048 #endif //SEG_MAPPING_TABLE
17049 #else //MULTIPLE_HEAPS
17050 UNREFERENCED_PARAMETER(o);
17052 #endif //MULTIPLE_HEAPS
17056 gc_heap* gc_heap::heap_of_gc (uint8_t* o)
17058 #ifdef MULTIPLE_HEAPS
17060 return g_heaps [0];
17061 #ifdef SEG_MAPPING_TABLE
17062 gc_heap* hp = seg_mapping_table_heap_of_gc (o);
17063 return (hp ? hp : g_heaps[0]);
17064 #else //SEG_MAPPING_TABLE
17065 ptrdiff_t delta = 0;
17066 heap_segment* seg = segment_of (o, delta);
17067 return (seg ? heap_segment_heap (seg) : g_heaps [0]);
17068 #endif //SEG_MAPPING_TABLE
17069 #else //MULTIPLE_HEAPS
17070 UNREFERENCED_PARAMETER(o);
17072 #endif //MULTIPLE_HEAPS
17075 #ifdef INTERIOR_POINTERS
17076 // will find all heap objects (large and small)
17077 uint8_t* gc_heap::find_object (uint8_t* interior, uint8_t* low)
17079 if (!gen0_bricks_cleared)
17081 #ifdef MULTIPLE_HEAPS
17082 assert (!"Should have already been done in server GC");
17083 #endif //MULTIPLE_HEAPS
17084 gen0_bricks_cleared = TRUE;
17085 //initialize brick table for gen 0
17086 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
17087 b < brick_of (align_on_brick
17088 (heap_segment_allocated (ephemeral_heap_segment)));
17094 #ifdef FFIND_OBJECT
17095 //indicate that in the future this needs to be done during allocation
17096 #ifdef MULTIPLE_HEAPS
17097 gen0_must_clear_bricks = FFIND_DECAY*gc_heap::n_heaps;
17099 gen0_must_clear_bricks = FFIND_DECAY;
17100 #endif //MULTIPLE_HEAPS
17101 #endif //FFIND_OBJECT
17103 int brick_entry = get_brick_entry(brick_of (interior));
17104 if (brick_entry == 0)
17106 // this is a pointer to a large object
17107 heap_segment* seg = find_segment_per_heap (interior, FALSE);
17109 #ifdef FEATURE_CONSERVATIVE_GC
17110 && (GCConfig::GetConservativeGC() || interior <= heap_segment_allocated(seg))
17114 // If interior falls within the first free object at the beginning of a generation,
17115 // we don't have brick entry for it, and we may incorrectly treat it as on large object heap.
17116 int align_const = get_alignment_constant (heap_segment_read_only_p (seg)
17117 #ifdef FEATURE_CONSERVATIVE_GC
17118 || (GCConfig::GetConservativeGC() && !heap_segment_loh_p (seg))
17121 //int align_const = get_alignment_constant (heap_segment_read_only_p (seg));
17122 assert (interior < heap_segment_allocated (seg));
17124 uint8_t* o = heap_segment_mem (seg);
17125 while (o < heap_segment_allocated (seg))
17127 uint8_t* next_o = o + Align (size (o), align_const);
17128 assert (next_o > o);
17129 if ((o <= interior) && (interior < next_o))
17140 else if (interior >= low)
17142 heap_segment* seg = find_segment_per_heap (interior, TRUE);
17145 #ifdef FEATURE_CONSERVATIVE_GC
17146 if (interior >= heap_segment_allocated (seg))
17149 assert (interior < heap_segment_allocated (seg));
17151 uint8_t* o = find_first_object (interior, heap_segment_mem (seg));
17162 gc_heap::find_object_for_relocation (uint8_t* interior, uint8_t* low, uint8_t* high)
17164 uint8_t* old_address = interior;
17165 if (!((old_address >= low) && (old_address < high)))
17168 size_t brick = brick_of (old_address);
17169 int brick_entry = brick_table [ brick ];
17170 if (brick_entry != 0)
17174 while (brick_entry < 0)
17176 brick = (brick + brick_entry);
17177 brick_entry = brick_table [ brick ];
17179 uint8_t* old_loc = old_address;
17180 uint8_t* node = tree_search ((brick_address (brick) + brick_entry-1),
17182 if (node <= old_loc)
17187 brick_entry = brick_table [ brick ];
17193 //find the object by going along the plug
17195 while (o <= interior)
17197 uint8_t* next_o = o + Align (size (o));
17198 assert (next_o > o);
17199 if (next_o > interior)
17205 assert ((o <= interior) && ((o + Align (size (o))) > interior));
17210 // this is a pointer to a large object
17211 heap_segment* seg = find_segment_per_heap (interior, FALSE);
17214 assert (interior < heap_segment_allocated (seg));
17216 uint8_t* o = heap_segment_mem (seg);
17217 while (o < heap_segment_allocated (seg))
17219 uint8_t* next_o = o + Align (size (o));
17220 assert (next_o > o);
17221 if ((o < interior) && (interior < next_o))
17233 #else //INTERIOR_POINTERS
17235 uint8_t* gc_heap::find_object (uint8_t* o, uint8_t* low)
17239 #endif //INTERIOR_POINTERS
17242 #ifdef GC_CONFIG_DRIVEN
17243 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;} else {mark_list_index++;} if (slow > o) slow = o; if (shigh < o) shigh = o;}
17245 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;}if (slow > o) slow = o; if (shigh < o) shigh = o;}
17246 #endif //GC_CONFIG_DRIVEN
17248 #define m_boundary(o) {if (slow > o) slow = o; if (shigh < o) shigh = o;}
17251 #define m_boundary_fullgc(o) {if (slow > o) slow = o; if (shigh < o) shigh = o;}
17253 #define method_table(o) ((CObjectHeader*)(o))->GetMethodTable()
17256 BOOL gc_heap::gc_mark1 (uint8_t* o)
17258 BOOL marked = !marked (o);
17260 dprintf (3, ("*%Ix*, newly marked: %d", (size_t)o, marked));
17265 BOOL gc_heap::gc_mark (uint8_t* o, uint8_t* low, uint8_t* high)
17267 BOOL marked = FALSE;
17268 if ((o >= low) && (o < high))
17269 marked = gc_mark1 (o);
17270 #ifdef MULTIPLE_HEAPS
17274 gc_heap* hp = heap_of_gc (o);
17276 if ((o >= hp->gc_low) && (o < hp->gc_high))
17277 marked = gc_mark1 (o);
17280 snoop_stat.objects_checked_count++;
17284 snoop_stat.objects_marked_count++;
17288 snoop_stat.zero_ref_count++;
17291 #endif //SNOOP_STATS
17292 #endif //MULTIPLE_HEAPS
17296 #ifdef BACKGROUND_GC
17299 BOOL gc_heap::background_marked (uint8_t* o)
17301 return mark_array_marked (o);
17304 BOOL gc_heap::background_mark1 (uint8_t* o)
17306 BOOL to_mark = !mark_array_marked (o);
17308 dprintf (3, ("b*%Ix*b(%d)", (size_t)o, (to_mark ? 1 : 0)));
17311 mark_array_set_marked (o);
17312 dprintf (4, ("n*%Ix*n", (size_t)o));
17319 // TODO: we could consider filtering out NULL's here instead of going to
17320 // look for it on other heaps
17322 BOOL gc_heap::background_mark (uint8_t* o, uint8_t* low, uint8_t* high)
17324 BOOL marked = FALSE;
17325 if ((o >= low) && (o < high))
17326 marked = background_mark1 (o);
17327 #ifdef MULTIPLE_HEAPS
17331 gc_heap* hp = heap_of (o);
17333 if ((o >= hp->background_saved_lowest_address) && (o < hp->background_saved_highest_address))
17334 marked = background_mark1 (o);
17336 #endif //MULTIPLE_HEAPS
17340 #endif //BACKGROUND_GC
17343 uint8_t* gc_heap::next_end (heap_segment* seg, uint8_t* f)
17345 if (seg == ephemeral_heap_segment)
17348 return heap_segment_allocated (seg);
17351 #define new_start() {if (ppstop <= start) {break;} else {parm = start}}
17352 #define ignore_start 0
17353 #define use_start 1
17355 #define go_through_object(mt,o,size,parm,start,start_useful,limit,exp) \
17357 CGCDesc* map = CGCDesc::GetCGCDescFromMT((MethodTable*)(mt)); \
17358 CGCDescSeries* cur = map->GetHighestSeries(); \
17359 ptrdiff_t cnt = (ptrdiff_t) map->GetNumSeries(); \
17363 CGCDescSeries* last = map->GetLowestSeries(); \
17364 uint8_t** parm = 0; \
17367 assert (parm <= (uint8_t**)((o) + cur->GetSeriesOffset())); \
17368 parm = (uint8_t**)((o) + cur->GetSeriesOffset()); \
17369 uint8_t** ppstop = \
17370 (uint8_t**)((uint8_t*)parm + cur->GetSeriesSize() + (size));\
17371 if (!start_useful || (uint8_t*)ppstop > (start)) \
17373 if (start_useful && (uint8_t*)parm < (start)) parm = (uint8_t**)(start);\
17374 while (parm < ppstop) \
17382 } while (cur >= last); \
17386 /* Handle the repeating case - array of valuetypes */ \
17387 uint8_t** parm = (uint8_t**)((o) + cur->startoffset); \
17388 if (start_useful && start > (uint8_t*)parm) \
17390 ptrdiff_t cs = mt->RawGetComponentSize(); \
17391 parm = (uint8_t**)((uint8_t*)parm + (((start) - (uint8_t*)parm)/cs)*cs); \
17393 while ((uint8_t*)parm < ((o)+(size)-plug_skew)) \
17395 for (ptrdiff_t __i = 0; __i > cnt; __i--) \
17397 HALF_SIZE_T skip = cur->val_serie[__i].skip; \
17398 HALF_SIZE_T nptrs = cur->val_serie[__i].nptrs; \
17399 uint8_t** ppstop = parm + nptrs; \
17400 if (!start_useful || (uint8_t*)ppstop > (start)) \
17402 if (start_useful && (uint8_t*)parm < (start)) parm = (uint8_t**)(start); \
17407 } while (parm < ppstop); \
17409 parm = (uint8_t**)((uint8_t*)ppstop + skip); \
17415 #define go_through_object_nostart(mt,o,size,parm,exp) {go_through_object(mt,o,size,parm,o,ignore_start,(o + size),exp); }
17417 // 1 thing to note about this macro:
17418 // 1) you can use *parm safely but in general you don't want to use parm
17419 // because for the collectible types it's not an address on the managed heap.
17420 #ifndef COLLECTIBLE_CLASS
17421 #define go_through_object_cl(mt,o,size,parm,exp) \
17423 if (header(o)->ContainsPointers()) \
17425 go_through_object_nostart(mt,o,size,parm,exp); \
17428 #else //COLLECTIBLE_CLASS
17429 #define go_through_object_cl(mt,o,size,parm,exp) \
17431 if (header(o)->Collectible()) \
17433 uint8_t* class_obj = get_class_object (o); \
17434 uint8_t** parm = &class_obj; \
17435 do {exp} while (false); \
17437 if (header(o)->ContainsPointers()) \
17439 go_through_object_nostart(mt,o,size,parm,exp); \
17442 #endif //COLLECTIBLE_CLASS
17444 // This starts a plug. But mark_stack_tos isn't increased until set_pinned_info is called.
17445 void gc_heap::enque_pinned_plug (uint8_t* plug,
17446 BOOL save_pre_plug_info_p,
17447 uint8_t* last_object_in_last_plug)
17449 if (mark_stack_array_length <= mark_stack_tos)
17451 if (!grow_mark_stack (mark_stack_array, mark_stack_array_length, MARK_STACK_INITIAL_LENGTH))
17453 // we don't want to continue here due to security
17454 // risks. This happens very rarely and fixing it in the
17455 // way so that we can continue is a bit involved and will
17456 // not be done in Dev10.
17457 GCToEEInterface::HandleFatalError(CORINFO_EXCEPTION_GC);
17461 dprintf (3, ("enqueuing P #%Id(%Ix): %Ix. oldest: %Id, LO: %Ix, pre: %d",
17462 mark_stack_tos, &mark_stack_array[mark_stack_tos], plug, mark_stack_bos, last_object_in_last_plug, (save_pre_plug_info_p ? 1 : 0)));
17463 mark& m = mark_stack_array[mark_stack_tos];
17465 // Must be set now because if we have a short object we'll need the value of saved_pre_p.
17466 m.saved_pre_p = save_pre_plug_info_p;
17468 if (save_pre_plug_info_p)
17471 BOOL is_padded = is_plug_padded (last_object_in_last_plug);
17473 clear_plug_padded (last_object_in_last_plug);
17474 #endif //SHORT_PLUGS
17475 memcpy (&(m.saved_pre_plug), &(((plug_and_gap*)plug)[-1]), sizeof (gap_reloc_pair));
17478 set_plug_padded (last_object_in_last_plug);
17479 #endif //SHORT_PLUGS
17481 memcpy (&(m.saved_pre_plug_reloc), &(((plug_and_gap*)plug)[-1]), sizeof (gap_reloc_pair));
17483 // If the last object in the last plug is too short, it requires special handling.
17484 size_t last_obj_size = plug - last_object_in_last_plug;
17485 if (last_obj_size < min_pre_pin_obj_size)
17487 record_interesting_data_point (idp_pre_short);
17490 record_interesting_data_point (idp_pre_short_padded);
17491 #endif //SHORT_PLUGS
17492 dprintf (3, ("encountered a short object %Ix right before pinned plug %Ix!",
17493 last_object_in_last_plug, plug));
17494 // Need to set the short bit regardless of having refs or not because we need to
17495 // indicate that this object is not walkable.
17498 #ifdef COLLECTIBLE_CLASS
17499 if (is_collectible (last_object_in_last_plug))
17501 m.set_pre_short_collectible();
17503 #endif //COLLECTIBLE_CLASS
17505 if (contain_pointers (last_object_in_last_plug))
17507 dprintf (3, ("short object: %Ix(%Ix)", last_object_in_last_plug, last_obj_size));
17509 go_through_object_nostart (method_table(last_object_in_last_plug), last_object_in_last_plug, last_obj_size, pval,
17511 size_t gap_offset = (((size_t)pval - (size_t)(plug - sizeof (gap_reloc_pair) - plug_skew))) / sizeof (uint8_t*);
17512 dprintf (3, ("member: %Ix->%Ix, %Id ptrs from beginning of gap", (uint8_t*)pval, *pval, gap_offset));
17513 m.set_pre_short_bit (gap_offset);
17520 m.saved_post_p = FALSE;
17523 void gc_heap::save_post_plug_info (uint8_t* last_pinned_plug, uint8_t* last_object_in_last_plug, uint8_t* post_plug)
17525 UNREFERENCED_PARAMETER(last_pinned_plug);
17527 mark& m = mark_stack_array[mark_stack_tos - 1];
17528 assert (last_pinned_plug == m.first);
17529 m.saved_post_plug_info_start = (uint8_t*)&(((plug_and_gap*)post_plug)[-1]);
17532 BOOL is_padded = is_plug_padded (last_object_in_last_plug);
17534 clear_plug_padded (last_object_in_last_plug);
17535 #endif //SHORT_PLUGS
17536 memcpy (&(m.saved_post_plug), m.saved_post_plug_info_start, sizeof (gap_reloc_pair));
17539 set_plug_padded (last_object_in_last_plug);
17540 #endif //SHORT_PLUGS
17542 memcpy (&(m.saved_post_plug_reloc), m.saved_post_plug_info_start, sizeof (gap_reloc_pair));
17544 // This is important - we need to clear all bits here except the last one.
17545 m.saved_post_p = TRUE;
17548 m.saved_post_plug_debug.gap = 1;
17551 dprintf (3, ("PP %Ix has NP %Ix right after", last_pinned_plug, post_plug));
17553 size_t last_obj_size = post_plug - last_object_in_last_plug;
17554 if (last_obj_size < min_pre_pin_obj_size)
17556 dprintf (3, ("PP %Ix last obj %Ix is too short", last_pinned_plug, last_object_in_last_plug));
17557 record_interesting_data_point (idp_post_short);
17560 record_interesting_data_point (idp_post_short_padded);
17561 #endif //SHORT_PLUGS
17562 m.set_post_short();
17563 verify_pinned_queue_p = TRUE;
17565 #ifdef COLLECTIBLE_CLASS
17566 if (is_collectible (last_object_in_last_plug))
17568 m.set_post_short_collectible();
17570 #endif //COLLECTIBLE_CLASS
17572 if (contain_pointers (last_object_in_last_plug))
17574 dprintf (3, ("short object: %Ix(%Ix)", last_object_in_last_plug, last_obj_size));
17576 // TODO: since we won't be able to walk this object in relocation, we still need to
17577 // take care of collectible assemblies here.
17578 go_through_object_nostart (method_table(last_object_in_last_plug), last_object_in_last_plug, last_obj_size, pval,
17580 size_t gap_offset = (((size_t)pval - (size_t)(post_plug - sizeof (gap_reloc_pair) - plug_skew))) / sizeof (uint8_t*);
17581 dprintf (3, ("member: %Ix->%Ix, %Id ptrs from beginning of gap", (uint8_t*)pval, *pval, gap_offset));
17582 m.set_post_short_bit (gap_offset);
17591 __declspec(naked) void __fastcall Prefetch(void* addr)
17599 inline void Prefetch (void* addr)
17601 UNREFERENCED_PARAMETER(addr);
17606 VOLATILE(uint8_t*)& gc_heap::ref_mark_stack (gc_heap* hp, int index)
17608 return ((VOLATILE(uint8_t*)*)(hp->mark_stack_array))[index];
17611 #endif //MH_SC_MARK
17615 #define partial_object 3
17617 uint8_t* ref_from_slot (uint8_t* r)
17619 return (uint8_t*)((size_t)r & ~(stolen | partial));
17622 BOOL stolen_p (uint8_t* r)
17624 return (((size_t)r&2) && !((size_t)r&1));
17627 BOOL ready_p (uint8_t* r)
17629 return ((size_t)r != 1);
17632 BOOL partial_p (uint8_t* r)
17634 return (((size_t)r&1) && !((size_t)r&2));
17637 BOOL straight_ref_p (uint8_t* r)
17639 return (!stolen_p (r) && !partial_p (r));
17642 BOOL partial_object_p (uint8_t* r)
17644 return (((size_t)r & partial_object) == partial_object);
17647 BOOL ref_p (uint8_t* r)
17649 return (straight_ref_p (r) || partial_object_p (r));
17652 void gc_heap::mark_object_simple1 (uint8_t* oo, uint8_t* start THREAD_NUMBER_DCL)
17654 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_tos = (SERVER_SC_MARK_VOLATILE(uint8_t*)*)mark_stack_array;
17655 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_limit = (SERVER_SC_MARK_VOLATILE(uint8_t*)*)&mark_stack_array[mark_stack_array_length];
17656 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_base = mark_stack_tos;
17657 #ifdef SORT_MARK_STACK
17658 SERVER_SC_MARK_VOLATILE(uint8_t*)* sorted_tos = mark_stack_base;
17659 #endif //SORT_MARK_STACK
17661 // If we are doing a full GC we don't use mark list anyway so use m_boundary_fullgc that doesn't
17662 // update mark list.
17663 BOOL full_p = (settings.condemned_generation == max_generation);
17665 assert ((start >= oo) && (start < oo+size(oo)));
17668 *mark_stack_tos = oo;
17669 #endif //!MH_SC_MARK
17673 #ifdef MULTIPLE_HEAPS
17674 #else //MULTIPLE_HEAPS
17675 const int thread = 0;
17676 #endif //MULTIPLE_HEAPS
17678 if (oo && ((size_t)oo != 4))
17686 else if (!partial_p (oo) && ((s = size (oo)) < (partial_size_th*sizeof (uint8_t*))))
17688 BOOL overflow_p = FALSE;
17690 if (mark_stack_tos + (s) /sizeof (uint8_t*) >= (mark_stack_limit - 1))
17692 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
17693 if (mark_stack_tos + CGCDesc::GetNumPointers(method_table(oo), s, num_components) >= (mark_stack_limit - 1))
17699 if (overflow_p == FALSE)
17701 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
17703 go_through_object_cl (method_table(oo), oo, s, ppslot,
17705 uint8_t* o = *ppslot;
17707 if (gc_mark (o, gc_low, gc_high))
17711 m_boundary_fullgc (o);
17717 size_t obj_size = size (o);
17718 promoted_bytes (thread) += obj_size;
17719 if (contain_pointers_or_collectible (o))
17721 *(mark_stack_tos++) = o;
17729 dprintf(3,("mark stack overflow for object %Ix ", (size_t)oo));
17730 min_overflow_address = min (min_overflow_address, oo);
17731 max_overflow_address = max (max_overflow_address, oo);
17736 if (partial_p (oo))
17738 start = ref_from_slot (oo);
17739 oo = ref_from_slot (*(--mark_stack_tos));
17740 dprintf (4, ("oo: %Ix, start: %Ix\n", (size_t)oo, (size_t)start));
17741 assert ((oo < start) && (start < (oo + size (oo))));
17743 #ifdef COLLECTIBLE_CLASS
17746 // If there's a class object, push it now. We are guaranteed to have the slot since
17747 // we just popped one object off.
17748 if (is_collectible (oo))
17750 uint8_t* class_obj = get_class_object (oo);
17751 if (gc_mark (class_obj, gc_low, gc_high))
17755 m_boundary_fullgc (class_obj);
17759 m_boundary (class_obj);
17762 size_t obj_size = size (class_obj);
17763 promoted_bytes (thread) += obj_size;
17764 *(mark_stack_tos++) = class_obj;
17765 // The code below expects that the oo is still stored in the stack slot that was
17766 // just popped and it "pushes" it back just by incrementing the mark_stack_tos.
17767 // But the class_obj has just overwritten that stack slot and so the oo needs to
17768 // be stored to the new slot that's pointed to by the mark_stack_tos.
17769 *mark_stack_tos = oo;
17773 #endif //COLLECTIBLE_CLASS
17777 BOOL overflow_p = FALSE;
17779 if (mark_stack_tos + (num_partial_refs + 2) >= mark_stack_limit)
17783 if (overflow_p == FALSE)
17785 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
17787 //push the object and its current
17788 SERVER_SC_MARK_VOLATILE(uint8_t*)* place = ++mark_stack_tos;
17792 *(place) = (uint8_t*)partial;
17793 #endif //MH_SC_MARK
17794 int i = num_partial_refs;
17795 uint8_t* ref_to_continue = 0;
17797 go_through_object (method_table(oo), oo, s, ppslot,
17798 start, use_start, (oo + s),
17800 uint8_t* o = *ppslot;
17802 if (gc_mark (o, gc_low, gc_high))
17806 m_boundary_fullgc (o);
17812 size_t obj_size = size (o);
17813 promoted_bytes (thread) += obj_size;
17814 if (contain_pointers_or_collectible (o))
17816 *(mark_stack_tos++) = o;
17819 ref_to_continue = (uint8_t*)((size_t)(ppslot+1) | partial);
17828 //we are finished with this object
17829 assert (ref_to_continue == 0);
17831 assert ((*(place-1)) == (uint8_t*)0);
17834 #endif //MH_SC_MARK
17836 // shouldn't we decrease tos by 2 here??
17839 if (ref_to_continue)
17843 assert ((*(place-1)) == (uint8_t*)0);
17844 *(place-1) = (uint8_t*)((size_t)oo | partial_object);
17845 assert (((*place) == (uint8_t*)1) || ((*place) == (uint8_t*)2));
17846 #endif //MH_SC_MARK
17847 *place = ref_to_continue;
17852 dprintf(3,("mark stack overflow for object %Ix ", (size_t)oo));
17853 min_overflow_address = min (min_overflow_address, oo);
17854 max_overflow_address = max (max_overflow_address, oo);
17857 #ifdef SORT_MARK_STACK
17858 if (mark_stack_tos > sorted_tos + mark_stack_array_length/8)
17860 rqsort1 (sorted_tos, mark_stack_tos-1);
17861 sorted_tos = mark_stack_tos-1;
17863 #endif //SORT_MARK_STACK
17866 if (!(mark_stack_empty_p()))
17868 oo = *(--mark_stack_tos);
17871 #ifdef SORT_MARK_STACK
17872 sorted_tos = min ((size_t)sorted_tos, (size_t)mark_stack_tos);
17873 #endif //SORT_MARK_STACK
17881 BOOL same_numa_node_p (int hn1, int hn2)
17883 return (heap_select::find_numa_node_from_heap_no (hn1) == heap_select::find_numa_node_from_heap_no (hn2));
17886 int find_next_buddy_heap (int this_heap_number, int current_buddy, int n_heaps)
17888 int hn = (current_buddy+1)%n_heaps;
17889 while (hn != current_buddy)
17891 if ((this_heap_number != hn) && (same_numa_node_p (this_heap_number, hn)))
17893 hn = (hn+1)%n_heaps;
17895 return current_buddy;
17899 gc_heap::mark_steal()
17901 mark_stack_busy() = 0;
17902 //clear the mark stack in the snooping range
17903 for (int i = 0; i < max_snoop_level; i++)
17905 ((VOLATILE(uint8_t*)*)(mark_stack_array))[i] = 0;
17908 //pick the next heap as our buddy
17909 int thpn = find_next_buddy_heap (heap_number, heap_number, n_heaps);
17912 dprintf (SNOOP_LOG, ("(GC%d)heap%d: start snooping %d", settings.gc_index, heap_number, (heap_number+1)%n_heaps));
17913 uint32_t begin_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
17914 #endif //SNOOP_STATS
17916 int idle_loop_count = 0;
17917 int first_not_ready_level = 0;
17921 gc_heap* hp = g_heaps [thpn];
17922 int level = first_not_ready_level;
17923 first_not_ready_level = 0;
17925 while (check_next_mark_stack (hp) && (level < (max_snoop_level-1)))
17927 idle_loop_count = 0;
17929 snoop_stat.busy_count++;
17930 dprintf (SNOOP_LOG, ("heap%d: looking at next heap level %d stack contents: %Ix",
17931 heap_number, level, (int)((uint8_t**)(hp->mark_stack_array))[level]));
17932 #endif //SNOOP_STATS
17934 uint8_t* o = ref_mark_stack (hp, level);
17936 uint8_t* start = o;
17939 mark_stack_busy() = 1;
17941 BOOL success = TRUE;
17942 uint8_t* next = (ref_mark_stack (hp, level+1));
17945 if (((size_t)o > 4) && !partial_object_p (o))
17947 //this is a normal object, not a partial mark tuple
17948 //success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level), 0, o)==o);
17949 success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level), (uint8_t*)4, o)==o);
17951 snoop_stat.interlocked_count++;
17953 snoop_stat.normal_count++;
17954 #endif //SNOOP_STATS
17958 //it is a stolen entry, or beginning/ending of a partial mark
17961 snoop_stat.stolen_or_pm_count++;
17962 #endif //SNOOP_STATS
17966 else if (stolen_p (next))
17968 //ignore the stolen guy and go to the next level
17972 snoop_stat.stolen_entry_count++;
17973 #endif //SNOOP_STATS
17977 assert (partial_p (next));
17978 start = ref_from_slot (next);
17979 //re-read the object
17980 o = ref_from_slot (ref_mark_stack (hp, level));
17984 success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level+1), (uint8_t*)stolen, next)==next);
17986 snoop_stat.interlocked_count++;
17989 snoop_stat.partial_mark_parent_count++;
17991 #endif //SNOOP_STATS
17995 // stack is not ready, or o is completely different from the last time we read from this stack level.
17996 // go up 2 levels to steal children or totally unrelated objects.
17998 if (first_not_ready_level == 0)
18000 first_not_ready_level = level;
18004 snoop_stat.pm_not_ready_count++;
18005 #endif //SNOOP_STATS
18012 dprintf (SNOOP_LOG, ("heap%d: marking %Ix from %d [%d] tl:%dms",
18013 heap_number, (size_t)o, (heap_number+1)%n_heaps, level,
18014 (GCToOSInterface::GetLowPrecisionTimeStamp()-begin_tick)));
18015 uint32_t start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
18016 #endif //SNOOP_STATS
18018 mark_object_simple1 (o, start, heap_number);
18021 dprintf (SNOOP_LOG, ("heap%d: done marking %Ix from %d [%d] %dms tl:%dms",
18022 heap_number, (size_t)o, (heap_number+1)%n_heaps, level,
18023 (GCToOSInterface::GetLowPrecisionTimeStamp()-start_tick),(GCToOSInterface::GetLowPrecisionTimeStamp()-begin_tick)));
18024 #endif //SNOOP_STATS
18026 mark_stack_busy() = 0;
18028 //clear the mark stack in snooping range
18029 for (int i = 0; i < max_snoop_level; i++)
18031 if (((uint8_t**)mark_stack_array)[i] != 0)
18033 ((VOLATILE(uint8_t*)*)(mark_stack_array))[i] = 0;
18035 snoop_stat.stack_bottom_clear_count++;
18036 #endif //SNOOP_STATS
18042 mark_stack_busy() = 0;
18046 //slot is either partial or stolen
18050 if ((first_not_ready_level != 0) && hp->mark_stack_busy())
18054 if (!hp->mark_stack_busy())
18056 first_not_ready_level = 0;
18059 if ((idle_loop_count % (6) )==1)
18062 snoop_stat.switch_to_thread_count++;
18063 #endif //SNOOP_STATS
18064 GCToOSInterface::Sleep(1);
18066 int free_count = 1;
18068 snoop_stat.stack_idle_count++;
18069 //dprintf (SNOOP_LOG, ("heap%d: counting idle threads", heap_number));
18070 #endif //SNOOP_STATS
18071 for (int hpn = (heap_number+1)%n_heaps; hpn != heap_number;)
18073 if (!((g_heaps [hpn])->mark_stack_busy()))
18077 dprintf (SNOOP_LOG, ("heap%d: %d idle", heap_number, free_count));
18078 #endif //SNOOP_STATS
18080 else if (same_numa_node_p (hpn, heap_number) || ((idle_loop_count%1000))==999)
18085 hpn = (hpn+1)%n_heaps;
18088 if (free_count == n_heaps)
18097 BOOL gc_heap::check_next_mark_stack (gc_heap* next_heap)
18100 snoop_stat.check_level_count++;
18101 #endif //SNOOP_STATS
18102 return (next_heap->mark_stack_busy()>=1);
18104 #endif //MH_SC_MARK
18107 void gc_heap::print_snoop_stat()
18109 dprintf (1234, ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s",
18110 "heap", "check", "zero", "mark", "stole", "pstack", "nstack", "nonsk"));
18111 dprintf (1234, ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d",
18112 snoop_stat.heap_index,
18113 snoop_stat.objects_checked_count,
18114 snoop_stat.zero_ref_count,
18115 snoop_stat.objects_marked_count,
18116 snoop_stat.stolen_stack_count,
18117 snoop_stat.partial_stack_count,
18118 snoop_stat.normal_stack_count,
18119 snoop_stat.non_stack_count));
18120 dprintf (1234, ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s",
18121 "heap", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "clear"));
18122 dprintf (1234, ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
18123 snoop_stat.heap_index,
18124 snoop_stat.check_level_count,
18125 snoop_stat.busy_count,
18126 snoop_stat.interlocked_count,
18127 snoop_stat.partial_mark_parent_count,
18128 snoop_stat.stolen_or_pm_count,
18129 snoop_stat.stolen_entry_count,
18130 snoop_stat.pm_not_ready_count,
18131 snoop_stat.normal_count,
18132 snoop_stat.stack_bottom_clear_count));
18134 printf ("\n%4s | %8s | %8s | %8s | %8s | %8s\n",
18135 "heap", "check", "zero", "mark", "idle", "switch");
18136 printf ("%4d | %8d | %8d | %8d | %8d | %8d\n",
18137 snoop_stat.heap_index,
18138 snoop_stat.objects_checked_count,
18139 snoop_stat.zero_ref_count,
18140 snoop_stat.objects_marked_count,
18141 snoop_stat.stack_idle_count,
18142 snoop_stat.switch_to_thread_count);
18143 printf ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s\n",
18144 "heap", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "normal", "clear");
18145 printf ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
18146 snoop_stat.heap_index,
18147 snoop_stat.check_level_count,
18148 snoop_stat.busy_count,
18149 snoop_stat.interlocked_count,
18150 snoop_stat.partial_mark_parent_count,
18151 snoop_stat.stolen_or_pm_count,
18152 snoop_stat.stolen_entry_count,
18153 snoop_stat.pm_not_ready_count,
18154 snoop_stat.normal_count,
18155 snoop_stat.stack_bottom_clear_count);
18157 #endif //SNOOP_STATS
18159 #ifdef HEAP_ANALYZE
18161 gc_heap::ha_mark_object_simple (uint8_t** po THREAD_NUMBER_DCL)
18163 if (!internal_root_array)
18165 internal_root_array = new (nothrow) uint8_t* [internal_root_array_length];
18166 if (!internal_root_array)
18168 heap_analyze_success = FALSE;
18172 if (heap_analyze_success && (internal_root_array_length <= internal_root_array_index))
18174 size_t new_size = 2*internal_root_array_length;
18176 uint64_t available_physical = 0;
18177 get_memory_info (NULL, &available_physical);
18178 if (new_size > (size_t)(available_physical / 10))
18180 heap_analyze_success = FALSE;
18184 uint8_t** tmp = new (nothrow) uint8_t* [new_size];
18187 memcpy (tmp, internal_root_array,
18188 internal_root_array_length*sizeof (uint8_t*));
18189 delete[] internal_root_array;
18190 internal_root_array = tmp;
18191 internal_root_array_length = new_size;
18195 heap_analyze_success = FALSE;
18200 if (heap_analyze_success)
18202 PREFIX_ASSUME(internal_root_array_index < internal_root_array_length);
18204 uint8_t* ref = (uint8_t*)po;
18205 if (!current_obj ||
18206 !((ref >= current_obj) && (ref < (current_obj + current_obj_size))))
18208 gc_heap* hp = gc_heap::heap_of (ref);
18209 current_obj = hp->find_object (ref, hp->lowest_address);
18210 current_obj_size = size (current_obj);
18212 internal_root_array[internal_root_array_index] = current_obj;
18213 internal_root_array_index++;
18217 mark_object_simple (po THREAD_NUMBER_ARG);
18219 #endif //HEAP_ANALYZE
18221 //this method assumes that *po is in the [low. high[ range
18223 gc_heap::mark_object_simple (uint8_t** po THREAD_NUMBER_DCL)
18226 #ifdef MULTIPLE_HEAPS
18227 #else //MULTIPLE_HEAPS
18228 const int thread = 0;
18229 #endif //MULTIPLE_HEAPS
18232 snoop_stat.objects_checked_count++;
18233 #endif //SNOOP_STATS
18238 size_t s = size (o);
18239 promoted_bytes (thread) += s;
18241 go_through_object_cl (method_table(o), o, s, poo,
18243 uint8_t* oo = *poo;
18244 if (gc_mark (oo, gc_low, gc_high))
18247 size_t obj_size = size (oo);
18248 promoted_bytes (thread) += obj_size;
18250 if (contain_pointers_or_collectible (oo))
18251 mark_object_simple1 (oo, oo THREAD_NUMBER_ARG);
18261 uint8_t* gc_heap::mark_object (uint8_t* o THREAD_NUMBER_DCL)
18263 if ((o >= gc_low) && (o < gc_high))
18264 mark_object_simple (&o THREAD_NUMBER_ARG);
18265 #ifdef MULTIPLE_HEAPS
18269 gc_heap* hp = heap_of (o);
18271 if ((o >= hp->gc_low) && (o < hp->gc_high))
18272 mark_object_simple (&o THREAD_NUMBER_ARG);
18274 #endif //MULTIPLE_HEAPS
18279 #ifdef BACKGROUND_GC
18281 void gc_heap::background_mark_simple1 (uint8_t* oo THREAD_NUMBER_DCL)
18283 uint8_t** mark_stack_limit = &background_mark_stack_array[background_mark_stack_array_length];
18285 #ifdef SORT_MARK_STACK
18286 uint8_t** sorted_tos = background_mark_stack_array;
18287 #endif //SORT_MARK_STACK
18289 background_mark_stack_tos = background_mark_stack_array;
18293 #ifdef MULTIPLE_HEAPS
18294 #else //MULTIPLE_HEAPS
18295 const int thread = 0;
18296 #endif //MULTIPLE_HEAPS
18300 if ((((size_t)oo & 1) == 0) && ((s = size (oo)) < (partial_size_th*sizeof (uint8_t*))))
18302 BOOL overflow_p = FALSE;
18304 if (background_mark_stack_tos + (s) /sizeof (uint8_t*) >= (mark_stack_limit - 1))
18306 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
18307 size_t num_pointers = CGCDesc::GetNumPointers(method_table(oo), s, num_components);
18308 if (background_mark_stack_tos + num_pointers >= (mark_stack_limit - 1))
18310 dprintf (2, ("h%d: %Id left, obj (mt: %Ix) %Id ptrs",
18312 (size_t)(mark_stack_limit - 1 - background_mark_stack_tos),
18316 bgc_overflow_count++;
18321 if (overflow_p == FALSE)
18323 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18325 go_through_object_cl (method_table(oo), oo, s, ppslot,
18327 uint8_t* o = *ppslot;
18329 if (background_mark (o,
18330 background_saved_lowest_address,
18331 background_saved_highest_address))
18334 size_t obj_size = size (o);
18335 bpromoted_bytes (thread) += obj_size;
18336 if (contain_pointers_or_collectible (o))
18338 *(background_mark_stack_tos++) = o;
18347 dprintf (3,("mark stack overflow for object %Ix ", (size_t)oo));
18348 background_min_overflow_address = min (background_min_overflow_address, oo);
18349 background_max_overflow_address = max (background_max_overflow_address, oo);
18354 uint8_t* start = oo;
18355 if ((size_t)oo & 1)
18357 oo = (uint8_t*)((size_t)oo & ~1);
18358 start = *(--background_mark_stack_tos);
18359 dprintf (4, ("oo: %Ix, start: %Ix\n", (size_t)oo, (size_t)start));
18361 #ifdef COLLECTIBLE_CLASS
18364 // If there's a class object, push it now. We are guaranteed to have the slot since
18365 // we just popped one object off.
18366 if (is_collectible (oo))
18368 uint8_t* class_obj = get_class_object (oo);
18369 if (background_mark (class_obj,
18370 background_saved_lowest_address,
18371 background_saved_highest_address))
18373 size_t obj_size = size (class_obj);
18374 bpromoted_bytes (thread) += obj_size;
18376 *(background_mark_stack_tos++) = class_obj;
18380 #endif //COLLECTIBLE_CLASS
18384 BOOL overflow_p = FALSE;
18386 if (background_mark_stack_tos + (num_partial_refs + 2) >= mark_stack_limit)
18388 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
18389 size_t num_pointers = CGCDesc::GetNumPointers(method_table(oo), s, num_components);
18391 dprintf (2, ("h%d: PM: %Id left, obj %Ix (mt: %Ix) start: %Ix, total: %Id",
18393 (size_t)(mark_stack_limit - background_mark_stack_tos),
18399 bgc_overflow_count++;
18402 if (overflow_p == FALSE)
18404 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18406 //push the object and its current
18407 uint8_t** place = background_mark_stack_tos++;
18409 *(background_mark_stack_tos++) = (uint8_t*)((size_t)oo | 1);
18411 int i = num_partial_refs;
18413 go_through_object (method_table(oo), oo, s, ppslot,
18414 start, use_start, (oo + s),
18416 uint8_t* o = *ppslot;
18419 if (background_mark (o,
18420 background_saved_lowest_address,
18421 background_saved_highest_address))
18424 size_t obj_size = size (o);
18425 bpromoted_bytes (thread) += obj_size;
18426 if (contain_pointers_or_collectible (o))
18428 *(background_mark_stack_tos++) = o;
18432 *place = (uint8_t*)(ppslot+1);
18441 //we are finished with this object
18449 dprintf (3,("mark stack overflow for object %Ix ", (size_t)oo));
18450 background_min_overflow_address = min (background_min_overflow_address, oo);
18451 background_max_overflow_address = max (background_max_overflow_address, oo);
18455 #ifdef SORT_MARK_STACK
18456 if (background_mark_stack_tos > sorted_tos + mark_stack_array_length/8)
18458 rqsort1 (sorted_tos, background_mark_stack_tos-1);
18459 sorted_tos = background_mark_stack_tos-1;
18461 #endif //SORT_MARK_STACK
18465 if (!(background_mark_stack_tos == background_mark_stack_array))
18467 oo = *(--background_mark_stack_tos);
18469 #ifdef SORT_MARK_STACK
18470 sorted_tos = (uint8_t**)min ((size_t)sorted_tos, (size_t)background_mark_stack_tos);
18471 #endif //SORT_MARK_STACK
18477 assert (background_mark_stack_tos == background_mark_stack_array);
18482 //this version is different than the foreground GC because
18483 //it can't keep pointers to the inside of an object
18484 //while calling background_mark_simple1. The object could be moved
18485 //by an intervening foreground gc.
18486 //this method assumes that *po is in the [low. high[ range
18488 gc_heap::background_mark_simple (uint8_t* o THREAD_NUMBER_DCL)
18490 #ifdef MULTIPLE_HEAPS
18491 #else //MULTIPLE_HEAPS
18492 const int thread = 0;
18493 #endif //MULTIPLE_HEAPS
18495 dprintf (3, ("bmarking %Ix", o));
18497 if (background_mark1 (o))
18500 size_t s = size (o);
18501 bpromoted_bytes (thread) += s;
18503 if (contain_pointers_or_collectible (o))
18505 background_mark_simple1 (o THREAD_NUMBER_ARG);
18512 uint8_t* gc_heap::background_mark_object (uint8_t* o THREAD_NUMBER_DCL)
18514 if ((o >= background_saved_lowest_address) && (o < background_saved_highest_address))
18516 background_mark_simple (o THREAD_NUMBER_ARG);
18522 dprintf (3, ("or-%Ix", o));
18528 void gc_heap::background_verify_mark (Object*& object, ScanContext* sc, uint32_t flags)
18530 UNREFERENCED_PARAMETER(sc);
18532 assert (settings.concurrent);
18533 uint8_t* o = (uint8_t*)object;
18535 gc_heap* hp = gc_heap::heap_of (o);
18536 #ifdef INTERIOR_POINTERS
18537 if (flags & GC_CALL_INTERIOR)
18539 o = hp->find_object (o, background_saved_lowest_address);
18541 #endif //INTERIOR_POINTERS
18543 if (!background_object_marked (o, FALSE))
18549 void gc_heap::background_promote (Object** ppObject, ScanContext* sc, uint32_t flags)
18551 UNREFERENCED_PARAMETER(sc);
18552 //in order to save space on the array, mark the object,
18553 //knowing that it will be visited later
18554 assert (settings.concurrent);
18556 THREAD_NUMBER_FROM_CONTEXT;
18557 #ifndef MULTIPLE_HEAPS
18558 const int thread = 0;
18559 #endif //!MULTIPLE_HEAPS
18561 uint8_t* o = (uint8_t*)*ppObject;
18566 #ifdef DEBUG_DestroyedHandleValue
18567 // we can race with destroy handle during concurrent scan
18568 if (o == (uint8_t*)DEBUG_DestroyedHandleValue)
18570 #endif //DEBUG_DestroyedHandleValue
18574 gc_heap* hp = gc_heap::heap_of (o);
18576 if ((o < hp->background_saved_lowest_address) || (o >= hp->background_saved_highest_address))
18581 #ifdef INTERIOR_POINTERS
18582 if (flags & GC_CALL_INTERIOR)
18584 o = hp->find_object (o, hp->background_saved_lowest_address);
18588 #endif //INTERIOR_POINTERS
18590 #ifdef FEATURE_CONSERVATIVE_GC
18591 // For conservative GC, a value on stack may point to middle of a free object.
18592 // In this case, we don't need to promote the pointer.
18593 if (GCConfig::GetConservativeGC() && ((CObjectHeader*)o)->IsFree())
18597 #endif //FEATURE_CONSERVATIVE_GC
18600 ((CObjectHeader*)o)->Validate();
18603 dprintf (BGC_LOG, ("Background Promote %Ix", (size_t)o));
18605 //needs to be called before the marking because it is possible for a foreground
18606 //gc to take place during the mark and move the object
18607 STRESS_LOG3(LF_GC|LF_GCROOTS, LL_INFO1000000, " GCHeap::Promote: Promote GC Root *%p = %p MT = %pT", ppObject, o, o ? ((Object*) o)->GetGCSafeMethodTable() : NULL);
18609 hpt->background_mark_simple (o THREAD_NUMBER_ARG);
18612 //used by the ephemeral collection to scan the local background structures
18613 //containing references.
18615 gc_heap::scan_background_roots (promote_func* fn, int hn, ScanContext *pSC)
18621 pSC->thread_number = hn;
18623 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
18624 pSC->pCurrentDomain = 0;
18627 BOOL relocate_p = (fn == &GCHeap::Relocate);
18629 dprintf (3, ("Scanning background mark list"));
18632 size_t mark_list_finger = 0;
18633 while (mark_list_finger < c_mark_list_index)
18635 uint8_t** o = &c_mark_list [mark_list_finger];
18638 // We may not be able to calculate the size during relocate as POPO
18639 // may have written over the object.
18640 size_t s = size (*o);
18641 assert (Align (s) >= Align (min_obj_size));
18642 dprintf(3,("background root %Ix", (size_t)*o));
18644 (*fn) ((Object**)o, pSC, 0);
18645 mark_list_finger++;
18648 //scan the mark stack
18649 dprintf (3, ("Scanning background mark stack"));
18651 uint8_t** finger = background_mark_stack_array;
18652 while (finger < background_mark_stack_tos)
18654 if ((finger + 1) < background_mark_stack_tos)
18656 // We need to check for the partial mark case here.
18657 uint8_t* parent_obj = *(finger + 1);
18658 if ((size_t)parent_obj & 1)
18660 uint8_t* place = *finger;
18661 size_t place_offset = 0;
18662 uint8_t* real_parent_obj = (uint8_t*)((size_t)parent_obj & ~1);
18666 *(finger + 1) = real_parent_obj;
18667 place_offset = place - real_parent_obj;
18668 dprintf(3,("relocating background root %Ix", (size_t)real_parent_obj));
18669 (*fn) ((Object**)(finger + 1), pSC, 0);
18670 real_parent_obj = *(finger + 1);
18671 *finger = real_parent_obj + place_offset;
18672 *(finger + 1) = (uint8_t*)((size_t)real_parent_obj | 1);
18673 dprintf(3,("roots changed to %Ix, %Ix", *finger, *(finger + 1)));
18677 uint8_t** temp = &real_parent_obj;
18678 dprintf(3,("marking background root %Ix", (size_t)real_parent_obj));
18679 (*fn) ((Object**)temp, pSC, 0);
18686 dprintf(3,("background root %Ix", (size_t)*finger));
18687 (*fn) ((Object**)finger, pSC, 0);
18693 void gc_heap::background_mark_through_object (uint8_t* oo THREAD_NUMBER_DCL)
18695 if (contain_pointers (oo))
18697 size_t total_refs = 0;
18698 size_t s = size (oo);
18699 go_through_object_nostart (method_table(oo), oo, s, po,
18703 background_mark_object (o THREAD_NUMBER_ARG);
18707 dprintf (3,("Background marking through %Ix went through %Id refs",
18713 uint8_t* gc_heap::background_seg_end (heap_segment* seg, BOOL concurrent_p)
18715 if (concurrent_p && (seg == saved_overflow_ephemeral_seg))
18717 // for now we stop at where gen1 started when we started processing
18718 return background_min_soh_overflow_address;
18722 return heap_segment_allocated (seg);
18726 uint8_t* gc_heap::background_first_overflow (uint8_t* min_add,
18729 BOOL small_object_p)
18733 if (small_object_p)
18735 if (in_range_for_segment (min_add, seg))
18737 // min_add was the beginning of gen1 when we did the concurrent
18738 // overflow. Now we could be in a situation where min_add is
18739 // actually the same as allocated for that segment (because
18740 // we expanded heap), in which case we can not call
18741 // find first on this address or we will AV.
18742 if (min_add >= heap_segment_allocated (seg))
18748 if (concurrent_p &&
18749 ((seg == saved_overflow_ephemeral_seg) && (min_add >= background_min_soh_overflow_address)))
18751 return background_min_soh_overflow_address;
18755 o = find_first_object (min_add, heap_segment_mem (seg));
18762 o = max (heap_segment_mem (seg), min_add);
18766 void gc_heap::background_process_mark_overflow_internal (int condemned_gen_number,
18767 uint8_t* min_add, uint8_t* max_add,
18772 current_bgc_state = bgc_overflow_soh;
18775 size_t total_marked_objects = 0;
18777 #ifdef MULTIPLE_HEAPS
18778 int thread = heap_number;
18779 #endif //MULTIPLE_HEAPS
18781 exclusive_sync* loh_alloc_lock = 0;
18783 dprintf (2,("Processing Mark overflow [%Ix %Ix]", (size_t)min_add, (size_t)max_add));
18784 #ifdef MULTIPLE_HEAPS
18785 // We don't have each heap scan all heaps concurrently because we are worried about
18786 // multiple threads calling things like find_first_object.
18787 int h_start = (concurrent_p ? heap_number : 0);
18788 int h_end = (concurrent_p ? (heap_number + 1) : n_heaps);
18789 for (int hi = h_start; hi < h_end; hi++)
18791 gc_heap* hp = (concurrent_p ? this : g_heaps [(heap_number + hi) % n_heaps]);
18797 #endif //MULTIPLE_HEAPS
18798 BOOL small_object_segments = TRUE;
18799 int align_const = get_alignment_constant (small_object_segments);
18800 generation* gen = hp->generation_of (condemned_gen_number);
18801 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
18802 PREFIX_ASSUME(seg != NULL);
18803 loh_alloc_lock = hp->bgc_alloc_lock;
18805 uint8_t* o = hp->background_first_overflow (min_add,
18808 small_object_segments);
18812 while ((o < hp->background_seg_end (seg, concurrent_p)) && (o <= max_add))
18814 dprintf (3, ("considering %Ix", (size_t)o));
18818 if (concurrent_p && !small_object_segments)
18820 loh_alloc_lock->bgc_mark_set (o);
18822 if (((CObjectHeader*)o)->IsFree())
18824 s = unused_array_size (o);
18836 if (background_object_marked (o, FALSE) && contain_pointers_or_collectible (o))
18838 total_marked_objects++;
18839 go_through_object_cl (method_table(o), o, s, poo,
18840 uint8_t* oo = *poo;
18841 background_mark_object (oo THREAD_NUMBER_ARG);
18845 if (concurrent_p && !small_object_segments)
18847 loh_alloc_lock->bgc_mark_done ();
18850 o = o + Align (s, align_const);
18858 dprintf (2, ("went through overflow objects in segment %Ix (%d) (so far %Id marked)",
18859 heap_segment_mem (seg), (small_object_segments ? 0 : 1), total_marked_objects));
18861 if ((concurrent_p && (seg == hp->saved_overflow_ephemeral_seg)) ||
18862 (seg = heap_segment_next_in_range (seg)) == 0)
18864 if (small_object_segments)
18868 current_bgc_state = bgc_overflow_loh;
18871 dprintf (2, ("h%d: SOH: ov-mo: %Id", heap_number, total_marked_objects));
18872 fire_overflow_event (min_add, max_add, total_marked_objects, !small_object_segments);
18873 concurrent_print_time_delta (concurrent_p ? "Cov SOH" : "Nov SOH");
18874 total_marked_objects = 0;
18875 small_object_segments = FALSE;
18876 align_const = get_alignment_constant (small_object_segments);
18877 seg = heap_segment_in_range (generation_start_segment (hp->generation_of (max_generation+1)));
18879 PREFIX_ASSUME(seg != NULL);
18881 o = max (heap_segment_mem (seg), min_add);
18886 dprintf (GTC_LOG, ("h%d: LOH: ov-mo: %Id", heap_number, total_marked_objects));
18887 fire_overflow_event (min_add, max_add, total_marked_objects, !small_object_segments);
18893 o = hp->background_first_overflow (min_add,
18896 small_object_segments);
18903 BOOL gc_heap::background_process_mark_overflow (BOOL concurrent_p)
18905 BOOL grow_mark_array_p = TRUE;
18909 assert (!processed_soh_overflow_p);
18911 if ((background_max_overflow_address != 0) &&
18912 (background_min_overflow_address != MAX_PTR))
18914 // We have overflow to process but we know we can't process the ephemeral generations
18915 // now (we actually could process till the current gen1 start but since we are going to
18916 // make overflow per segment, for now I'll just stop at the saved gen1 start.
18917 saved_overflow_ephemeral_seg = ephemeral_heap_segment;
18918 background_max_soh_overflow_address = heap_segment_reserved (saved_overflow_ephemeral_seg);
18919 background_min_soh_overflow_address = generation_allocation_start (generation_of (max_generation-1));
18924 assert ((saved_overflow_ephemeral_seg == 0) ||
18925 ((background_max_soh_overflow_address != 0) &&
18926 (background_min_soh_overflow_address != MAX_PTR)));
18928 if (!processed_soh_overflow_p)
18930 // if there was no more overflow we just need to process what we didn't process
18931 // on the saved ephemeral segment.
18932 if ((background_max_overflow_address == 0) && (background_min_overflow_address == MAX_PTR))
18934 dprintf (2, ("final processing mark overflow - no more overflow since last time"));
18935 grow_mark_array_p = FALSE;
18938 background_min_overflow_address = min (background_min_overflow_address,
18939 background_min_soh_overflow_address);
18940 background_max_overflow_address = max (background_max_overflow_address,
18941 background_max_soh_overflow_address);
18942 processed_soh_overflow_p = TRUE;
18946 BOOL overflow_p = FALSE;
18948 if ((! ((background_max_overflow_address == 0)) ||
18949 ! ((background_min_overflow_address == MAX_PTR))))
18953 if (grow_mark_array_p)
18955 // Try to grow the array.
18956 size_t new_size = max (MARK_STACK_INITIAL_LENGTH, 2*background_mark_stack_array_length);
18958 if ((new_size * sizeof(mark)) > 100*1024)
18960 size_t new_max_size = (get_total_heap_size() / 10) / sizeof(mark);
18962 new_size = min(new_max_size, new_size);
18965 if ((background_mark_stack_array_length < new_size) &&
18966 ((new_size - background_mark_stack_array_length) > (background_mark_stack_array_length / 2)))
18968 dprintf (2, ("h%d: ov grow to %Id", heap_number, new_size));
18970 uint8_t** tmp = new (nothrow) uint8_t* [new_size];
18973 delete background_mark_stack_array;
18974 background_mark_stack_array = tmp;
18975 background_mark_stack_array_length = new_size;
18976 background_mark_stack_tos = background_mark_stack_array;
18982 grow_mark_array_p = TRUE;
18985 uint8_t* min_add = background_min_overflow_address;
18986 uint8_t* max_add = background_max_overflow_address;
18988 background_max_overflow_address = 0;
18989 background_min_overflow_address = MAX_PTR;
18991 background_process_mark_overflow_internal (max_generation, min_add, max_add, concurrent_p);
19001 #endif //BACKGROUND_GC
19004 void gc_heap::mark_through_object (uint8_t* oo, BOOL mark_class_object_p THREAD_NUMBER_DCL)
19006 #ifndef COLLECTIBLE_CLASS
19007 UNREFERENCED_PARAMETER(mark_class_object_p);
19008 BOOL to_mark_class_object = FALSE;
19009 #else //COLLECTIBLE_CLASS
19010 BOOL to_mark_class_object = (mark_class_object_p && (is_collectible(oo)));
19011 #endif //COLLECTIBLE_CLASS
19012 if (contain_pointers (oo) || to_mark_class_object)
19014 dprintf(3,( "Marking through %Ix", (size_t)oo));
19015 size_t s = size (oo);
19017 #ifdef COLLECTIBLE_CLASS
19018 if (to_mark_class_object)
19020 uint8_t* class_obj = get_class_object (oo);
19021 mark_object (class_obj THREAD_NUMBER_ARG);
19023 #endif //COLLECTIBLE_CLASS
19025 if (contain_pointers (oo))
19027 go_through_object_nostart (method_table(oo), oo, s, po,
19029 mark_object (o THREAD_NUMBER_ARG);
19035 size_t gc_heap::get_total_heap_size()
19037 size_t total_heap_size = 0;
19039 #ifdef MULTIPLE_HEAPS
19042 for (hn = 0; hn < gc_heap::n_heaps; hn++)
19044 gc_heap* hp2 = gc_heap::g_heaps [hn];
19045 total_heap_size += hp2->generation_size (max_generation + 1) + hp2->generation_sizes (hp2->generation_of (max_generation));
19048 total_heap_size = generation_size (max_generation + 1) + generation_sizes (generation_of (max_generation));
19049 #endif //MULTIPLE_HEAPS
19051 return total_heap_size;
19054 size_t gc_heap::get_total_fragmentation()
19056 size_t total_fragmentation = 0;
19058 #ifdef MULTIPLE_HEAPS
19059 for (int i = 0; i < gc_heap::n_heaps; i++)
19061 gc_heap* hp = gc_heap::g_heaps[i];
19062 #else //MULTIPLE_HEAPS
19064 gc_heap* hp = pGenGCHeap;
19065 #endif //MULTIPLE_HEAPS
19066 for (int i = 0; i <= (max_generation + 1); i++)
19068 generation* gen = hp->generation_of (i);
19069 total_fragmentation += (generation_free_list_space (gen) + generation_free_obj_space (gen));
19073 return total_fragmentation;
19076 size_t gc_heap::committed_size()
19078 generation* gen = generation_of (max_generation);
19079 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
19080 size_t total_committed = 0;
19084 total_committed += heap_segment_committed (seg) - (uint8_t*)seg;
19086 seg = heap_segment_next (seg);
19089 if (gen != large_object_generation)
19091 gen = generation_of (max_generation + 1);
19092 seg = generation_start_segment (gen);
19099 return total_committed;
19102 size_t gc_heap::get_total_committed_size()
19104 size_t total_committed = 0;
19106 #ifdef MULTIPLE_HEAPS
19109 for (hn = 0; hn < gc_heap::n_heaps; hn++)
19111 gc_heap* hp = gc_heap::g_heaps [hn];
19112 total_committed += hp->committed_size();
19115 total_committed = committed_size();
19116 #endif //MULTIPLE_HEAPS
19118 return total_committed;
19121 void gc_heap::get_memory_info (uint32_t* memory_load,
19122 uint64_t* available_physical,
19123 uint64_t* available_page_file)
19125 GCToOSInterface::GetMemoryStatus(memory_load, available_physical, available_page_file);
19128 void fire_mark_event (int heap_num, int root_type, size_t bytes_marked)
19130 dprintf (DT_LOG_0, ("-----------[%d]mark %d: %Id", heap_num, root_type, bytes_marked));
19131 FIRE_EVENT(GCMarkWithType, heap_num, root_type, bytes_marked);
19134 //returns TRUE is an overflow happened.
19135 BOOL gc_heap::process_mark_overflow(int condemned_gen_number)
19137 size_t last_promoted_bytes = promoted_bytes (heap_number);
19138 BOOL overflow_p = FALSE;
19140 if ((! (max_overflow_address == 0) ||
19141 ! (min_overflow_address == MAX_PTR)))
19144 // Try to grow the array.
19146 max (MARK_STACK_INITIAL_LENGTH, 2*mark_stack_array_length);
19148 if ((new_size * sizeof(mark)) > 100*1024)
19150 size_t new_max_size = (get_total_heap_size() / 10) / sizeof(mark);
19152 new_size = min(new_max_size, new_size);
19155 if ((mark_stack_array_length < new_size) &&
19156 ((new_size - mark_stack_array_length) > (mark_stack_array_length / 2)))
19158 mark* tmp = new (nothrow) mark [new_size];
19161 delete mark_stack_array;
19162 mark_stack_array = tmp;
19163 mark_stack_array_length = new_size;
19167 uint8_t* min_add = min_overflow_address;
19168 uint8_t* max_add = max_overflow_address;
19169 max_overflow_address = 0;
19170 min_overflow_address = MAX_PTR;
19171 process_mark_overflow_internal (condemned_gen_number, min_add, max_add);
19175 size_t current_promoted_bytes = promoted_bytes (heap_number);
19177 if (current_promoted_bytes != last_promoted_bytes)
19178 fire_mark_event (heap_number, ETW::GC_ROOT_OVERFLOW, (current_promoted_bytes - last_promoted_bytes));
19182 void gc_heap::process_mark_overflow_internal (int condemned_gen_number,
19183 uint8_t* min_add, uint8_t* max_add)
19185 #ifdef MULTIPLE_HEAPS
19186 int thread = heap_number;
19187 #endif //MULTIPLE_HEAPS
19188 BOOL full_p = (condemned_gen_number == max_generation);
19190 dprintf(3,("Processing Mark overflow [%Ix %Ix]", (size_t)min_add, (size_t)max_add));
19191 #ifdef MULTIPLE_HEAPS
19192 for (int hi = 0; hi < n_heaps; hi++)
19194 gc_heap* hp = g_heaps [(heap_number + hi) % n_heaps];
19200 #endif //MULTIPLE_HEAPS
19201 BOOL small_object_segments = TRUE;
19202 int align_const = get_alignment_constant (small_object_segments);
19203 generation* gen = hp->generation_of (condemned_gen_number);
19204 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
19206 PREFIX_ASSUME(seg != NULL);
19207 uint8_t* o = max (heap_segment_mem (seg), min_add);
19210 uint8_t* end = heap_segment_allocated (seg);
19212 while ((o < end) && (o <= max_add))
19214 assert ((min_add <= o) && (max_add >= o));
19215 dprintf (3, ("considering %Ix", (size_t)o));
19218 mark_through_object (o, TRUE THREAD_NUMBER_ARG);
19221 o = o + Align (size (o), align_const);
19224 if (( seg = heap_segment_next_in_range (seg)) == 0)
19226 if (small_object_segments && full_p)
19228 small_object_segments = FALSE;
19229 align_const = get_alignment_constant (small_object_segments);
19230 seg = heap_segment_in_range (generation_start_segment (hp->generation_of (max_generation+1)));
19232 PREFIX_ASSUME(seg != NULL);
19234 o = max (heap_segment_mem (seg), min_add);
19244 o = max (heap_segment_mem (seg), min_add);
19251 // Scanning for promotion for dependent handles need special handling. Because the primary holds a strong
19252 // reference to the secondary (when the primary itself is reachable) and this can cause a cascading series of
19253 // promotions (the secondary of one handle is or promotes the primary of another) we might need to perform the
19254 // promotion scan multiple times.
19255 // This helper encapsulates the logic to complete all dependent handle promotions when running a server GC. It
19256 // also has the effect of processing any mark stack overflow.
19258 #ifdef MULTIPLE_HEAPS
19259 // When multiple heaps are enabled we have must utilize a more complex algorithm in order to keep all the GC
19260 // worker threads synchronized. The algorithms are sufficiently divergent that we have different
19261 // implementations based on whether MULTIPLE_HEAPS is defined or not.
19263 // Define some static variables used for synchronization in the method below. These should really be defined
19264 // locally but MSVC complains when the VOLATILE macro is expanded into an instantiation of the Volatile class.
19266 // A note about the synchronization used within this method. Communication between the worker threads is
19267 // achieved via two shared booleans (defined below). These both act as latches that are transitioned only from
19268 // false -> true by unsynchronized code. They are only read or reset to false by a single thread under the
19269 // protection of a join.
19270 static VOLATILE(BOOL) s_fUnpromotedHandles = FALSE;
19271 static VOLATILE(BOOL) s_fUnscannedPromotions = FALSE;
19272 static VOLATILE(BOOL) s_fScanRequired;
19273 void gc_heap::scan_dependent_handles (int condemned_gen_number, ScanContext *sc, BOOL initial_scan_p)
19275 // Whenever we call this method there may have been preceding object promotions. So set
19276 // s_fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
19277 // based on the how the scanning proceeded).
19278 s_fUnscannedPromotions = TRUE;
19280 // We don't know how many times we need to loop yet. In particular we can't base the loop condition on
19281 // the state of this thread's portion of the dependent handle table. That's because promotions on other
19282 // threads could cause handle promotions to become necessary here. Even if there are definitely no more
19283 // promotions possible in this thread's handles, we still have to stay in lock-step with those worker
19284 // threads that haven't finished yet (each GC worker thread has to join exactly the same number of times
19285 // as all the others or they'll get out of step).
19288 // The various worker threads are all currently racing in this code. We need to work out if at least
19289 // one of them think they have work to do this cycle. Each thread needs to rescan its portion of the
19290 // dependent handle table when both of the following conditions apply:
19291 // 1) At least one (arbitrary) object might have been promoted since the last scan (because if this
19292 // object happens to correspond to a primary in one of our handles we might potentially have to
19293 // promote the associated secondary).
19294 // 2) The table for this thread has at least one handle with a secondary that isn't promoted yet.
19296 // The first condition is represented by s_fUnscannedPromotions. This is always non-zero for the first
19297 // iteration of this loop (see comment above) and in subsequent cycles each thread updates this
19298 // whenever a mark stack overflow occurs or scanning their dependent handles results in a secondary
19299 // being promoted. This value is cleared back to zero in a synchronized fashion in the join that
19300 // follows below. Note that we can't read this outside of the join since on any iteration apart from
19301 // the first threads will be racing between reading this value and completing their previous
19302 // iteration's table scan.
19304 // The second condition is tracked by the dependent handle code itself on a per worker thread basis
19305 // (and updated by the GcDhReScan() method). We call GcDhUnpromotedHandlesExist() on each thread to
19306 // determine the local value and collect the results into the s_fUnpromotedHandles variable in what is
19307 // effectively an OR operation. As per s_fUnscannedPromotions we can't read the final result until
19308 // we're safely joined.
19309 if (GCScan::GcDhUnpromotedHandlesExist(sc))
19310 s_fUnpromotedHandles = TRUE;
19312 // Synchronize all the threads so we can read our state variables safely. The shared variable
19313 // s_fScanRequired, indicating whether we should scan the tables or terminate the loop, will be set by
19314 // a single thread inside the join.
19315 gc_t_join.join(this, gc_join_scan_dependent_handles);
19316 if (gc_t_join.joined())
19318 // We're synchronized so it's safe to read our shared state variables. We update another shared
19319 // variable to indicate to all threads whether we'll be scanning for another cycle or terminating
19320 // the loop. We scan if there has been at least one object promotion since last time and at least
19321 // one thread has a dependent handle table with a potential handle promotion possible.
19322 s_fScanRequired = s_fUnscannedPromotions && s_fUnpromotedHandles;
19324 // Reset our shared state variables (ready to be set again on this scan or with a good initial
19325 // value for the next call if we're terminating the loop).
19326 s_fUnscannedPromotions = FALSE;
19327 s_fUnpromotedHandles = FALSE;
19329 if (!s_fScanRequired)
19331 // We're terminating the loop. Perform any last operations that require single threaded access.
19332 if (!initial_scan_p)
19334 // On the second invocation we reconcile all mark overflow ranges across the heaps. This can help
19335 // load balance if some of the heaps have an abnormally large workload.
19336 uint8_t* all_heaps_max = 0;
19337 uint8_t* all_heaps_min = MAX_PTR;
19339 for (i = 0; i < n_heaps; i++)
19341 if (all_heaps_max < g_heaps[i]->max_overflow_address)
19342 all_heaps_max = g_heaps[i]->max_overflow_address;
19343 if (all_heaps_min > g_heaps[i]->min_overflow_address)
19344 all_heaps_min = g_heaps[i]->min_overflow_address;
19346 for (i = 0; i < n_heaps; i++)
19348 g_heaps[i]->max_overflow_address = all_heaps_max;
19349 g_heaps[i]->min_overflow_address = all_heaps_min;
19354 // Restart all the workers.
19355 dprintf(3, ("Starting all gc thread mark stack overflow processing"));
19356 gc_t_join.restart();
19359 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
19360 // being visible. If there really was an overflow (process_mark_overflow returns true) then set the
19361 // global flag indicating that at least one object promotion may have occurred (the usual comment
19362 // about races applies). (Note it's OK to set this flag even if we're about to terminate the loop and
19363 // exit the method since we unconditionally set this variable on method entry anyway).
19364 if (process_mark_overflow(condemned_gen_number))
19365 s_fUnscannedPromotions = TRUE;
19367 // If we decided that no scan was required we can terminate the loop now.
19368 if (!s_fScanRequired)
19371 // Otherwise we must join with the other workers to ensure that all mark stack overflows have been
19372 // processed before we start scanning dependent handle tables (if overflows remain while we scan we
19373 // could miss noting the promotion of some primary objects).
19374 gc_t_join.join(this, gc_join_rescan_dependent_handles);
19375 if (gc_t_join.joined())
19377 // Restart all the workers.
19378 dprintf(3, ("Starting all gc thread for dependent handle promotion"));
19379 gc_t_join.restart();
19382 // If the portion of the dependent handle table managed by this worker has handles that could still be
19383 // promoted perform a rescan. If the rescan resulted in at least one promotion note this fact since it
19384 // could require a rescan of handles on this or other workers.
19385 if (GCScan::GcDhUnpromotedHandlesExist(sc))
19386 if (GCScan::GcDhReScan(sc))
19387 s_fUnscannedPromotions = TRUE;
19390 #else //MULTIPLE_HEAPS
19391 // Non-multiple heap version of scan_dependent_handles: much simpler without the need to keep multiple worker
19392 // threads synchronized.
19393 void gc_heap::scan_dependent_handles (int condemned_gen_number, ScanContext *sc, BOOL initial_scan_p)
19395 UNREFERENCED_PARAMETER(initial_scan_p);
19397 // Whenever we call this method there may have been preceding object promotions. So set
19398 // fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
19399 // based on the how the scanning proceeded).
19400 bool fUnscannedPromotions = true;
19402 // Loop until there are either no more dependent handles that can have their secondary promoted or we've
19403 // managed to perform a scan without promoting anything new.
19404 while (GCScan::GcDhUnpromotedHandlesExist(sc) && fUnscannedPromotions)
19406 // On each iteration of the loop start with the assumption that no further objects have been promoted.
19407 fUnscannedPromotions = false;
19409 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
19410 // being visible. If there was an overflow (process_mark_overflow returned true) then additional
19411 // objects now appear to be promoted and we should set the flag.
19412 if (process_mark_overflow(condemned_gen_number))
19413 fUnscannedPromotions = true;
19415 // Perform the scan and set the flag if any promotions resulted.
19416 if (GCScan::GcDhReScan(sc))
19417 fUnscannedPromotions = true;
19420 // Process any mark stack overflow that may have resulted from scanning handles (or if we didn't need to
19421 // scan any handles at all this is the processing of overflows that may have occurred prior to this method
19423 process_mark_overflow(condemned_gen_number);
19425 #endif //MULTIPLE_HEAPS
19427 void gc_heap::mark_phase (int condemned_gen_number, BOOL mark_only_p)
19429 assert (settings.concurrent == FALSE);
19432 sc.thread_number = heap_number;
19433 sc.promotion = TRUE;
19434 sc.concurrent = FALSE;
19436 dprintf(2,("---- Mark Phase condemning %d ----", condemned_gen_number));
19437 BOOL full_p = (condemned_gen_number == max_generation);
19442 start = GetCycleCount32();
19445 int gen_to_init = condemned_gen_number;
19446 if (condemned_gen_number == max_generation)
19448 gen_to_init = max_generation + 1;
19450 for (int gen_idx = 0; gen_idx <= gen_to_init; gen_idx++)
19452 dynamic_data* dd = dynamic_data_of (gen_idx);
19453 dd_begin_data_size (dd) = generation_size (gen_idx) -
19454 dd_fragmentation (dd) -
19455 Align (size (generation_allocation_start (generation_of (gen_idx))));
19456 dprintf (2, ("begin data size for gen%d is %Id", gen_idx, dd_begin_data_size (dd)));
19457 dd_survived_size (dd) = 0;
19458 dd_pinned_survived_size (dd) = 0;
19459 dd_artificial_pinned_survived_size (dd) = 0;
19460 dd_added_pinned_size (dd) = 0;
19462 dd_padding_size (dd) = 0;
19463 #endif //SHORT_PLUGS
19464 #if defined (RESPECT_LARGE_ALIGNMENT) || defined (FEATURE_STRUCTALIGN)
19465 dd_num_npinned_plugs (dd) = 0;
19466 #endif //RESPECT_LARGE_ALIGNMENT || FEATURE_STRUCTALIGN
19469 #ifdef FFIND_OBJECT
19470 if (gen0_must_clear_bricks > 0)
19471 gen0_must_clear_bricks--;
19472 #endif //FFIND_OBJECT
19474 size_t last_promoted_bytes = 0;
19476 promoted_bytes (heap_number) = 0;
19477 reset_mark_stack();
19480 memset (&snoop_stat, 0, sizeof(snoop_stat));
19481 snoop_stat.heap_index = heap_number;
19482 #endif //SNOOP_STATS
19487 //initialize the mark stack
19488 for (int i = 0; i < max_snoop_level; i++)
19490 ((uint8_t**)(mark_stack_array))[i] = 0;
19493 mark_stack_busy() = 1;
19495 #endif //MH_SC_MARK
19497 static uint32_t num_sizedrefs = 0;
19500 static BOOL do_mark_steal_p = FALSE;
19501 #endif //MH_SC_MARK
19503 #ifdef MULTIPLE_HEAPS
19504 gc_t_join.join(this, gc_join_begin_mark_phase);
19505 if (gc_t_join.joined())
19507 #endif //MULTIPLE_HEAPS
19509 num_sizedrefs = GCToEEInterface::GetTotalNumSizedRefHandles();
19511 #ifdef MULTIPLE_HEAPS
19516 size_t total_heap_size = get_total_heap_size();
19518 if (total_heap_size > (100 * 1024 * 1024))
19520 do_mark_steal_p = TRUE;
19524 do_mark_steal_p = FALSE;
19529 do_mark_steal_p = FALSE;
19531 #endif //MH_SC_MARK
19533 gc_t_join.restart();
19535 #endif //MULTIPLE_HEAPS
19540 //set up the mark lists from g_mark_list
19541 assert (g_mark_list);
19542 #ifdef MULTIPLE_HEAPS
19543 mark_list = &g_mark_list [heap_number*mark_list_size];
19545 mark_list = g_mark_list;
19546 #endif //MULTIPLE_HEAPS
19547 //dont use the mark list for full gc
19548 //because multiple segments are more complex to handle and the list
19549 //is likely to overflow
19550 if (condemned_gen_number != max_generation)
19551 mark_list_end = &mark_list [mark_list_size-1];
19553 mark_list_end = &mark_list [0];
19554 mark_list_index = &mark_list [0];
19557 shigh = (uint8_t*) 0;
19560 //%type% category = quote (mark);
19562 if ((condemned_gen_number == max_generation) && (num_sizedrefs > 0))
19564 GCScan::GcScanSizedRefs(GCHeap::Promote, condemned_gen_number, max_generation, &sc);
19565 fire_mark_event (heap_number, ETW::GC_ROOT_SIZEDREF, (promoted_bytes (heap_number) - last_promoted_bytes));
19566 last_promoted_bytes = promoted_bytes (heap_number);
19568 #ifdef MULTIPLE_HEAPS
19569 gc_t_join.join(this, gc_join_scan_sizedref_done);
19570 if (gc_t_join.joined())
19572 dprintf(3, ("Done with marking all sized refs. Starting all gc thread for marking other strong roots"));
19573 gc_t_join.restart();
19575 #endif //MULTIPLE_HEAPS
19578 dprintf(3,("Marking Roots"));
19580 GCScan::GcScanRoots(GCHeap::Promote,
19581 condemned_gen_number, max_generation,
19584 fire_mark_event (heap_number, ETW::GC_ROOT_STACK, (promoted_bytes (heap_number) - last_promoted_bytes));
19585 last_promoted_bytes = promoted_bytes (heap_number);
19587 #ifdef BACKGROUND_GC
19588 if (recursive_gc_sync::background_running_p())
19590 scan_background_roots (GCHeap::Promote, heap_number, &sc);
19592 #endif //BACKGROUND_GC
19594 #ifdef FEATURE_PREMORTEM_FINALIZATION
19595 dprintf(3, ("Marking finalization data"));
19596 finalize_queue->GcScanRoots(GCHeap::Promote, heap_number, 0);
19597 #endif // FEATURE_PREMORTEM_FINALIZATION
19599 fire_mark_event (heap_number, ETW::GC_ROOT_FQ, (promoted_bytes (heap_number) - last_promoted_bytes));
19600 last_promoted_bytes = promoted_bytes (heap_number);
19605 dprintf(3,("Marking handle table"));
19606 GCScan::GcScanHandles(GCHeap::Promote,
19607 condemned_gen_number, max_generation,
19609 fire_mark_event (heap_number, ETW::GC_ROOT_HANDLES, (promoted_bytes (heap_number) - last_promoted_bytes));
19610 last_promoted_bytes = promoted_bytes (heap_number);
19614 size_t promoted_before_cards = promoted_bytes (heap_number);
19617 dprintf (3, ("before cards: %Id", promoted_before_cards));
19621 #ifdef MULTIPLE_HEAPS
19622 if (gc_t_join.r_join(this, gc_r_join_update_card_bundle))
19624 #endif //MULTIPLE_HEAPS
19626 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
19627 // If we are manually managing card bundles, every write to the card table should already be
19628 // accounted for in the card bundle table so there's nothing to update here.
19629 update_card_table_bundle();
19631 if (card_bundles_enabled())
19633 verify_card_bundles();
19636 #ifdef MULTIPLE_HEAPS
19637 gc_t_join.r_restart();
19639 #endif //MULTIPLE_HEAPS
19640 #endif //CARD_BUNDLE
19642 card_fn mark_object_fn = &gc_heap::mark_object_simple;
19643 #ifdef HEAP_ANALYZE
19644 heap_analyze_success = TRUE;
19645 if (heap_analyze_enabled)
19647 internal_root_array_index = 0;
19649 current_obj_size = 0;
19650 mark_object_fn = &gc_heap::ha_mark_object_simple;
19652 #endif //HEAP_ANALYZE
19654 dprintf(3,("Marking cross generation pointers"));
19655 mark_through_cards_for_segments (mark_object_fn, FALSE);
19657 dprintf(3,("Marking cross generation pointers for large objects"));
19658 mark_through_cards_for_large_objects (mark_object_fn, FALSE);
19660 dprintf (3, ("marked by cards: %Id",
19661 (promoted_bytes (heap_number) - promoted_before_cards)));
19662 fire_mark_event (heap_number, ETW::GC_ROOT_OLDER, (promoted_bytes (heap_number) - last_promoted_bytes));
19663 last_promoted_bytes = promoted_bytes (heap_number);
19668 if (do_mark_steal_p)
19672 #endif //MH_SC_MARK
19674 // Dependent handles need to be scanned with a special algorithm (see the header comment on
19675 // scan_dependent_handles for more detail). We perform an initial scan without synchronizing with other
19676 // worker threads or processing any mark stack overflow. This is not guaranteed to complete the operation
19677 // but in a common case (where there are no dependent handles that are due to be collected) it allows us
19678 // to optimize away further scans. The call to scan_dependent_handles is what will cycle through more
19679 // iterations if required and will also perform processing of any mark stack overflow once the dependent
19680 // handle table has been fully promoted.
19681 GCScan::GcDhInitialScan(GCHeap::Promote, condemned_gen_number, max_generation, &sc);
19682 scan_dependent_handles(condemned_gen_number, &sc, true);
19684 #ifdef MULTIPLE_HEAPS
19685 dprintf(3, ("Joining for short weak handle scan"));
19686 gc_t_join.join(this, gc_join_null_dead_short_weak);
19687 if (gc_t_join.joined())
19688 #endif //MULTIPLE_HEAPS
19690 #ifdef HEAP_ANALYZE
19691 heap_analyze_enabled = FALSE;
19692 GCToEEInterface::AnalyzeSurvivorsFinished(condemned_gen_number);
19693 #endif // HEAP_ANALYZE
19694 GCToEEInterface::AfterGcScanRoots (condemned_gen_number, max_generation, &sc);
19696 #ifdef MULTIPLE_HEAPS
19699 // we used r_join and need to reinitialize states for it here.
19700 gc_t_join.r_init();
19703 //start all threads on the roots.
19704 dprintf(3, ("Starting all gc thread for short weak handle scan"));
19705 gc_t_join.restart();
19706 #endif //MULTIPLE_HEAPS
19710 // null out the target of short weakref that were not promoted.
19711 GCScan::GcShortWeakPtrScan(GCHeap::Promote, condemned_gen_number, max_generation,&sc);
19713 // MTHTS: keep by single thread
19714 #ifdef MULTIPLE_HEAPS
19715 dprintf(3, ("Joining for finalization"));
19716 gc_t_join.join(this, gc_join_scan_finalization);
19717 if (gc_t_join.joined())
19718 #endif //MULTIPLE_HEAPS
19721 #ifdef MULTIPLE_HEAPS
19722 //start all threads on the roots.
19723 dprintf(3, ("Starting all gc thread for Finalization"));
19724 gc_t_join.restart();
19725 #endif //MULTIPLE_HEAPS
19728 //Handle finalization.
19729 size_t promoted_bytes_live = promoted_bytes (heap_number);
19731 #ifdef FEATURE_PREMORTEM_FINALIZATION
19732 dprintf (3, ("Finalize marking"));
19733 finalize_queue->ScanForFinalization (GCHeap::Promote, condemned_gen_number, mark_only_p, __this);
19735 GCToEEInterface::DiagWalkFReachableObjects(__this);
19736 #endif // FEATURE_PREMORTEM_FINALIZATION
19738 // Scan dependent handles again to promote any secondaries associated with primaries that were promoted
19739 // for finalization. As before scan_dependent_handles will also process any mark stack overflow.
19740 scan_dependent_handles(condemned_gen_number, &sc, false);
19742 #ifdef MULTIPLE_HEAPS
19743 dprintf(3, ("Joining for weak pointer deletion"));
19744 gc_t_join.join(this, gc_join_null_dead_long_weak);
19745 if (gc_t_join.joined())
19747 //start all threads on the roots.
19748 dprintf(3, ("Starting all gc thread for weak pointer deletion"));
19749 gc_t_join.restart();
19751 #endif //MULTIPLE_HEAPS
19753 // null out the target of long weakref that were not promoted.
19754 GCScan::GcWeakPtrScan (GCHeap::Promote, condemned_gen_number, max_generation, &sc);
19756 // MTHTS: keep by single thread
19757 #ifdef MULTIPLE_HEAPS
19759 #ifdef PARALLEL_MARK_LIST_SORT
19760 // unsigned long start = GetCycleCount32();
19762 // printf("sort_mark_list took %u cycles\n", GetCycleCount32() - start);
19763 #endif //PARALLEL_MARK_LIST_SORT
19766 dprintf (3, ("Joining for sync block cache entry scanning"));
19767 gc_t_join.join(this, gc_join_null_dead_syncblk);
19768 if (gc_t_join.joined())
19769 #endif //MULTIPLE_HEAPS
19771 // scan for deleted entries in the syncblk cache
19772 GCScan::GcWeakPtrScanBySingleThread (condemned_gen_number, max_generation, &sc);
19774 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
19777 size_t promoted_all_heaps = 0;
19778 #ifdef MULTIPLE_HEAPS
19779 for (int i = 0; i < n_heaps; i++)
19781 promoted_all_heaps += promoted_bytes (i);
19784 promoted_all_heaps = promoted_bytes (heap_number);
19785 #endif //MULTIPLE_HEAPS
19786 SystemDomain::RecordTotalSurvivedBytes (promoted_all_heaps);
19788 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
19790 #ifdef MULTIPLE_HEAPS
19793 #ifndef PARALLEL_MARK_LIST_SORT
19794 //compact g_mark_list and sort it.
19795 combine_mark_lists();
19796 #endif //PARALLEL_MARK_LIST_SORT
19799 //decide on promotion
19800 if (!settings.promotion)
19803 for (int n = 0; n <= condemned_gen_number;n++)
19805 m += (size_t)(dd_min_size (dynamic_data_of (n))*(n+1)*0.1);
19808 for (int i = 0; i < n_heaps; i++)
19810 dynamic_data* dd = g_heaps[i]->dynamic_data_of (min (condemned_gen_number +1,
19812 size_t older_gen_size = (dd_current_size (dd) +
19813 (dd_desired_allocation (dd) -
19814 dd_new_allocation (dd)));
19816 if ((m > (older_gen_size)) ||
19817 (promoted_bytes (i) > m))
19819 settings.promotion = TRUE;
19825 if (do_mark_steal_p)
19827 size_t objects_checked_count = 0;
19828 size_t zero_ref_count = 0;
19829 size_t objects_marked_count = 0;
19830 size_t check_level_count = 0;
19831 size_t busy_count = 0;
19832 size_t interlocked_count = 0;
19833 size_t partial_mark_parent_count = 0;
19834 size_t stolen_or_pm_count = 0;
19835 size_t stolen_entry_count = 0;
19836 size_t pm_not_ready_count = 0;
19837 size_t normal_count = 0;
19838 size_t stack_bottom_clear_count = 0;
19840 for (int i = 0; i < n_heaps; i++)
19842 gc_heap* hp = g_heaps[i];
19843 hp->print_snoop_stat();
19844 objects_checked_count += hp->snoop_stat.objects_checked_count;
19845 zero_ref_count += hp->snoop_stat.zero_ref_count;
19846 objects_marked_count += hp->snoop_stat.objects_marked_count;
19847 check_level_count += hp->snoop_stat.check_level_count;
19848 busy_count += hp->snoop_stat.busy_count;
19849 interlocked_count += hp->snoop_stat.interlocked_count;
19850 partial_mark_parent_count += hp->snoop_stat.partial_mark_parent_count;
19851 stolen_or_pm_count += hp->snoop_stat.stolen_or_pm_count;
19852 stolen_entry_count += hp->snoop_stat.stolen_entry_count;
19853 pm_not_ready_count += hp->snoop_stat.pm_not_ready_count;
19854 normal_count += hp->snoop_stat.normal_count;
19855 stack_bottom_clear_count += hp->snoop_stat.stack_bottom_clear_count;
19860 printf ("-------total stats-------\n");
19861 printf ("%8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s\n",
19862 "checked", "zero", "marked", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "normal", "clear");
19863 printf ("%8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
19864 objects_checked_count,
19866 objects_marked_count,
19870 partial_mark_parent_count,
19871 stolen_or_pm_count,
19872 stolen_entry_count,
19873 pm_not_ready_count,
19875 stack_bottom_clear_count);
19877 #endif //SNOOP_STATS
19879 //start all threads.
19880 dprintf(3, ("Starting all threads for end of mark phase"));
19881 gc_t_join.restart();
19882 #else //MULTIPLE_HEAPS
19884 //decide on promotion
19885 if (!settings.promotion)
19888 for (int n = 0; n <= condemned_gen_number;n++)
19890 m += (size_t)(dd_min_size (dynamic_data_of (n))*(n+1)*0.06);
19892 dynamic_data* dd = dynamic_data_of (min (condemned_gen_number +1,
19894 size_t older_gen_size = (dd_current_size (dd) +
19895 (dd_desired_allocation (dd) -
19896 dd_new_allocation (dd)));
19898 dprintf (2, ("promotion threshold: %Id, promoted bytes: %Id size n+1: %Id",
19899 m, promoted_bytes (heap_number), older_gen_size));
19901 if ((m > older_gen_size) ||
19902 (promoted_bytes (heap_number) > m))
19904 settings.promotion = TRUE;
19908 #endif //MULTIPLE_HEAPS
19911 #ifdef MULTIPLE_HEAPS
19913 #ifdef PARALLEL_MARK_LIST_SORT
19914 // start = GetCycleCount32();
19915 merge_mark_lists();
19916 // printf("merge_mark_lists took %u cycles\n", GetCycleCount32() - start);
19917 #endif //PARALLEL_MARK_LIST_SORT
19919 #endif //MULTIPLE_HEAPS
19921 #ifdef BACKGROUND_GC
19922 total_promoted_bytes = promoted_bytes (heap_number);
19923 #endif //BACKGROUND_GC
19925 promoted_bytes (heap_number) -= promoted_bytes_live;
19928 finish = GetCycleCount32();
19929 mark_time = finish - start;
19932 dprintf(2,("---- End of mark phase ----"));
19936 void gc_heap::pin_object (uint8_t* o, uint8_t** ppObject, uint8_t* low, uint8_t* high)
19938 dprintf (3, ("Pinning %Ix", (size_t)o));
19939 if ((o >= low) && (o < high))
19941 dprintf(3,("^%Ix^", (size_t)o));
19944 #ifdef FEATURE_EVENT_TRACE
19945 if(EVENT_ENABLED(PinObjectAtGCTime))
19947 fire_etw_pin_object_event(o, ppObject);
19949 #endif // FEATURE_EVENT_TRACE
19951 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
19952 num_pinned_objects++;
19953 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
19957 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
19958 size_t gc_heap::get_total_pinned_objects()
19960 #ifdef MULTIPLE_HEAPS
19961 size_t total_num_pinned_objects = 0;
19962 for (int i = 0; i < gc_heap::n_heaps; i++)
19964 gc_heap* hp = gc_heap::g_heaps[i];
19965 total_num_pinned_objects += hp->num_pinned_objects;
19967 return total_num_pinned_objects;
19968 #else //MULTIPLE_HEAPS
19969 return num_pinned_objects;
19970 #endif //MULTIPLE_HEAPS
19972 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
19974 void gc_heap::reset_mark_stack ()
19976 reset_pinned_queue();
19977 max_overflow_address = 0;
19978 min_overflow_address = MAX_PTR;
19981 #ifdef FEATURE_STRUCTALIGN
19983 // The word with left child, right child, and align info is laid out as follows:
19985 // | upper short word | lower short word |
19986 // |<------------> <----->|<------------> <----->|
19987 // | left child info hi| right child info lo|
19988 // x86: | 10 bits 6 bits| 10 bits 6 bits|
19990 // where left/right child are signed values and concat(info hi, info lo) is unsigned.
19992 // The "align info" encodes two numbers: the required alignment (a power of two)
19993 // and the misalignment (the number of machine words the destination address needs
19994 // to be adjusted by to provide alignment - so this number is always smaller than
19995 // the required alignment). Thus, the two can be represented as the "logical or"
19996 // of the two numbers. Note that the actual pad is computed from the misalignment
19997 // by adding the alignment iff the misalignment is non-zero and less than min_obj_size.
20000 // The number of bits in a brick.
20001 #if defined (_TARGET_AMD64_)
20002 #define brick_bits (12)
20004 #define brick_bits (11)
20005 #endif //_TARGET_AMD64_
20006 C_ASSERT(brick_size == (1 << brick_bits));
20008 // The number of bits needed to represent the offset to a child node.
20009 // "brick_bits + 1" allows us to represent a signed offset within a brick.
20010 #define child_bits (brick_bits + 1 - LOG2_PTRSIZE)
20012 // The number of bits in each of the pad hi, pad lo fields.
20013 #define pad_bits (sizeof(short) * 8 - child_bits)
20015 #define child_from_short(w) (((signed short)(w) / (1 << (pad_bits - LOG2_PTRSIZE))) & ~((1 << LOG2_PTRSIZE) - 1))
20016 #define pad_mask ((1 << pad_bits) - 1)
20017 #define pad_from_short(w) ((size_t)(w) & pad_mask)
20018 #else // FEATURE_STRUCTALIGN
20019 #define child_from_short(w) (w)
20020 #endif // FEATURE_STRUCTALIGN
20023 short node_left_child(uint8_t* node)
20025 return child_from_short(((plug_and_pair*)node)[-1].m_pair.left);
20029 void set_node_left_child(uint8_t* node, ptrdiff_t val)
20031 assert (val > -(ptrdiff_t)brick_size);
20032 assert (val < (ptrdiff_t)brick_size);
20033 assert (Aligned (val));
20034 #ifdef FEATURE_STRUCTALIGN
20035 size_t pad = pad_from_short(((plug_and_pair*)node)[-1].m_pair.left);
20036 ((plug_and_pair*)node)[-1].m_pair.left = ((short)val << (pad_bits - LOG2_PTRSIZE)) | (short)pad;
20037 #else // FEATURE_STRUCTALIGN
20038 ((plug_and_pair*)node)[-1].m_pair.left = (short)val;
20039 #endif // FEATURE_STRUCTALIGN
20040 assert (node_left_child (node) == val);
20044 short node_right_child(uint8_t* node)
20046 return child_from_short(((plug_and_pair*)node)[-1].m_pair.right);
20050 void set_node_right_child(uint8_t* node, ptrdiff_t val)
20052 assert (val > -(ptrdiff_t)brick_size);
20053 assert (val < (ptrdiff_t)brick_size);
20054 assert (Aligned (val));
20055 #ifdef FEATURE_STRUCTALIGN
20056 size_t pad = pad_from_short(((plug_and_pair*)node)[-1].m_pair.right);
20057 ((plug_and_pair*)node)[-1].m_pair.right = ((short)val << (pad_bits - LOG2_PTRSIZE)) | (short)pad;
20058 #else // FEATURE_STRUCTALIGN
20059 ((plug_and_pair*)node)[-1].m_pair.right = (short)val;
20060 #endif // FEATURE_STRUCTALIGN
20061 assert (node_right_child (node) == val);
20064 #ifdef FEATURE_STRUCTALIGN
20065 void node_aligninfo (uint8_t* node, int& requiredAlignment, ptrdiff_t& pad)
20067 // Extract the single-number aligninfo from the fields.
20068 short left = ((plug_and_pair*)node)[-1].m_pair.left;
20069 short right = ((plug_and_pair*)node)[-1].m_pair.right;
20070 ptrdiff_t pad_shifted = (pad_from_short(left) << pad_bits) | pad_from_short(right);
20071 ptrdiff_t aligninfo = pad_shifted * DATA_ALIGNMENT;
20073 // Replicate the topmost bit into all lower bits.
20074 ptrdiff_t x = aligninfo;
20080 // Clear all bits but the highest.
20081 requiredAlignment = (int)(x ^ (x >> 1));
20082 pad = aligninfo - requiredAlignment;
20083 pad += AdjustmentForMinPadSize(pad, requiredAlignment);
20087 ptrdiff_t node_alignpad (uint8_t* node)
20089 int requiredAlignment;
20090 ptrdiff_t alignpad;
20091 node_aligninfo (node, requiredAlignment, alignpad);
20095 void clear_node_aligninfo (uint8_t* node)
20097 ((plug_and_pair*)node)[-1].m_pair.left &= ~0 << pad_bits;
20098 ((plug_and_pair*)node)[-1].m_pair.right &= ~0 << pad_bits;
20101 void set_node_aligninfo (uint8_t* node, int requiredAlignment, ptrdiff_t pad)
20103 // Encode the alignment requirement and alignment offset as a single number
20104 // as described above.
20105 ptrdiff_t aligninfo = (size_t)requiredAlignment + (pad & (requiredAlignment-1));
20106 assert (Aligned (aligninfo));
20107 ptrdiff_t aligninfo_shifted = aligninfo / DATA_ALIGNMENT;
20108 assert (aligninfo_shifted < (1 << (pad_bits + pad_bits)));
20110 ptrdiff_t hi = aligninfo_shifted >> pad_bits;
20111 assert (pad_from_short(((plug_and_gap*)node)[-1].m_pair.left) == 0);
20112 ((plug_and_pair*)node)[-1].m_pair.left |= hi;
20114 ptrdiff_t lo = aligninfo_shifted & pad_mask;
20115 assert (pad_from_short(((plug_and_gap*)node)[-1].m_pair.right) == 0);
20116 ((plug_and_pair*)node)[-1].m_pair.right |= lo;
20119 int requiredAlignment2;
20121 node_aligninfo (node, requiredAlignment2, pad2);
20122 assert (requiredAlignment == requiredAlignment2);
20123 assert (pad == pad2);
20126 #endif // FEATURE_STRUCTALIGN
20129 void loh_set_node_relocation_distance(uint8_t* node, ptrdiff_t val)
20131 ptrdiff_t* place = &(((loh_obj_and_pad*)node)[-1].reloc);
20136 ptrdiff_t loh_node_relocation_distance(uint8_t* node)
20138 return (((loh_obj_and_pad*)node)[-1].reloc);
20142 ptrdiff_t node_relocation_distance (uint8_t* node)
20144 return (((plug_and_reloc*)(node))[-1].reloc & ~3);
20148 void set_node_relocation_distance(uint8_t* node, ptrdiff_t val)
20150 assert (val == (val & ~3));
20151 ptrdiff_t* place = &(((plug_and_reloc*)node)[-1].reloc);
20152 //clear the left bit and the relocation field
20158 #define node_left_p(node) (((plug_and_reloc*)(node))[-1].reloc & 2)
20160 #define set_node_left(node) ((plug_and_reloc*)(node))[-1].reloc |= 2;
20162 #ifndef FEATURE_STRUCTALIGN
20163 void set_node_realigned(uint8_t* node)
20165 ((plug_and_reloc*)(node))[-1].reloc |= 1;
20168 void clear_node_realigned(uint8_t* node)
20170 #ifdef RESPECT_LARGE_ALIGNMENT
20171 ((plug_and_reloc*)(node))[-1].reloc &= ~1;
20173 UNREFERENCED_PARAMETER(node);
20174 #endif //RESPECT_LARGE_ALIGNMENT
20176 #endif // FEATURE_STRUCTALIGN
20179 size_t node_gap_size (uint8_t* node)
20181 return ((plug_and_gap *)node)[-1].gap;
20184 void set_gap_size (uint8_t* node, size_t size)
20186 assert (Aligned (size));
20188 // clear the 2 uint32_t used by the node.
20189 ((plug_and_gap *)node)[-1].reloc = 0;
20190 ((plug_and_gap *)node)[-1].lr =0;
20191 ((plug_and_gap *)node)[-1].gap = size;
20193 assert ((size == 0 )||(size >= sizeof(plug_and_reloc)));
20197 uint8_t* gc_heap::insert_node (uint8_t* new_node, size_t sequence_number,
20198 uint8_t* tree, uint8_t* last_node)
20200 dprintf (3, ("IN: %Ix(%Ix), T: %Ix(%Ix), L: %Ix(%Ix) [%Ix]",
20201 (size_t)new_node, brick_of(new_node),
20202 (size_t)tree, brick_of(tree),
20203 (size_t)last_node, brick_of(last_node),
20205 if (power_of_two_p (sequence_number))
20207 set_node_left_child (new_node, (tree - new_node));
20208 dprintf (3, ("NT: %Ix, LC->%Ix", (size_t)new_node, (tree - new_node)));
20213 if (oddp (sequence_number))
20215 set_node_right_child (last_node, (new_node - last_node));
20216 dprintf (3, ("%Ix RC->%Ix", last_node, (new_node - last_node)));
20220 uint8_t* earlier_node = tree;
20221 size_t imax = logcount(sequence_number) - 2;
20222 for (size_t i = 0; i != imax; i++)
20224 earlier_node = earlier_node + node_right_child (earlier_node);
20226 int tmp_offset = node_right_child (earlier_node);
20227 assert (tmp_offset); // should never be empty
20228 set_node_left_child (new_node, ((earlier_node + tmp_offset ) - new_node));
20229 set_node_right_child (earlier_node, (new_node - earlier_node));
20231 dprintf (3, ("%Ix LC->%Ix, %Ix RC->%Ix",
20232 new_node, ((earlier_node + tmp_offset ) - new_node),
20233 earlier_node, (new_node - earlier_node)));
20239 size_t gc_heap::update_brick_table (uint8_t* tree, size_t current_brick,
20240 uint8_t* x, uint8_t* plug_end)
20242 dprintf (3, ("tree: %Ix, current b: %Ix, x: %Ix, plug_end: %Ix",
20243 tree, current_brick, x, plug_end));
20247 dprintf (3, ("b- %Ix->%Ix pointing to tree %Ix",
20248 current_brick, (size_t)(tree - brick_address (current_brick)), tree));
20249 set_brick (current_brick, (tree - brick_address (current_brick)));
20253 dprintf (3, ("b- %Ix->-1", current_brick));
20254 set_brick (current_brick, -1);
20256 size_t b = 1 + current_brick;
20257 ptrdiff_t offset = 0;
20258 size_t last_br = brick_of (plug_end-1);
20259 current_brick = brick_of (x-1);
20260 dprintf (3, ("ubt: %Ix->%Ix]->%Ix]", b, last_br, current_brick));
20261 while (b <= current_brick)
20265 set_brick (b, --offset);
20273 return brick_of (x);
20276 void gc_heap::plan_generation_start (generation* gen, generation* consing_gen, uint8_t* next_plug_to_allocate)
20279 // We should never demote big plugs to gen0.
20280 if (gen == youngest_generation)
20282 heap_segment* seg = ephemeral_heap_segment;
20283 size_t mark_stack_large_bos = mark_stack_bos;
20284 size_t large_plug_pos = 0;
20285 while (mark_stack_large_bos < mark_stack_tos)
20287 if (mark_stack_array[mark_stack_large_bos].len > demotion_plug_len_th)
20289 while (mark_stack_bos <= mark_stack_large_bos)
20291 size_t entry = deque_pinned_plug();
20292 size_t len = pinned_len (pinned_plug_of (entry));
20293 uint8_t* plug = pinned_plug (pinned_plug_of(entry));
20294 if (len > demotion_plug_len_th)
20296 dprintf (2, ("ps(%d): S %Ix (%Id)(%Ix)", gen->gen_num, plug, len, (plug+len)));
20298 pinned_len (pinned_plug_of (entry)) = plug - generation_allocation_pointer (consing_gen);
20299 assert(mark_stack_array[entry].len == 0 ||
20300 mark_stack_array[entry].len >= Align(min_obj_size));
20301 generation_allocation_pointer (consing_gen) = plug + len;
20302 generation_allocation_limit (consing_gen) = heap_segment_plan_allocated (seg);
20303 set_allocator_next_pin (consing_gen);
20307 mark_stack_large_bos++;
20312 generation_plan_allocation_start (gen) =
20313 allocate_in_condemned_generations (consing_gen, Align (min_obj_size), -1);
20314 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
20315 size_t allocation_left = (size_t)(generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
20316 if (next_plug_to_allocate)
20318 size_t dist_to_next_plug = (size_t)(next_plug_to_allocate - generation_allocation_pointer (consing_gen));
20319 if (allocation_left > dist_to_next_plug)
20321 allocation_left = dist_to_next_plug;
20324 if (allocation_left < Align (min_obj_size))
20326 generation_plan_allocation_start_size (gen) += allocation_left;
20327 generation_allocation_pointer (consing_gen) += allocation_left;
20330 dprintf (1, ("plan alloc gen%d(%Ix) start at %Ix (ptr: %Ix, limit: %Ix, next: %Ix)", gen->gen_num,
20331 generation_plan_allocation_start (gen),
20332 generation_plan_allocation_start_size (gen),
20333 generation_allocation_pointer (consing_gen), generation_allocation_limit (consing_gen),
20334 next_plug_to_allocate));
20337 void gc_heap::realloc_plan_generation_start (generation* gen, generation* consing_gen)
20339 BOOL adjacentp = FALSE;
20341 generation_plan_allocation_start (gen) =
20342 allocate_in_expanded_heap (consing_gen, Align(min_obj_size), adjacentp, 0,
20345 #endif //SHORT_PLUGS
20346 FALSE, -1 REQD_ALIGN_AND_OFFSET_ARG);
20348 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
20349 size_t allocation_left = (size_t)(generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
20350 if ((allocation_left < Align (min_obj_size)) &&
20351 (generation_allocation_limit (consing_gen)!=heap_segment_plan_allocated (generation_allocation_segment (consing_gen))))
20353 generation_plan_allocation_start_size (gen) += allocation_left;
20354 generation_allocation_pointer (consing_gen) += allocation_left;
20357 dprintf (1, ("plan re-alloc gen%d start at %Ix (ptr: %Ix, limit: %Ix)", gen->gen_num,
20358 generation_plan_allocation_start (consing_gen),
20359 generation_allocation_pointer (consing_gen),
20360 generation_allocation_limit (consing_gen)));
20363 void gc_heap::plan_generation_starts (generation*& consing_gen)
20365 //make sure that every generation has a planned allocation start
20366 int gen_number = settings.condemned_generation;
20367 while (gen_number >= 0)
20369 if (gen_number < max_generation)
20371 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
20373 generation* gen = generation_of (gen_number);
20374 if (0 == generation_plan_allocation_start (gen))
20376 plan_generation_start (gen, consing_gen, 0);
20377 assert (generation_plan_allocation_start (gen));
20381 // now we know the planned allocation size
20382 heap_segment_plan_allocated (ephemeral_heap_segment) =
20383 generation_allocation_pointer (consing_gen);
20386 void gc_heap::advance_pins_for_demotion (generation* gen)
20388 uint8_t* original_youngest_start = generation_allocation_start (youngest_generation);
20389 heap_segment* seg = ephemeral_heap_segment;
20391 if ((!(pinned_plug_que_empty_p())))
20393 size_t gen1_pinned_promoted = generation_pinned_allocation_compact_size (generation_of (max_generation));
20394 size_t gen1_pins_left = dd_pinned_survived_size (dynamic_data_of (max_generation - 1)) - gen1_pinned_promoted;
20395 size_t total_space_to_skip = last_gen1_pin_end - generation_allocation_pointer (gen);
20396 float pin_frag_ratio = (float)gen1_pins_left / (float)total_space_to_skip;
20397 float pin_surv_ratio = (float)gen1_pins_left / (float)(dd_survived_size (dynamic_data_of (max_generation - 1)));
20398 if ((pin_frag_ratio > 0.15) && (pin_surv_ratio > 0.30))
20400 while (!pinned_plug_que_empty_p() &&
20401 (pinned_plug (oldest_pin()) < original_youngest_start))
20403 size_t entry = deque_pinned_plug();
20404 size_t len = pinned_len (pinned_plug_of (entry));
20405 uint8_t* plug = pinned_plug (pinned_plug_of(entry));
20406 pinned_len (pinned_plug_of (entry)) = plug - generation_allocation_pointer (gen);
20407 assert(mark_stack_array[entry].len == 0 ||
20408 mark_stack_array[entry].len >= Align(min_obj_size));
20409 generation_allocation_pointer (gen) = plug + len;
20410 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20411 set_allocator_next_pin (gen);
20413 //Add the size of the pinned plug to the right pinned allocations
20414 //find out which gen this pinned plug came from
20415 int frgn = object_gennum (plug);
20416 if ((frgn != (int)max_generation) && settings.promotion)
20418 int togn = object_gennum_plan (plug);
20419 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
20422 generation_pinned_allocation_compact_size (generation_of (togn)) += len;
20426 dprintf (2, ("skipping gap %d, pin %Ix (%Id)",
20427 pinned_len (pinned_plug_of (entry)), plug, len));
20430 dprintf (2, ("ad_p_d: PL: %Id, SL: %Id, pfr: %d, psr: %d",
20431 gen1_pins_left, total_space_to_skip, (int)(pin_frag_ratio*100), (int)(pin_surv_ratio*100)));
20435 void gc_heap::process_ephemeral_boundaries (uint8_t* x,
20436 int& active_new_gen_number,
20437 int& active_old_gen_number,
20438 generation*& consing_gen,
20439 BOOL& allocate_in_condemned)
20442 if ((active_old_gen_number > 0) &&
20443 (x >= generation_allocation_start (generation_of (active_old_gen_number - 1))))
20445 dprintf (1, ("crossing gen%d, x is %Ix", active_old_gen_number - 1, x));
20447 if (!pinned_plug_que_empty_p())
20449 dprintf (1, ("oldest pin: %Ix(%Id)",
20450 pinned_plug (oldest_pin()),
20451 (x - pinned_plug (oldest_pin()))));
20454 if (active_old_gen_number <= (settings.promotion ? (max_generation - 1) : max_generation))
20456 active_new_gen_number--;
20459 active_old_gen_number--;
20460 assert ((!settings.promotion) || (active_new_gen_number>0));
20462 if (active_new_gen_number == (max_generation - 1))
20464 #ifdef FREE_USAGE_STATS
20465 if (settings.condemned_generation == max_generation)
20467 // We need to do this before we skip the rest of the pinned plugs.
20468 generation* gen_2 = generation_of (max_generation);
20469 generation* gen_1 = generation_of (max_generation - 1);
20471 size_t total_num_pinned_free_spaces_left = 0;
20473 // We are about to allocate gen1, check to see how efficient fitting in gen2 pinned free spaces is.
20474 for (int j = 0; j < NUM_GEN_POWER2; j++)
20476 dprintf (1, ("[h%d][#%Id]2^%d: current: %Id, S: 2: %Id, 1: %Id(%Id)",
20480 gen_2->gen_current_pinned_free_spaces[j],
20481 gen_2->gen_plugs[j], gen_1->gen_plugs[j],
20482 (gen_2->gen_plugs[j] + gen_1->gen_plugs[j])));
20484 total_num_pinned_free_spaces_left += gen_2->gen_current_pinned_free_spaces[j];
20487 float pinned_free_list_efficiency = 0;
20488 size_t total_pinned_free_space = generation_allocated_in_pinned_free (gen_2) + generation_pinned_free_obj_space (gen_2);
20489 if (total_pinned_free_space != 0)
20491 pinned_free_list_efficiency = (float)(generation_allocated_in_pinned_free (gen_2)) / (float)total_pinned_free_space;
20494 dprintf (1, ("[h%d] gen2 allocated %Id bytes with %Id bytes pinned free spaces (effi: %d%%), %Id (%Id) left",
20496 generation_allocated_in_pinned_free (gen_2),
20497 total_pinned_free_space,
20498 (int)(pinned_free_list_efficiency * 100),
20499 generation_pinned_free_obj_space (gen_2),
20500 total_num_pinned_free_spaces_left));
20502 #endif //FREE_USAGE_STATS
20504 //Go past all of the pinned plugs for this generation.
20505 while (!pinned_plug_que_empty_p() &&
20506 (!in_range_for_segment ((pinned_plug (oldest_pin())), ephemeral_heap_segment)))
20508 size_t entry = deque_pinned_plug();
20509 mark* m = pinned_plug_of (entry);
20510 uint8_t* plug = pinned_plug (m);
20511 size_t len = pinned_len (m);
20512 // detect pinned block in different segment (later) than
20513 // allocation segment, skip those until the oldest pin is in the ephemeral seg.
20514 // adjust the allocation segment along the way (at the end it will
20515 // be the ephemeral segment.
20516 heap_segment* nseg = heap_segment_in_range (generation_allocation_segment (consing_gen));
20518 PREFIX_ASSUME(nseg != NULL);
20520 while (!((plug >= generation_allocation_pointer (consing_gen))&&
20521 (plug < heap_segment_allocated (nseg))))
20523 //adjust the end of the segment to be the end of the plug
20524 assert (generation_allocation_pointer (consing_gen)>=
20525 heap_segment_mem (nseg));
20526 assert (generation_allocation_pointer (consing_gen)<=
20527 heap_segment_committed (nseg));
20529 heap_segment_plan_allocated (nseg) =
20530 generation_allocation_pointer (consing_gen);
20531 //switch allocation segment
20532 nseg = heap_segment_next_rw (nseg);
20533 generation_allocation_segment (consing_gen) = nseg;
20534 //reset the allocation pointer and limits
20535 generation_allocation_pointer (consing_gen) =
20536 heap_segment_mem (nseg);
20538 set_new_pin_info (m, generation_allocation_pointer (consing_gen));
20539 assert(pinned_len(m) == 0 || pinned_len(m) >= Align(min_obj_size));
20540 generation_allocation_pointer (consing_gen) = plug + len;
20541 generation_allocation_limit (consing_gen) =
20542 generation_allocation_pointer (consing_gen);
20544 allocate_in_condemned = TRUE;
20545 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
20548 if (active_new_gen_number != max_generation)
20550 if (active_new_gen_number == (max_generation - 1))
20552 maxgen_pinned_compact_before_advance = generation_pinned_allocation_compact_size (generation_of (max_generation));
20553 if (!demote_gen1_p)
20554 advance_pins_for_demotion (consing_gen);
20557 plan_generation_start (generation_of (active_new_gen_number), consing_gen, x);
20559 dprintf (1, ("process eph: allocated gen%d start at %Ix",
20560 active_new_gen_number,
20561 generation_plan_allocation_start (generation_of (active_new_gen_number))));
20563 if ((demotion_low == MAX_PTR) && !pinned_plug_que_empty_p())
20565 uint8_t* pplug = pinned_plug (oldest_pin());
20566 if (object_gennum (pplug) > 0)
20568 demotion_low = pplug;
20569 dprintf (3, ("process eph: dlow->%Ix", demotion_low));
20573 assert (generation_plan_allocation_start (generation_of (active_new_gen_number)));
20581 void gc_heap::seg_clear_mark_bits (heap_segment* seg)
20583 uint8_t* o = heap_segment_mem (seg);
20584 while (o < heap_segment_allocated (seg))
20590 o = o + Align (size (o));
20594 #ifdef FEATURE_BASICFREEZE
20595 void gc_heap::sweep_ro_segments (heap_segment* start_seg)
20597 //go through all of the segment in range and reset the mark bit
20598 //TODO works only on small object segments
20600 heap_segment* seg = start_seg;
20604 if (heap_segment_read_only_p (seg) &&
20605 heap_segment_in_range_p (seg))
20607 #ifdef BACKGROUND_GC
20608 if (settings.concurrent)
20610 seg_clear_mark_array_bits_soh (seg);
20614 seg_clear_mark_bits (seg);
20616 #else //BACKGROUND_GC
20619 if(gc_can_use_concurrent)
20621 clear_mark_array (max (heap_segment_mem (seg), lowest_address),
20622 min (heap_segment_allocated (seg), highest_address),
20623 FALSE); // read_only segments need the mark clear
20626 seg_clear_mark_bits (seg);
20627 #endif //MARK_ARRAY
20629 #endif //BACKGROUND_GC
20631 seg = heap_segment_next (seg);
20634 #endif // FEATURE_BASICFREEZE
20636 #ifdef FEATURE_LOH_COMPACTION
20638 BOOL gc_heap::loh_pinned_plug_que_empty_p()
20640 return (loh_pinned_queue_bos == loh_pinned_queue_tos);
20643 void gc_heap::loh_set_allocator_next_pin()
20645 if (!(loh_pinned_plug_que_empty_p()))
20647 mark* oldest_entry = loh_oldest_pin();
20648 uint8_t* plug = pinned_plug (oldest_entry);
20649 generation* gen = large_object_generation;
20650 if ((plug >= generation_allocation_pointer (gen)) &&
20651 (plug < generation_allocation_limit (gen)))
20653 generation_allocation_limit (gen) = pinned_plug (oldest_entry);
20656 assert (!((plug < generation_allocation_pointer (gen)) &&
20657 (plug >= heap_segment_mem (generation_allocation_segment (gen)))));
20661 size_t gc_heap::loh_deque_pinned_plug ()
20663 size_t m = loh_pinned_queue_bos;
20664 loh_pinned_queue_bos++;
20669 mark* gc_heap::loh_pinned_plug_of (size_t bos)
20671 return &loh_pinned_queue[bos];
20675 mark* gc_heap::loh_oldest_pin()
20677 return loh_pinned_plug_of (loh_pinned_queue_bos);
20680 // If we can't grow the queue, then don't compact.
20681 BOOL gc_heap::loh_enque_pinned_plug (uint8_t* plug, size_t len)
20683 assert(len >= Align(min_obj_size, get_alignment_constant (FALSE)));
20685 if (loh_pinned_queue_length <= loh_pinned_queue_tos)
20687 if (!grow_mark_stack (loh_pinned_queue, loh_pinned_queue_length, LOH_PIN_QUEUE_LENGTH))
20692 dprintf (3, (" P: %Ix(%Id)", plug, len));
20693 mark& m = loh_pinned_queue[loh_pinned_queue_tos];
20696 loh_pinned_queue_tos++;
20697 loh_set_allocator_next_pin();
20702 BOOL gc_heap::loh_size_fit_p (size_t size, uint8_t* alloc_pointer, uint8_t* alloc_limit)
20704 dprintf (1235, ("trying to fit %Id(%Id) between %Ix and %Ix (%Id)",
20706 (2* AlignQword (loh_padding_obj_size) + size),
20709 (alloc_limit - alloc_pointer)));
20711 return ((alloc_pointer + 2* AlignQword (loh_padding_obj_size) + size) <= alloc_limit);
20714 uint8_t* gc_heap::loh_allocate_in_condemned (uint8_t* old_loc, size_t size)
20716 UNREFERENCED_PARAMETER(old_loc);
20718 generation* gen = large_object_generation;
20719 dprintf (1235, ("E: p:%Ix, l:%Ix, s: %Id",
20720 generation_allocation_pointer (gen),
20721 generation_allocation_limit (gen),
20726 heap_segment* seg = generation_allocation_segment (gen);
20727 if (!(loh_size_fit_p (size, generation_allocation_pointer (gen), generation_allocation_limit (gen))))
20729 if ((!(loh_pinned_plug_que_empty_p()) &&
20730 (generation_allocation_limit (gen) ==
20731 pinned_plug (loh_oldest_pin()))))
20733 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
20734 size_t len = pinned_len (m);
20735 uint8_t* plug = pinned_plug (m);
20736 dprintf (1235, ("AIC: %Ix->%Ix(%Id)", generation_allocation_pointer (gen), plug, plug - generation_allocation_pointer (gen)));
20737 pinned_len (m) = plug - generation_allocation_pointer (gen);
20738 generation_allocation_pointer (gen) = plug + len;
20740 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20741 loh_set_allocator_next_pin();
20742 dprintf (1235, ("s: p: %Ix, l: %Ix (%Id)",
20743 generation_allocation_pointer (gen),
20744 generation_allocation_limit (gen),
20745 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20750 if (generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))
20752 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20753 dprintf (1235, ("l->pa(%Ix)", generation_allocation_limit (gen)));
20757 if (heap_segment_plan_allocated (seg) != heap_segment_committed (seg))
20759 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
20760 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20761 dprintf (1235, ("l->c(%Ix)", generation_allocation_limit (gen)));
20765 if (loh_size_fit_p (size, generation_allocation_pointer (gen), heap_segment_reserved (seg)) &&
20766 (grow_heap_segment (seg, (generation_allocation_pointer (gen) + size + 2* AlignQword (loh_padding_obj_size)))))
20768 dprintf (1235, ("growing seg from %Ix to %Ix\n", heap_segment_committed (seg),
20769 (generation_allocation_pointer (gen) + size)));
20771 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
20772 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20774 dprintf (1235, ("g: p: %Ix, l: %Ix (%Id)",
20775 generation_allocation_pointer (gen),
20776 generation_allocation_limit (gen),
20777 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20781 heap_segment* next_seg = heap_segment_next (seg);
20782 assert (generation_allocation_pointer (gen)>=
20783 heap_segment_mem (seg));
20784 // Verify that all pinned plugs for this segment are consumed
20785 if (!loh_pinned_plug_que_empty_p() &&
20786 ((pinned_plug (loh_oldest_pin()) <
20787 heap_segment_allocated (seg)) &&
20788 (pinned_plug (loh_oldest_pin()) >=
20789 generation_allocation_pointer (gen))))
20791 LOG((LF_GC, LL_INFO10, "remaining pinned plug %Ix while leaving segment on allocation",
20792 pinned_plug (loh_oldest_pin())));
20793 dprintf (1236, ("queue empty: %d", loh_pinned_plug_que_empty_p()));
20796 assert (generation_allocation_pointer (gen)>=
20797 heap_segment_mem (seg));
20798 assert (generation_allocation_pointer (gen)<=
20799 heap_segment_committed (seg));
20800 heap_segment_plan_allocated (seg) = generation_allocation_pointer (gen);
20804 // for LOH do we want to try starting from the first LOH every time though?
20805 generation_allocation_segment (gen) = next_seg;
20806 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
20807 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
20809 dprintf (1235, ("n: p: %Ix, l: %Ix (%Id)",
20810 generation_allocation_pointer (gen),
20811 generation_allocation_limit (gen),
20812 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20816 dprintf (1, ("We ran out of space compacting, shouldn't happen"));
20822 loh_set_allocator_next_pin();
20824 dprintf (1235, ("r: p: %Ix, l: %Ix (%Id)",
20825 generation_allocation_pointer (gen),
20826 generation_allocation_limit (gen),
20827 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20834 assert (generation_allocation_pointer (gen)>=
20835 heap_segment_mem (generation_allocation_segment (gen)));
20836 uint8_t* result = generation_allocation_pointer (gen);
20837 size_t loh_pad = AlignQword (loh_padding_obj_size);
20839 generation_allocation_pointer (gen) += size + loh_pad;
20840 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
20842 dprintf (1235, ("p: %Ix, l: %Ix (%Id)",
20843 generation_allocation_pointer (gen),
20844 generation_allocation_limit (gen),
20845 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20847 assert (result + loh_pad);
20848 return result + loh_pad;
20852 BOOL gc_heap::should_compact_loh()
20854 return (loh_compaction_always_p || (loh_compaction_mode != loh_compaction_default));
20858 void gc_heap::check_loh_compact_mode (BOOL all_heaps_compacted_p)
20860 if (settings.loh_compaction && (loh_compaction_mode == loh_compaction_once))
20862 if (all_heaps_compacted_p)
20864 // If the compaction mode says to compact once and we are going to compact LOH,
20865 // we need to revert it back to no compaction.
20866 loh_compaction_mode = loh_compaction_default;
20871 BOOL gc_heap::plan_loh()
20873 if (!loh_pinned_queue)
20875 loh_pinned_queue = new (nothrow) (mark [LOH_PIN_QUEUE_LENGTH]);
20876 if (!loh_pinned_queue)
20878 dprintf (1, ("Cannot allocate the LOH pinned queue (%Id bytes), no compaction",
20879 LOH_PIN_QUEUE_LENGTH * sizeof (mark)));
20883 loh_pinned_queue_length = LOH_PIN_QUEUE_LENGTH;
20886 if (heap_number == 0)
20887 loh_pinned_queue_decay = LOH_PIN_DECAY;
20889 loh_pinned_queue_tos = 0;
20890 loh_pinned_queue_bos = 0;
20892 generation* gen = large_object_generation;
20893 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
20894 PREFIX_ASSUME(start_seg != NULL);
20895 heap_segment* seg = start_seg;
20896 uint8_t* o = generation_allocation_start (gen);
20898 dprintf (1235, ("before GC LOH size: %Id, free list: %Id, free obj: %Id\n",
20899 generation_size (max_generation + 1),
20900 generation_free_list_space (gen),
20901 generation_free_obj_space (gen)));
20905 heap_segment_plan_allocated (seg) = heap_segment_mem (seg);
20906 seg = heap_segment_next (seg);
20911 //Skip the generation gap object
20912 o = o + AlignQword (size (o));
20913 // We don't need to ever realloc gen3 start so don't touch it.
20914 heap_segment_plan_allocated (seg) = o;
20915 generation_allocation_pointer (gen) = o;
20916 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
20917 generation_allocation_segment (gen) = start_seg;
20919 uint8_t* free_space_start = o;
20920 uint8_t* free_space_end = o;
20921 uint8_t* new_address = 0;
20925 if (o >= heap_segment_allocated (seg))
20927 seg = heap_segment_next (seg);
20933 o = heap_segment_mem (seg);
20938 free_space_end = o;
20939 size_t size = AlignQword (size (o));
20940 dprintf (1235, ("%Ix(%Id) M", o, size));
20944 // We don't clear the pinned bit yet so we can check in
20945 // compact phase how big a free object we should allocate
20946 // in front of the pinned object. We use the reloc address
20947 // field to store this.
20948 if (!loh_enque_pinned_plug (o, size))
20956 new_address = loh_allocate_in_condemned (o, size);
20959 loh_set_node_relocation_distance (o, (new_address - o));
20960 dprintf (1235, ("lobj %Ix-%Ix -> %Ix-%Ix (%Id)", o, (o + size), new_address, (new_address + size), (new_address - o)));
20963 free_space_start = o;
20964 if (o < heap_segment_allocated (seg))
20966 assert (!marked (o));
20971 while (o < heap_segment_allocated (seg) && !marked (o))
20973 dprintf (1235, ("%Ix(%Id) F (%d)", o, AlignQword (size (o)), ((method_table (o) == g_gc_pFreeObjectMethodTable) ? 1 : 0)));
20974 o = o + AlignQword (size (o));
20979 while (!loh_pinned_plug_que_empty_p())
20981 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
20982 size_t len = pinned_len (m);
20983 uint8_t* plug = pinned_plug (m);
20985 // detect pinned block in different segment (later) than
20986 // allocation segment
20987 heap_segment* nseg = heap_segment_rw (generation_allocation_segment (gen));
20989 while ((plug < generation_allocation_pointer (gen)) ||
20990 (plug >= heap_segment_allocated (nseg)))
20992 assert ((plug < heap_segment_mem (nseg)) ||
20993 (plug > heap_segment_reserved (nseg)));
20994 //adjust the end of the segment to be the end of the plug
20995 assert (generation_allocation_pointer (gen)>=
20996 heap_segment_mem (nseg));
20997 assert (generation_allocation_pointer (gen)<=
20998 heap_segment_committed (nseg));
21000 heap_segment_plan_allocated (nseg) =
21001 generation_allocation_pointer (gen);
21002 //switch allocation segment
21003 nseg = heap_segment_next_rw (nseg);
21004 generation_allocation_segment (gen) = nseg;
21005 //reset the allocation pointer and limits
21006 generation_allocation_pointer (gen) =
21007 heap_segment_mem (nseg);
21010 dprintf (1235, ("SP: %Ix->%Ix(%Id)", generation_allocation_pointer (gen), plug, plug - generation_allocation_pointer (gen)));
21011 pinned_len (m) = plug - generation_allocation_pointer (gen);
21012 generation_allocation_pointer (gen) = plug + len;
21015 heap_segment_plan_allocated (generation_allocation_segment (gen)) = generation_allocation_pointer (gen);
21016 generation_allocation_pointer (gen) = 0;
21017 generation_allocation_limit (gen) = 0;
21022 void gc_heap::compact_loh()
21024 assert (should_compact_loh());
21026 generation* gen = large_object_generation;
21027 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
21028 PREFIX_ASSUME(start_seg != NULL);
21029 heap_segment* seg = start_seg;
21030 heap_segment* prev_seg = 0;
21031 uint8_t* o = generation_allocation_start (gen);
21033 //Skip the generation gap object
21034 o = o + AlignQword (size (o));
21035 // We don't need to ever realloc gen3 start so don't touch it.
21036 uint8_t* free_space_start = o;
21037 uint8_t* free_space_end = o;
21038 generation_allocator (gen)->clear();
21039 generation_free_list_space (gen) = 0;
21040 generation_free_obj_space (gen) = 0;
21042 loh_pinned_queue_bos = 0;
21046 if (o >= heap_segment_allocated (seg))
21048 heap_segment* next_seg = heap_segment_next (seg);
21050 if ((heap_segment_plan_allocated (seg) == heap_segment_mem (seg)) &&
21051 (seg != start_seg) && !heap_segment_read_only_p (seg))
21053 dprintf (3, ("Preparing empty large segment %Ix", (size_t)seg));
21055 heap_segment_next (prev_seg) = next_seg;
21056 heap_segment_next (seg) = freeable_large_heap_segment;
21057 freeable_large_heap_segment = seg;
21061 if (!heap_segment_read_only_p (seg))
21063 // We grew the segment to accommodate allocations.
21064 if (heap_segment_plan_allocated (seg) > heap_segment_allocated (seg))
21066 if ((heap_segment_plan_allocated (seg) - plug_skew) > heap_segment_used (seg))
21068 heap_segment_used (seg) = heap_segment_plan_allocated (seg) - plug_skew;
21072 heap_segment_allocated (seg) = heap_segment_plan_allocated (seg);
21073 dprintf (3, ("Trimming seg to %Ix[", heap_segment_allocated (seg)));
21074 decommit_heap_segment_pages (seg, 0);
21075 dprintf (1236, ("CLOH: seg: %Ix, alloc: %Ix, used: %Ix, committed: %Ix",
21077 heap_segment_allocated (seg),
21078 heap_segment_used (seg),
21079 heap_segment_committed (seg)));
21080 //heap_segment_used (seg) = heap_segment_allocated (seg) - plug_skew;
21081 dprintf (1236, ("CLOH: used is set to %Ix", heap_segment_used (seg)));
21091 o = heap_segment_mem (seg);
21097 free_space_end = o;
21098 size_t size = AlignQword (size (o));
21101 uint8_t* reloc = o;
21106 // We are relying on the fact the pinned objects are always looked at in the same order
21107 // in plan phase and in compact phase.
21108 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
21109 uint8_t* plug = pinned_plug (m);
21110 assert (plug == o);
21112 loh_pad = pinned_len (m);
21117 loh_pad = AlignQword (loh_padding_obj_size);
21119 reloc += loh_node_relocation_distance (o);
21120 gcmemcopy (reloc, o, size, TRUE);
21123 thread_gap ((reloc - loh_pad), loh_pad, gen);
21126 free_space_start = o;
21127 if (o < heap_segment_allocated (seg))
21129 assert (!marked (o));
21134 while (o < heap_segment_allocated (seg) && !marked (o))
21136 o = o + AlignQword (size (o));
21141 assert (loh_pinned_plug_que_empty_p());
21143 dprintf (1235, ("after GC LOH size: %Id, free list: %Id, free obj: %Id\n\n",
21144 generation_size (max_generation + 1),
21145 generation_free_list_space (gen),
21146 generation_free_obj_space (gen)));
21149 void gc_heap::relocate_in_loh_compact()
21151 generation* gen = large_object_generation;
21152 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
21153 uint8_t* o = generation_allocation_start (gen);
21155 //Skip the generation gap object
21156 o = o + AlignQword (size (o));
21158 relocate_args args;
21160 args.high = gc_high;
21161 args.last_plug = 0;
21165 if (o >= heap_segment_allocated (seg))
21167 seg = heap_segment_next (seg);
21173 o = heap_segment_mem (seg);
21178 size_t size = AlignQword (size (o));
21180 check_class_object_demotion (o);
21181 if (contain_pointers (o))
21183 go_through_object_nostart (method_table (o), o, size(o), pval,
21185 reloc_survivor_helper (pval);
21190 if (o < heap_segment_allocated (seg))
21192 assert (!marked (o));
21197 while (o < heap_segment_allocated (seg) && !marked (o))
21199 o = o + AlignQword (size (o));
21204 dprintf (1235, ("after GC LOH size: %Id, free list: %Id, free obj: %Id\n\n",
21205 generation_size (max_generation + 1),
21206 generation_free_list_space (gen),
21207 generation_free_obj_space (gen)));
21210 void gc_heap::walk_relocation_for_loh (void* profiling_context, record_surv_fn fn)
21212 generation* gen = large_object_generation;
21213 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
21214 uint8_t* o = generation_allocation_start (gen);
21216 //Skip the generation gap object
21217 o = o + AlignQword (size (o));
21221 if (o >= heap_segment_allocated (seg))
21223 seg = heap_segment_next (seg);
21229 o = heap_segment_mem (seg);
21234 size_t size = AlignQword (size (o));
21236 ptrdiff_t reloc = loh_node_relocation_distance (o);
21238 STRESS_LOG_PLUG_MOVE(o, (o + size), -reloc);
21240 fn (o, (o + size), reloc, profiling_context, !!settings.compaction, false);
21243 if (o < heap_segment_allocated (seg))
21245 assert (!marked (o));
21250 while (o < heap_segment_allocated (seg) && !marked (o))
21252 o = o + AlignQword (size (o));
21258 BOOL gc_heap::loh_object_p (uint8_t* o)
21260 #ifdef MULTIPLE_HEAPS
21261 gc_heap* hp = gc_heap::g_heaps [0];
21262 int brick_entry = hp->brick_table[hp->brick_of (o)];
21263 #else //MULTIPLE_HEAPS
21264 int brick_entry = brick_table[brick_of (o)];
21265 #endif //MULTIPLE_HEAPS
21267 return (brick_entry == 0);
21269 #endif //FEATURE_LOH_COMPACTION
21271 void gc_heap::convert_to_pinned_plug (BOOL& last_npinned_plug_p,
21272 BOOL& last_pinned_plug_p,
21273 BOOL& pinned_plug_p,
21275 size_t& artificial_pinned_size)
21277 last_npinned_plug_p = FALSE;
21278 last_pinned_plug_p = TRUE;
21279 pinned_plug_p = TRUE;
21280 artificial_pinned_size = ps;
21283 // Because we have the artificial pinning, we can't guarantee that pinned and npinned
21284 // plugs are always interleaved.
21285 void gc_heap::store_plug_gap_info (uint8_t* plug_start,
21287 BOOL& last_npinned_plug_p,
21288 BOOL& last_pinned_plug_p,
21289 uint8_t*& last_pinned_plug,
21290 BOOL& pinned_plug_p,
21291 uint8_t* last_object_in_last_plug,
21292 BOOL& merge_with_last_pin_p,
21293 // this is only for verification purpose
21294 size_t last_plug_len)
21296 UNREFERENCED_PARAMETER(last_plug_len);
21298 if (!last_npinned_plug_p && !last_pinned_plug_p)
21300 //dprintf (3, ("last full plug end: %Ix, full plug start: %Ix", plug_end, plug_start));
21301 dprintf (3, ("Free: %Ix", (plug_start - plug_end)));
21302 assert ((plug_start == plug_end) || ((size_t)(plug_start - plug_end) >= Align (min_obj_size)));
21303 set_gap_size (plug_start, plug_start - plug_end);
21306 if (pinned (plug_start))
21308 BOOL save_pre_plug_info_p = FALSE;
21310 if (last_npinned_plug_p || last_pinned_plug_p)
21312 //if (last_plug_len == Align (min_obj_size))
21314 // dprintf (3, ("debugging only - last npinned plug is min, check to see if it's correct"));
21315 // GCToOSInterface::DebugBreak();
21317 save_pre_plug_info_p = TRUE;
21320 pinned_plug_p = TRUE;
21321 last_npinned_plug_p = FALSE;
21323 if (last_pinned_plug_p)
21325 dprintf (3, ("last plug %Ix was also pinned, should merge", last_pinned_plug));
21326 merge_with_last_pin_p = TRUE;
21330 last_pinned_plug_p = TRUE;
21331 last_pinned_plug = plug_start;
21333 enque_pinned_plug (last_pinned_plug, save_pre_plug_info_p, last_object_in_last_plug);
21335 if (save_pre_plug_info_p)
21337 set_gap_size (plug_start, sizeof (gap_reloc_pair));
21343 if (last_pinned_plug_p)
21345 //if (Align (last_plug_len) < min_pre_pin_obj_size)
21347 // dprintf (3, ("debugging only - last pinned plug is min, check to see if it's correct"));
21348 // GCToOSInterface::DebugBreak();
21351 save_post_plug_info (last_pinned_plug, last_object_in_last_plug, plug_start);
21352 set_gap_size (plug_start, sizeof (gap_reloc_pair));
21354 verify_pins_with_post_plug_info("after saving post plug info");
21356 last_npinned_plug_p = TRUE;
21357 last_pinned_plug_p = FALSE;
21361 void gc_heap::record_interesting_data_point (interesting_data_point idp)
21363 #ifdef GC_CONFIG_DRIVEN
21364 (interesting_data_per_gc[idp])++;
21366 UNREFERENCED_PARAMETER(idp);
21367 #endif //GC_CONFIG_DRIVEN
21371 #pragma warning(push)
21372 #pragma warning(disable:21000) // Suppress PREFast warning about overly large function
21374 void gc_heap::plan_phase (int condemned_gen_number)
21376 size_t old_gen2_allocated = 0;
21377 size_t old_gen2_size = 0;
21379 if (condemned_gen_number == (max_generation - 1))
21381 old_gen2_allocated = generation_free_list_allocated (generation_of (max_generation));
21382 old_gen2_size = generation_size (max_generation);
21385 assert (settings.concurrent == FALSE);
21387 // %type% category = quote (plan);
21391 start = GetCycleCount32();
21394 dprintf (2,("---- Plan Phase ---- Condemned generation %d, promotion: %d",
21395 condemned_gen_number, settings.promotion ? 1 : 0));
21397 generation* condemned_gen1 = generation_of (condemned_gen_number);
21400 BOOL use_mark_list = FALSE;
21401 uint8_t** mark_list_next = &mark_list[0];
21402 #ifdef GC_CONFIG_DRIVEN
21403 dprintf (3, ("total number of marked objects: %Id (%Id)",
21404 (mark_list_index - &mark_list[0]), ((mark_list_end - &mark_list[0]))));
21406 dprintf (3, ("mark_list length: %Id",
21407 (mark_list_index - &mark_list[0])));
21408 #endif //GC_CONFIG_DRIVEN
21410 if ((condemned_gen_number < max_generation) &&
21411 (mark_list_index <= mark_list_end)
21412 #ifdef BACKGROUND_GC
21413 && (!recursive_gc_sync::background_running_p())
21414 #endif //BACKGROUND_GC
21417 #ifndef MULTIPLE_HEAPS
21418 _sort (&mark_list[0], mark_list_index-1, 0);
21419 //printf ("using mark list at GC #%d", dd_collection_count (dynamic_data_of (0)));
21420 //verify_qsort_array (&mark_list[0], mark_list_index-1);
21421 #endif //!MULTIPLE_HEAPS
21422 use_mark_list = TRUE;
21423 get_gc_data_per_heap()->set_mechanism_bit (gc_mark_list_bit);
21427 dprintf (3, ("mark_list not used"));
21432 #ifdef FEATURE_BASICFREEZE
21433 if ((generation_start_segment (condemned_gen1) != ephemeral_heap_segment) &&
21434 ro_segments_in_range)
21436 sweep_ro_segments (generation_start_segment (condemned_gen1));
21438 #endif // FEATURE_BASICFREEZE
21440 #ifndef MULTIPLE_HEAPS
21441 if (shigh != (uint8_t*)0)
21443 heap_segment* seg = heap_segment_rw (generation_start_segment (condemned_gen1));
21445 PREFIX_ASSUME(seg != NULL);
21447 heap_segment* fseg = seg;
21450 if (slow > heap_segment_mem (seg) &&
21451 slow < heap_segment_reserved (seg))
21455 uint8_t* o = generation_allocation_start (condemned_gen1) +
21456 Align (size (generation_allocation_start (condemned_gen1)));
21459 assert ((slow - o) >= (int)Align (min_obj_size));
21460 #ifdef BACKGROUND_GC
21461 if (current_c_gc_state == c_gc_state_marking)
21463 bgc_clear_batch_mark_array_bits (o, slow);
21465 #endif //BACKGROUND_GC
21466 make_unused_array (o, slow - o);
21471 assert (condemned_gen_number == max_generation);
21472 make_unused_array (heap_segment_mem (seg),
21473 slow - heap_segment_mem (seg));
21476 if (in_range_for_segment (shigh, seg))
21478 #ifdef BACKGROUND_GC
21479 if (current_c_gc_state == c_gc_state_marking)
21481 bgc_clear_batch_mark_array_bits ((shigh + Align (size (shigh))), heap_segment_allocated (seg));
21483 #endif //BACKGROUND_GC
21484 heap_segment_allocated (seg) = shigh + Align (size (shigh));
21486 // test if the segment is in the range of [slow, shigh]
21487 if (!((heap_segment_reserved (seg) >= slow) &&
21488 (heap_segment_mem (seg) <= shigh)))
21490 // shorten it to minimum
21491 heap_segment_allocated (seg) = heap_segment_mem (seg);
21493 seg = heap_segment_next_rw (seg);
21498 heap_segment* seg = heap_segment_rw (generation_start_segment (condemned_gen1));
21500 PREFIX_ASSUME(seg != NULL);
21502 heap_segment* sseg = seg;
21505 // shorten it to minimum
21508 // no survivors make all generations look empty
21509 uint8_t* o = generation_allocation_start (condemned_gen1) +
21510 Align (size (generation_allocation_start (condemned_gen1)));
21511 #ifdef BACKGROUND_GC
21512 if (current_c_gc_state == c_gc_state_marking)
21514 bgc_clear_batch_mark_array_bits (o, heap_segment_allocated (seg));
21516 #endif //BACKGROUND_GC
21517 heap_segment_allocated (seg) = o;
21521 assert (condemned_gen_number == max_generation);
21522 #ifdef BACKGROUND_GC
21523 if (current_c_gc_state == c_gc_state_marking)
21525 bgc_clear_batch_mark_array_bits (heap_segment_mem (seg), heap_segment_allocated (seg));
21527 #endif //BACKGROUND_GC
21528 heap_segment_allocated (seg) = heap_segment_mem (seg);
21530 seg = heap_segment_next_rw (seg);
21534 #endif //MULTIPLE_HEAPS
21536 heap_segment* seg1 = heap_segment_rw (generation_start_segment (condemned_gen1));
21538 PREFIX_ASSUME(seg1 != NULL);
21540 uint8_t* end = heap_segment_allocated (seg1);
21541 uint8_t* first_condemned_address = generation_allocation_start (condemned_gen1);
21542 uint8_t* x = first_condemned_address;
21544 assert (!marked (x));
21545 uint8_t* plug_end = x;
21547 size_t sequence_number = 0;
21548 uint8_t* last_node = 0;
21549 size_t current_brick = brick_of (x);
21550 BOOL allocate_in_condemned = ((condemned_gen_number == max_generation)||
21551 (settings.promotion == FALSE));
21552 int active_old_gen_number = condemned_gen_number;
21553 int active_new_gen_number = (allocate_in_condemned ? condemned_gen_number:
21554 (1 + condemned_gen_number));
21555 generation* older_gen = 0;
21556 generation* consing_gen = condemned_gen1;
21557 alloc_list r_free_list [MAX_BUCKET_COUNT];
21559 size_t r_free_list_space = 0;
21560 size_t r_free_obj_space = 0;
21561 size_t r_older_gen_free_list_allocated = 0;
21562 size_t r_older_gen_condemned_allocated = 0;
21563 size_t r_older_gen_end_seg_allocated = 0;
21564 uint8_t* r_allocation_pointer = 0;
21565 uint8_t* r_allocation_limit = 0;
21566 uint8_t* r_allocation_start_region = 0;
21567 heap_segment* r_allocation_segment = 0;
21568 #ifdef FREE_USAGE_STATS
21569 size_t r_older_gen_free_space[NUM_GEN_POWER2];
21570 #endif //FREE_USAGE_STATS
21572 if ((condemned_gen_number < max_generation))
21574 older_gen = generation_of (min (max_generation, 1 + condemned_gen_number));
21575 generation_allocator (older_gen)->copy_to_alloc_list (r_free_list);
21577 r_free_list_space = generation_free_list_space (older_gen);
21578 r_free_obj_space = generation_free_obj_space (older_gen);
21579 #ifdef FREE_USAGE_STATS
21580 memcpy (r_older_gen_free_space, older_gen->gen_free_spaces, sizeof (r_older_gen_free_space));
21581 #endif //FREE_USAGE_STATS
21582 generation_allocate_end_seg_p (older_gen) = FALSE;
21583 r_older_gen_free_list_allocated = generation_free_list_allocated (older_gen);
21584 r_older_gen_condemned_allocated = generation_condemned_allocated (older_gen);
21585 r_older_gen_end_seg_allocated = generation_end_seg_allocated (older_gen);
21586 r_allocation_limit = generation_allocation_limit (older_gen);
21587 r_allocation_pointer = generation_allocation_pointer (older_gen);
21588 r_allocation_start_region = generation_allocation_context_start_region (older_gen);
21589 r_allocation_segment = generation_allocation_segment (older_gen);
21590 heap_segment* start_seg = heap_segment_rw (generation_start_segment (older_gen));
21592 PREFIX_ASSUME(start_seg != NULL);
21594 if (start_seg != ephemeral_heap_segment)
21596 assert (condemned_gen_number == (max_generation - 1));
21597 while (start_seg && (start_seg != ephemeral_heap_segment))
21599 assert (heap_segment_allocated (start_seg) >=
21600 heap_segment_mem (start_seg));
21601 assert (heap_segment_allocated (start_seg) <=
21602 heap_segment_reserved (start_seg));
21603 heap_segment_plan_allocated (start_seg) =
21604 heap_segment_allocated (start_seg);
21605 start_seg = heap_segment_next_rw (start_seg);
21610 //reset all of the segment allocated sizes
21612 heap_segment* seg2 = heap_segment_rw (generation_start_segment (condemned_gen1));
21614 PREFIX_ASSUME(seg2 != NULL);
21618 heap_segment_plan_allocated (seg2) =
21619 heap_segment_mem (seg2);
21620 seg2 = heap_segment_next_rw (seg2);
21623 int condemned_gn = condemned_gen_number;
21625 int bottom_gen = 0;
21626 init_free_and_plug();
21628 while (condemned_gn >= bottom_gen)
21630 generation* condemned_gen2 = generation_of (condemned_gn);
21631 generation_allocator (condemned_gen2)->clear();
21632 generation_free_list_space (condemned_gen2) = 0;
21633 generation_free_obj_space (condemned_gen2) = 0;
21634 generation_allocation_size (condemned_gen2) = 0;
21635 generation_condemned_allocated (condemned_gen2) = 0;
21636 generation_pinned_allocated (condemned_gen2) = 0;
21637 generation_free_list_allocated(condemned_gen2) = 0;
21638 generation_end_seg_allocated (condemned_gen2) = 0;
21639 generation_pinned_allocation_sweep_size (condemned_gen2) = 0;
21640 generation_pinned_allocation_compact_size (condemned_gen2) = 0;
21641 #ifdef FREE_USAGE_STATS
21642 generation_pinned_free_obj_space (condemned_gen2) = 0;
21643 generation_allocated_in_pinned_free (condemned_gen2) = 0;
21644 generation_allocated_since_last_pin (condemned_gen2) = 0;
21645 #endif //FREE_USAGE_STATS
21646 generation_plan_allocation_start (condemned_gen2) = 0;
21647 generation_allocation_segment (condemned_gen2) =
21648 heap_segment_rw (generation_start_segment (condemned_gen2));
21650 PREFIX_ASSUME(generation_allocation_segment(condemned_gen2) != NULL);
21652 if (generation_start_segment (condemned_gen2) != ephemeral_heap_segment)
21654 generation_allocation_pointer (condemned_gen2) =
21655 heap_segment_mem (generation_allocation_segment (condemned_gen2));
21659 generation_allocation_pointer (condemned_gen2) = generation_allocation_start (condemned_gen2);
21662 generation_allocation_limit (condemned_gen2) = generation_allocation_pointer (condemned_gen2);
21663 generation_allocation_context_start_region (condemned_gen2) = generation_allocation_pointer (condemned_gen2);
21668 BOOL allocate_first_generation_start = FALSE;
21670 if (allocate_in_condemned)
21672 allocate_first_generation_start = TRUE;
21675 dprintf(3,( " From %Ix to %Ix", (size_t)x, (size_t)end));
21677 demotion_low = MAX_PTR;
21678 demotion_high = heap_segment_allocated (ephemeral_heap_segment);
21680 // If we are doing a gen1 only because of cards, it means we should not demote any pinned plugs
21681 // from gen1. They should get promoted to gen2.
21682 demote_gen1_p = !(settings.promotion &&
21683 (settings.condemned_generation == (max_generation - 1)) &&
21684 gen_to_condemn_reasons.is_only_condition (gen_low_card_p));
21686 total_ephemeral_size = 0;
21688 print_free_and_plug ("BP");
21690 for (int gen_idx = 0; gen_idx <= max_generation; gen_idx++)
21692 generation* temp_gen = generation_of (gen_idx);
21694 dprintf (2, ("gen%d start %Ix, plan start %Ix",
21696 generation_allocation_start (temp_gen),
21697 generation_plan_allocation_start (temp_gen)));
21700 BOOL fire_pinned_plug_events_p = ETW_EVENT_ENABLED(MICROSOFT_WINDOWS_DOTNETRUNTIME_PRIVATE_PROVIDER_Context, PinPlugAtGCTime);
21701 size_t last_plug_len = 0;
21708 assert (heap_segment_allocated (seg1) == end);
21709 heap_segment_allocated (seg1) = plug_end;
21711 current_brick = update_brick_table (tree, current_brick, x, plug_end);
21712 dprintf (3, ("end of seg: new tree, sequence# 0"));
21713 sequence_number = 0;
21716 if (heap_segment_next_rw (seg1))
21718 seg1 = heap_segment_next_rw (seg1);
21719 end = heap_segment_allocated (seg1);
21720 plug_end = x = heap_segment_mem (seg1);
21721 current_brick = brick_of (x);
21722 dprintf(3,( " From %Ix to %Ix", (size_t)x, (size_t)end));
21731 BOOL last_npinned_plug_p = FALSE;
21732 BOOL last_pinned_plug_p = FALSE;
21734 // last_pinned_plug is the beginning of the last pinned plug. If we merge a plug into a pinned
21735 // plug we do not change the value of last_pinned_plug. This happens with artificially pinned plugs -
21736 // it can be merged with a previous pinned plug and a pinned plug after it can be merged with it.
21737 uint8_t* last_pinned_plug = 0;
21738 size_t num_pinned_plugs_in_plug = 0;
21740 uint8_t* last_object_in_plug = 0;
21742 while ((x < end) && marked (x))
21744 uint8_t* plug_start = x;
21745 uint8_t* saved_plug_end = plug_end;
21746 BOOL pinned_plug_p = FALSE;
21747 BOOL npin_before_pin_p = FALSE;
21748 BOOL saved_last_npinned_plug_p = last_npinned_plug_p;
21749 uint8_t* saved_last_object_in_plug = last_object_in_plug;
21750 BOOL merge_with_last_pin_p = FALSE;
21752 size_t added_pinning_size = 0;
21753 size_t artificial_pinned_size = 0;
21755 store_plug_gap_info (plug_start, plug_end, last_npinned_plug_p, last_pinned_plug_p,
21756 last_pinned_plug, pinned_plug_p, last_object_in_plug,
21757 merge_with_last_pin_p, last_plug_len);
21759 #ifdef FEATURE_STRUCTALIGN
21760 int requiredAlignment = ((CObjectHeader*)plug_start)->GetRequiredAlignment();
21761 size_t alignmentOffset = OBJECT_ALIGNMENT_OFFSET;
21762 #endif // FEATURE_STRUCTALIGN
21766 while ((xl < end) && marked (xl) && (pinned (xl) == pinned_plug_p))
21773 #ifdef FEATURE_STRUCTALIGN
21776 int obj_requiredAlignment = ((CObjectHeader*)xl)->GetRequiredAlignment();
21777 if (obj_requiredAlignment > requiredAlignment)
21779 requiredAlignment = obj_requiredAlignment;
21780 alignmentOffset = xl - plug_start + OBJECT_ALIGNMENT_OFFSET;
21783 #endif // FEATURE_STRUCTALIGN
21787 dprintf(4, ("+%Ix+", (size_t)xl));
21788 assert ((size (xl) > 0));
21789 assert ((size (xl) <= LARGE_OBJECT_SIZE));
21791 last_object_in_plug = xl;
21793 xl = xl + Align (size (xl));
21797 BOOL next_object_marked_p = ((xl < end) && marked (xl));
21801 // If it is pinned we need to extend to the next marked object as we can't use part of
21802 // a pinned object to make the artificial gap (unless the last 3 ptr sized words are all
21803 // references but for now I am just using the next non pinned object for that).
21804 if (next_object_marked_p)
21807 last_object_in_plug = xl;
21808 size_t extra_size = Align (size (xl));
21809 xl = xl + extra_size;
21810 added_pinning_size = extra_size;
21815 if (next_object_marked_p)
21816 npin_before_pin_p = TRUE;
21819 assert (xl <= end);
21822 dprintf (3, ( "%Ix[", (size_t)x));
21824 size_t ps = plug_end - plug_start;
21825 last_plug_len = ps;
21826 dprintf (3, ( "%Ix[(%Ix)", (size_t)x, ps));
21827 uint8_t* new_address = 0;
21829 if (!pinned_plug_p)
21831 if (allocate_in_condemned &&
21832 (settings.condemned_generation == max_generation) &&
21833 (ps > OS_PAGE_SIZE))
21835 ptrdiff_t reloc = plug_start - generation_allocation_pointer (consing_gen);
21836 //reloc should >=0 except when we relocate
21837 //across segments and the dest seg is higher then the src
21839 if ((ps > (8*OS_PAGE_SIZE)) &&
21841 ((size_t)reloc < (ps/16)))
21843 dprintf (3, ("Pinning %Ix; reloc would have been: %Ix",
21844 (size_t)plug_start, reloc));
21845 // The last plug couldn't have been a npinned plug or it would have
21846 // included this plug.
21847 assert (!saved_last_npinned_plug_p);
21849 if (last_pinned_plug)
21851 dprintf (3, ("artificially pinned plug merged with last pinned plug"));
21852 merge_with_last_pin_p = TRUE;
21856 enque_pinned_plug (plug_start, FALSE, 0);
21857 last_pinned_plug = plug_start;
21860 convert_to_pinned_plug (last_npinned_plug_p, last_pinned_plug_p, pinned_plug_p,
21861 ps, artificial_pinned_size);
21866 if (allocate_first_generation_start)
21868 allocate_first_generation_start = FALSE;
21869 plan_generation_start (condemned_gen1, consing_gen, plug_start);
21870 assert (generation_plan_allocation_start (condemned_gen1));
21873 if (seg1 == ephemeral_heap_segment)
21875 process_ephemeral_boundaries (plug_start, active_new_gen_number,
21876 active_old_gen_number,
21878 allocate_in_condemned);
21881 dprintf (3, ("adding %Id to gen%d surv", ps, active_old_gen_number));
21883 dynamic_data* dd_active_old = dynamic_data_of (active_old_gen_number);
21884 dd_survived_size (dd_active_old) += ps;
21886 BOOL convert_to_pinned_p = FALSE;
21888 if (!pinned_plug_p)
21890 #if defined (RESPECT_LARGE_ALIGNMENT) || defined (FEATURE_STRUCTALIGN)
21891 dd_num_npinned_plugs (dd_active_old)++;
21892 #endif //RESPECT_LARGE_ALIGNMENT || FEATURE_STRUCTALIGN
21894 add_gen_plug (active_old_gen_number, ps);
21896 if (allocate_in_condemned)
21898 verify_pins_with_post_plug_info("before aic");
21901 allocate_in_condemned_generations (consing_gen,
21903 active_old_gen_number,
21905 &convert_to_pinned_p,
21906 (npin_before_pin_p ? plug_end : 0),
21908 #endif //SHORT_PLUGS
21909 plug_start REQD_ALIGN_AND_OFFSET_ARG);
21910 verify_pins_with_post_plug_info("after aic");
21914 new_address = allocate_in_older_generation (older_gen, ps, active_old_gen_number, plug_start REQD_ALIGN_AND_OFFSET_ARG);
21916 if (new_address != 0)
21918 if (settings.condemned_generation == (max_generation - 1))
21920 dprintf (3, (" NA: %Ix-%Ix -> %Ix, %Ix (%Ix)",
21921 plug_start, plug_end,
21922 (size_t)new_address, (size_t)new_address + (plug_end - plug_start),
21923 (size_t)(plug_end - plug_start)));
21928 allocate_in_condemned = TRUE;
21930 new_address = allocate_in_condemned_generations (consing_gen, ps, active_old_gen_number,
21932 &convert_to_pinned_p,
21933 (npin_before_pin_p ? plug_end : 0),
21935 #endif //SHORT_PLUGS
21936 plug_start REQD_ALIGN_AND_OFFSET_ARG);
21940 if (convert_to_pinned_p)
21942 assert (last_npinned_plug_p != FALSE);
21943 assert (last_pinned_plug_p == FALSE);
21944 convert_to_pinned_plug (last_npinned_plug_p, last_pinned_plug_p, pinned_plug_p,
21945 ps, artificial_pinned_size);
21946 enque_pinned_plug (plug_start, FALSE, 0);
21947 last_pinned_plug = plug_start;
21953 //verify that we are at then end of the ephemeral segment
21954 assert (generation_allocation_segment (consing_gen) ==
21955 ephemeral_heap_segment);
21956 //verify that we are near the end
21957 assert ((generation_allocation_pointer (consing_gen) + Align (ps)) <
21958 heap_segment_allocated (ephemeral_heap_segment));
21959 assert ((generation_allocation_pointer (consing_gen) + Align (ps)) >
21960 (heap_segment_allocated (ephemeral_heap_segment) + Align (min_obj_size)));
21964 #ifdef SIMPLE_DPRINTF
21965 dprintf (3, ("(%Ix)[%Ix->%Ix, NA: [%Ix(%Id), %Ix[: %Ix(%d)",
21966 (size_t)(node_gap_size (plug_start)),
21967 plug_start, plug_end, (size_t)new_address, (size_t)(plug_start - new_address),
21968 (size_t)new_address + ps, ps,
21969 (is_plug_padded (plug_start) ? 1 : 0)));
21970 #endif //SIMPLE_DPRINTF
21973 if (is_plug_padded (plug_start))
21975 dprintf (3, ("%Ix was padded", plug_start));
21976 dd_padding_size (dd_active_old) += Align (min_obj_size);
21978 #endif //SHORT_PLUGS
21985 if (fire_pinned_plug_events_p)
21986 FireEtwPinPlugAtGCTime(plug_start, plug_end,
21987 (merge_with_last_pin_p ? 0 : (uint8_t*)node_gap_size (plug_start)),
21988 GetClrInstanceId());
21990 if (merge_with_last_pin_p)
21992 merge_with_last_pinned_plug (last_pinned_plug, ps);
21996 assert (last_pinned_plug == plug_start);
21997 set_pinned_info (plug_start, ps, consing_gen);
22000 new_address = plug_start;
22002 dprintf (3, ( "(%Ix)PP: [%Ix, %Ix[%Ix](m:%d)",
22003 (size_t)(node_gap_size (plug_start)), (size_t)plug_start,
22004 (size_t)plug_end, ps,
22005 (merge_with_last_pin_p ? 1 : 0)));
22007 dprintf (3, ("adding %Id to gen%d pinned surv", plug_end - plug_start, active_old_gen_number));
22008 dd_pinned_survived_size (dd_active_old) += plug_end - plug_start;
22009 dd_added_pinned_size (dd_active_old) += added_pinning_size;
22010 dd_artificial_pinned_survived_size (dd_active_old) += artificial_pinned_size;
22012 if (!demote_gen1_p && (active_old_gen_number == (max_generation - 1)))
22014 last_gen1_pin_end = plug_end;
22019 // detect forward allocation in the same segment
22020 assert (!((new_address > plug_start) &&
22021 (new_address < heap_segment_reserved (seg1))));
22024 if (!merge_with_last_pin_p)
22026 if (current_brick != brick_of (plug_start))
22028 current_brick = update_brick_table (tree, current_brick, plug_start, saved_plug_end);
22029 sequence_number = 0;
22033 set_node_relocation_distance (plug_start, (new_address - plug_start));
22034 if (last_node && (node_relocation_distance (last_node) ==
22035 (node_relocation_distance (plug_start) +
22036 node_gap_size (plug_start))))
22038 //dprintf(3,( " Lb"));
22039 dprintf (3, ("%Ix Lb", plug_start));
22040 set_node_left (plug_start);
22042 if (0 == sequence_number)
22044 dprintf (2, ("sn: 0, tree is set to %Ix", plug_start));
22048 verify_pins_with_post_plug_info("before insert node");
22050 tree = insert_node (plug_start, ++sequence_number, tree, last_node);
22051 dprintf (3, ("tree is %Ix (b: %Ix) after insert_node", tree, brick_of (tree)));
22052 last_node = plug_start;
22055 // If we detect if the last plug is pinned plug right before us, we should save this gap info
22056 if (!pinned_plug_p)
22058 if (mark_stack_tos > 0)
22060 mark& m = mark_stack_array[mark_stack_tos - 1];
22061 if (m.has_post_plug_info())
22063 uint8_t* post_plug_info_start = m.saved_post_plug_info_start;
22064 size_t* current_plug_gap_start = (size_t*)(plug_start - sizeof (plug_and_gap));
22065 if ((uint8_t*)current_plug_gap_start == post_plug_info_start)
22067 dprintf (3, ("Ginfo: %Ix, %Ix, %Ix",
22068 *current_plug_gap_start, *(current_plug_gap_start + 1),
22069 *(current_plug_gap_start + 2)));
22070 memcpy (&(m.saved_post_plug_debug), current_plug_gap_start, sizeof (gap_reloc_pair));
22077 verify_pins_with_post_plug_info("after insert node");
22081 if (num_pinned_plugs_in_plug > 1)
22083 dprintf (3, ("more than %Id pinned plugs in this plug", num_pinned_plugs_in_plug));
22090 while ((mark_list_next < mark_list_index) &&
22091 (*mark_list_next <= x))
22095 if ((mark_list_next < mark_list_index)
22096 #ifdef MULTIPLE_HEAPS
22097 && (*mark_list_next < end) //for multiple segments
22098 #endif //MULTIPLE_HEAPS
22100 x = *mark_list_next;
22108 #ifdef BACKGROUND_GC
22109 if (current_c_gc_state == c_gc_state_marking)
22111 assert (recursive_gc_sync::background_running_p());
22112 while ((xl < end) && !marked (xl))
22114 dprintf (4, ("-%Ix-", (size_t)xl));
22115 assert ((size (xl) > 0));
22116 background_object_marked (xl, TRUE);
22117 xl = xl + Align (size (xl));
22122 #endif //BACKGROUND_GC
22124 while ((xl < end) && !marked (xl))
22126 dprintf (4, ("-%Ix-", (size_t)xl));
22127 assert ((size (xl) > 0));
22128 xl = xl + Align (size (xl));
22132 assert (xl <= end);
22138 while (!pinned_plug_que_empty_p())
22140 if (settings.promotion)
22142 uint8_t* pplug = pinned_plug (oldest_pin());
22143 if (in_range_for_segment (pplug, ephemeral_heap_segment))
22145 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
22146 //allocate all of the generation gaps
22147 while (active_new_gen_number > 0)
22149 active_new_gen_number--;
22151 if (active_new_gen_number == (max_generation - 1))
22153 maxgen_pinned_compact_before_advance = generation_pinned_allocation_compact_size (generation_of (max_generation));
22154 if (!demote_gen1_p)
22155 advance_pins_for_demotion (consing_gen);
22158 generation* gen = generation_of (active_new_gen_number);
22159 plan_generation_start (gen, consing_gen, 0);
22161 if (demotion_low == MAX_PTR)
22163 demotion_low = pplug;
22164 dprintf (3, ("end plan: dlow->%Ix", demotion_low));
22167 dprintf (2, ("(%d)gen%d plan start: %Ix",
22168 heap_number, active_new_gen_number, (size_t)generation_plan_allocation_start (gen)));
22169 assert (generation_plan_allocation_start (gen));
22174 if (pinned_plug_que_empty_p())
22177 size_t entry = deque_pinned_plug();
22178 mark* m = pinned_plug_of (entry);
22179 uint8_t* plug = pinned_plug (m);
22180 size_t len = pinned_len (m);
22182 // detect pinned block in different segment (later) than
22183 // allocation segment
22184 heap_segment* nseg = heap_segment_rw (generation_allocation_segment (consing_gen));
22186 while ((plug < generation_allocation_pointer (consing_gen)) ||
22187 (plug >= heap_segment_allocated (nseg)))
22189 assert ((plug < heap_segment_mem (nseg)) ||
22190 (plug > heap_segment_reserved (nseg)));
22191 //adjust the end of the segment to be the end of the plug
22192 assert (generation_allocation_pointer (consing_gen)>=
22193 heap_segment_mem (nseg));
22194 assert (generation_allocation_pointer (consing_gen)<=
22195 heap_segment_committed (nseg));
22197 heap_segment_plan_allocated (nseg) =
22198 generation_allocation_pointer (consing_gen);
22199 //switch allocation segment
22200 nseg = heap_segment_next_rw (nseg);
22201 generation_allocation_segment (consing_gen) = nseg;
22202 //reset the allocation pointer and limits
22203 generation_allocation_pointer (consing_gen) =
22204 heap_segment_mem (nseg);
22207 set_new_pin_info (m, generation_allocation_pointer (consing_gen));
22208 dprintf (2, ("pin %Ix b: %Ix->%Ix", plug, brick_of (plug),
22209 (size_t)(brick_table[brick_of (plug)])));
22211 generation_allocation_pointer (consing_gen) = plug + len;
22212 generation_allocation_limit (consing_gen) =
22213 generation_allocation_pointer (consing_gen);
22214 //Add the size of the pinned plug to the right pinned allocations
22215 //find out which gen this pinned plug came from
22216 int frgn = object_gennum (plug);
22217 if ((frgn != (int)max_generation) && settings.promotion)
22219 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
22224 plan_generation_starts (consing_gen);
22225 print_free_and_plug ("AP");
22228 #ifdef SIMPLE_DPRINTF
22229 for (int gen_idx = 0; gen_idx <= max_generation; gen_idx++)
22231 generation* temp_gen = generation_of (gen_idx);
22232 dynamic_data* temp_dd = dynamic_data_of (gen_idx);
22234 int added_pinning_ratio = 0;
22235 int artificial_pinned_ratio = 0;
22237 if (dd_pinned_survived_size (temp_dd) != 0)
22239 added_pinning_ratio = (int)((float)dd_added_pinned_size (temp_dd) * 100 / (float)dd_pinned_survived_size (temp_dd));
22240 artificial_pinned_ratio = (int)((float)dd_artificial_pinned_survived_size (temp_dd) * 100 / (float)dd_pinned_survived_size (temp_dd));
22243 size_t padding_size =
22245 dd_padding_size (temp_dd);
22248 #endif //SHORT_PLUGS
22249 dprintf (1, ("gen%d: %Ix, %Ix(%Id), NON PIN alloc: %Id, pin com: %Id, sweep: %Id, surv: %Id, pinsurv: %Id(%d%% added, %d%% art), np surv: %Id, pad: %Id",
22251 generation_allocation_start (temp_gen),
22252 generation_plan_allocation_start (temp_gen),
22253 (size_t)(generation_plan_allocation_start (temp_gen) - generation_allocation_start (temp_gen)),
22254 generation_allocation_size (temp_gen),
22255 generation_pinned_allocation_compact_size (temp_gen),
22256 generation_pinned_allocation_sweep_size (temp_gen),
22257 dd_survived_size (temp_dd),
22258 dd_pinned_survived_size (temp_dd),
22259 added_pinning_ratio,
22260 artificial_pinned_ratio,
22261 (dd_survived_size (temp_dd) - dd_pinned_survived_size (temp_dd)),
22264 #endif //SIMPLE_DPRINTF
22267 if (settings.condemned_generation == (max_generation - 1 ))
22269 size_t plan_gen2_size = generation_plan_size (max_generation);
22270 size_t growth = plan_gen2_size - old_gen2_size;
22274 dprintf (1, ("gen2 grew %Id (end seg alloc: %Id, gen1 c alloc: %Id",
22275 growth, generation_end_seg_allocated (generation_of (max_generation)),
22276 generation_condemned_allocated (generation_of (max_generation - 1))));
22280 dprintf (1, ("gen2 shrank %Id (end seg alloc: %Id, gen1 c alloc: %Id",
22281 (old_gen2_size - plan_gen2_size), generation_end_seg_allocated (generation_of (max_generation)),
22282 generation_condemned_allocated (generation_of (max_generation - 1))));
22285 generation* older_gen = generation_of (settings.condemned_generation + 1);
22286 size_t rejected_free_space = generation_free_obj_space (older_gen) - r_free_obj_space;
22287 size_t free_list_allocated = generation_free_list_allocated (older_gen) - r_older_gen_free_list_allocated;
22288 size_t end_seg_allocated = generation_end_seg_allocated (older_gen) - r_older_gen_end_seg_allocated;
22289 size_t condemned_allocated = generation_condemned_allocated (older_gen) - r_older_gen_condemned_allocated;
22291 dprintf (1, ("older gen's free alloc: %Id->%Id, seg alloc: %Id->%Id, condemned alloc: %Id->%Id",
22292 r_older_gen_free_list_allocated, generation_free_list_allocated (older_gen),
22293 r_older_gen_end_seg_allocated, generation_end_seg_allocated (older_gen),
22294 r_older_gen_condemned_allocated, generation_condemned_allocated (older_gen)));
22296 dprintf (1, ("this GC did %Id free list alloc(%Id bytes free space rejected), %Id seg alloc and %Id condemned alloc, gen1 condemned alloc is %Id",
22297 free_list_allocated, rejected_free_space, end_seg_allocated,
22298 condemned_allocated, generation_condemned_allocated (generation_of (settings.condemned_generation))));
22300 maxgen_size_increase* maxgen_size_info = &(get_gc_data_per_heap()->maxgen_size_info);
22301 maxgen_size_info->free_list_allocated = free_list_allocated;
22302 maxgen_size_info->free_list_rejected = rejected_free_space;
22303 maxgen_size_info->end_seg_allocated = end_seg_allocated;
22304 maxgen_size_info->condemned_allocated = condemned_allocated;
22305 maxgen_size_info->pinned_allocated = maxgen_pinned_compact_before_advance;
22306 maxgen_size_info->pinned_allocated_advance = generation_pinned_allocation_compact_size (generation_of (max_generation)) - maxgen_pinned_compact_before_advance;
22308 #ifdef FREE_USAGE_STATS
22309 int free_list_efficiency = 0;
22310 if ((free_list_allocated + rejected_free_space) != 0)
22311 free_list_efficiency = (int)(((float) (free_list_allocated) / (float)(free_list_allocated + rejected_free_space)) * (float)100);
22313 int running_free_list_efficiency = (int)(generation_allocator_efficiency(older_gen)*100);
22315 dprintf (1, ("gen%d free list alloc effi: %d%%, current effi: %d%%",
22316 older_gen->gen_num,
22317 free_list_efficiency, running_free_list_efficiency));
22319 dprintf (1, ("gen2 free list change"));
22320 for (int j = 0; j < NUM_GEN_POWER2; j++)
22322 dprintf (1, ("[h%d][#%Id]: 2^%d: F: %Id->%Id(%Id), P: %Id",
22325 (j + 10), r_older_gen_free_space[j], older_gen->gen_free_spaces[j],
22326 (ptrdiff_t)(r_older_gen_free_space[j] - older_gen->gen_free_spaces[j]),
22327 (generation_of(max_generation - 1))->gen_plugs[j]));
22329 #endif //FREE_USAGE_STATS
22332 size_t fragmentation =
22333 generation_fragmentation (generation_of (condemned_gen_number),
22335 heap_segment_allocated (ephemeral_heap_segment));
22337 dprintf (2,("Fragmentation: %Id", fragmentation));
22338 dprintf (2,("---- End of Plan phase ----"));
22341 finish = GetCycleCount32();
22342 plan_time = finish - start;
22345 // We may update write barrier code. We assume here EE has been suspended if we are on a GC thread.
22346 assert(IsGCInProgress());
22348 BOOL should_expand = FALSE;
22349 BOOL should_compact= FALSE;
22350 ephemeral_promotion = FALSE;
22353 if ((!settings.concurrent) &&
22354 ((condemned_gen_number < max_generation) &&
22355 ((settings.gen0_reduction_count > 0) || (settings.entry_memory_load >= 95))))
22357 dprintf (2, ("gen0 reduction count is %d, condemning %d, mem load %d",
22358 settings.gen0_reduction_count,
22359 condemned_gen_number,
22360 settings.entry_memory_load));
22361 should_compact = TRUE;
22363 get_gc_data_per_heap()->set_mechanism (gc_heap_compact,
22364 ((settings.gen0_reduction_count > 0) ? compact_fragmented_gen0 : compact_high_mem_load));
22366 if ((condemned_gen_number >= (max_generation - 1)) &&
22367 dt_low_ephemeral_space_p (tuning_deciding_expansion))
22369 dprintf (2, ("Not enough space for all ephemeral generations with compaction"));
22370 should_expand = TRUE;
22376 should_compact = decide_on_compacting (condemned_gen_number, fragmentation, should_expand);
22381 #ifdef FEATURE_LOH_COMPACTION
22382 loh_compacted_p = FALSE;
22383 #endif //FEATURE_LOH_COMPACTION
22385 if (condemned_gen_number == max_generation)
22387 #ifdef FEATURE_LOH_COMPACTION
22388 if (settings.loh_compaction)
22392 should_compact = TRUE;
22393 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_loh_forced);
22394 loh_compacted_p = TRUE;
22399 if ((heap_number == 0) && (loh_pinned_queue))
22401 loh_pinned_queue_decay--;
22403 if (!loh_pinned_queue_decay)
22405 delete loh_pinned_queue;
22406 loh_pinned_queue = 0;
22411 if (!loh_compacted_p)
22412 #endif //FEATURE_LOH_COMPACTION
22414 GCToEEInterface::DiagWalkLOHSurvivors(__this);
22415 sweep_large_objects();
22420 settings.loh_compaction = FALSE;
22423 #ifdef MULTIPLE_HEAPS
22425 new_heap_segment = NULL;
22427 if (should_compact && should_expand)
22428 gc_policy = policy_expand;
22429 else if (should_compact)
22430 gc_policy = policy_compact;
22432 gc_policy = policy_sweep;
22434 //vote for result of should_compact
22435 dprintf (3, ("Joining for compaction decision"));
22436 gc_t_join.join(this, gc_join_decide_on_compaction);
22437 if (gc_t_join.joined())
22439 //safe place to delete large heap segments
22440 if (condemned_gen_number == max_generation)
22442 for (int i = 0; i < n_heaps; i++)
22444 g_heaps [i]->rearrange_large_heap_segments ();
22448 settings.demotion = FALSE;
22449 int pol_max = policy_sweep;
22450 #ifdef GC_CONFIG_DRIVEN
22451 BOOL is_compaction_mandatory = FALSE;
22452 #endif //GC_CONFIG_DRIVEN
22455 for (i = 0; i < n_heaps; i++)
22457 if (pol_max < g_heaps[i]->gc_policy)
22458 pol_max = policy_compact;
22459 // set the demotion flag is any of the heap has demotion
22460 if (g_heaps[i]->demotion_high >= g_heaps[i]->demotion_low)
22462 (g_heaps[i]->get_gc_data_per_heap())->set_mechanism_bit (gc_demotion_bit);
22463 settings.demotion = TRUE;
22466 #ifdef GC_CONFIG_DRIVEN
22467 if (!is_compaction_mandatory)
22469 int compact_reason = (g_heaps[i]->get_gc_data_per_heap())->get_mechanism (gc_heap_compact);
22470 if (compact_reason >= 0)
22472 if (gc_heap_compact_reason_mandatory_p[compact_reason])
22473 is_compaction_mandatory = TRUE;
22476 #endif //GC_CONFIG_DRIVEN
22479 #ifdef GC_CONFIG_DRIVEN
22480 if (!is_compaction_mandatory)
22482 // If compaction is not mandatory we can feel free to change it to a sweeping GC.
22483 // Note that we may want to change this to only checking every so often instead of every single GC.
22484 if (should_do_sweeping_gc (pol_max >= policy_compact))
22486 pol_max = policy_sweep;
22490 if (pol_max == policy_sweep)
22491 pol_max = policy_compact;
22494 #endif //GC_CONFIG_DRIVEN
22496 for (i = 0; i < n_heaps; i++)
22498 if (pol_max > g_heaps[i]->gc_policy)
22499 g_heaps[i]->gc_policy = pol_max;
22500 //get the segment while we are serialized
22501 if (g_heaps[i]->gc_policy == policy_expand)
22503 g_heaps[i]->new_heap_segment =
22504 g_heaps[i]->soh_get_segment_to_expand();
22505 if (!g_heaps[i]->new_heap_segment)
22507 set_expand_in_full_gc (condemned_gen_number);
22508 //we are out of memory, cancel the expansion
22509 g_heaps[i]->gc_policy = policy_compact;
22514 BOOL is_full_compacting_gc = FALSE;
22516 if ((gc_policy >= policy_compact) && (condemned_gen_number == max_generation))
22518 full_gc_counts[gc_type_compacting]++;
22519 is_full_compacting_gc = TRUE;
22522 for (i = 0; i < n_heaps; i++)
22524 //copy the card and brick tables
22525 if (g_gc_card_table!= g_heaps[i]->card_table)
22527 g_heaps[i]->copy_brick_card_table();
22530 if (is_full_compacting_gc)
22532 g_heaps[i]->loh_alloc_since_cg = 0;
22536 //start all threads on the roots.
22537 dprintf(3, ("Starting all gc threads after compaction decision"));
22538 gc_t_join.restart();
22541 //reset the local variable accordingly
22542 should_compact = (gc_policy >= policy_compact);
22543 should_expand = (gc_policy >= policy_expand);
22545 #else //MULTIPLE_HEAPS
22547 //safe place to delete large heap segments
22548 if (condemned_gen_number == max_generation)
22550 rearrange_large_heap_segments ();
22553 settings.demotion = ((demotion_high >= demotion_low) ? TRUE : FALSE);
22554 if (settings.demotion)
22555 get_gc_data_per_heap()->set_mechanism_bit (gc_demotion_bit);
22557 #ifdef GC_CONFIG_DRIVEN
22558 BOOL is_compaction_mandatory = FALSE;
22559 int compact_reason = get_gc_data_per_heap()->get_mechanism (gc_heap_compact);
22560 if (compact_reason >= 0)
22561 is_compaction_mandatory = gc_heap_compact_reason_mandatory_p[compact_reason];
22563 if (!is_compaction_mandatory)
22565 if (should_do_sweeping_gc (should_compact))
22566 should_compact = FALSE;
22568 should_compact = TRUE;
22570 #endif //GC_CONFIG_DRIVEN
22572 if (should_compact && (condemned_gen_number == max_generation))
22574 full_gc_counts[gc_type_compacting]++;
22575 loh_alloc_since_cg = 0;
22577 #endif //MULTIPLE_HEAPS
22579 if (should_compact)
22581 dprintf (2,( "**** Doing Compacting GC ****"));
22585 #ifndef MULTIPLE_HEAPS
22586 heap_segment* new_heap_segment = soh_get_segment_to_expand();
22587 #endif //!MULTIPLE_HEAPS
22588 if (new_heap_segment)
22590 consing_gen = expand_heap(condemned_gen_number,
22595 // If we couldn't get a new segment, or we were able to
22596 // reserve one but no space to commit, we couldn't
22598 if (ephemeral_heap_segment != new_heap_segment)
22600 set_expand_in_full_gc (condemned_gen_number);
22601 should_expand = FALSE;
22604 generation_allocation_limit (condemned_gen1) =
22605 generation_allocation_pointer (condemned_gen1);
22606 if ((condemned_gen_number < max_generation))
22608 generation_allocator (older_gen)->commit_alloc_list_changes();
22610 // Fix the allocation area of the older generation
22611 fix_older_allocation_area (older_gen);
22613 assert (generation_allocation_segment (consing_gen) ==
22614 ephemeral_heap_segment);
22616 GCToEEInterface::DiagWalkSurvivors(__this);
22618 relocate_phase (condemned_gen_number, first_condemned_address);
22619 compact_phase (condemned_gen_number, first_condemned_address,
22620 (!settings.demotion && settings.promotion));
22621 fix_generation_bounds (condemned_gen_number, consing_gen);
22622 assert (generation_allocation_limit (youngest_generation) ==
22623 generation_allocation_pointer (youngest_generation));
22624 if (condemned_gen_number >= (max_generation -1))
22626 #ifdef MULTIPLE_HEAPS
22627 // this needs be serialized just because we have one
22628 // segment_standby_list/seg_table for all heaps. We should make it at least
22629 // so that when hoarding is not on we don't need this join because
22630 // decommitting memory can take a long time.
22631 //must serialize on deleting segments
22632 gc_t_join.join(this, gc_join_rearrange_segs_compaction);
22633 if (gc_t_join.joined())
22635 for (int i = 0; i < n_heaps; i++)
22637 g_heaps[i]->rearrange_heap_segments(TRUE);
22639 gc_t_join.restart();
22642 rearrange_heap_segments(TRUE);
22643 #endif //MULTIPLE_HEAPS
22647 //fix the start_segment for the ephemeral generations
22648 for (int i = 0; i < max_generation; i++)
22650 generation* gen = generation_of (i);
22651 generation_start_segment (gen) = ephemeral_heap_segment;
22652 generation_allocation_segment (gen) = ephemeral_heap_segment;
22658 #ifdef FEATURE_PREMORTEM_FINALIZATION
22659 finalize_queue->UpdatePromotedGenerations (condemned_gen_number,
22660 (!settings.demotion && settings.promotion));
22661 #endif // FEATURE_PREMORTEM_FINALIZATION
22663 #ifdef MULTIPLE_HEAPS
22664 dprintf(3, ("Joining after end of compaction"));
22665 gc_t_join.join(this, gc_join_adjust_handle_age_compact);
22666 if (gc_t_join.joined())
22667 #endif //MULTIPLE_HEAPS
22669 #ifdef MULTIPLE_HEAPS
22670 //join all threads to make sure they are synchronized
22671 dprintf(3, ("Restarting after Promotion granted"));
22672 gc_t_join.restart();
22673 #endif //MULTIPLE_HEAPS
22677 sc.thread_number = heap_number;
22678 sc.promotion = FALSE;
22679 sc.concurrent = FALSE;
22680 // new generations bounds are set can call this guy
22681 if (settings.promotion && !settings.demotion)
22683 dprintf (2, ("Promoting EE roots for gen %d",
22684 condemned_gen_number));
22685 GCScan::GcPromotionsGranted(condemned_gen_number,
22686 max_generation, &sc);
22688 else if (settings.demotion)
22690 dprintf (2, ("Demoting EE roots for gen %d",
22691 condemned_gen_number));
22692 GCScan::GcDemote (condemned_gen_number, max_generation, &sc);
22697 gen0_big_free_spaces = 0;
22699 reset_pinned_queue_bos();
22700 unsigned int gen_number = min (max_generation, 1 + condemned_gen_number);
22701 generation* gen = generation_of (gen_number);
22702 uint8_t* low = generation_allocation_start (generation_of (gen_number-1));
22703 uint8_t* high = heap_segment_allocated (ephemeral_heap_segment);
22705 while (!pinned_plug_que_empty_p())
22707 mark* m = pinned_plug_of (deque_pinned_plug());
22708 size_t len = pinned_len (m);
22709 uint8_t* arr = (pinned_plug (m) - len);
22710 dprintf(3,("free [%Ix %Ix[ pin",
22711 (size_t)arr, (size_t)arr + len));
22714 assert (len >= Align (min_obj_size));
22715 make_unused_array (arr, len);
22716 // fix fully contained bricks + first one
22717 // if the array goes beyond the first brick
22718 size_t start_brick = brick_of (arr);
22719 size_t end_brick = brick_of (arr + len);
22720 if (end_brick != start_brick)
22723 ("Fixing bricks [%Ix, %Ix[ to point to unused array %Ix",
22724 start_brick, end_brick, (size_t)arr));
22725 set_brick (start_brick,
22726 arr - brick_address (start_brick));
22727 size_t brick = start_brick+1;
22728 while (brick < end_brick)
22730 set_brick (brick, start_brick - brick);
22735 //when we take an old segment to make the new
22736 //ephemeral segment. we can have a bunch of
22737 //pinned plugs out of order going to the new ephemeral seg
22738 //and then the next plugs go back to max_generation
22739 if ((heap_segment_mem (ephemeral_heap_segment) <= arr) &&
22740 (heap_segment_reserved (ephemeral_heap_segment) > arr))
22743 while ((low <= arr) && (high > arr))
22746 assert ((gen_number >= 1) || (demotion_low != MAX_PTR) ||
22747 settings.demotion || !settings.promotion);
22748 dprintf (3, ("new free list generation %d", gen_number));
22750 gen = generation_of (gen_number);
22751 if (gen_number >= 1)
22752 low = generation_allocation_start (generation_of (gen_number-1));
22759 dprintf (3, ("new free list generation %d", max_generation));
22760 gen_number = max_generation;
22761 gen = generation_of (gen_number);
22764 dprintf(3,("threading it into generation %d", gen_number));
22765 thread_gap (arr, len, gen);
22766 add_gen_free (gen_number, len);
22772 for (int x = 0; x <= max_generation; x++)
22774 assert (generation_allocation_start (generation_of (x)));
22778 if (!settings.demotion && settings.promotion)
22780 //clear card for generation 1. generation 0 is empty
22781 clear_card_for_addresses (
22782 generation_allocation_start (generation_of (1)),
22783 generation_allocation_start (generation_of (0)));
22785 if (settings.promotion && !settings.demotion)
22787 uint8_t* start = generation_allocation_start (youngest_generation);
22788 MAYBE_UNUSED_VAR(start);
22789 assert (heap_segment_allocated (ephemeral_heap_segment) ==
22790 (start + Align (size (start))));
22795 //force promotion for sweep
22796 settings.promotion = TRUE;
22797 settings.compaction = FALSE;
22800 sc.thread_number = heap_number;
22801 sc.promotion = FALSE;
22802 sc.concurrent = FALSE;
22804 dprintf (2, ("**** Doing Mark and Sweep GC****"));
22806 if ((condemned_gen_number < max_generation))
22808 generation_allocator (older_gen)->copy_from_alloc_list (r_free_list);
22809 generation_free_list_space (older_gen) = r_free_list_space;
22810 generation_free_obj_space (older_gen) = r_free_obj_space;
22811 generation_free_list_allocated (older_gen) = r_older_gen_free_list_allocated;
22812 generation_end_seg_allocated (older_gen) = r_older_gen_end_seg_allocated;
22813 generation_condemned_allocated (older_gen) = r_older_gen_condemned_allocated;
22814 generation_allocation_limit (older_gen) = r_allocation_limit;
22815 generation_allocation_pointer (older_gen) = r_allocation_pointer;
22816 generation_allocation_context_start_region (older_gen) = r_allocation_start_region;
22817 generation_allocation_segment (older_gen) = r_allocation_segment;
22820 if ((condemned_gen_number < max_generation))
22822 // Fix the allocation area of the older generation
22823 fix_older_allocation_area (older_gen);
22826 GCToEEInterface::DiagWalkSurvivors(__this);
22828 gen0_big_free_spaces = 0;
22829 make_free_lists (condemned_gen_number);
22830 recover_saved_pinned_info();
22832 #ifdef FEATURE_PREMORTEM_FINALIZATION
22833 finalize_queue->UpdatePromotedGenerations (condemned_gen_number, TRUE);
22834 #endif // FEATURE_PREMORTEM_FINALIZATION
22835 // MTHTS: leave single thread for HT processing on plan_phase
22836 #ifdef MULTIPLE_HEAPS
22837 dprintf(3, ("Joining after end of sweep"));
22838 gc_t_join.join(this, gc_join_adjust_handle_age_sweep);
22839 if (gc_t_join.joined())
22840 #endif //MULTIPLE_HEAPS
22842 GCScan::GcPromotionsGranted(condemned_gen_number,
22843 max_generation, &sc);
22844 if (condemned_gen_number >= (max_generation -1))
22846 #ifdef MULTIPLE_HEAPS
22847 for (int i = 0; i < n_heaps; i++)
22849 g_heaps[i]->rearrange_heap_segments(FALSE);
22852 rearrange_heap_segments(FALSE);
22853 #endif //MULTIPLE_HEAPS
22856 #ifdef MULTIPLE_HEAPS
22857 //join all threads to make sure they are synchronized
22858 dprintf(3, ("Restarting after Promotion granted"));
22859 gc_t_join.restart();
22860 #endif //MULTIPLE_HEAPS
22864 for (int x = 0; x <= max_generation; x++)
22866 assert (generation_allocation_start (generation_of (x)));
22870 //clear card for generation 1. generation 0 is empty
22871 clear_card_for_addresses (
22872 generation_allocation_start (generation_of (1)),
22873 generation_allocation_start (generation_of (0)));
22874 assert ((heap_segment_allocated (ephemeral_heap_segment) ==
22875 (generation_allocation_start (youngest_generation) +
22876 Align (min_obj_size))));
22879 //verify_partial();
22882 #pragma warning(pop)
22886 /*****************************
22887 Called after compact phase to fix all generation gaps
22888 ********************************/
22889 void gc_heap::fix_generation_bounds (int condemned_gen_number,
22890 generation* consing_gen)
22892 UNREFERENCED_PARAMETER(consing_gen);
22894 assert (generation_allocation_segment (consing_gen) ==
22895 ephemeral_heap_segment);
22897 //assign the planned allocation start to the generation
22898 int gen_number = condemned_gen_number;
22899 int bottom_gen = 0;
22901 while (gen_number >= bottom_gen)
22903 generation* gen = generation_of (gen_number);
22904 dprintf(3,("Fixing generation pointers for %Ix", gen_number));
22905 if ((gen_number < max_generation) && ephemeral_promotion)
22907 make_unused_array (saved_ephemeral_plan_start[gen_number],
22908 saved_ephemeral_plan_start_size[gen_number]);
22910 reset_allocation_pointers (gen, generation_plan_allocation_start (gen));
22911 make_unused_array (generation_allocation_start (gen), generation_plan_allocation_start_size (gen));
22912 dprintf(3,(" start %Ix", (size_t)generation_allocation_start (gen)));
22915 #ifdef MULTIPLE_HEAPS
22916 if (ephemeral_promotion)
22918 //we are creating a generation fault. set the cards.
22919 // and we are only doing this for multiple heaps because in the single heap scenario the
22920 // new ephemeral generations will be empty and there'll be no need to set cards for the
22921 // old ephemeral generations that got promoted into max_generation.
22922 ptrdiff_t delta = 0;
22923 #ifdef SEG_MAPPING_TABLE
22924 heap_segment* old_ephemeral_seg = seg_mapping_table_segment_of (saved_ephemeral_plan_start[max_generation-1]);
22925 #else //SEG_MAPPING_TABLE
22926 heap_segment* old_ephemeral_seg = segment_of (saved_ephemeral_plan_start[max_generation-1], delta);
22927 #endif //SEG_MAPPING_TABLE
22929 assert (in_range_for_segment (saved_ephemeral_plan_start[max_generation-1], old_ephemeral_seg));
22930 size_t end_card = card_of (align_on_card (heap_segment_plan_allocated (old_ephemeral_seg)));
22931 size_t card = card_of (saved_ephemeral_plan_start[max_generation-1]);
22932 while (card != end_card)
22938 #endif //MULTIPLE_HEAPS
22940 alloc_allocated = heap_segment_plan_allocated(ephemeral_heap_segment);
22941 //reset the allocated size
22942 uint8_t* start = generation_allocation_start (youngest_generation);
22943 MAYBE_UNUSED_VAR(start);
22944 if (settings.promotion && !settings.demotion)
22946 assert ((start + Align (size (start))) ==
22947 heap_segment_plan_allocated(ephemeral_heap_segment));
22950 heap_segment_allocated(ephemeral_heap_segment)=
22951 heap_segment_plan_allocated(ephemeral_heap_segment);
22955 uint8_t* gc_heap::generation_limit (int gen_number)
22957 if (settings.promotion)
22959 if (gen_number <= 1)
22960 return heap_segment_reserved (ephemeral_heap_segment);
22962 return generation_allocation_start (generation_of ((gen_number - 2)));
22966 if (gen_number <= 0)
22967 return heap_segment_reserved (ephemeral_heap_segment);
22969 return generation_allocation_start (generation_of ((gen_number - 1)));
22973 BOOL gc_heap::ensure_gap_allocation (int condemned_gen_number)
22975 uint8_t* start = heap_segment_allocated (ephemeral_heap_segment);
22976 size_t size = Align (min_obj_size)*(condemned_gen_number+1);
22977 assert ((start + size) <=
22978 heap_segment_reserved (ephemeral_heap_segment));
22979 if ((start + size) >
22980 heap_segment_committed (ephemeral_heap_segment))
22982 if (!grow_heap_segment (ephemeral_heap_segment, start + size))
22990 uint8_t* gc_heap::allocate_at_end (size_t size)
22992 uint8_t* start = heap_segment_allocated (ephemeral_heap_segment);
22993 size = Align (size);
22994 uint8_t* result = start;
22995 // only called to allocate a min obj so can't overflow here.
22996 assert ((start + size) <=
22997 heap_segment_reserved (ephemeral_heap_segment));
22998 //ensure_gap_allocation took care of it
22999 assert ((start + size) <=
23000 heap_segment_committed (ephemeral_heap_segment));
23001 heap_segment_allocated (ephemeral_heap_segment) += size;
23006 void gc_heap::make_free_lists (int condemned_gen_number)
23011 start = GetCycleCount32();
23014 //Promotion has to happen in sweep case.
23015 assert (settings.promotion);
23017 generation* condemned_gen = generation_of (condemned_gen_number);
23018 uint8_t* start_address = generation_allocation_start (condemned_gen);
23020 size_t current_brick = brick_of (start_address);
23021 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
23023 PREFIX_ASSUME(current_heap_segment != NULL);
23025 uint8_t* end_address = heap_segment_allocated (current_heap_segment);
23026 size_t end_brick = brick_of (end_address-1);
23027 make_free_args args;
23028 args.free_list_gen_number = min (max_generation, 1 + condemned_gen_number);
23029 args.current_gen_limit = (((condemned_gen_number == max_generation)) ?
23031 (generation_limit (args.free_list_gen_number)));
23032 args.free_list_gen = generation_of (args.free_list_gen_number);
23033 args.highest_plug = 0;
23035 if ((start_address < end_address) ||
23036 (condemned_gen_number == max_generation))
23040 if ((current_brick > end_brick))
23042 if (args.current_gen_limit == MAX_PTR)
23044 //We had an empty segment
23045 //need to allocate the generation start
23047 generation* gen = generation_of (max_generation);
23049 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
23051 PREFIX_ASSUME(start_seg != NULL);
23053 uint8_t* gap = heap_segment_mem (start_seg);
23055 generation_allocation_start (gen) = gap;
23056 heap_segment_allocated (start_seg) = gap + Align (min_obj_size);
23057 make_unused_array (gap, Align (min_obj_size));
23058 reset_allocation_pointers (gen, gap);
23059 dprintf (3, ("Start segment empty, fixing generation start of %d to: %Ix",
23060 max_generation, (size_t)gap));
23061 args.current_gen_limit = generation_limit (args.free_list_gen_number);
23063 if (heap_segment_next_rw (current_heap_segment))
23065 current_heap_segment = heap_segment_next_rw (current_heap_segment);
23066 current_brick = brick_of (heap_segment_mem (current_heap_segment));
23067 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
23077 int brick_entry = brick_table [ current_brick ];
23078 if ((brick_entry >= 0))
23080 make_free_list_in_brick (brick_address (current_brick) + brick_entry-1, &args);
23081 dprintf(3,("Fixing brick entry %Ix to %Ix",
23082 current_brick, (size_t)args.highest_plug));
23083 set_brick (current_brick,
23084 (args.highest_plug - brick_address (current_brick)));
23088 if ((brick_entry > -32768))
23092 ptrdiff_t offset = brick_of (args.highest_plug) - current_brick;
23093 if ((brick_entry != -32767) && (! ((offset == brick_entry))))
23095 assert ((brick_entry == -1));
23098 //init to -1 for faster find_first_object
23099 set_brick (current_brick, -1);
23107 int bottom_gen = 0;
23108 args.free_list_gen_number--;
23109 while (args.free_list_gen_number >= bottom_gen)
23112 generation* gen2 = generation_of (args.free_list_gen_number);
23113 gap = allocate_at_end (Align(min_obj_size));
23114 generation_allocation_start (gen2) = gap;
23115 reset_allocation_pointers (gen2, gap);
23116 dprintf(3,("Fixing generation start of %d to: %Ix",
23117 args.free_list_gen_number, (size_t)gap));
23118 PREFIX_ASSUME(gap != NULL);
23119 make_unused_array (gap, Align (min_obj_size));
23121 args.free_list_gen_number--;
23124 //reset the allocated size
23125 uint8_t* start2 = generation_allocation_start (youngest_generation);
23126 alloc_allocated = start2 + Align (size (start2));
23130 finish = GetCycleCount32();
23131 sweep_time = finish - start;
23135 void gc_heap::make_free_list_in_brick (uint8_t* tree, make_free_args* args)
23137 assert ((tree != NULL));
23139 int right_node = node_right_child (tree);
23140 int left_node = node_left_child (tree);
23141 args->highest_plug = 0;
23144 if (! (0 == left_node))
23146 make_free_list_in_brick (tree + left_node, args);
23150 uint8_t* plug = tree;
23151 size_t gap_size = node_gap_size (tree);
23152 uint8_t* gap = (plug - gap_size);
23153 dprintf (3,("Making free list %Ix len %d in %d",
23154 //dprintf (3,("F: %Ix len %Ix in %d",
23155 (size_t)gap, gap_size, args->free_list_gen_number));
23156 args->highest_plug = tree;
23158 if (is_plug_padded (plug))
23160 dprintf (3, ("%Ix padded", plug));
23161 clear_plug_padded (plug);
23163 #endif //SHORT_PLUGS
23166 if ((args->current_gen_limit == MAX_PTR) ||
23167 ((plug >= args->current_gen_limit) &&
23168 ephemeral_pointer_p (plug)))
23170 dprintf(3,(" Crossing Generation boundary at %Ix",
23171 (size_t)args->current_gen_limit));
23172 if (!(args->current_gen_limit == MAX_PTR))
23174 args->free_list_gen_number--;
23175 args->free_list_gen = generation_of (args->free_list_gen_number);
23177 dprintf(3,( " Fixing generation start of %d to: %Ix",
23178 args->free_list_gen_number, (size_t)gap));
23180 reset_allocation_pointers (args->free_list_gen, gap);
23181 args->current_gen_limit = generation_limit (args->free_list_gen_number);
23183 if ((gap_size >= (2*Align (min_obj_size))))
23185 dprintf(3,(" Splitting the gap in two %Id left",
23187 make_unused_array (gap, Align(min_obj_size));
23188 gap_size = (gap_size - Align(min_obj_size));
23189 gap = (gap + Align(min_obj_size));
23193 make_unused_array (gap, gap_size);
23200 thread_gap (gap, gap_size, args->free_list_gen);
23201 add_gen_free (args->free_list_gen->gen_num, gap_size);
23203 if (! (0 == right_node))
23205 make_free_list_in_brick (tree + right_node, args);
23211 void gc_heap::thread_gap (uint8_t* gap_start, size_t size, generation* gen)
23213 assert (generation_allocation_start (gen));
23216 if ((gen->gen_num == 0) && (size > CLR_SIZE))
23218 gen0_big_free_spaces += size;
23221 assert ((heap_segment_rw (generation_start_segment (gen))!=
23222 ephemeral_heap_segment) ||
23223 (gap_start > generation_allocation_start (gen)));
23224 // The beginning of a segment gap is not aligned
23225 assert (size >= Align (min_obj_size));
23226 make_unused_array (gap_start, size,
23227 (!settings.concurrent && (gen != youngest_generation)),
23228 (gen->gen_num == max_generation));
23229 dprintf (3, ("fr: [%Ix, %Ix[", (size_t)gap_start, (size_t)gap_start+size));
23231 if ((size >= min_free_list))
23233 generation_free_list_space (gen) += size;
23234 generation_allocator (gen)->thread_item (gap_start, size);
23238 generation_free_obj_space (gen) += size;
23243 void gc_heap::loh_thread_gap_front (uint8_t* gap_start, size_t size, generation* gen)
23245 assert (generation_allocation_start (gen));
23246 if (size >= min_free_list)
23248 generation_free_list_space (gen) += size;
23249 generation_allocator (gen)->thread_item_front (gap_start, size);
23253 void gc_heap::make_unused_array (uint8_t* x, size_t size, BOOL clearp, BOOL resetp)
23255 dprintf (3, ("Making unused array [%Ix, %Ix[",
23256 (size_t)x, (size_t)(x+size)));
23257 assert (size >= Align (min_obj_size));
23259 //#if defined (VERIFY_HEAP) && defined (BACKGROUND_GC)
23260 // check_batch_mark_array_bits (x, x+size);
23261 //#endif //VERIFY_HEAP && BACKGROUND_GC
23264 reset_memory (x, size);
23266 ((CObjectHeader*)x)->SetFree(size);
23271 #error "This won't work on big endian platforms"
23274 size_t size_as_object = (uint32_t)(size - free_object_base_size) + free_object_base_size;
23276 if (size_as_object < size)
23279 // If the size is more than 4GB, we need to create multiple objects because of
23280 // the Array::m_NumComponents is uint32_t and the high 32 bits of unused array
23281 // size is ignored in regular object size computation.
23283 uint8_t * tmp = x + size_as_object;
23284 size_t remaining_size = size - size_as_object;
23286 while (remaining_size > UINT32_MAX)
23288 // Make sure that there will be at least Align(min_obj_size) left
23289 size_t current_size = UINT32_MAX - get_alignment_constant (FALSE)
23290 - Align (min_obj_size, get_alignment_constant (FALSE));
23292 ((CObjectHeader*)tmp)->SetFree(current_size);
23294 remaining_size -= current_size;
23295 tmp += current_size;
23298 ((CObjectHeader*)tmp)->SetFree(remaining_size);
23303 clear_card_for_addresses (x, x + Align(size));
23306 // Clear memory set by make_unused_array.
23307 void gc_heap::clear_unused_array (uint8_t* x, size_t size)
23309 // Also clear the sync block
23310 *(((PTR_PTR)x)-1) = 0;
23312 ((CObjectHeader*)x)->UnsetFree();
23317 #error "This won't work on big endian platforms"
23320 // The memory could have been cleared in the meantime. We have to mirror the algorithm
23321 // from make_unused_array since we cannot depend on the object sizes in memory.
23322 size_t size_as_object = (uint32_t)(size - free_object_base_size) + free_object_base_size;
23324 if (size_as_object < size)
23326 uint8_t * tmp = x + size_as_object;
23327 size_t remaining_size = size - size_as_object;
23329 while (remaining_size > UINT32_MAX)
23331 size_t current_size = UINT32_MAX - get_alignment_constant (FALSE)
23332 - Align (min_obj_size, get_alignment_constant (FALSE));
23334 ((CObjectHeader*)tmp)->UnsetFree();
23336 remaining_size -= current_size;
23337 tmp += current_size;
23340 ((CObjectHeader*)tmp)->UnsetFree();
23343 UNREFERENCED_PARAMETER(size);
23348 uint8_t* tree_search (uint8_t* tree, uint8_t* old_address)
23350 uint8_t* candidate = 0;
23354 if (tree < old_address)
23356 if ((cn = node_right_child (tree)) != 0)
23358 assert (candidate < tree);
23361 Prefetch (tree - 8);
23367 else if (tree > old_address)
23369 if ((cn = node_left_child (tree)) != 0)
23372 Prefetch (tree - 8);
23380 if (tree <= old_address)
23382 else if (candidate)
23388 #ifdef FEATURE_BASICFREEZE
23389 bool gc_heap::frozen_object_p (Object* obj)
23391 heap_segment* pSegment = gc_heap::find_segment ((uint8_t*)obj, FALSE);
23392 _ASSERTE(pSegment);
23394 return heap_segment_read_only_p(pSegment);
23396 #endif // FEATURE_BASICFREEZE
23398 #ifdef FEATURE_REDHAWK
23399 // TODO: this was added on RH, we have not done perf runs to see if this is the right
23400 // thing to do for other versions of the CLR.
23402 #endif // FEATURE_REDHAWK
23403 void gc_heap::relocate_address (uint8_t** pold_address THREAD_NUMBER_DCL)
23405 uint8_t* old_address = *pold_address;
23406 if (!((old_address >= gc_low) && (old_address < gc_high)))
23407 #ifdef MULTIPLE_HEAPS
23409 UNREFERENCED_PARAMETER(thread);
23410 if (old_address == 0)
23412 gc_heap* hp = heap_of (old_address);
23413 if ((hp == this) ||
23414 !((old_address >= hp->gc_low) && (old_address < hp->gc_high)))
23417 #else //MULTIPLE_HEAPS
23419 #endif //MULTIPLE_HEAPS
23420 // delta translates old_address into address_gc (old_address);
23421 size_t brick = brick_of (old_address);
23422 int brick_entry = brick_table [ brick ];
23423 uint8_t* new_address = old_address;
23424 if (! ((brick_entry == 0)))
23428 while (brick_entry < 0)
23430 brick = (brick + brick_entry);
23431 brick_entry = brick_table [ brick ];
23433 uint8_t* old_loc = old_address;
23435 uint8_t* node = tree_search ((brick_address (brick) + brick_entry-1),
23437 if ((node <= old_loc))
23438 new_address = (old_address + node_relocation_distance (node));
23441 if (node_left_p (node))
23443 dprintf(3,(" L: %Ix", (size_t)node));
23444 new_address = (old_address +
23445 (node_relocation_distance (node) +
23446 node_gap_size (node)));
23451 brick_entry = brick_table [ brick ];
23457 *pold_address = new_address;
23461 #ifdef FEATURE_LOH_COMPACTION
23462 if (loh_compacted_p
23463 #ifdef FEATURE_BASICFREEZE
23464 && !frozen_object_p((Object*)old_address)
23465 #endif // FEATURE_BASICFREEZE
23468 *pold_address = old_address + loh_node_relocation_distance (old_address);
23471 #endif //FEATURE_LOH_COMPACTION
23473 *pold_address = new_address;
23478 gc_heap::check_class_object_demotion (uint8_t* obj)
23480 #ifdef COLLECTIBLE_CLASS
23481 if (is_collectible(obj))
23483 check_class_object_demotion_internal (obj);
23486 UNREFERENCED_PARAMETER(obj);
23487 #endif //COLLECTIBLE_CLASS
23490 #ifdef COLLECTIBLE_CLASS
23492 gc_heap::check_class_object_demotion_internal (uint8_t* obj)
23494 if (settings.demotion)
23496 #ifdef MULTIPLE_HEAPS
23497 // We set the card without checking the demotion range 'cause at this point
23498 // the handle that points to the loader allocator object may or may not have
23499 // been relocated by other GC threads.
23500 set_card (card_of (obj));
23503 uint8_t* class_obj = get_class_object (obj);
23504 dprintf (3, ("%Ix: got classobj %Ix", obj, class_obj));
23505 uint8_t* temp_class_obj = class_obj;
23506 uint8_t** temp = &temp_class_obj;
23507 relocate_address (temp THREAD_NUMBER_ARG);
23509 check_demotion_helper (temp, obj);
23510 #endif //MULTIPLE_HEAPS
23514 #endif //COLLECTIBLE_CLASS
23517 gc_heap::check_demotion_helper (uint8_t** pval, uint8_t* parent_obj)
23519 // detect if we are demoting an object
23520 if ((*pval < demotion_high) &&
23521 (*pval >= demotion_low))
23523 dprintf(3, ("setting card %Ix:%Ix",
23524 card_of((uint8_t*)pval),
23527 set_card (card_of (parent_obj));
23529 #ifdef MULTIPLE_HEAPS
23530 else if (settings.demotion)
23532 dprintf (4, ("Demotion active, computing heap_of object"));
23533 gc_heap* hp = heap_of (*pval);
23534 if ((*pval < hp->demotion_high) &&
23535 (*pval >= hp->demotion_low))
23537 dprintf(3, ("setting card %Ix:%Ix",
23538 card_of((uint8_t*)pval),
23541 set_card (card_of (parent_obj));
23544 #endif //MULTIPLE_HEAPS
23548 gc_heap::reloc_survivor_helper (uint8_t** pval)
23551 relocate_address (pval THREAD_NUMBER_ARG);
23553 check_demotion_helper (pval, (uint8_t*)pval);
23557 gc_heap::relocate_obj_helper (uint8_t* x, size_t s)
23560 if (contain_pointers (x))
23562 dprintf (3, ("$%Ix$", (size_t)x));
23564 go_through_object_nostart (method_table(x), x, s, pval,
23566 uint8_t* child = *pval;
23567 reloc_survivor_helper (pval);
23570 dprintf (3, ("%Ix->%Ix->%Ix", (uint8_t*)pval, child, *pval));
23575 check_class_object_demotion (x);
23579 void gc_heap::reloc_ref_in_shortened_obj (uint8_t** address_to_set_card, uint8_t** address_to_reloc)
23583 uint8_t* old_val = (address_to_reloc ? *address_to_reloc : 0);
23584 relocate_address (address_to_reloc THREAD_NUMBER_ARG);
23585 if (address_to_reloc)
23587 dprintf (3, ("SR %Ix: %Ix->%Ix", (uint8_t*)address_to_reloc, old_val, *address_to_reloc));
23590 //check_demotion_helper (current_saved_info_to_relocate, (uint8_t*)pval);
23591 uint8_t* relocated_addr = *address_to_reloc;
23592 if ((relocated_addr < demotion_high) &&
23593 (relocated_addr >= demotion_low))
23595 dprintf (3, ("set card for location %Ix(%Ix)",
23596 (size_t)address_to_set_card, card_of((uint8_t*)address_to_set_card)));
23598 set_card (card_of ((uint8_t*)address_to_set_card));
23600 #ifdef MULTIPLE_HEAPS
23601 else if (settings.demotion)
23603 gc_heap* hp = heap_of (relocated_addr);
23604 if ((relocated_addr < hp->demotion_high) &&
23605 (relocated_addr >= hp->demotion_low))
23607 dprintf (3, ("%Ix on h%d, set card for location %Ix(%Ix)",
23608 relocated_addr, hp->heap_number, (size_t)address_to_set_card, card_of((uint8_t*)address_to_set_card)));
23610 set_card (card_of ((uint8_t*)address_to_set_card));
23613 #endif //MULTIPLE_HEAPS
23616 void gc_heap::relocate_pre_plug_info (mark* pinned_plug_entry)
23619 uint8_t* plug = pinned_plug (pinned_plug_entry);
23620 uint8_t* pre_plug_start = plug - sizeof (plug_and_gap);
23621 // Note that we need to add one ptr size here otherwise we may not be able to find the relocated
23622 // address. Consider this scenario:
23623 // gen1 start | 3-ptr sized NP | PP
23625 // If we are asking for the reloc address of 0x10 we will AV in relocate_address because
23626 // the first plug we saw in the brick is 0x18 which means 0x10 will cause us to go back a brick
23627 // which is 0, and then we'll AV in tree_search when we try to do node_right_child (tree).
23628 pre_plug_start += sizeof (uint8_t*);
23629 uint8_t** old_address = &pre_plug_start;
23631 uint8_t* old_val = (old_address ? *old_address : 0);
23632 relocate_address (old_address THREAD_NUMBER_ARG);
23635 dprintf (3, ("PreR %Ix: %Ix->%Ix, set reloc: %Ix",
23636 (uint8_t*)old_address, old_val, *old_address, (pre_plug_start - sizeof (uint8_t*))));
23639 pinned_plug_entry->set_pre_plug_info_reloc_start (pre_plug_start - sizeof (uint8_t*));
23643 void gc_heap::relocate_shortened_obj_helper (uint8_t* x, size_t s, uint8_t* end, mark* pinned_plug_entry, BOOL is_pinned)
23646 uint8_t* plug = pinned_plug (pinned_plug_entry);
23650 //// Temporary - we just wanna make sure we are doing things right when padding is needed.
23651 //if ((x + s) < plug)
23653 // dprintf (3, ("obj %Ix needed padding: end %Ix is %d bytes from pinned obj %Ix",
23654 // x, (x + s), (plug- (x + s)), plug));
23655 // GCToOSInterface::DebugBreak();
23658 relocate_pre_plug_info (pinned_plug_entry);
23661 verify_pins_with_post_plug_info("after relocate_pre_plug_info");
23663 uint8_t* saved_plug_info_start = 0;
23664 uint8_t** saved_info_to_relocate = 0;
23668 saved_plug_info_start = (uint8_t*)(pinned_plug_entry->get_post_plug_info_start());
23669 saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_post_plug_reloc_info());
23673 saved_plug_info_start = (plug - sizeof (plug_and_gap));
23674 saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_pre_plug_reloc_info());
23677 uint8_t** current_saved_info_to_relocate = 0;
23678 uint8_t* child = 0;
23680 dprintf (3, ("x: %Ix, pp: %Ix, end: %Ix", x, plug, end));
23682 if (contain_pointers (x))
23684 dprintf (3,("$%Ix$", (size_t)x));
23686 go_through_object_nostart (method_table(x), x, s, pval,
23688 dprintf (3, ("obj %Ix, member: %Ix->%Ix", x, (uint8_t*)pval, *pval));
23690 if ((uint8_t*)pval >= end)
23692 current_saved_info_to_relocate = saved_info_to_relocate + ((uint8_t*)pval - saved_plug_info_start) / sizeof (uint8_t**);
23693 child = *current_saved_info_to_relocate;
23694 reloc_ref_in_shortened_obj (pval, current_saved_info_to_relocate);
23695 dprintf (3, ("last part: R-%Ix(saved: %Ix)->%Ix ->%Ix",
23696 (uint8_t*)pval, current_saved_info_to_relocate, child, *current_saved_info_to_relocate));
23700 reloc_survivor_helper (pval);
23705 check_class_object_demotion (x);
23708 void gc_heap::relocate_survivor_helper (uint8_t* plug, uint8_t* plug_end)
23711 while (x < plug_end)
23713 size_t s = size (x);
23714 uint8_t* next_obj = x + Align (s);
23715 Prefetch (next_obj);
23716 relocate_obj_helper (x, s);
23722 // if we expanded, right now we are not handling it as We are not saving the new reloc info.
23723 void gc_heap::verify_pins_with_post_plug_info (const char* msg)
23725 #if defined (_DEBUG) && defined (VERIFY_HEAP)
23726 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
23728 if (!verify_pinned_queue_p)
23731 if (settings.heap_expansion)
23734 for (size_t i = 0; i < mark_stack_tos; i++)
23736 mark& m = mark_stack_array[i];
23738 mark* pinned_plug_entry = pinned_plug_of(i);
23740 if (pinned_plug_entry->has_post_plug_info() &&
23741 pinned_plug_entry->post_short_p() &&
23742 (pinned_plug_entry->saved_post_plug_debug.gap != 1))
23744 uint8_t* next_obj = pinned_plug_entry->get_post_plug_info_start() + sizeof (plug_and_gap);
23745 // object after pin
23746 dprintf (3, ("OFP: %Ix, G: %Ix, R: %Ix, LC: %d, RC: %d",
23747 next_obj, node_gap_size (next_obj), node_relocation_distance (next_obj),
23748 (int)node_left_child (next_obj), (int)node_right_child (next_obj)));
23750 size_t* post_plug_debug = (size_t*)(&m.saved_post_plug_debug);
23752 if (node_gap_size (next_obj) != *post_plug_debug)
23754 dprintf (3, ("obj: %Ix gap should be %Ix but it is %Ix",
23755 next_obj, *post_plug_debug, (size_t)(node_gap_size (next_obj))));
23759 // can't do node_relocation_distance here as it clears the left bit.
23760 //if (node_relocation_distance (next_obj) != *post_plug_debug)
23761 if (*((size_t*)(next_obj - 3 * sizeof (size_t))) != *post_plug_debug)
23763 dprintf (3, ("obj: %Ix reloc should be %Ix but it is %Ix",
23764 next_obj, *post_plug_debug, (size_t)(node_relocation_distance (next_obj))));
23767 if (node_left_child (next_obj) > 0)
23769 dprintf (3, ("obj: %Ix, vLC: %d\n", next_obj, (int)(node_left_child (next_obj))));
23775 dprintf (3, ("%s verified", msg));
23777 #else // _DEBUG && VERIFY_HEAP
23778 UNREFERENCED_PARAMETER(msg);
23779 #endif // _DEBUG && VERIFY_HEAP
23782 #ifdef COLLECTIBLE_CLASS
23783 // We don't want to burn another ptr size space for pinned plugs to record this so just
23784 // set the card unconditionally for collectible objects if we are demoting.
23786 gc_heap::unconditional_set_card_collectible (uint8_t* obj)
23788 if (settings.demotion)
23790 set_card (card_of (obj));
23793 #endif //COLLECTIBLE_CLASS
23795 void gc_heap::relocate_shortened_survivor_helper (uint8_t* plug, uint8_t* plug_end, mark* pinned_plug_entry)
23798 uint8_t* p_plug = pinned_plug (pinned_plug_entry);
23799 BOOL is_pinned = (plug == p_plug);
23800 BOOL check_short_obj_p = (is_pinned ? pinned_plug_entry->post_short_p() : pinned_plug_entry->pre_short_p());
23802 plug_end += sizeof (gap_reloc_pair);
23804 //dprintf (3, ("%s %Ix is shortened, and last object %s overwritten", (is_pinned ? "PP" : "NP"), plug, (check_short_obj_p ? "is" : "is not")));
23805 dprintf (3, ("%s %Ix-%Ix short, LO: %s OW", (is_pinned ? "PP" : "NP"), plug, plug_end, (check_short_obj_p ? "is" : "is not")));
23807 verify_pins_with_post_plug_info("begin reloc short surv");
23809 while (x < plug_end)
23811 if (check_short_obj_p && ((plug_end - x) < min_pre_pin_obj_size))
23813 dprintf (3, ("last obj %Ix is short", x));
23817 #ifdef COLLECTIBLE_CLASS
23818 if (pinned_plug_entry->post_short_collectible_p())
23819 unconditional_set_card_collectible (x);
23820 #endif //COLLECTIBLE_CLASS
23822 // Relocate the saved references based on bits set.
23823 uint8_t** saved_plug_info_start = (uint8_t**)(pinned_plug_entry->get_post_plug_info_start());
23824 uint8_t** saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_post_plug_reloc_info());
23825 for (size_t i = 0; i < pinned_plug_entry->get_max_short_bits(); i++)
23827 if (pinned_plug_entry->post_short_bit_p (i))
23829 reloc_ref_in_shortened_obj ((saved_plug_info_start + i), (saved_info_to_relocate + i));
23835 #ifdef COLLECTIBLE_CLASS
23836 if (pinned_plug_entry->pre_short_collectible_p())
23837 unconditional_set_card_collectible (x);
23838 #endif //COLLECTIBLE_CLASS
23840 relocate_pre_plug_info (pinned_plug_entry);
23842 // Relocate the saved references based on bits set.
23843 uint8_t** saved_plug_info_start = (uint8_t**)(p_plug - sizeof (plug_and_gap));
23844 uint8_t** saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_pre_plug_reloc_info());
23845 for (size_t i = 0; i < pinned_plug_entry->get_max_short_bits(); i++)
23847 if (pinned_plug_entry->pre_short_bit_p (i))
23849 reloc_ref_in_shortened_obj ((saved_plug_info_start + i), (saved_info_to_relocate + i));
23857 size_t s = size (x);
23858 uint8_t* next_obj = x + Align (s);
23859 Prefetch (next_obj);
23861 if (next_obj >= plug_end)
23863 dprintf (3, ("object %Ix is at the end of the plug %Ix->%Ix",
23864 next_obj, plug, plug_end));
23866 verify_pins_with_post_plug_info("before reloc short obj");
23868 relocate_shortened_obj_helper (x, s, (x + Align (s) - sizeof (plug_and_gap)), pinned_plug_entry, is_pinned);
23872 relocate_obj_helper (x, s);
23879 verify_pins_with_post_plug_info("end reloc short surv");
23882 void gc_heap::relocate_survivors_in_plug (uint8_t* plug, uint8_t* plug_end,
23883 BOOL check_last_object_p,
23884 mark* pinned_plug_entry)
23886 //dprintf(3,("Relocating pointers in Plug [%Ix,%Ix[", (size_t)plug, (size_t)plug_end));
23887 dprintf (3,("RP: [%Ix,%Ix[", (size_t)plug, (size_t)plug_end));
23889 if (check_last_object_p)
23891 relocate_shortened_survivor_helper (plug, plug_end, pinned_plug_entry);
23895 relocate_survivor_helper (plug, plug_end);
23899 void gc_heap::relocate_survivors_in_brick (uint8_t* tree, relocate_args* args)
23901 assert ((tree != NULL));
23903 dprintf (3, ("tree: %Ix, args->last_plug: %Ix, left: %Ix, right: %Ix, gap(t): %Ix",
23904 tree, args->last_plug,
23905 (tree + node_left_child (tree)),
23906 (tree + node_right_child (tree)),
23907 node_gap_size (tree)));
23909 if (node_left_child (tree))
23911 relocate_survivors_in_brick (tree + node_left_child (tree), args);
23914 uint8_t* plug = tree;
23915 BOOL has_post_plug_info_p = FALSE;
23916 BOOL has_pre_plug_info_p = FALSE;
23918 if (tree == oldest_pinned_plug)
23920 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
23921 &has_post_plug_info_p);
23922 assert (tree == pinned_plug (args->pinned_plug_entry));
23924 dprintf (3, ("tree is the oldest pin: %Ix", tree));
23926 if (args->last_plug)
23928 size_t gap_size = node_gap_size (tree);
23929 uint8_t* gap = (plug - gap_size);
23930 dprintf (3, ("tree: %Ix, gap: %Ix (%Ix)", tree, gap, gap_size));
23931 assert (gap_size >= Align (min_obj_size));
23932 uint8_t* last_plug_end = gap;
23934 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
23937 relocate_survivors_in_plug (args->last_plug, last_plug_end, check_last_object_p, args->pinned_plug_entry);
23942 assert (!has_pre_plug_info_p);
23945 args->last_plug = plug;
23946 args->is_shortened = has_post_plug_info_p;
23947 if (has_post_plug_info_p)
23949 dprintf (3, ("setting %Ix as shortened", plug));
23951 dprintf (3, ("last_plug: %Ix(shortened: %d)", plug, (args->is_shortened ? 1 : 0)));
23953 if (node_right_child (tree))
23955 relocate_survivors_in_brick (tree + node_right_child (tree), args);
23960 void gc_heap::update_oldest_pinned_plug()
23962 oldest_pinned_plug = (pinned_plug_que_empty_p() ? 0 : pinned_plug (oldest_pin()));
23965 void gc_heap::relocate_survivors (int condemned_gen_number,
23966 uint8_t* first_condemned_address)
23968 generation* condemned_gen = generation_of (condemned_gen_number);
23969 uint8_t* start_address = first_condemned_address;
23970 size_t current_brick = brick_of (start_address);
23971 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
23973 PREFIX_ASSUME(current_heap_segment != NULL);
23975 uint8_t* end_address = 0;
23977 reset_pinned_queue_bos();
23978 update_oldest_pinned_plug();
23980 end_address = heap_segment_allocated (current_heap_segment);
23982 size_t end_brick = brick_of (end_address - 1);
23983 relocate_args args;
23985 args.high = gc_high;
23986 args.is_shortened = FALSE;
23987 args.pinned_plug_entry = 0;
23988 args.last_plug = 0;
23991 if (current_brick > end_brick)
23993 if (args.last_plug)
23996 assert (!(args.is_shortened));
23997 relocate_survivors_in_plug (args.last_plug,
23998 heap_segment_allocated (current_heap_segment),
24000 args.pinned_plug_entry);
24003 args.last_plug = 0;
24006 if (heap_segment_next_rw (current_heap_segment))
24008 current_heap_segment = heap_segment_next_rw (current_heap_segment);
24009 current_brick = brick_of (heap_segment_mem (current_heap_segment));
24010 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24019 int brick_entry = brick_table [ current_brick ];
24021 if (brick_entry >= 0)
24023 relocate_survivors_in_brick (brick_address (current_brick) +
24032 void gc_heap::walk_plug (uint8_t* plug, size_t size, BOOL check_last_object_p, walk_relocate_args* args)
24034 if (check_last_object_p)
24036 size += sizeof (gap_reloc_pair);
24037 mark* entry = args->pinned_plug_entry;
24039 if (args->is_shortened)
24041 assert (entry->has_post_plug_info());
24042 entry->swap_post_plug_and_saved_for_profiler();
24046 assert (entry->has_pre_plug_info());
24047 entry->swap_pre_plug_and_saved_for_profiler();
24051 ptrdiff_t last_plug_relocation = node_relocation_distance (plug);
24052 STRESS_LOG_PLUG_MOVE(plug, (plug + size), -last_plug_relocation);
24053 ptrdiff_t reloc = settings.compaction ? last_plug_relocation : 0;
24055 (args->fn) (plug, (plug + size), reloc, args->profiling_context, !!settings.compaction, false);
24057 if (check_last_object_p)
24059 mark* entry = args->pinned_plug_entry;
24061 if (args->is_shortened)
24063 entry->swap_post_plug_and_saved_for_profiler();
24067 entry->swap_pre_plug_and_saved_for_profiler();
24072 void gc_heap::walk_relocation_in_brick (uint8_t* tree, walk_relocate_args* args)
24074 assert ((tree != NULL));
24075 if (node_left_child (tree))
24077 walk_relocation_in_brick (tree + node_left_child (tree), args);
24080 uint8_t* plug = tree;
24081 BOOL has_pre_plug_info_p = FALSE;
24082 BOOL has_post_plug_info_p = FALSE;
24084 if (tree == oldest_pinned_plug)
24086 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
24087 &has_post_plug_info_p);
24088 assert (tree == pinned_plug (args->pinned_plug_entry));
24091 if (args->last_plug != 0)
24093 size_t gap_size = node_gap_size (tree);
24094 uint8_t* gap = (plug - gap_size);
24095 uint8_t* last_plug_end = gap;
24096 size_t last_plug_size = (last_plug_end - args->last_plug);
24097 dprintf (3, ("tree: %Ix, last_plug: %Ix, gap: %Ix(%Ix), last_plug_end: %Ix, size: %Ix",
24098 tree, args->last_plug, gap, gap_size, last_plug_end, last_plug_size));
24100 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
24101 if (!check_last_object_p)
24103 assert (last_plug_size >= Align (min_obj_size));
24106 walk_plug (args->last_plug, last_plug_size, check_last_object_p, args);
24110 assert (!has_pre_plug_info_p);
24113 dprintf (3, ("set args last plug to plug: %Ix", plug));
24114 args->last_plug = plug;
24115 args->is_shortened = has_post_plug_info_p;
24117 if (node_right_child (tree))
24119 walk_relocation_in_brick (tree + node_right_child (tree), args);
24123 void gc_heap::walk_relocation (void* profiling_context, record_surv_fn fn)
24125 generation* condemned_gen = generation_of (settings.condemned_generation);
24126 uint8_t* start_address = generation_allocation_start (condemned_gen);
24127 size_t current_brick = brick_of (start_address);
24128 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
24130 PREFIX_ASSUME(current_heap_segment != NULL);
24132 reset_pinned_queue_bos();
24133 update_oldest_pinned_plug();
24134 size_t end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24135 walk_relocate_args args;
24136 args.is_shortened = FALSE;
24137 args.pinned_plug_entry = 0;
24138 args.last_plug = 0;
24139 args.profiling_context = profiling_context;
24144 if (current_brick > end_brick)
24146 if (args.last_plug)
24148 walk_plug (args.last_plug,
24149 (heap_segment_allocated (current_heap_segment) - args.last_plug),
24152 args.last_plug = 0;
24154 if (heap_segment_next_rw (current_heap_segment))
24156 current_heap_segment = heap_segment_next_rw (current_heap_segment);
24157 current_brick = brick_of (heap_segment_mem (current_heap_segment));
24158 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24167 int brick_entry = brick_table [ current_brick ];
24168 if (brick_entry >= 0)
24170 walk_relocation_in_brick (brick_address (current_brick) +
24179 void gc_heap::walk_survivors (record_surv_fn fn, void* context, walk_surv_type type)
24181 if (type == walk_for_gc)
24182 walk_survivors_relocation (context, fn);
24183 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24184 else if (type == walk_for_bgc)
24185 walk_survivors_for_bgc (context, fn);
24186 #endif //BACKGROUND_GC && FEATURE_EVENT_TRACE
24187 else if (type == walk_for_loh)
24188 walk_survivors_for_loh (context, fn);
24190 assert (!"unknown type!");
24193 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24194 void gc_heap::walk_survivors_for_bgc (void* profiling_context, record_surv_fn fn)
24196 // This should only be called for BGCs
24197 assert(settings.concurrent);
24199 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
24201 BOOL small_object_segments = TRUE;
24202 int align_const = get_alignment_constant (small_object_segments);
24208 if (small_object_segments)
24210 //switch to large segment
24211 small_object_segments = FALSE;
24213 align_const = get_alignment_constant (small_object_segments);
24214 seg = heap_segment_rw (generation_start_segment (large_object_generation));
24216 PREFIX_ASSUME(seg != NULL);
24224 uint8_t* o = heap_segment_mem (seg);
24225 uint8_t* end = heap_segment_allocated (seg);
24229 if (method_table(o) == g_gc_pFreeObjectMethodTable)
24231 o += Align (size (o), align_const);
24235 // It's survived. Make a fake plug, starting at o,
24236 // and send the event
24238 uint8_t* plug_start = o;
24240 while (method_table(o) != g_gc_pFreeObjectMethodTable)
24242 o += Align (size (o), align_const);
24249 uint8_t* plug_end = o;
24253 0, // Reloc distance == 0 as this is non-compacting
24255 false, // Non-compacting
24259 seg = heap_segment_next (seg);
24262 #endif // defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24264 void gc_heap::relocate_phase (int condemned_gen_number,
24265 uint8_t* first_condemned_address)
24268 sc.thread_number = heap_number;
24269 sc.promotion = FALSE;
24270 sc.concurrent = FALSE;
24276 start = GetCycleCount32();
24279 // %type% category = quote (relocate);
24280 dprintf (2,("---- Relocate phase -----"));
24282 #ifdef MULTIPLE_HEAPS
24283 //join all threads to make sure they are synchronized
24284 dprintf(3, ("Joining after end of plan"));
24285 gc_t_join.join(this, gc_join_begin_relocate_phase);
24286 if (gc_t_join.joined())
24287 #endif //MULTIPLE_HEAPS
24290 #ifdef MULTIPLE_HEAPS
24292 //join all threads to make sure they are synchronized
24293 dprintf(3, ("Restarting for relocation"));
24294 gc_t_join.restart();
24295 #endif //MULTIPLE_HEAPS
24298 dprintf(3,("Relocating roots"));
24299 GCScan::GcScanRoots(GCHeap::Relocate,
24300 condemned_gen_number, max_generation, &sc);
24302 verify_pins_with_post_plug_info("after reloc stack");
24304 #ifdef BACKGROUND_GC
24305 if (recursive_gc_sync::background_running_p())
24307 scan_background_roots (GCHeap::Relocate, heap_number, &sc);
24309 #endif //BACKGROUND_GC
24311 if (condemned_gen_number != max_generation)
24313 dprintf(3,("Relocating cross generation pointers"));
24314 mark_through_cards_for_segments (&gc_heap::relocate_address, TRUE);
24315 verify_pins_with_post_plug_info("after reloc cards");
24317 if (condemned_gen_number != max_generation)
24319 dprintf(3,("Relocating cross generation pointers for large objects"));
24320 mark_through_cards_for_large_objects (&gc_heap::relocate_address, TRUE);
24324 #ifdef FEATURE_LOH_COMPACTION
24325 if (loh_compacted_p)
24327 assert (settings.condemned_generation == max_generation);
24328 relocate_in_loh_compact();
24331 #endif //FEATURE_LOH_COMPACTION
24333 relocate_in_large_objects ();
24337 dprintf(3,("Relocating survivors"));
24338 relocate_survivors (condemned_gen_number,
24339 first_condemned_address);
24342 #ifdef FEATURE_PREMORTEM_FINALIZATION
24343 dprintf(3,("Relocating finalization data"));
24344 finalize_queue->RelocateFinalizationData (condemned_gen_number,
24346 #endif // FEATURE_PREMORTEM_FINALIZATION
24351 dprintf(3,("Relocating handle table"));
24352 GCScan::GcScanHandles(GCHeap::Relocate,
24353 condemned_gen_number, max_generation, &sc);
24356 #ifdef MULTIPLE_HEAPS
24357 //join all threads to make sure they are synchronized
24358 dprintf(3, ("Joining after end of relocation"));
24359 gc_t_join.join(this, gc_join_relocate_phase_done);
24361 #endif //MULTIPLE_HEAPS
24364 finish = GetCycleCount32();
24365 reloc_time = finish - start;
24368 dprintf(2,( "---- End of Relocate phase ----"));
24371 // This compares to see if tree is the current pinned plug and returns info
24372 // for this pinned plug. Also advances the pinned queue if that's the case.
24374 // We don't change the values of the plug info if tree is not the same as
24375 // the current pinned plug - the caller is responsible for setting the right
24376 // values to begin with.
24378 // POPO TODO: We are keeping this temporarily as this is also used by realloc
24379 // where it passes FALSE to deque_p, change it to use the same optimization
24380 // as relocate. Not as essential since realloc is already a slow path.
24381 mark* gc_heap::get_next_pinned_entry (uint8_t* tree,
24382 BOOL* has_pre_plug_info_p,
24383 BOOL* has_post_plug_info_p,
24386 if (!pinned_plug_que_empty_p())
24388 mark* oldest_entry = oldest_pin();
24389 uint8_t* oldest_plug = pinned_plug (oldest_entry);
24390 if (tree == oldest_plug)
24392 *has_pre_plug_info_p = oldest_entry->has_pre_plug_info();
24393 *has_post_plug_info_p = oldest_entry->has_post_plug_info();
24397 deque_pinned_plug();
24400 dprintf (3, ("found a pinned plug %Ix, pre: %d, post: %d",
24402 (*has_pre_plug_info_p ? 1 : 0),
24403 (*has_post_plug_info_p ? 1 : 0)));
24405 return oldest_entry;
24412 // This also deques the oldest entry and update the oldest plug
24413 mark* gc_heap::get_oldest_pinned_entry (BOOL* has_pre_plug_info_p,
24414 BOOL* has_post_plug_info_p)
24416 mark* oldest_entry = oldest_pin();
24417 *has_pre_plug_info_p = oldest_entry->has_pre_plug_info();
24418 *has_post_plug_info_p = oldest_entry->has_post_plug_info();
24420 deque_pinned_plug();
24421 update_oldest_pinned_plug();
24422 return oldest_entry;
24426 void gc_heap::copy_cards_range (uint8_t* dest, uint8_t* src, size_t len, BOOL copy_cards_p)
24429 copy_cards_for_addresses (dest, src, len);
24431 clear_card_for_addresses (dest, dest + len);
24434 // POPO TODO: We should actually just recover the artifically made gaps here..because when we copy
24435 // we always copy the earlier plugs first which means we won't need the gap sizes anymore. This way
24436 // we won't need to individually recover each overwritten part of plugs.
24438 void gc_heap::gcmemcopy (uint8_t* dest, uint8_t* src, size_t len, BOOL copy_cards_p)
24442 #ifdef BACKGROUND_GC
24443 if (current_c_gc_state == c_gc_state_marking)
24445 //TODO: should look to see whether we should consider changing this
24446 // to copy a consecutive region of the mark array instead.
24447 copy_mark_bits_for_addresses (dest, src, len);
24449 #endif //BACKGROUND_GC
24450 //dprintf(3,(" Memcopy [%Ix->%Ix, %Ix->%Ix[", (size_t)src, (size_t)dest, (size_t)src+len, (size_t)dest+len));
24451 dprintf(3,(" mc: [%Ix->%Ix, %Ix->%Ix[", (size_t)src, (size_t)dest, (size_t)src+len, (size_t)dest+len));
24452 memcopy (dest - plug_skew, src - plug_skew, len);
24453 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
24454 if (SoftwareWriteWatch::IsEnabledForGCHeap())
24456 // The ranges [src - plug_kew .. src[ and [src + len - plug_skew .. src + len[ are ObjHeaders, which don't have GC
24457 // references, and are not relevant for write watch. The latter range actually corresponds to the ObjHeader for the
24458 // object at (src + len), so it can be ignored anyway.
24459 SoftwareWriteWatch::SetDirtyRegion(dest, len - plug_skew);
24461 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
24462 copy_cards_range (dest, src, len, copy_cards_p);
24466 void gc_heap::compact_plug (uint8_t* plug, size_t size, BOOL check_last_object_p, compact_args* args)
24469 uint8_t* reloc_plug = plug + args->last_plug_relocation;
24471 if (check_last_object_p)
24473 size += sizeof (gap_reloc_pair);
24474 mark* entry = args->pinned_plug_entry;
24476 if (args->is_shortened)
24478 assert (entry->has_post_plug_info());
24479 entry->swap_post_plug_and_saved();
24483 assert (entry->has_pre_plug_info());
24484 entry->swap_pre_plug_and_saved();
24488 int old_brick_entry = brick_table [brick_of (plug)];
24490 assert (node_relocation_distance (plug) == args->last_plug_relocation);
24492 #ifdef FEATURE_STRUCTALIGN
24493 ptrdiff_t alignpad = node_alignpad(plug);
24496 make_unused_array (reloc_plug - alignpad, alignpad);
24497 if (brick_of (reloc_plug - alignpad) != brick_of (reloc_plug))
24499 // The alignment padding is straddling one or more bricks;
24500 // it has to be the last "object" of its first brick.
24501 fix_brick_to_highest (reloc_plug - alignpad, reloc_plug);
24504 #else // FEATURE_STRUCTALIGN
24505 size_t unused_arr_size = 0;
24506 BOOL already_padded_p = FALSE;
24508 if (is_plug_padded (plug))
24510 already_padded_p = TRUE;
24511 clear_plug_padded (plug);
24512 unused_arr_size = Align (min_obj_size);
24514 #endif //SHORT_PLUGS
24515 if (node_realigned (plug))
24517 unused_arr_size += switch_alignment_size (already_padded_p);
24520 if (unused_arr_size != 0)
24522 make_unused_array (reloc_plug - unused_arr_size, unused_arr_size);
24524 if (brick_of (reloc_plug - unused_arr_size) != brick_of (reloc_plug))
24526 dprintf (3, ("fix B for padding: %Id: %Ix->%Ix",
24527 unused_arr_size, (reloc_plug - unused_arr_size), reloc_plug));
24528 // The alignment padding is straddling one or more bricks;
24529 // it has to be the last "object" of its first brick.
24530 fix_brick_to_highest (reloc_plug - unused_arr_size, reloc_plug);
24533 #endif // FEATURE_STRUCTALIGN
24536 if (is_plug_padded (plug))
24538 make_unused_array (reloc_plug - Align (min_obj_size), Align (min_obj_size));
24540 if (brick_of (reloc_plug - Align (min_obj_size)) != brick_of (reloc_plug))
24542 // The alignment padding is straddling one or more bricks;
24543 // it has to be the last "object" of its first brick.
24544 fix_brick_to_highest (reloc_plug - Align (min_obj_size), reloc_plug);
24547 #endif //SHORT_PLUGS
24549 gcmemcopy (reloc_plug, plug, size, args->copy_cards_p);
24551 if (args->check_gennum_p)
24553 int src_gennum = args->src_gennum;
24554 if (src_gennum == -1)
24556 src_gennum = object_gennum (plug);
24559 int dest_gennum = object_gennum_plan (reloc_plug);
24561 if (src_gennum < dest_gennum)
24563 generation_allocation_size (generation_of (dest_gennum)) += size;
24567 size_t current_reloc_brick = args->current_compacted_brick;
24569 if (brick_of (reloc_plug) != current_reloc_brick)
24571 dprintf (3, ("last reloc B: %Ix, current reloc B: %Ix",
24572 current_reloc_brick, brick_of (reloc_plug)));
24574 if (args->before_last_plug)
24576 dprintf (3,(" fixing last brick %Ix to point to last plug %Ix(%Ix)",
24577 current_reloc_brick,
24578 args->before_last_plug,
24579 (args->before_last_plug - brick_address (current_reloc_brick))));
24582 set_brick (current_reloc_brick,
24583 args->before_last_plug - brick_address (current_reloc_brick));
24586 current_reloc_brick = brick_of (reloc_plug);
24588 size_t end_brick = brick_of (reloc_plug + size-1);
24589 if (end_brick != current_reloc_brick)
24591 // The plug is straddling one or more bricks
24592 // It has to be the last plug of its first brick
24593 dprintf (3,("plug spanning multiple bricks, fixing first brick %Ix to %Ix(%Ix)",
24594 current_reloc_brick, (size_t)reloc_plug,
24595 (reloc_plug - brick_address (current_reloc_brick))));
24598 set_brick (current_reloc_brick,
24599 reloc_plug - brick_address (current_reloc_brick));
24601 // update all intervening brick
24602 size_t brick = current_reloc_brick + 1;
24603 dprintf (3,("setting intervening bricks %Ix->%Ix to -1",
24604 brick, (end_brick - 1)));
24605 while (brick < end_brick)
24607 set_brick (brick, -1);
24610 // code last brick offset as a plug address
24611 args->before_last_plug = brick_address (end_brick) -1;
24612 current_reloc_brick = end_brick;
24613 dprintf (3, ("setting before last to %Ix, last brick to %Ix",
24614 args->before_last_plug, current_reloc_brick));
24618 dprintf (3, ("still in the same brick: %Ix", end_brick));
24619 args->before_last_plug = reloc_plug;
24621 args->current_compacted_brick = current_reloc_brick;
24623 if (check_last_object_p)
24625 mark* entry = args->pinned_plug_entry;
24627 if (args->is_shortened)
24629 entry->swap_post_plug_and_saved();
24633 entry->swap_pre_plug_and_saved();
24638 void gc_heap::compact_in_brick (uint8_t* tree, compact_args* args)
24640 assert (tree != NULL);
24641 int left_node = node_left_child (tree);
24642 int right_node = node_right_child (tree);
24643 ptrdiff_t relocation = node_relocation_distance (tree);
24649 dprintf (3, ("B: L: %d->%Ix", left_node, (tree + left_node)));
24650 compact_in_brick ((tree + left_node), args);
24653 uint8_t* plug = tree;
24654 BOOL has_pre_plug_info_p = FALSE;
24655 BOOL has_post_plug_info_p = FALSE;
24657 if (tree == oldest_pinned_plug)
24659 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
24660 &has_post_plug_info_p);
24661 assert (tree == pinned_plug (args->pinned_plug_entry));
24664 if (args->last_plug != 0)
24666 size_t gap_size = node_gap_size (tree);
24667 uint8_t* gap = (plug - gap_size);
24668 uint8_t* last_plug_end = gap;
24669 size_t last_plug_size = (last_plug_end - args->last_plug);
24670 dprintf (3, ("tree: %Ix, last_plug: %Ix, gap: %Ix(%Ix), last_plug_end: %Ix, size: %Ix",
24671 tree, args->last_plug, gap, gap_size, last_plug_end, last_plug_size));
24673 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
24674 if (!check_last_object_p)
24676 assert (last_plug_size >= Align (min_obj_size));
24679 compact_plug (args->last_plug, last_plug_size, check_last_object_p, args);
24683 assert (!has_pre_plug_info_p);
24686 dprintf (3, ("set args last plug to plug: %Ix, reloc: %Ix", plug, relocation));
24687 args->last_plug = plug;
24688 args->last_plug_relocation = relocation;
24689 args->is_shortened = has_post_plug_info_p;
24693 dprintf (3, ("B: R: %d->%Ix", right_node, (tree + right_node)));
24694 compact_in_brick ((tree + right_node), args);
24698 void gc_heap::recover_saved_pinned_info()
24700 reset_pinned_queue_bos();
24702 while (!(pinned_plug_que_empty_p()))
24704 mark* oldest_entry = oldest_pin();
24705 oldest_entry->recover_plug_info();
24706 #ifdef GC_CONFIG_DRIVEN
24707 if (oldest_entry->has_pre_plug_info() && oldest_entry->has_post_plug_info())
24708 record_interesting_data_point (idp_pre_and_post_pin);
24709 else if (oldest_entry->has_pre_plug_info())
24710 record_interesting_data_point (idp_pre_pin);
24711 else if (oldest_entry->has_post_plug_info())
24712 record_interesting_data_point (idp_post_pin);
24713 #endif //GC_CONFIG_DRIVEN
24715 deque_pinned_plug();
24719 void gc_heap::compact_phase (int condemned_gen_number,
24720 uint8_t* first_condemned_address,
24723 // %type% category = quote (compact);
24727 start = GetCycleCount32();
24729 generation* condemned_gen = generation_of (condemned_gen_number);
24730 uint8_t* start_address = first_condemned_address;
24731 size_t current_brick = brick_of (start_address);
24732 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
24734 PREFIX_ASSUME(current_heap_segment != NULL);
24736 reset_pinned_queue_bos();
24737 update_oldest_pinned_plug();
24739 BOOL reused_seg = expand_reused_seg_p();
24742 for (int i = 1; i <= max_generation; i++)
24744 generation_allocation_size (generation_of (i)) = 0;
24748 uint8_t* end_address = heap_segment_allocated (current_heap_segment);
24750 size_t end_brick = brick_of (end_address-1);
24752 args.last_plug = 0;
24753 args.before_last_plug = 0;
24754 args.current_compacted_brick = ~((size_t)1);
24755 args.is_shortened = FALSE;
24756 args.pinned_plug_entry = 0;
24757 args.copy_cards_p = (condemned_gen_number >= 1) || !clear_cards;
24758 args.check_gennum_p = reused_seg;
24759 if (args.check_gennum_p)
24761 args.src_gennum = ((current_heap_segment == ephemeral_heap_segment) ? -1 : 2);
24764 dprintf (2,("---- Compact Phase: %Ix(%Ix)----",
24765 first_condemned_address, brick_of (first_condemned_address)));
24767 #ifdef MULTIPLE_HEAPS
24769 if (gc_t_join.joined())
24771 #endif //MULTIPLE_HEAPS
24773 #ifdef MULTIPLE_HEAPS
24774 dprintf(3, ("Restarting for compaction"));
24775 gc_t_join.restart();
24777 #endif //MULTIPLE_HEAPS
24779 reset_pinned_queue_bos();
24781 #ifdef FEATURE_LOH_COMPACTION
24782 if (loh_compacted_p)
24786 #endif //FEATURE_LOH_COMPACTION
24788 if ((start_address < end_address) ||
24789 (condemned_gen_number == max_generation))
24793 if (current_brick > end_brick)
24795 if (args.last_plug != 0)
24797 dprintf (3, ("compacting last plug: %Ix", args.last_plug))
24798 compact_plug (args.last_plug,
24799 (heap_segment_allocated (current_heap_segment) - args.last_plug),
24804 if (heap_segment_next_rw (current_heap_segment))
24806 current_heap_segment = heap_segment_next_rw (current_heap_segment);
24807 current_brick = brick_of (heap_segment_mem (current_heap_segment));
24808 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24809 args.last_plug = 0;
24810 if (args.check_gennum_p)
24812 args.src_gennum = ((current_heap_segment == ephemeral_heap_segment) ? -1 : 2);
24818 if (args.before_last_plug !=0)
24820 dprintf (3, ("Fixing last brick %Ix to point to plug %Ix",
24821 args.current_compacted_brick, (size_t)args.before_last_plug));
24822 assert (args.current_compacted_brick != ~1u);
24823 set_brick (args.current_compacted_brick,
24824 args.before_last_plug - brick_address (args.current_compacted_brick));
24830 int brick_entry = brick_table [ current_brick ];
24831 dprintf (3, ("B: %Ix(%Ix)->%Ix",
24832 current_brick, (size_t)brick_entry, (brick_address (current_brick) + brick_entry - 1)));
24834 if (brick_entry >= 0)
24836 compact_in_brick ((brick_address (current_brick) + brick_entry -1),
24845 recover_saved_pinned_info();
24848 finish = GetCycleCount32();
24849 compact_time = finish - start;
24852 concurrent_print_time_delta ("compact end");
24854 dprintf(2,("---- End of Compact phase ----"));
24857 #ifdef MULTIPLE_HEAPS
24860 #pragma warning(push)
24861 #pragma warning(disable:4702) // C4702: unreachable code: gc_thread_function may not return
24863 void gc_heap::gc_thread_stub (void* arg)
24865 gc_heap* heap = (gc_heap*)arg;
24866 if (!gc_thread_no_affinitize_p)
24868 GCThreadAffinity affinity;
24869 affinity.Group = GCThreadAffinity::None;
24870 affinity.Processor = GCThreadAffinity::None;
24872 // We are about to set affinity for GC threads. It is a good place to set up NUMA and
24873 // CPU groups because the process mask, processor number, and group number are all
24874 // readily available.
24875 if (GCToOSInterface::CanEnableGCCPUGroups())
24876 set_thread_group_affinity_for_heap(heap->heap_number, &affinity);
24878 set_thread_affinity_mask_for_heap(heap->heap_number, &affinity);
24880 if (!GCToOSInterface::SetThreadAffinity(&affinity))
24882 dprintf(1, ("Failed to set thread affinity for server GC thread"));
24886 // server GC threads run at a higher priority than normal.
24887 GCToOSInterface::BoostThreadPriority();
24888 _alloca (256*heap->heap_number);
24889 heap->gc_thread_function();
24892 #pragma warning(pop)
24895 #endif //MULTIPLE_HEAPS
24897 #ifdef BACKGROUND_GC
24900 #pragma warning(push)
24901 #pragma warning(disable:4702) // C4702: unreachable code: gc_thread_function may not return
24903 void gc_heap::bgc_thread_stub (void* arg)
24905 gc_heap* heap = (gc_heap*)arg;
24906 heap->bgc_thread = GCToEEInterface::GetThread();
24907 assert(heap->bgc_thread != nullptr);
24908 heap->bgc_thread_function();
24911 #pragma warning(pop)
24914 #endif //BACKGROUND_GC
24916 /*------------------ Background GC ----------------------------*/
24918 #ifdef BACKGROUND_GC
24920 void gc_heap::background_drain_mark_list (int thread)
24922 UNREFERENCED_PARAMETER(thread);
24924 size_t saved_c_mark_list_index = c_mark_list_index;
24926 if (saved_c_mark_list_index)
24928 concurrent_print_time_delta ("SML");
24930 while (c_mark_list_index != 0)
24932 size_t current_index = c_mark_list_index - 1;
24933 uint8_t* o = c_mark_list [current_index];
24934 background_mark_object (o THREAD_NUMBER_ARG);
24935 c_mark_list_index--;
24937 if (saved_c_mark_list_index)
24940 concurrent_print_time_delta ("EML");
24943 fire_drain_mark_list_event (saved_c_mark_list_index);
24947 // The background GC version of scan_dependent_handles (see that method for a more in-depth comment).
24948 #ifdef MULTIPLE_HEAPS
24949 // Since we only scan dependent handles while we are stopped we'll never interfere with FGCs scanning
24950 // them. So we can use the same static variables.
24951 void gc_heap::background_scan_dependent_handles (ScanContext *sc)
24953 // Whenever we call this method there may have been preceding object promotions. So set
24954 // s_fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
24955 // based on the how the scanning proceeded).
24956 s_fUnscannedPromotions = TRUE;
24958 // We don't know how many times we need to loop yet. In particular we can't base the loop condition on
24959 // the state of this thread's portion of the dependent handle table. That's because promotions on other
24960 // threads could cause handle promotions to become necessary here. Even if there are definitely no more
24961 // promotions possible in this thread's handles, we still have to stay in lock-step with those worker
24962 // threads that haven't finished yet (each GC worker thread has to join exactly the same number of times
24963 // as all the others or they'll get out of step).
24966 // The various worker threads are all currently racing in this code. We need to work out if at least
24967 // one of them think they have work to do this cycle. Each thread needs to rescan its portion of the
24968 // dependent handle table when both of the following conditions apply:
24969 // 1) At least one (arbitrary) object might have been promoted since the last scan (because if this
24970 // object happens to correspond to a primary in one of our handles we might potentially have to
24971 // promote the associated secondary).
24972 // 2) The table for this thread has at least one handle with a secondary that isn't promoted yet.
24974 // The first condition is represented by s_fUnscannedPromotions. This is always non-zero for the first
24975 // iteration of this loop (see comment above) and in subsequent cycles each thread updates this
24976 // whenever a mark stack overflow occurs or scanning their dependent handles results in a secondary
24977 // being promoted. This value is cleared back to zero in a synchronized fashion in the join that
24978 // follows below. Note that we can't read this outside of the join since on any iteration apart from
24979 // the first threads will be racing between reading this value and completing their previous
24980 // iteration's table scan.
24982 // The second condition is tracked by the dependent handle code itself on a per worker thread basis
24983 // (and updated by the GcDhReScan() method). We call GcDhUnpromotedHandlesExist() on each thread to
24984 // determine the local value and collect the results into the s_fUnpromotedHandles variable in what is
24985 // effectively an OR operation. As per s_fUnscannedPromotions we can't read the final result until
24986 // we're safely joined.
24987 if (GCScan::GcDhUnpromotedHandlesExist(sc))
24988 s_fUnpromotedHandles = TRUE;
24990 // Synchronize all the threads so we can read our state variables safely. The following shared
24991 // variable (indicating whether we should scan the tables or terminate the loop) will be set by a
24992 // single thread inside the join.
24993 bgc_t_join.join(this, gc_join_scan_dependent_handles);
24994 if (bgc_t_join.joined())
24996 // We're synchronized so it's safe to read our shared state variables. We update another shared
24997 // variable to indicate to all threads whether we'll be scanning for another cycle or terminating
24998 // the loop. We scan if there has been at least one object promotion since last time and at least
24999 // one thread has a dependent handle table with a potential handle promotion possible.
25000 s_fScanRequired = s_fUnscannedPromotions && s_fUnpromotedHandles;
25002 // Reset our shared state variables (ready to be set again on this scan or with a good initial
25003 // value for the next call if we're terminating the loop).
25004 s_fUnscannedPromotions = FALSE;
25005 s_fUnpromotedHandles = FALSE;
25007 if (!s_fScanRequired)
25009 uint8_t* all_heaps_max = 0;
25010 uint8_t* all_heaps_min = MAX_PTR;
25012 for (i = 0; i < n_heaps; i++)
25014 if (all_heaps_max < g_heaps[i]->background_max_overflow_address)
25015 all_heaps_max = g_heaps[i]->background_max_overflow_address;
25016 if (all_heaps_min > g_heaps[i]->background_min_overflow_address)
25017 all_heaps_min = g_heaps[i]->background_min_overflow_address;
25019 for (i = 0; i < n_heaps; i++)
25021 g_heaps[i]->background_max_overflow_address = all_heaps_max;
25022 g_heaps[i]->background_min_overflow_address = all_heaps_min;
25026 // Restart all the workers.
25027 dprintf(2, ("Starting all gc thread mark stack overflow processing"));
25028 bgc_t_join.restart();
25031 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
25032 // being visible. If there really was an overflow (process_mark_overflow returns true) then set the
25033 // global flag indicating that at least one object promotion may have occurred (the usual comment
25034 // about races applies). (Note it's OK to set this flag even if we're about to terminate the loop and
25035 // exit the method since we unconditionally set this variable on method entry anyway).
25036 if (background_process_mark_overflow (sc->concurrent))
25037 s_fUnscannedPromotions = TRUE;
25039 // If we decided that no scan was required we can terminate the loop now.
25040 if (!s_fScanRequired)
25043 // Otherwise we must join with the other workers to ensure that all mark stack overflows have been
25044 // processed before we start scanning dependent handle tables (if overflows remain while we scan we
25045 // could miss noting the promotion of some primary objects).
25046 bgc_t_join.join(this, gc_join_rescan_dependent_handles);
25047 if (bgc_t_join.joined())
25049 // Restart all the workers.
25050 dprintf(3, ("Starting all gc thread for dependent handle promotion"));
25051 bgc_t_join.restart();
25054 // If the portion of the dependent handle table managed by this worker has handles that could still be
25055 // promoted perform a rescan. If the rescan resulted in at least one promotion note this fact since it
25056 // could require a rescan of handles on this or other workers.
25057 if (GCScan::GcDhUnpromotedHandlesExist(sc))
25058 if (GCScan::GcDhReScan(sc))
25059 s_fUnscannedPromotions = TRUE;
25063 void gc_heap::background_scan_dependent_handles (ScanContext *sc)
25065 // Whenever we call this method there may have been preceding object promotions. So set
25066 // fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
25067 // based on the how the scanning proceeded).
25068 bool fUnscannedPromotions = true;
25070 // Scan dependent handles repeatedly until there are no further promotions that can be made or we made a
25071 // scan without performing any new promotions.
25072 while (GCScan::GcDhUnpromotedHandlesExist(sc) && fUnscannedPromotions)
25074 // On each iteration of the loop start with the assumption that no further objects have been promoted.
25075 fUnscannedPromotions = false;
25077 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
25078 // being visible. If there was an overflow (background_process_mark_overflow returned true) then
25079 // additional objects now appear to be promoted and we should set the flag.
25080 if (background_process_mark_overflow (sc->concurrent))
25081 fUnscannedPromotions = true;
25083 // Perform the scan and set the flag if any promotions resulted.
25084 if (GCScan::GcDhReScan (sc))
25085 fUnscannedPromotions = true;
25088 // Perform a last processing of any overflowed mark stack.
25089 background_process_mark_overflow (sc->concurrent);
25091 #endif //MULTIPLE_HEAPS
25093 void gc_heap::recover_bgc_settings()
25095 if ((settings.condemned_generation < max_generation) && recursive_gc_sync::background_running_p())
25097 dprintf (2, ("restoring bgc settings"));
25098 settings = saved_bgc_settings;
25099 GCHeap::GcCondemnedGeneration = gc_heap::settings.condemned_generation;
25103 void gc_heap::allow_fgc()
25105 assert (bgc_thread == GCToEEInterface::GetThread());
25106 bool bToggleGC = false;
25108 if (g_fSuspensionPending > 0)
25110 bToggleGC = GCToEEInterface::EnablePreemptiveGC();
25113 GCToEEInterface::DisablePreemptiveGC();
25118 BOOL gc_heap::should_commit_mark_array()
25120 return (recursive_gc_sync::background_running_p() || (current_bgc_state == bgc_initialized));
25123 void gc_heap::clear_commit_flag()
25125 generation* gen = generation_of (max_generation);
25126 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
25131 if (gen != large_object_generation)
25133 gen = large_object_generation;
25134 seg = heap_segment_in_range (generation_start_segment (gen));
25142 if (seg->flags & heap_segment_flags_ma_committed)
25144 seg->flags &= ~heap_segment_flags_ma_committed;
25147 if (seg->flags & heap_segment_flags_ma_pcommitted)
25149 seg->flags &= ~heap_segment_flags_ma_pcommitted;
25152 seg = heap_segment_next (seg);
25156 void gc_heap::clear_commit_flag_global()
25158 #ifdef MULTIPLE_HEAPS
25159 for (int i = 0; i < n_heaps; i++)
25161 g_heaps[i]->clear_commit_flag();
25164 clear_commit_flag();
25165 #endif //MULTIPLE_HEAPS
25168 void gc_heap::verify_mark_array_cleared (uint8_t* begin, uint8_t* end, uint32_t* mark_array_addr)
25171 size_t markw = mark_word_of (begin);
25172 size_t markw_end = mark_word_of (end);
25174 while (markw < markw_end)
25176 if (mark_array_addr[markw])
25178 dprintf (1, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
25179 markw, mark_array_addr[markw], mark_word_address (markw)));
25185 UNREFERENCED_PARAMETER(begin);
25186 UNREFERENCED_PARAMETER(end);
25187 UNREFERENCED_PARAMETER(mark_array_addr);
25191 void gc_heap::verify_mark_array_cleared (heap_segment* seg, uint32_t* mark_array_addr)
25193 verify_mark_array_cleared (heap_segment_mem (seg), heap_segment_reserved (seg), mark_array_addr);
25196 BOOL gc_heap::commit_mark_array_new_seg (gc_heap* hp,
25198 uint32_t* new_card_table,
25199 uint8_t* new_lowest_address)
25201 UNREFERENCED_PARAMETER(hp); // compiler bug? -- this *is*, indeed, referenced
25203 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
25204 uint8_t* end = heap_segment_reserved (seg);
25206 uint8_t* lowest = hp->background_saved_lowest_address;
25207 uint8_t* highest = hp->background_saved_highest_address;
25209 uint8_t* commit_start = NULL;
25210 uint8_t* commit_end = NULL;
25211 size_t commit_flag = 0;
25213 if ((highest >= start) &&
25216 if ((start >= lowest) && (end <= highest))
25218 dprintf (GC_TABLE_LOG, ("completely in bgc range: seg %Ix-%Ix, bgc: %Ix-%Ix",
25219 start, end, lowest, highest));
25220 commit_flag = heap_segment_flags_ma_committed;
25224 dprintf (GC_TABLE_LOG, ("partially in bgc range: seg %Ix-%Ix, bgc: %Ix-%Ix",
25225 start, end, lowest, highest));
25226 commit_flag = heap_segment_flags_ma_pcommitted;
25229 commit_start = max (lowest, start);
25230 commit_end = min (highest, end);
25232 if (!commit_mark_array_by_range (commit_start, commit_end, hp->mark_array))
25237 if (new_card_table == 0)
25239 new_card_table = g_gc_card_table;
25242 if (hp->card_table != new_card_table)
25244 if (new_lowest_address == 0)
25246 new_lowest_address = g_gc_lowest_address;
25249 uint32_t* ct = &new_card_table[card_word (gcard_of (new_lowest_address))];
25250 uint32_t* ma = (uint32_t*)((uint8_t*)card_table_mark_array (ct) - size_mark_array_of (0, new_lowest_address));
25252 dprintf (GC_TABLE_LOG, ("table realloc-ed: %Ix->%Ix, MA: %Ix->%Ix",
25253 hp->card_table, new_card_table,
25254 hp->mark_array, ma));
25256 if (!commit_mark_array_by_range (commit_start, commit_end, ma))
25262 seg->flags |= commit_flag;
25268 BOOL gc_heap::commit_mark_array_by_range (uint8_t* begin, uint8_t* end, uint32_t* mark_array_addr)
25270 size_t beg_word = mark_word_of (begin);
25271 size_t end_word = mark_word_of (align_on_mark_word (end));
25272 uint8_t* commit_start = align_lower_page ((uint8_t*)&mark_array_addr[beg_word]);
25273 uint8_t* commit_end = align_on_page ((uint8_t*)&mark_array_addr[end_word]);
25274 size_t size = (size_t)(commit_end - commit_start);
25276 #ifdef SIMPLE_DPRINTF
25277 dprintf (GC_TABLE_LOG, ("range: %Ix->%Ix mark word: %Ix->%Ix(%Id), mark array: %Ix->%Ix(%Id), commit %Ix->%Ix(%Id)",
25279 beg_word, end_word,
25280 (end_word - beg_word) * sizeof (uint32_t),
25281 &mark_array_addr[beg_word],
25282 &mark_array_addr[end_word],
25283 (size_t)(&mark_array_addr[end_word] - &mark_array_addr[beg_word]),
25284 commit_start, commit_end,
25286 #endif //SIMPLE_DPRINTF
25288 if (GCToOSInterface::VirtualCommit (commit_start, size))
25290 // We can only verify the mark array is cleared from begin to end, the first and the last
25291 // page aren't necessarily all cleared 'cause they could be used by other segments or
25293 verify_mark_array_cleared (begin, end, mark_array_addr);
25298 dprintf (GC_TABLE_LOG, ("failed to commit %Id bytes", (end_word - beg_word) * sizeof (uint32_t)));
25303 BOOL gc_heap::commit_mark_array_with_check (heap_segment* seg, uint32_t* new_mark_array_addr)
25305 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
25306 uint8_t* end = heap_segment_reserved (seg);
25308 #ifdef MULTIPLE_HEAPS
25309 uint8_t* lowest = heap_segment_heap (seg)->background_saved_lowest_address;
25310 uint8_t* highest = heap_segment_heap (seg)->background_saved_highest_address;
25312 uint8_t* lowest = background_saved_lowest_address;
25313 uint8_t* highest = background_saved_highest_address;
25314 #endif //MULTIPLE_HEAPS
25316 if ((highest >= start) &&
25319 start = max (lowest, start);
25320 end = min (highest, end);
25321 if (!commit_mark_array_by_range (start, end, new_mark_array_addr))
25330 BOOL gc_heap::commit_mark_array_by_seg (heap_segment* seg, uint32_t* mark_array_addr)
25332 dprintf (GC_TABLE_LOG, ("seg: %Ix->%Ix; MA: %Ix",
25334 heap_segment_reserved (seg),
25336 uint8_t* start = (heap_segment_read_only_p (seg) ? heap_segment_mem (seg) : (uint8_t*)seg);
25338 return commit_mark_array_by_range (start, heap_segment_reserved (seg), mark_array_addr);
25341 BOOL gc_heap::commit_mark_array_bgc_init (uint32_t* mark_array_addr)
25343 UNREFERENCED_PARAMETER(mark_array_addr);
25345 dprintf (GC_TABLE_LOG, ("BGC init commit: lowest: %Ix, highest: %Ix, mark_array: %Ix",
25346 lowest_address, highest_address, mark_array));
25348 generation* gen = generation_of (max_generation);
25349 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
25354 if (gen != large_object_generation)
25356 gen = large_object_generation;
25357 seg = heap_segment_in_range (generation_start_segment (gen));
25365 dprintf (GC_TABLE_LOG, ("seg: %Ix, flags: %Id", seg, seg->flags));
25367 if (!(seg->flags & heap_segment_flags_ma_committed))
25369 // For ro segments they could always be only partially in range so we'd
25370 // be calling this at the beginning of every BGC. We are not making this
25371 // more efficient right now - ro segments are currently only used by redhawk.
25372 if (heap_segment_read_only_p (seg))
25374 if ((heap_segment_mem (seg) >= lowest_address) &&
25375 (heap_segment_reserved (seg) <= highest_address))
25377 if (commit_mark_array_by_seg (seg, mark_array))
25379 seg->flags |= heap_segment_flags_ma_committed;
25388 uint8_t* start = max (lowest_address, heap_segment_mem (seg));
25389 uint8_t* end = min (highest_address, heap_segment_reserved (seg));
25390 if (commit_mark_array_by_range (start, end, mark_array))
25392 seg->flags |= heap_segment_flags_ma_pcommitted;
25402 // For normal segments they are by design completely in range so just
25403 // commit the whole mark array for each seg.
25404 if (commit_mark_array_by_seg (seg, mark_array))
25406 if (seg->flags & heap_segment_flags_ma_pcommitted)
25408 seg->flags &= ~heap_segment_flags_ma_pcommitted;
25410 seg->flags |= heap_segment_flags_ma_committed;
25419 seg = heap_segment_next (seg);
25425 // This function doesn't check the commit flag since it's for a new array -
25426 // the mark_array flag for these segments will remain the same.
25427 BOOL gc_heap::commit_new_mark_array (uint32_t* new_mark_array_addr)
25429 dprintf (GC_TABLE_LOG, ("commiting existing segs on MA %Ix", new_mark_array_addr));
25430 generation* gen = generation_of (max_generation);
25431 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
25436 if (gen != large_object_generation)
25438 gen = large_object_generation;
25439 seg = heap_segment_in_range (generation_start_segment (gen));
25447 if (!commit_mark_array_with_check (seg, new_mark_array_addr))
25452 seg = heap_segment_next (seg);
25455 #ifdef MULTIPLE_HEAPS
25456 if (new_heap_segment)
25458 if (!commit_mark_array_with_check (new_heap_segment, new_mark_array_addr))
25463 #endif //MULTIPLE_HEAPS
25468 BOOL gc_heap::commit_new_mark_array_global (uint32_t* new_mark_array)
25470 #ifdef MULTIPLE_HEAPS
25471 for (int i = 0; i < n_heaps; i++)
25473 if (!g_heaps[i]->commit_new_mark_array (new_mark_array))
25479 if (!commit_new_mark_array (new_mark_array))
25483 #endif //MULTIPLE_HEAPS
25488 void gc_heap::decommit_mark_array_by_seg (heap_segment* seg)
25490 // if BGC is disabled (the finalize watchdog does this at shutdown), the mark array could have
25491 // been set to NULL.
25492 if (mark_array == NULL)
25497 dprintf (GC_TABLE_LOG, ("decommitting seg %Ix(%Ix), MA: %Ix", seg, seg->flags, mark_array));
25499 size_t flags = seg->flags;
25501 if ((flags & heap_segment_flags_ma_committed) ||
25502 (flags & heap_segment_flags_ma_pcommitted))
25504 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
25505 uint8_t* end = heap_segment_reserved (seg);
25507 if (flags & heap_segment_flags_ma_pcommitted)
25509 start = max (lowest_address, start);
25510 end = min (highest_address, end);
25513 size_t beg_word = mark_word_of (start);
25514 size_t end_word = mark_word_of (align_on_mark_word (end));
25515 uint8_t* decommit_start = align_on_page ((uint8_t*)&mark_array[beg_word]);
25516 uint8_t* decommit_end = align_lower_page ((uint8_t*)&mark_array[end_word]);
25517 size_t size = (size_t)(decommit_end - decommit_start);
25519 #ifdef SIMPLE_DPRINTF
25520 dprintf (GC_TABLE_LOG, ("seg: %Ix mark word: %Ix->%Ix(%Id), mark array: %Ix->%Ix(%Id), decommit %Ix->%Ix(%Id)",
25522 beg_word, end_word,
25523 (end_word - beg_word) * sizeof (uint32_t),
25524 &mark_array[beg_word],
25525 &mark_array[end_word],
25526 (size_t)(&mark_array[end_word] - &mark_array[beg_word]),
25527 decommit_start, decommit_end,
25529 #endif //SIMPLE_DPRINTF
25531 if (decommit_start < decommit_end)
25533 if (!GCToOSInterface::VirtualDecommit (decommit_start, size))
25535 dprintf (GC_TABLE_LOG, ("GCToOSInterface::VirtualDecommit on %Ix for %Id bytes failed",
25536 decommit_start, size));
25537 assert (!"decommit failed");
25541 dprintf (GC_TABLE_LOG, ("decommited [%Ix for address [%Ix", beg_word, seg));
25545 void gc_heap::background_mark_phase ()
25547 verify_mark_array_cleared();
25550 sc.thread_number = heap_number;
25551 sc.promotion = TRUE;
25552 sc.concurrent = FALSE;
25555 BOOL cooperative_mode = TRUE;
25556 #ifndef MULTIPLE_HEAPS
25557 const int thread = heap_number;
25558 #endif //!MULTIPLE_HEAPS
25560 dprintf(2,("-(GC%d)BMark-", VolatileLoad(&settings.gc_index)));
25562 assert (settings.concurrent);
25567 start = GetCycleCount32();
25570 #ifdef FFIND_OBJECT
25571 if (gen0_must_clear_bricks > 0)
25572 gen0_must_clear_bricks--;
25573 #endif //FFIND_OBJECT
25575 background_soh_alloc_count = 0;
25576 background_loh_alloc_count = 0;
25577 bgc_overflow_count = 0;
25579 bpromoted_bytes (heap_number) = 0;
25580 static uint32_t num_sizedrefs = 0;
25582 background_min_overflow_address = MAX_PTR;
25583 background_max_overflow_address = 0;
25584 background_min_soh_overflow_address = MAX_PTR;
25585 background_max_soh_overflow_address = 0;
25586 processed_soh_overflow_p = FALSE;
25589 //set up the mark lists from g_mark_list
25590 assert (g_mark_list);
25591 mark_list = g_mark_list;
25592 //dont use the mark list for full gc
25593 //because multiple segments are more complex to handle and the list
25594 //is likely to overflow
25595 mark_list_end = &mark_list [0];
25596 mark_list_index = &mark_list [0];
25598 c_mark_list_index = 0;
25600 shigh = (uint8_t*) 0;
25603 generation* gen = generation_of (max_generation);
25605 dprintf(3,("BGC: stack marking"));
25606 sc.concurrent = TRUE;
25608 GCScan::GcScanRoots(background_promote_callback,
25609 max_generation, max_generation,
25614 dprintf(3,("BGC: finalization marking"));
25615 finalize_queue->GcScanRoots(background_promote_callback, heap_number, 0);
25618 size_t total_loh_size = generation_size (max_generation + 1);
25619 bgc_begin_loh_size = total_loh_size;
25620 bgc_alloc_spin_loh = 0;
25621 bgc_loh_size_increased = 0;
25622 bgc_loh_allocated_in_free = 0;
25623 size_t total_soh_size = generation_sizes (generation_of (max_generation));
25625 dprintf (GTC_LOG, ("BM: h%d: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
25628 //concurrent_print_time_delta ("copying stack roots");
25629 concurrent_print_time_delta ("CS");
25631 FIRE_EVENT(BGC1stNonConEnd);
25633 expanded_in_fgc = FALSE;
25634 saved_overflow_ephemeral_seg = 0;
25635 current_bgc_state = bgc_reset_ww;
25637 // we don't need a join here - just whichever thread that gets here
25638 // first can change the states and call restart_vm.
25639 // this is not true - we can't let the EE run when we are scanning stack.
25640 // since we now allow reset ww to run concurrently and have a join for it,
25641 // we can do restart ee on the 1st thread that got here. Make sure we handle the
25642 // sizedref handles correctly.
25643 #ifdef MULTIPLE_HEAPS
25644 bgc_t_join.join(this, gc_join_restart_ee);
25645 if (bgc_t_join.joined())
25646 #endif //MULTIPLE_HEAPS
25648 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25649 // Resetting write watch for software write watch is pretty fast, much faster than for hardware write watch. Reset
25650 // can be done while the runtime is suspended or after the runtime is restarted, the preference was to reset while
25651 // the runtime is suspended. The reset for hardware write watch is done after the runtime is restarted below.
25653 concurrent_print_time_delta ("CRWW begin");
25655 #ifdef MULTIPLE_HEAPS
25656 for (int i = 0; i < n_heaps; i++)
25658 g_heaps[i]->reset_write_watch (FALSE);
25661 reset_write_watch (FALSE);
25662 #endif //MULTIPLE_HEAPS
25664 concurrent_print_time_delta ("CRWW");
25665 #endif //WRITE_WATCH
25666 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25668 num_sizedrefs = GCToEEInterface::GetTotalNumSizedRefHandles();
25670 // this c_write is not really necessary because restart_vm
25671 // has an instruction that will flush the cpu cache (interlocked
25672 // or whatever) but we don't want to rely on that.
25673 dprintf (BGC_LOG, ("setting cm_in_progress"));
25674 c_write (cm_in_progress, TRUE);
25676 //restart all thread, doing the marking from the array
25677 assert (dont_restart_ee_p);
25678 dont_restart_ee_p = FALSE;
25681 GCToOSInterface::YieldThread (0);
25682 #ifdef MULTIPLE_HEAPS
25683 dprintf(3, ("Starting all gc threads for gc"));
25684 bgc_t_join.restart();
25685 #endif //MULTIPLE_HEAPS
25688 #ifdef MULTIPLE_HEAPS
25689 bgc_t_join.join(this, gc_join_after_reset);
25690 if (bgc_t_join.joined())
25691 #endif //MULTIPLE_HEAPS
25693 disable_preemptive (true);
25695 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25696 // When software write watch is enabled, resetting write watch is done while the runtime is suspended above. The
25697 // post-reset call to revisit_written_pages is only necessary for concurrent reset_write_watch, to discard dirtied
25698 // pages during the concurrent reset.
25701 concurrent_print_time_delta ("CRWW begin");
25703 #ifdef MULTIPLE_HEAPS
25704 for (int i = 0; i < n_heaps; i++)
25706 g_heaps[i]->reset_write_watch (TRUE);
25709 reset_write_watch (TRUE);
25710 #endif //MULTIPLE_HEAPS
25712 concurrent_print_time_delta ("CRWW");
25713 #endif //WRITE_WATCH
25715 #ifdef MULTIPLE_HEAPS
25716 for (int i = 0; i < n_heaps; i++)
25718 g_heaps[i]->revisit_written_pages (TRUE, TRUE);
25721 revisit_written_pages (TRUE, TRUE);
25722 #endif //MULTIPLE_HEAPS
25724 concurrent_print_time_delta ("CRW");
25725 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25727 #ifdef MULTIPLE_HEAPS
25728 for (int i = 0; i < n_heaps; i++)
25730 g_heaps[i]->current_bgc_state = bgc_mark_handles;
25733 current_bgc_state = bgc_mark_handles;
25734 #endif //MULTIPLE_HEAPS
25736 current_c_gc_state = c_gc_state_marking;
25738 enable_preemptive ();
25740 #ifdef MULTIPLE_HEAPS
25741 dprintf(3, ("Joining BGC threads after resetting writewatch"));
25742 bgc_t_join.restart();
25743 #endif //MULTIPLE_HEAPS
25746 disable_preemptive (true);
25748 if (num_sizedrefs > 0)
25750 GCScan::GcScanSizedRefs(background_promote, max_generation, max_generation, &sc);
25752 enable_preemptive ();
25754 #ifdef MULTIPLE_HEAPS
25755 bgc_t_join.join(this, gc_join_scan_sizedref_done);
25756 if (bgc_t_join.joined())
25758 dprintf(3, ("Done with marking all sized refs. Starting all bgc thread for marking other strong roots"));
25759 bgc_t_join.restart();
25761 #endif //MULTIPLE_HEAPS
25763 disable_preemptive (true);
25766 dprintf (3,("BGC: handle table marking"));
25767 GCScan::GcScanHandles(background_promote,
25768 max_generation, max_generation,
25770 //concurrent_print_time_delta ("concurrent marking handle table");
25771 concurrent_print_time_delta ("CRH");
25773 current_bgc_state = bgc_mark_stack;
25774 dprintf (2,("concurrent draining mark list"));
25775 background_drain_mark_list (thread);
25776 //concurrent_print_time_delta ("concurrent marking stack roots");
25777 concurrent_print_time_delta ("CRS");
25779 dprintf (2,("concurrent revisiting dirtied pages"));
25780 revisit_written_pages (TRUE);
25781 revisit_written_pages (TRUE);
25782 //concurrent_print_time_delta ("concurrent marking dirtied pages on LOH");
25783 concurrent_print_time_delta ("CRre");
25785 enable_preemptive ();
25787 #ifdef MULTIPLE_HEAPS
25788 bgc_t_join.join(this, gc_join_concurrent_overflow);
25789 if (bgc_t_join.joined())
25791 uint8_t* all_heaps_max = 0;
25792 uint8_t* all_heaps_min = MAX_PTR;
25794 for (i = 0; i < n_heaps; i++)
25796 dprintf (3, ("heap %d overflow max is %Ix, min is %Ix",
25798 g_heaps[i]->background_max_overflow_address,
25799 g_heaps[i]->background_min_overflow_address));
25800 if (all_heaps_max < g_heaps[i]->background_max_overflow_address)
25801 all_heaps_max = g_heaps[i]->background_max_overflow_address;
25802 if (all_heaps_min > g_heaps[i]->background_min_overflow_address)
25803 all_heaps_min = g_heaps[i]->background_min_overflow_address;
25805 for (i = 0; i < n_heaps; i++)
25807 g_heaps[i]->background_max_overflow_address = all_heaps_max;
25808 g_heaps[i]->background_min_overflow_address = all_heaps_min;
25810 dprintf(3, ("Starting all bgc threads after updating the overflow info"));
25811 bgc_t_join.restart();
25813 #endif //MULTIPLE_HEAPS
25815 disable_preemptive (true);
25817 dprintf (2, ("before CRov count: %d", bgc_overflow_count));
25818 bgc_overflow_count = 0;
25819 background_process_mark_overflow (TRUE);
25820 dprintf (2, ("after CRov count: %d", bgc_overflow_count));
25821 bgc_overflow_count = 0;
25822 //concurrent_print_time_delta ("concurrent processing mark overflow");
25823 concurrent_print_time_delta ("CRov");
25825 // Stop all threads, crawl all stacks and revisit changed pages.
25826 FIRE_EVENT(BGC1stConEnd);
25828 dprintf (2, ("Stopping the EE"));
25830 enable_preemptive ();
25832 #ifdef MULTIPLE_HEAPS
25833 bgc_t_join.join(this, gc_join_suspend_ee);
25834 if (bgc_t_join.joined())
25836 bgc_threads_sync_event.Reset();
25838 dprintf(3, ("Joining BGC threads for non concurrent final marking"));
25839 bgc_t_join.restart();
25841 #endif //MULTIPLE_HEAPS
25843 if (heap_number == 0)
25845 enter_spin_lock (&gc_lock);
25849 bgc_threads_sync_event.Set();
25853 bgc_threads_sync_event.Wait(INFINITE, FALSE);
25854 dprintf (2, ("bgc_threads_sync_event is signalled"));
25857 assert (settings.concurrent);
25858 assert (settings.condemned_generation == max_generation);
25860 dprintf (2, ("clearing cm_in_progress"));
25861 c_write (cm_in_progress, FALSE);
25863 bgc_alloc_lock->check();
25865 current_bgc_state = bgc_final_marking;
25867 //concurrent_print_time_delta ("concurrent marking ended");
25868 concurrent_print_time_delta ("CR");
25870 FIRE_EVENT(BGC2ndNonConBegin);
25872 mark_absorb_new_alloc();
25874 // We need a join here 'cause find_object would complain if the gen0
25875 // bricks of another heap haven't been fixed up. So we need to make sure
25876 // that every heap's gen0 bricks are fixed up before we proceed.
25877 #ifdef MULTIPLE_HEAPS
25878 bgc_t_join.join(this, gc_join_after_absorb);
25879 if (bgc_t_join.joined())
25881 dprintf(3, ("Joining BGC threads after absorb"));
25882 bgc_t_join.restart();
25884 #endif //MULTIPLE_HEAPS
25886 // give VM a chance to do work
25887 GCToEEInterface::GcBeforeBGCSweepWork();
25889 //reset the flag, indicating that the EE no longer expect concurrent
25891 sc.concurrent = FALSE;
25893 total_loh_size = generation_size (max_generation + 1);
25894 total_soh_size = generation_sizes (generation_of (max_generation));
25896 dprintf (GTC_LOG, ("FM: h%d: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
25898 dprintf (2, ("nonconcurrent marking stack roots"));
25899 GCScan::GcScanRoots(background_promote,
25900 max_generation, max_generation,
25902 //concurrent_print_time_delta ("nonconcurrent marking stack roots");
25903 concurrent_print_time_delta ("NRS");
25905 // finalize_queue->EnterFinalizeLock();
25906 finalize_queue->GcScanRoots(background_promote, heap_number, 0);
25907 // finalize_queue->LeaveFinalizeLock();
25909 dprintf (2, ("nonconcurrent marking handle table"));
25910 GCScan::GcScanHandles(background_promote,
25911 max_generation, max_generation,
25913 //concurrent_print_time_delta ("nonconcurrent marking handle table");
25914 concurrent_print_time_delta ("NRH");
25916 dprintf (2,("---- (GC%d)final going through written pages ----", VolatileLoad(&settings.gc_index)));
25917 revisit_written_pages (FALSE);
25918 //concurrent_print_time_delta ("nonconcurrent revisit dirtied pages on LOH");
25919 concurrent_print_time_delta ("NRre LOH");
25921 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25922 #ifdef MULTIPLE_HEAPS
25923 bgc_t_join.join(this, gc_join_disable_software_write_watch);
25924 if (bgc_t_join.joined())
25925 #endif // MULTIPLE_HEAPS
25927 // The runtime is suspended, and we will be doing a final query of dirty pages, so pause tracking written pages to
25928 // avoid further perf penalty after the runtime is restarted
25929 SoftwareWriteWatch::DisableForGCHeap();
25931 #ifdef MULTIPLE_HEAPS
25932 dprintf(3, ("Restarting BGC threads after disabling software write watch"));
25933 bgc_t_join.restart();
25934 #endif // MULTIPLE_HEAPS
25936 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25938 dprintf (2, ("before NR 1st Hov count: %d", bgc_overflow_count));
25939 bgc_overflow_count = 0;
25941 // Dependent handles need to be scanned with a special algorithm (see the header comment on
25942 // scan_dependent_handles for more detail). We perform an initial scan without processing any mark
25943 // stack overflow. This is not guaranteed to complete the operation but in a common case (where there
25944 // are no dependent handles that are due to be collected) it allows us to optimize away further scans.
25945 // The call to background_scan_dependent_handles is what will cycle through more iterations if
25946 // required and will also perform processing of any mark stack overflow once the dependent handle
25947 // table has been fully promoted.
25948 dprintf (2, ("1st dependent handle scan and process mark overflow"));
25949 GCScan::GcDhInitialScan(background_promote, max_generation, max_generation, &sc);
25950 background_scan_dependent_handles (&sc);
25951 //concurrent_print_time_delta ("1st nonconcurrent dependent handle scan and process mark overflow");
25952 concurrent_print_time_delta ("NR 1st Hov");
25954 dprintf (2, ("after NR 1st Hov count: %d", bgc_overflow_count));
25955 bgc_overflow_count = 0;
25957 #ifdef MULTIPLE_HEAPS
25958 bgc_t_join.join(this, gc_join_null_dead_short_weak);
25959 if (bgc_t_join.joined())
25960 #endif //MULTIPLE_HEAPS
25962 GCToEEInterface::AfterGcScanRoots (max_generation, max_generation, &sc);
25964 #ifdef MULTIPLE_HEAPS
25965 dprintf(3, ("Joining BGC threads for short weak handle scan"));
25966 bgc_t_join.restart();
25967 #endif //MULTIPLE_HEAPS
25970 // null out the target of short weakref that were not promoted.
25971 GCScan::GcShortWeakPtrScan(background_promote, max_generation, max_generation,&sc);
25973 //concurrent_print_time_delta ("bgc GcShortWeakPtrScan");
25974 concurrent_print_time_delta ("NR GcShortWeakPtrScan");
25978 #ifdef MULTIPLE_HEAPS
25979 bgc_t_join.join(this, gc_join_scan_finalization);
25980 if (bgc_t_join.joined())
25982 dprintf(3, ("Joining BGC threads for finalization"));
25983 bgc_t_join.restart();
25985 #endif //MULTIPLE_HEAPS
25987 //Handle finalization.
25988 dprintf(3,("Marking finalization data"));
25989 //concurrent_print_time_delta ("bgc joined to mark finalization");
25990 concurrent_print_time_delta ("NRj");
25992 // finalize_queue->EnterFinalizeLock();
25993 finalize_queue->ScanForFinalization (background_promote, max_generation, FALSE, __this);
25994 // finalize_queue->LeaveFinalizeLock();
25996 concurrent_print_time_delta ("NRF");
25999 dprintf (2, ("before NR 2nd Hov count: %d", bgc_overflow_count));
26000 bgc_overflow_count = 0;
26002 // Scan dependent handles again to promote any secondaries associated with primaries that were promoted
26003 // for finalization. As before background_scan_dependent_handles will also process any mark stack
26005 dprintf (2, ("2nd dependent handle scan and process mark overflow"));
26006 background_scan_dependent_handles (&sc);
26007 //concurrent_print_time_delta ("2nd nonconcurrent dependent handle scan and process mark overflow");
26008 concurrent_print_time_delta ("NR 2nd Hov");
26010 #ifdef MULTIPLE_HEAPS
26011 bgc_t_join.join(this, gc_join_null_dead_long_weak);
26012 if (bgc_t_join.joined())
26014 dprintf(2, ("Joining BGC threads for weak pointer deletion"));
26015 bgc_t_join.restart();
26017 #endif //MULTIPLE_HEAPS
26019 // null out the target of long weakref that were not promoted.
26020 GCScan::GcWeakPtrScan (background_promote, max_generation, max_generation, &sc);
26021 concurrent_print_time_delta ("NR GcWeakPtrScan");
26023 #ifdef MULTIPLE_HEAPS
26024 bgc_t_join.join(this, gc_join_null_dead_syncblk);
26025 if (bgc_t_join.joined())
26026 #endif //MULTIPLE_HEAPS
26028 dprintf (2, ("calling GcWeakPtrScanBySingleThread"));
26029 // scan for deleted entries in the syncblk cache
26030 GCScan::GcWeakPtrScanBySingleThread (max_generation, max_generation, &sc);
26031 concurrent_print_time_delta ("NR GcWeakPtrScanBySingleThread");
26032 #ifdef MULTIPLE_HEAPS
26033 dprintf(2, ("Starting BGC threads for end of background mark phase"));
26034 bgc_t_join.restart();
26035 #endif //MULTIPLE_HEAPS
26038 gen0_bricks_cleared = FALSE;
26040 dprintf (2, ("end of bgc mark: loh: %d, soh: %d",
26041 generation_size (max_generation + 1),
26042 generation_sizes (generation_of (max_generation))));
26044 for (int gen_idx = max_generation; gen_idx <= (max_generation + 1); gen_idx++)
26046 generation* gen = generation_of (gen_idx);
26047 dynamic_data* dd = dynamic_data_of (gen_idx);
26048 dd_begin_data_size (dd) = generation_size (gen_idx) -
26049 (generation_free_list_space (gen) + generation_free_obj_space (gen)) -
26050 Align (size (generation_allocation_start (gen)));
26051 dd_survived_size (dd) = 0;
26052 dd_pinned_survived_size (dd) = 0;
26053 dd_artificial_pinned_survived_size (dd) = 0;
26054 dd_added_pinned_size (dd) = 0;
26057 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
26058 PREFIX_ASSUME(seg != NULL);
26062 seg->flags &= ~heap_segment_flags_swept;
26064 if (heap_segment_allocated (seg) == heap_segment_mem (seg))
26066 // This can't happen...
26070 if (seg == ephemeral_heap_segment)
26072 heap_segment_background_allocated (seg) = generation_allocation_start (generation_of (max_generation - 1));
26076 heap_segment_background_allocated (seg) = heap_segment_allocated (seg);
26079 dprintf (2, ("seg %Ix background allocated is %Ix",
26080 heap_segment_mem (seg),
26081 heap_segment_background_allocated (seg)));
26082 seg = heap_segment_next_rw (seg);
26085 // We need to void alloc contexts here 'cause while background_ephemeral_sweep is running
26086 // we can't let the user code consume the left over parts in these alloc contexts.
26087 repair_allocation_contexts (FALSE);
26090 finish = GetCycleCount32();
26091 mark_time = finish - start;
26094 dprintf (2, ("end of bgc mark: gen2 free list space: %d, free obj space: %d",
26095 generation_free_list_space (generation_of (max_generation)),
26096 generation_free_obj_space (generation_of (max_generation))));
26098 dprintf(2,("---- (GC%d)End of background mark phase ----", VolatileLoad(&settings.gc_index)));
26102 gc_heap::suspend_EE ()
26104 dprintf (2, ("suspend_EE"));
26105 #ifdef MULTIPLE_HEAPS
26106 gc_heap* hp = gc_heap::g_heaps[0];
26107 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26109 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26110 #endif //MULTIPLE_HEAPS
26113 #ifdef MULTIPLE_HEAPS
26115 gc_heap::bgc_suspend_EE ()
26117 for (int i = 0; i < n_heaps; i++)
26119 gc_heap::g_heaps[i]->reset_gc_done();
26122 dprintf (2, ("bgc_suspend_EE"));
26123 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26125 gc_started = FALSE;
26126 for (int i = 0; i < n_heaps; i++)
26128 gc_heap::g_heaps[i]->set_gc_done();
26133 gc_heap::bgc_suspend_EE ()
26137 dprintf (2, ("bgc_suspend_EE"));
26138 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26139 gc_started = FALSE;
26142 #endif //MULTIPLE_HEAPS
26145 gc_heap::restart_EE ()
26147 dprintf (2, ("restart_EE"));
26148 #ifdef MULTIPLE_HEAPS
26149 GCToEEInterface::RestartEE(FALSE);
26151 GCToEEInterface::RestartEE(FALSE);
26152 #endif //MULTIPLE_HEAPS
26155 inline uint8_t* gc_heap::high_page ( heap_segment* seg, BOOL concurrent_p)
26159 uint8_t* end = ((seg == ephemeral_heap_segment) ?
26160 generation_allocation_start (generation_of (max_generation-1)) :
26161 heap_segment_allocated (seg));
26162 return align_lower_page (end);
26166 return heap_segment_allocated (seg);
26170 void gc_heap::revisit_written_page (uint8_t* page,
26174 uint8_t*& last_page,
26175 uint8_t*& last_object,
26176 BOOL large_objects_p,
26177 size_t& num_marked_objects)
26179 UNREFERENCED_PARAMETER(seg);
26181 uint8_t* start_address = page;
26183 int align_const = get_alignment_constant (!large_objects_p);
26184 uint8_t* high_address = end;
26185 uint8_t* current_lowest_address = background_saved_lowest_address;
26186 uint8_t* current_highest_address = background_saved_highest_address;
26187 BOOL no_more_loop_p = FALSE;
26190 #ifndef MULTIPLE_HEAPS
26191 const int thread = heap_number;
26192 #endif //!MULTIPLE_HEAPS
26194 if (large_objects_p)
26200 if (((last_page + WRITE_WATCH_UNIT_SIZE) == page)
26201 || (start_address <= last_object))
26207 o = find_first_object (start_address, last_object);
26208 // We can visit the same object again, but on a different page.
26209 assert (o >= last_object);
26213 dprintf (3,("page %Ix start: %Ix, %Ix[ ",
26214 (size_t)page, (size_t)o,
26215 (size_t)(min (high_address, page + WRITE_WATCH_UNIT_SIZE))));
26217 while (o < (min (high_address, page + WRITE_WATCH_UNIT_SIZE)))
26221 if (concurrent_p && large_objects_p)
26223 bgc_alloc_lock->bgc_mark_set (o);
26225 if (((CObjectHeader*)o)->IsFree())
26227 s = unused_array_size (o);
26239 dprintf (3,("Considering object %Ix(%s)", (size_t)o, (background_object_marked (o, FALSE) ? "bm" : "nbm")));
26241 assert (Align (s) >= Align (min_obj_size));
26243 uint8_t* next_o = o + Align (s, align_const);
26245 if (next_o >= start_address)
26247 #ifdef MULTIPLE_HEAPS
26250 // We set last_object here for SVR BGC here because SVR BGC has more than
26251 // one GC thread. When we have more than one GC thread we would run into this
26252 // situation if we skipped unmarked objects:
26253 // bgc thread 1 calls GWW, and detect object X not marked so it would skip it
26255 // bgc thread 2 marks X and all its current children.
26256 // user thread comes along and dirties more (and later) pages in X.
26257 // bgc thread 1 calls GWW again and gets those later pages but it will not mark anything
26258 // on them because it had already skipped X. We need to detect that this object is now
26259 // marked and mark the children on the dirtied pages.
26260 // In the future if we have less BGC threads than we have heaps we should add
26261 // the check to the number of BGC threads.
26264 #endif //MULTIPLE_HEAPS
26266 if (contain_pointers (o) &&
26267 (!((o >= current_lowest_address) && (o < current_highest_address)) ||
26268 background_marked (o)))
26270 dprintf (3, ("going through %Ix", (size_t)o));
26271 go_through_object (method_table(o), o, s, poo, start_address, use_start, (o + s),
26272 if ((uint8_t*)poo >= min (high_address, page + WRITE_WATCH_UNIT_SIZE))
26274 no_more_loop_p = TRUE;
26277 uint8_t* oo = *poo;
26279 num_marked_objects++;
26280 background_mark_object (oo THREAD_NUMBER_ARG);
26285 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP // see comment below
26287 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26288 ((CObjectHeader*)o)->IsFree() &&
26289 (next_o > min (high_address, page + WRITE_WATCH_UNIT_SIZE)))
26291 // We need to not skip the object here because of this corner scenario:
26292 // A large object was being allocated during BGC mark so we first made it
26293 // into a free object, then cleared its memory. In this loop we would detect
26294 // that it's a free object which normally we would skip. But by the next time
26295 // we call GetWriteWatch we could still be on this object and the object had
26296 // been made into a valid object and some of its memory was changed. We need
26297 // to be sure to process those written pages so we can't skip the object just
26300 // Similarly, when using software write watch, don't advance last_object when
26301 // the current object is a free object that spans beyond the current page or
26302 // high_address. Software write watch acquires gc_lock before the concurrent
26303 // GetWriteWatch() call during revisit_written_pages(). A foreground GC may
26304 // happen at that point and allocate from this free region, so when
26305 // revisit_written_pages() continues, it cannot skip now-valid objects in this
26307 no_more_loop_p = TRUE;
26312 if (concurrent_p && large_objects_p)
26314 bgc_alloc_lock->bgc_mark_done ();
26316 if (no_more_loop_p)
26323 #ifdef MULTIPLE_HEAPS
26326 assert (last_object < (min (high_address, page + WRITE_WATCH_UNIT_SIZE)));
26329 #endif //MULTIPLE_HEAPS
26334 dprintf (3,("Last object: %Ix", (size_t)last_object));
26335 last_page = align_write_watch_lower_page (o);
26338 // When reset_only_p is TRUE, we should only reset pages that are in range
26339 // because we need to consider the segments or part of segments that were
26340 // allocated out of range all live.
26341 void gc_heap::revisit_written_pages (BOOL concurrent_p, BOOL reset_only_p)
26344 if (concurrent_p && !reset_only_p)
26346 current_bgc_state = bgc_revisit_soh;
26349 size_t total_dirtied_pages = 0;
26350 size_t total_marked_objects = 0;
26352 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
26354 PREFIX_ASSUME(seg != NULL);
26356 bool reset_watch_state = !!concurrent_p;
26357 bool is_runtime_suspended = !concurrent_p;
26358 BOOL small_object_segments = TRUE;
26359 int align_const = get_alignment_constant (small_object_segments);
26365 if (small_object_segments)
26367 //switch to large segment
26368 if (concurrent_p && !reset_only_p)
26370 current_bgc_state = bgc_revisit_loh;
26375 dprintf (GTC_LOG, ("h%d: SOH: dp:%Id; mo: %Id", heap_number, total_dirtied_pages, total_marked_objects));
26376 fire_revisit_event (total_dirtied_pages, total_marked_objects, !small_object_segments);
26377 concurrent_print_time_delta (concurrent_p ? "CR SOH" : "NR SOH");
26378 total_dirtied_pages = 0;
26379 total_marked_objects = 0;
26382 small_object_segments = FALSE;
26383 //concurrent_print_time_delta (concurrent_p ? "concurrent marking dirtied pages on SOH" : "nonconcurrent marking dirtied pages on SOH");
26385 dprintf (3, ("now revisiting large object segments"));
26386 align_const = get_alignment_constant (small_object_segments);
26387 seg = heap_segment_rw (generation_start_segment (large_object_generation));
26389 PREFIX_ASSUME(seg != NULL);
26397 dprintf (GTC_LOG, ("h%d: tdp: %Id", heap_number, total_dirtied_pages));
26401 dprintf (GTC_LOG, ("h%d: LOH: dp:%Id; mo: %Id", heap_number, total_dirtied_pages, total_marked_objects));
26402 fire_revisit_event (total_dirtied_pages, total_marked_objects, !small_object_segments);
26407 uint8_t* base_address = (uint8_t*)heap_segment_mem (seg);
26408 //we need to truncate to the base of the page because
26409 //some newly allocated could exist beyond heap_segment_allocated
26410 //and if we reset the last page write watch status,
26411 // they wouldn't be guaranteed to be visited -> gc hole.
26412 uintptr_t bcount = array_size;
26413 uint8_t* last_page = 0;
26414 uint8_t* last_object = heap_segment_mem (seg);
26415 uint8_t* high_address = 0;
26417 BOOL skip_seg_p = FALSE;
26421 if ((heap_segment_mem (seg) >= background_saved_lowest_address) ||
26422 (heap_segment_reserved (seg) <= background_saved_highest_address))
26424 dprintf (3, ("h%d: sseg: %Ix(-%Ix)", heap_number,
26425 heap_segment_mem (seg), heap_segment_reserved (seg)));
26432 dprintf (3, ("looking at seg %Ix", (size_t)last_object));
26436 base_address = max (base_address, background_saved_lowest_address);
26437 dprintf (3, ("h%d: reset only starting %Ix", heap_number, base_address));
26440 dprintf (3, ("h%d: starting: %Ix, seg %Ix-%Ix", heap_number, base_address,
26441 heap_segment_mem (seg), heap_segment_reserved (seg)));
26448 high_address = ((seg == ephemeral_heap_segment) ? alloc_allocated : heap_segment_allocated (seg));
26449 high_address = min (high_address, background_saved_highest_address);
26453 high_address = high_page (seg, concurrent_p);
26456 if ((base_address < high_address) &&
26457 (bcount >= array_size))
26459 ptrdiff_t region_size = high_address - base_address;
26460 dprintf (3, ("h%d: gw: [%Ix(%Id)", heap_number, (size_t)base_address, (size_t)region_size));
26462 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26463 // When the runtime is not suspended, it's possible for the table to be resized concurrently with the scan
26464 // for dirty pages below. Prevent that by synchronizing with grow_brick_card_tables(). When the runtime is
26465 // suspended, it's ok to scan for dirty pages concurrently from multiple background GC threads for disjoint
26467 if (!is_runtime_suspended)
26469 enter_spin_lock(&gc_lock);
26471 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26473 get_write_watch_for_gc_heap (reset_watch_state, base_address, region_size,
26474 (void**)background_written_addresses,
26475 &bcount, is_runtime_suspended);
26477 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26478 if (!is_runtime_suspended)
26480 leave_spin_lock(&gc_lock);
26482 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26486 total_dirtied_pages += bcount;
26488 dprintf (3, ("Found %d pages [%Ix, %Ix[",
26489 bcount, (size_t)base_address, (size_t)high_address));
26494 for (unsigned i = 0; i < bcount; i++)
26496 uint8_t* page = (uint8_t*)background_written_addresses[i];
26497 dprintf (3, ("looking at page %d at %Ix(h: %Ix)", i,
26498 (size_t)page, (size_t)high_address));
26499 if (page < high_address)
26501 //search for marked objects in the page
26502 revisit_written_page (page, high_address, concurrent_p,
26503 seg, last_page, last_object,
26504 !small_object_segments,
26505 total_marked_objects);
26509 dprintf (3, ("page %d at %Ix is >= %Ix!", i, (size_t)page, (size_t)high_address));
26510 assert (!"page shouldn't have exceeded limit");
26515 if (bcount >= array_size){
26516 base_address = background_written_addresses [array_size-1] + WRITE_WATCH_UNIT_SIZE;
26517 bcount = array_size;
26527 seg = heap_segment_next_rw (seg);
26530 #endif //WRITE_WATCH
26533 void gc_heap::background_grow_c_mark_list()
26535 assert (c_mark_list_index >= c_mark_list_length);
26536 BOOL should_drain_p = FALSE;
26538 #ifndef MULTIPLE_HEAPS
26539 const int thread = heap_number;
26540 #endif //!MULTIPLE_HEAPS
26542 dprintf (2, ("stack copy buffer overflow"));
26543 uint8_t** new_c_mark_list = 0;
26546 if (c_mark_list_length >= (SIZE_T_MAX / (2 * sizeof (uint8_t*))))
26548 should_drain_p = TRUE;
26552 new_c_mark_list = new (nothrow) uint8_t*[c_mark_list_length*2];
26553 if (new_c_mark_list == 0)
26555 should_drain_p = TRUE;
26559 if (should_drain_p)
26562 dprintf (2, ("No more memory for the stacks copy, draining.."));
26563 //drain the list by marking its elements
26564 background_drain_mark_list (thread);
26568 assert (new_c_mark_list);
26569 memcpy (new_c_mark_list, c_mark_list, c_mark_list_length*sizeof(uint8_t*));
26570 c_mark_list_length = c_mark_list_length*2;
26571 delete c_mark_list;
26572 c_mark_list = new_c_mark_list;
26576 void gc_heap::background_promote_callback (Object** ppObject, ScanContext* sc,
26579 UNREFERENCED_PARAMETER(sc);
26580 //in order to save space on the array, mark the object,
26581 //knowing that it will be visited later
26582 assert (settings.concurrent);
26584 THREAD_NUMBER_FROM_CONTEXT;
26585 #ifndef MULTIPLE_HEAPS
26586 const int thread = 0;
26587 #endif //!MULTIPLE_HEAPS
26589 uint8_t* o = (uint8_t*)*ppObject;
26596 gc_heap* hp = gc_heap::heap_of (o);
26598 if ((o < hp->background_saved_lowest_address) || (o >= hp->background_saved_highest_address))
26603 #ifdef INTERIOR_POINTERS
26604 if (flags & GC_CALL_INTERIOR)
26606 o = hp->find_object (o, hp->background_saved_lowest_address);
26610 #endif //INTERIOR_POINTERS
26612 #ifdef FEATURE_CONSERVATIVE_GC
26613 // For conservative GC, a value on stack may point to middle of a free object.
26614 // In this case, we don't need to promote the pointer.
26615 if (GCConfig::GetConservativeGC() && ((CObjectHeader*)o)->IsFree())
26619 #endif //FEATURE_CONSERVATIVE_GC
26622 ((CObjectHeader*)o)->Validate();
26625 dprintf (3, ("Concurrent Background Promote %Ix", (size_t)o));
26626 if (o && (size (o) > LARGE_OBJECT_SIZE))
26628 dprintf (3, ("Brc %Ix", (size_t)o));
26631 if (hpt->c_mark_list_index >= hpt->c_mark_list_length)
26633 hpt->background_grow_c_mark_list();
26635 dprintf (3, ("pushing %08x into mark_list", (size_t)o));
26636 hpt->c_mark_list [hpt->c_mark_list_index++] = o;
26638 STRESS_LOG3(LF_GC|LF_GCROOTS, LL_INFO1000000, " GCHeap::Background Promote: Promote GC Root *%p = %p MT = %pT", ppObject, o, o ? ((Object*) o)->GetGCSafeMethodTable() : NULL);
26641 void gc_heap::mark_absorb_new_alloc()
26643 fix_allocation_contexts (FALSE);
26645 gen0_bricks_cleared = FALSE;
26647 clear_gen0_bricks();
26650 BOOL gc_heap::prepare_bgc_thread(gc_heap* gh)
26652 BOOL success = FALSE;
26653 BOOL thread_created = FALSE;
26654 dprintf (2, ("Preparing gc thread"));
26655 gh->bgc_threads_timeout_cs.Enter();
26656 if (!(gh->bgc_thread_running))
26658 dprintf (2, ("GC thread not runnning"));
26659 if ((gh->bgc_thread == 0) && create_bgc_thread(gh))
26662 thread_created = TRUE;
26667 dprintf (3, ("GC thread already running"));
26670 gh->bgc_threads_timeout_cs.Leave();
26673 FIRE_EVENT(GCCreateConcurrentThread_V1);
26678 BOOL gc_heap::create_bgc_thread(gc_heap* gh)
26680 assert (background_gc_done_event.IsValid());
26682 //dprintf (2, ("Creating BGC thread"));
26684 gh->bgc_thread_running = GCToEEInterface::CreateThread(gh->bgc_thread_stub, gh, true, ".NET Background GC");
26685 return gh->bgc_thread_running;
26688 BOOL gc_heap::create_bgc_threads_support (int number_of_heaps)
26691 dprintf (3, ("Creating concurrent GC thread for the first time"));
26692 if (!background_gc_done_event.CreateManualEventNoThrow(TRUE))
26696 if (!bgc_threads_sync_event.CreateManualEventNoThrow(FALSE))
26700 if (!ee_proceed_event.CreateAutoEventNoThrow(FALSE))
26704 if (!bgc_start_event.CreateManualEventNoThrow(FALSE))
26709 #ifdef MULTIPLE_HEAPS
26710 bgc_t_join.init (number_of_heaps, join_flavor_bgc);
26712 UNREFERENCED_PARAMETER(number_of_heaps);
26713 #endif //MULTIPLE_HEAPS
26721 if (background_gc_done_event.IsValid())
26723 background_gc_done_event.CloseEvent();
26725 if (bgc_threads_sync_event.IsValid())
26727 bgc_threads_sync_event.CloseEvent();
26729 if (ee_proceed_event.IsValid())
26731 ee_proceed_event.CloseEvent();
26733 if (bgc_start_event.IsValid())
26735 bgc_start_event.CloseEvent();
26742 BOOL gc_heap::create_bgc_thread_support()
26747 if (!gc_lh_block_event.CreateManualEventNoThrow(FALSE))
26752 //needs to have room for enough smallest objects fitting on a page
26753 parr = new (nothrow) uint8_t*[1 + OS_PAGE_SIZE / MIN_OBJECT_SIZE];
26759 make_c_mark_list (parr);
26767 if (gc_lh_block_event.IsValid())
26769 gc_lh_block_event.CloseEvent();
26776 int gc_heap::check_for_ephemeral_alloc()
26778 int gen = ((settings.reason == reason_oos_soh) ? (max_generation - 1) : -1);
26782 #ifdef MULTIPLE_HEAPS
26783 for (int heap_index = 0; heap_index < n_heaps; heap_index++)
26784 #endif //MULTIPLE_HEAPS
26786 for (int i = 0; i <= (max_generation - 1); i++)
26788 #ifdef MULTIPLE_HEAPS
26789 if (g_heaps[heap_index]->get_new_allocation (i) <= 0)
26791 if (get_new_allocation (i) <= 0)
26792 #endif //MULTIPLE_HEAPS
26794 gen = max (gen, i);
26805 // Wait for gc to finish sequential part
26806 void gc_heap::wait_to_proceed()
26808 assert (background_gc_done_event.IsValid());
26809 assert (bgc_start_event.IsValid());
26811 user_thread_wait(&ee_proceed_event, FALSE);
26814 // Start a new concurrent gc
26815 void gc_heap::start_c_gc()
26817 assert (background_gc_done_event.IsValid());
26818 assert (bgc_start_event.IsValid());
26820 //Need to make sure that the gc thread is in the right place.
26821 background_gc_done_event.Wait(INFINITE, FALSE);
26822 background_gc_done_event.Reset();
26823 bgc_start_event.Set();
26826 void gc_heap::do_background_gc()
26828 dprintf (2, ("starting a BGC"));
26829 #ifdef MULTIPLE_HEAPS
26830 for (int i = 0; i < n_heaps; i++)
26832 g_heaps[i]->init_background_gc();
26835 init_background_gc();
26836 #endif //MULTIPLE_HEAPS
26837 //start the background gc
26840 //wait until we get restarted by the BGC.
26844 void gc_heap::kill_gc_thread()
26846 //assert (settings.concurrent == FALSE);
26848 // We are doing a two-stage shutdown now.
26849 // In the first stage, we do minimum work, and call ExitProcess at the end.
26850 // In the secodn stage, we have the Loader lock and only one thread is
26851 // alive. Hence we do not need to kill gc thread.
26852 background_gc_done_event.CloseEvent();
26853 gc_lh_block_event.CloseEvent();
26854 bgc_start_event.CloseEvent();
26855 bgc_threads_timeout_cs.Destroy();
26857 recursive_gc_sync::shutdown();
26860 void gc_heap::bgc_thread_function()
26862 assert (background_gc_done_event.IsValid());
26863 assert (bgc_start_event.IsValid());
26865 dprintf (3, ("gc_thread thread starting..."));
26867 BOOL do_exit = FALSE;
26869 bool cooperative_mode = true;
26870 bgc_thread_id.SetToCurrentThread();
26871 dprintf (1, ("bgc_thread_id is set to %x", (uint32_t)GCToOSInterface::GetCurrentThreadIdForLogging()));
26874 // Wait for work to do...
26875 dprintf (3, ("bgc thread: waiting..."));
26877 cooperative_mode = enable_preemptive ();
26878 //current_thread->m_fPreemptiveGCDisabled = 0;
26880 uint32_t result = bgc_start_event.Wait(
26882 #ifdef MULTIPLE_HEAPS
26886 #endif //MULTIPLE_HEAPS
26888 #ifdef MULTIPLE_HEAPS
26892 #endif //MULTIPLE_HEAPS
26895 dprintf (2, ("gc thread: finished waiting"));
26897 // not calling disable_preemptive here 'cause we
26898 // can't wait for GC complete here - RestartEE will be called
26899 // when we've done the init work.
26901 if (result == WAIT_TIMEOUT)
26903 // Should join the bgc threads and terminate all of them
26905 dprintf (1, ("GC thread timeout"));
26906 bgc_threads_timeout_cs.Enter();
26907 if (!keep_bgc_threads_p)
26909 dprintf (2, ("GC thread exiting"));
26910 bgc_thread_running = FALSE;
26912 bgc_thread_id.Clear();
26915 bgc_threads_timeout_cs.Leave();
26920 dprintf (3, ("GC thread needed, not exiting"));
26924 // if we signal the thread with no concurrent work to do -> exit
26925 if (!settings.concurrent)
26927 dprintf (3, ("no concurrent GC needed, exiting"));
26933 recursive_gc_sync::begin_background();
26934 dprintf (2, ("beginning of bgc: gen2 FL: %d, FO: %d, frag: %d",
26935 generation_free_list_space (generation_of (max_generation)),
26936 generation_free_obj_space (generation_of (max_generation)),
26937 dd_fragmentation (dynamic_data_of (max_generation))));
26941 current_bgc_state = bgc_not_in_process;
26944 //trace_gc = FALSE;
26947 enable_preemptive ();
26948 #ifdef MULTIPLE_HEAPS
26949 bgc_t_join.join(this, gc_join_done);
26950 if (bgc_t_join.joined())
26951 #endif //MULTIPLE_HEAPS
26953 enter_spin_lock (&gc_lock);
26954 dprintf (SPINLOCK_LOG, ("bgc Egc"));
26956 bgc_start_event.Reset();
26958 #ifdef MULTIPLE_HEAPS
26959 for (int gen = max_generation; gen <= (max_generation + 1); gen++)
26961 size_t desired_per_heap = 0;
26962 size_t total_desired = 0;
26965 for (int i = 0; i < n_heaps; i++)
26968 dd = hp->dynamic_data_of (gen);
26969 size_t temp_total_desired = total_desired + dd_desired_allocation (dd);
26970 if (temp_total_desired < total_desired)
26973 total_desired = (size_t)MAX_PTR;
26976 total_desired = temp_total_desired;
26979 desired_per_heap = Align ((total_desired/n_heaps), get_alignment_constant (FALSE));
26981 for (int i = 0; i < n_heaps; i++)
26983 hp = gc_heap::g_heaps[i];
26984 dd = hp->dynamic_data_of (gen);
26985 dd_desired_allocation (dd) = desired_per_heap;
26986 dd_gc_new_allocation (dd) = desired_per_heap;
26987 dd_new_allocation (dd) = desired_per_heap;
26990 #endif //MULTIPLE_HEAPS
26991 #ifdef MULTIPLE_HEAPS
26993 #endif //MULTIPLE_HEAPS
26995 c_write (settings.concurrent, FALSE);
26996 recursive_gc_sync::end_background();
26997 keep_bgc_threads_p = FALSE;
26998 background_gc_done_event.Set();
27000 dprintf (SPINLOCK_LOG, ("bgc Lgc"));
27001 leave_spin_lock (&gc_lock);
27002 #ifdef MULTIPLE_HEAPS
27003 dprintf(1, ("End of BGC - starting all BGC threads"));
27004 bgc_t_join.restart();
27005 #endif //MULTIPLE_HEAPS
27007 // We can't disable preempt here because there might've been a GC already
27008 // started and decided to do a BGC and waiting for a BGC thread to restart
27009 // vm. That GC will be waiting in wait_to_proceed and we are waiting for it
27010 // to restart the VM so we deadlock.
27011 //gc_heap::disable_preemptive (current_thread, TRUE);
27014 FIRE_EVENT(GCTerminateConcurrentThread_V1);
27016 dprintf (3, ("bgc_thread thread exiting"));
27020 #endif //BACKGROUND_GC
27022 //Clear the cards [start_card, end_card[
27023 void gc_heap::clear_cards (size_t start_card, size_t end_card)
27025 if (start_card < end_card)
27027 size_t start_word = card_word (start_card);
27028 size_t end_word = card_word (end_card);
27029 if (start_word < end_word)
27031 // Figure out the bit positions of the cards within their words
27032 unsigned bits = card_bit (start_card);
27033 card_table [start_word] &= lowbits (~0, bits);
27034 for (size_t i = start_word+1; i < end_word; i++)
27035 card_table [i] = 0;
27036 bits = card_bit (end_card);
27037 // Don't write beyond end_card (and possibly uncommitted card table space).
27040 card_table [end_word] &= highbits (~0, bits);
27045 // If the start and end cards are in the same word, just clear the appropriate card
27046 // bits in that word.
27047 card_table [start_word] &= (lowbits (~0, card_bit (start_card)) |
27048 highbits (~0, card_bit (end_card)));
27050 #ifdef VERYSLOWDEBUG
27051 size_t card = start_card;
27052 while (card < end_card)
27054 assert (! (card_set_p (card)));
27057 #endif //VERYSLOWDEBUG
27058 dprintf (3,("Cleared cards [%Ix:%Ix, %Ix:%Ix[",
27059 start_card, (size_t)card_address (start_card),
27060 end_card, (size_t)card_address (end_card)));
27064 void gc_heap::clear_card_for_addresses (uint8_t* start_address, uint8_t* end_address)
27066 size_t start_card = card_of (align_on_card (start_address));
27067 size_t end_card = card_of (align_lower_card (end_address));
27068 clear_cards (start_card, end_card);
27071 // copy [srccard, ...[ to [dst_card, end_card[
27072 // This will set the same bit twice. Can be optimized.
27074 void gc_heap::copy_cards (size_t dst_card,
27079 // If the range is empty, this function is a no-op - with the subtlety that
27080 // either of the accesses card_table[srcwrd] or card_table[dstwrd] could be
27081 // outside the committed region. To avoid the access, leave early.
27082 if (!(dst_card < end_card))
27085 unsigned int srcbit = card_bit (src_card);
27086 unsigned int dstbit = card_bit (dst_card);
27087 size_t srcwrd = card_word (src_card);
27088 size_t dstwrd = card_word (dst_card);
27089 unsigned int srctmp = card_table[srcwrd];
27090 unsigned int dsttmp = card_table[dstwrd];
27092 for (size_t card = dst_card; card < end_card; card++)
27094 if (srctmp & (1 << srcbit))
27095 dsttmp |= 1 << dstbit;
27097 dsttmp &= ~(1 << dstbit);
27099 if (!(++srcbit % 32))
27101 srctmp = card_table[++srcwrd];
27107 if (srctmp & (1 << srcbit))
27108 dsttmp |= 1 << dstbit;
27111 if (!(++dstbit % 32))
27113 card_table[dstwrd] = dsttmp;
27115 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27118 card_bundle_set(cardw_card_bundle(dstwrd));
27123 dsttmp = card_table[dstwrd];
27128 card_table[dstwrd] = dsttmp;
27130 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27133 card_bundle_set(cardw_card_bundle(dstwrd));
27138 void gc_heap::copy_cards_for_addresses (uint8_t* dest, uint8_t* src, size_t len)
27140 ptrdiff_t relocation_distance = src - dest;
27141 size_t start_dest_card = card_of (align_on_card (dest));
27142 size_t end_dest_card = card_of (dest + len - 1);
27143 size_t dest_card = start_dest_card;
27144 size_t src_card = card_of (card_address (dest_card)+relocation_distance);
27145 dprintf (3,("Copying cards [%Ix:%Ix->%Ix:%Ix, ",
27146 src_card, (size_t)src, dest_card, (size_t)dest));
27147 dprintf (3,(" %Ix->%Ix:%Ix[",
27148 (size_t)src+len, end_dest_card, (size_t)dest+len));
27150 dprintf (3, ("dest: %Ix, src: %Ix, len: %Ix, reloc: %Ix, align_on_card(dest) is %Ix",
27151 dest, src, len, relocation_distance, (align_on_card (dest))));
27153 dprintf (3, ("start_dest_card: %Ix (address: %Ix), end_dest_card: %Ix(addr: %Ix), card_of (dest): %Ix",
27154 start_dest_card, card_address (start_dest_card), end_dest_card, card_address (end_dest_card), card_of (dest)));
27156 //First card has two boundaries
27157 if (start_dest_card != card_of (dest))
27159 if ((card_of (card_address (start_dest_card) + relocation_distance) <= card_of (src + len - 1))&&
27160 card_set_p (card_of (card_address (start_dest_card) + relocation_distance)))
27162 dprintf (3, ("card_address (start_dest_card) + reloc is %Ix, card: %Ix(set), src+len-1: %Ix, card: %Ix",
27163 (card_address (start_dest_card) + relocation_distance),
27164 card_of (card_address (start_dest_card) + relocation_distance),
27166 card_of (src + len - 1)));
27168 dprintf (3, ("setting card: %Ix", card_of (dest)));
27169 set_card (card_of (dest));
27173 if (card_set_p (card_of (src)))
27174 set_card (card_of (dest));
27177 copy_cards (dest_card, src_card, end_dest_card,
27178 ((dest - align_lower_card (dest)) != (src - align_lower_card (src))));
27180 //Last card has two boundaries.
27181 if ((card_of (card_address (end_dest_card) + relocation_distance) >= card_of (src)) &&
27182 card_set_p (card_of (card_address (end_dest_card) + relocation_distance)))
27184 dprintf (3, ("card_address (end_dest_card) + reloc is %Ix, card: %Ix(set), src: %Ix, card: %Ix",
27185 (card_address (end_dest_card) + relocation_distance),
27186 card_of (card_address (end_dest_card) + relocation_distance),
27190 dprintf (3, ("setting card: %Ix", end_dest_card));
27191 set_card (end_dest_card);
27194 if (card_set_p (card_of (src + len - 1)))
27195 set_card (end_dest_card);
27197 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27198 card_bundles_set(cardw_card_bundle(card_word(card_of(dest))), cardw_card_bundle(align_cardw_on_bundle(card_word(end_dest_card))));
27202 #ifdef BACKGROUND_GC
27203 // this does not need the Interlocked version of mark_array_set_marked.
27204 void gc_heap::copy_mark_bits_for_addresses (uint8_t* dest, uint8_t* src, size_t len)
27206 dprintf (3, ("Copying mark_bits for addresses [%Ix->%Ix, %Ix->%Ix[",
27207 (size_t)src, (size_t)dest,
27208 (size_t)src+len, (size_t)dest+len));
27210 uint8_t* src_o = src;
27212 uint8_t* src_end = src + len;
27213 int align_const = get_alignment_constant (TRUE);
27214 ptrdiff_t reloc = dest - src;
27216 while (src_o < src_end)
27218 uint8_t* next_o = src_o + Align (size (src_o), align_const);
27220 if (background_object_marked (src_o, TRUE))
27222 dest_o = src_o + reloc;
27224 //if (background_object_marked (dest_o, FALSE))
27226 // dprintf (3, ("*%Ix shouldn't have already been marked!", (size_t)(dest_o)));
27227 // FATAL_GC_ERROR();
27230 background_mark (dest_o,
27231 background_saved_lowest_address,
27232 background_saved_highest_address);
27233 dprintf (3, ("bc*%Ix*bc, b*%Ix*b", (size_t)src_o, (size_t)(dest_o)));
27239 #endif //BACKGROUND_GC
27241 void gc_heap::fix_brick_to_highest (uint8_t* o, uint8_t* next_o)
27243 size_t new_current_brick = brick_of (o);
27244 set_brick (new_current_brick,
27245 (o - brick_address (new_current_brick)));
27246 size_t b = 1 + new_current_brick;
27247 size_t limit = brick_of (next_o);
27248 //dprintf(3,(" fixing brick %Ix to point to object %Ix, till %Ix(%Ix)",
27249 dprintf(3,("b:%Ix->%Ix-%Ix",
27250 new_current_brick, (size_t)o, (size_t)next_o));
27253 set_brick (b,(new_current_brick - b));
27258 // start can not be >= heap_segment_allocated for the segment.
27259 uint8_t* gc_heap::find_first_object (uint8_t* start, uint8_t* first_object)
27261 size_t brick = brick_of (start);
27263 //last_object == null -> no search shortcut needed
27264 if ((brick == brick_of (first_object) || (start <= first_object)))
27270 ptrdiff_t min_brick = (ptrdiff_t)brick_of (first_object);
27271 ptrdiff_t prev_brick = (ptrdiff_t)brick - 1;
27272 int brick_entry = 0;
27275 if (prev_brick < min_brick)
27279 if ((brick_entry = get_brick_entry(prev_brick)) >= 0)
27283 assert (! ((brick_entry == 0)));
27284 prev_brick = (brick_entry + prev_brick);
27287 o = ((prev_brick < min_brick) ? first_object :
27288 brick_address (prev_brick) + brick_entry - 1);
27289 assert (o <= start);
27292 assert (Align (size (o)) >= Align (min_obj_size));
27293 uint8_t* next_o = o + Align (size (o));
27294 size_t curr_cl = (size_t)next_o / brick_size;
27295 size_t min_cl = (size_t)first_object / brick_size;
27297 //dprintf (3,( "Looking for intersection with %Ix from %Ix", (size_t)start, (size_t)o));
27299 unsigned int n_o = 1;
27302 uint8_t* next_b = min (align_lower_brick (next_o) + brick_size, start+1);
27304 while (next_o <= start)
27312 assert (Align (size (o)) >= Align (min_obj_size));
27313 next_o = o + Align (size (o));
27315 }while (next_o < next_b);
27317 if (((size_t)next_o / brick_size) != curr_cl)
27319 if (curr_cl >= min_cl)
27321 fix_brick_to_highest (o, next_o);
27323 curr_cl = (size_t) next_o / brick_size;
27325 next_b = min (align_lower_brick (next_o) + brick_size, start+1);
27328 size_t bo = brick_of (o);
27329 //dprintf (3, ("Looked at %Id objects, fixing brick [%Ix-[%Ix",
27330 dprintf (3, ("%Id o, [%Ix-[%Ix",
27334 set_brick (bo, (o - brick_address(bo)));
27349 // Find the first non-zero card word between cardw and cardw_end.
27350 // The index of the word we find is returned in cardw.
27351 BOOL gc_heap::find_card_dword (size_t& cardw, size_t cardw_end)
27353 dprintf (3, ("gc: %d, find_card_dword cardw: %Ix, cardw_end: %Ix",
27354 dd_collection_count (dynamic_data_of (0)), cardw, cardw_end));
27356 if (card_bundles_enabled())
27358 size_t cardb = cardw_card_bundle (cardw);
27359 size_t end_cardb = cardw_card_bundle (align_cardw_on_bundle (cardw_end));
27362 // Find a non-zero bundle
27363 while ((cardb < end_cardb) && (card_bundle_set_p (cardb) == 0))
27368 if (cardb == end_cardb)
27371 // We found a bundle, so go through its words and find a non-zero card word
27372 uint32_t* card_word = &card_table[max(card_bundle_cardw (cardb),cardw)];
27373 uint32_t* card_word_end = &card_table[min(card_bundle_cardw (cardb+1),cardw_end)];
27374 while ((card_word < card_word_end) && !(*card_word))
27379 if (card_word != card_word_end)
27381 cardw = (card_word - &card_table[0]);
27384 else if ((cardw <= card_bundle_cardw (cardb)) &&
27385 (card_word == &card_table [card_bundle_cardw (cardb+1)]))
27387 // a whole bundle was explored and is empty
27388 dprintf (3, ("gc: %d, find_card_dword clear bundle: %Ix cardw:[%Ix,%Ix[",
27389 dd_collection_count (dynamic_data_of (0)),
27390 cardb, card_bundle_cardw (cardb),
27391 card_bundle_cardw (cardb+1)));
27392 card_bundle_clear (cardb);
27400 uint32_t* card_word = &card_table[cardw];
27401 uint32_t* card_word_end = &card_table [cardw_end];
27403 while (card_word < card_word_end)
27405 if (*card_word != 0)
27407 cardw = (card_word - &card_table [0]);
27418 #endif //CARD_BUNDLE
27420 // Find cards that are set between two points in a card table.
27422 // card_table : The card table.
27423 // card : [in/out] As input, the card to start searching from.
27424 // As output, the first card that's set.
27425 // card_word_end : The card word at which to stop looking.
27426 // end_card : [out] The last card which is set.
27427 BOOL gc_heap::find_card(uint32_t* card_table,
27429 size_t card_word_end,
27432 uint32_t* last_card_word;
27433 uint32_t card_word_value;
27434 uint32_t bit_position;
27436 // Find the first card which is set
27437 last_card_word = &card_table [card_word (card)];
27438 bit_position = card_bit (card);
27439 card_word_value = (*last_card_word) >> bit_position;
27440 if (!card_word_value)
27444 // Using the card bundle, go through the remaining card words between here and
27445 // card_word_end until we find one that is non-zero.
27446 size_t lcw = card_word(card) + 1;
27447 if (gc_heap::find_card_dword (lcw, card_word_end) == FALSE)
27453 last_card_word = &card_table [lcw];
27454 card_word_value = *last_card_word;
27457 #else //CARD_BUNDLE
27458 // Go through the remaining card words between here and card_word_end until we find
27459 // one that is non-zero.
27464 while ((last_card_word < &card_table [card_word_end]) && !(*last_card_word));
27466 if (last_card_word < &card_table [card_word_end])
27468 card_word_value = *last_card_word;
27472 // We failed to find any non-zero card words before we got to card_word_end
27475 #endif //CARD_BUNDLE
27478 // Look for the lowest bit set
27479 if (card_word_value)
27481 while (!(card_word_value & 1))
27484 card_word_value = card_word_value / 2;
27488 // card is the card word index * card size + the bit index within the card
27489 card = (last_card_word - &card_table[0]) * card_word_width + bit_position;
27493 // Keep going until we get to an un-set card.
27495 card_word_value = card_word_value / 2;
27497 // If we reach the end of the card word and haven't hit a 0 yet, start going
27498 // card word by card word until we get to one that's not fully set (0xFFFF...)
27499 // or we reach card_word_end.
27500 if ((bit_position == card_word_width) && (last_card_word < &card_table [card_word_end]))
27504 card_word_value = *(++last_card_word);
27505 } while ((last_card_word < &card_table [card_word_end]) &&
27508 (card_word_value == (1 << card_word_width)-1)
27510 // if left shift count >= width of type,
27511 // gcc reports error.
27512 (card_word_value == ~0u)
27517 } while (card_word_value & 1);
27519 end_card = (last_card_word - &card_table [0])* card_word_width + bit_position;
27521 //dprintf (3, ("find_card: [%Ix, %Ix[ set", card, end_card));
27522 dprintf (3, ("fc: [%Ix, %Ix[", card, end_card));
27527 //because of heap expansion, computing end is complicated.
27528 uint8_t* compute_next_end (heap_segment* seg, uint8_t* low)
27530 if ((low >= heap_segment_mem (seg)) &&
27531 (low < heap_segment_allocated (seg)))
27534 return heap_segment_allocated (seg);
27538 gc_heap::compute_next_boundary (uint8_t* low, int gen_number,
27541 UNREFERENCED_PARAMETER(low);
27543 //when relocating, the fault line is the plan start of the younger
27544 //generation because the generation is promoted.
27545 if (relocating && (gen_number == (settings.condemned_generation + 1)))
27547 generation* gen = generation_of (gen_number - 1);
27548 uint8_t* gen_alloc = generation_plan_allocation_start (gen);
27549 assert (gen_alloc);
27554 assert (gen_number > settings.condemned_generation);
27555 return generation_allocation_start (generation_of (gen_number - 1 ));
27561 gc_heap::keep_card_live (uint8_t* o, size_t& n_gen,
27562 size_t& cg_pointers_found)
27565 if ((gc_low <= o) && (gc_high > o))
27569 #ifdef MULTIPLE_HEAPS
27572 gc_heap* hp = heap_of (o);
27575 if ((hp->gc_low <= o) &&
27582 #endif //MULTIPLE_HEAPS
27583 cg_pointers_found ++;
27584 dprintf (4, ("keep card live for %Ix", o));
27588 gc_heap::mark_through_cards_helper (uint8_t** poo, size_t& n_gen,
27589 size_t& cg_pointers_found,
27590 card_fn fn, uint8_t* nhigh,
27591 uint8_t* next_boundary)
27594 if ((gc_low <= *poo) && (gc_high > *poo))
27597 call_fn(fn) (poo THREAD_NUMBER_ARG);
27599 #ifdef MULTIPLE_HEAPS
27602 gc_heap* hp = heap_of_gc (*poo);
27605 if ((hp->gc_low <= *poo) &&
27606 (hp->gc_high > *poo))
27609 call_fn(fn) (poo THREAD_NUMBER_ARG);
27611 if ((fn == &gc_heap::relocate_address) ||
27612 ((hp->ephemeral_low <= *poo) &&
27613 (hp->ephemeral_high > *poo)))
27615 cg_pointers_found++;
27619 #endif //MULTIPLE_HEAPS
27620 if ((next_boundary <= *poo) && (nhigh > *poo))
27622 cg_pointers_found ++;
27623 dprintf (4, ("cg pointer %Ix found, %Id so far",
27624 (size_t)*poo, cg_pointers_found ));
27629 BOOL gc_heap::card_transition (uint8_t* po, uint8_t* end, size_t card_word_end,
27630 size_t& cg_pointers_found,
27631 size_t& n_eph, size_t& n_card_set,
27632 size_t& card, size_t& end_card,
27633 BOOL& foundp, uint8_t*& start_address,
27634 uint8_t*& limit, size_t& n_cards_cleared)
27636 dprintf (3, ("pointer %Ix past card %Ix", (size_t)po, (size_t)card));
27637 dprintf (3, ("ct: %Id cg", cg_pointers_found));
27638 BOOL passed_end_card_p = FALSE;
27641 if (cg_pointers_found == 0)
27643 //dprintf(3,(" Clearing cards [%Ix, %Ix[ ",
27644 dprintf(3,(" CC [%Ix, %Ix[ ",
27645 (size_t)card_address(card), (size_t)po));
27646 clear_cards (card, card_of(po));
27647 n_card_set -= (card_of (po) - card);
27648 n_cards_cleared += (card_of (po) - card);
27651 n_eph +=cg_pointers_found;
27652 cg_pointers_found = 0;
27653 card = card_of (po);
27654 if (card >= end_card)
27656 passed_end_card_p = TRUE;
27657 dprintf (3, ("card %Ix exceeding end_card %Ix",
27658 (size_t)card, (size_t)end_card));
27659 foundp = find_card (card_table, card, card_word_end, end_card);
27662 n_card_set+= end_card - card;
27663 start_address = card_address (card);
27664 dprintf (3, ("NewC: %Ix, start: %Ix, end: %Ix",
27665 (size_t)card, (size_t)start_address,
27666 (size_t)card_address (end_card)));
27668 limit = min (end, card_address (end_card));
27670 assert (!((limit == card_address (end_card))&&
27671 card_set_p (end_card)));
27674 return passed_end_card_p;
27677 void gc_heap::mark_through_cards_for_segments (card_fn fn, BOOL relocating)
27679 #ifdef BACKGROUND_GC
27680 dprintf (3, ("current_sweep_pos is %Ix, saved_sweep_ephemeral_seg is %Ix(%Ix)",
27681 current_sweep_pos, saved_sweep_ephemeral_seg, saved_sweep_ephemeral_start));
27683 heap_segment* soh_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
27684 PREFIX_ASSUME(soh_seg != NULL);
27688 dprintf (3, ("seg %Ix, bgc_alloc: %Ix, alloc: %Ix",
27690 heap_segment_background_allocated (soh_seg),
27691 heap_segment_allocated (soh_seg)));
27693 soh_seg = heap_segment_next_rw (soh_seg);
27695 #endif //BACKGROUND_GC
27697 uint8_t* low = gc_low;
27698 uint8_t* high = gc_high;
27699 size_t end_card = 0;
27701 generation* oldest_gen = generation_of (max_generation);
27702 int curr_gen_number = max_generation;
27703 uint8_t* gen_boundary = generation_allocation_start(generation_of(curr_gen_number - 1));
27704 uint8_t* next_boundary = compute_next_boundary(gc_low, curr_gen_number, relocating);
27706 heap_segment* seg = heap_segment_rw (generation_start_segment (oldest_gen));
27707 PREFIX_ASSUME(seg != NULL);
27709 uint8_t* beg = generation_allocation_start (oldest_gen);
27710 uint8_t* end = compute_next_end (seg, low);
27711 uint8_t* last_object = beg;
27713 size_t cg_pointers_found = 0;
27715 size_t card_word_end = (card_of (align_on_card_word (end)) / card_word_width);
27719 size_t n_card_set = 0;
27720 uint8_t* nhigh = (relocating ? heap_segment_plan_allocated (ephemeral_heap_segment) : high);
27722 BOOL foundp = FALSE;
27723 uint8_t* start_address = 0;
27724 uint8_t* limit = 0;
27725 size_t card = card_of (beg);
27726 #ifdef BACKGROUND_GC
27727 BOOL consider_bgc_mark_p = FALSE;
27728 BOOL check_current_sweep_p = FALSE;
27729 BOOL check_saved_sweep_p = FALSE;
27730 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
27731 #endif //BACKGROUND_GC
27733 dprintf(3, ("CMs: %Ix->%Ix", (size_t)beg, (size_t)end));
27734 size_t total_cards_cleared = 0;
27738 if (card_of(last_object) > card)
27740 // cg means cross-generational
27741 dprintf (3, ("Found %Id cg pointers", cg_pointers_found));
27742 if (cg_pointers_found == 0)
27744 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card), (size_t)last_object));
27745 clear_cards (card, card_of(last_object));
27746 n_card_set -= (card_of (last_object) - card);
27747 total_cards_cleared += (card_of (last_object) - card);
27750 n_eph += cg_pointers_found;
27751 cg_pointers_found = 0;
27752 card = card_of (last_object);
27755 if (card >= end_card)
27757 // Find the first card that's set (between card and card_word_end)
27758 foundp = find_card(card_table, card, card_word_end, end_card);
27761 // We found card(s) set.
27762 n_card_set += end_card - card;
27763 start_address = max (beg, card_address (card));
27766 limit = min (end, card_address (end_card));
27769 if (!foundp || (last_object >= end) || (card_address (card) >= end))
27771 if (foundp && (cg_pointers_found == 0))
27773 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card),
27775 clear_cards (card, card_of (end));
27776 n_card_set -= (card_of (end) - card);
27777 total_cards_cleared += (card_of (end) - card);
27780 n_eph += cg_pointers_found;
27781 cg_pointers_found = 0;
27783 if ((seg = heap_segment_next_in_range (seg)) != 0)
27785 #ifdef BACKGROUND_GC
27786 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
27787 #endif //BACKGROUND_GC
27788 beg = heap_segment_mem (seg);
27789 end = compute_next_end (seg, low);
27790 card_word_end = card_of (align_on_card_word (end)) / card_word_width;
27791 card = card_of (beg);
27802 // We've found a card and will now go through the objects in it.
27803 assert (card_set_p (card));
27805 uint8_t* o = last_object;
27806 o = find_first_object (start_address, last_object);
27807 // Never visit an object twice.
27808 assert (o >= last_object);
27810 //dprintf(3,("Considering card %Ix start object: %Ix, %Ix[ boundary: %Ix",
27811 dprintf(3, ("c: %Ix, o: %Ix, l: %Ix[ boundary: %Ix",
27812 card, (size_t)o, (size_t)limit, (size_t)gen_boundary));
27816 assert (Align (size (o)) >= Align (min_obj_size));
27817 size_t s = size (o);
27819 uint8_t* next_o = o + Align (s);
27822 if ((o >= gen_boundary) &&
27823 (seg == ephemeral_heap_segment))
27825 dprintf (3, ("switching gen boundary %Ix", (size_t)gen_boundary));
27827 assert ((curr_gen_number > 0));
27828 gen_boundary = generation_allocation_start
27829 (generation_of (curr_gen_number - 1));
27830 next_boundary = (compute_next_boundary
27831 (low, curr_gen_number, relocating));
27834 dprintf (4, ("|%Ix|", (size_t)o));
27836 if (next_o < start_address)
27841 #ifdef BACKGROUND_GC
27842 if (!fgc_should_consider_object (o, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p))
27846 #endif //BACKGROUND_GC
27848 #ifdef COLLECTIBLE_CLASS
27849 if (is_collectible(o))
27851 BOOL passed_end_card_p = FALSE;
27853 if (card_of (o) > card)
27855 passed_end_card_p = card_transition (o, end, card_word_end,
27859 foundp, start_address,
27860 limit, total_cards_cleared);
27863 if ((!passed_end_card_p || foundp) && (card_of (o) == card))
27865 // card is valid and it covers the head of the object
27866 if (fn == &gc_heap::relocate_address)
27868 keep_card_live (o, n_gen, cg_pointers_found);
27872 uint8_t* class_obj = get_class_object (o);
27873 mark_through_cards_helper (&class_obj, n_gen,
27874 cg_pointers_found, fn,
27875 nhigh, next_boundary);
27879 if (passed_end_card_p)
27881 if (foundp && (card_address (card) < next_o))
27883 goto go_through_refs;
27885 else if (foundp && (start_address < limit))
27887 next_o = find_first_object (start_address, o);
27896 #endif //COLLECTIBLE_CLASS
27898 if (contain_pointers (o))
27900 dprintf(3,("Going through %Ix start_address: %Ix", (size_t)o, (size_t)start_address));
27903 dprintf (4, ("normal object path"));
27905 (method_table(o), o, s, poo,
27906 start_address, use_start, (o + s),
27908 dprintf (4, ("<%Ix>:%Ix", (size_t)poo, (size_t)*poo));
27909 if (card_of ((uint8_t*)poo) > card)
27911 BOOL passed_end_card_p = card_transition ((uint8_t*)poo, end,
27916 foundp, start_address,
27917 limit, total_cards_cleared);
27919 if (passed_end_card_p)
27921 if (foundp && (card_address (card) < next_o))
27925 if (ppstop <= (uint8_t**)start_address)
27927 else if (poo < (uint8_t**)start_address)
27928 {poo = (uint8_t**)start_address;}
27931 else if (foundp && (start_address < limit))
27933 next_o = find_first_object (start_address, o);
27941 mark_through_cards_helper (poo, n_gen,
27942 cg_pointers_found, fn,
27943 nhigh, next_boundary);
27950 if (((size_t)next_o / brick_size) != ((size_t) o / brick_size))
27952 if (brick_table [brick_of (o)] <0)
27953 fix_brick_to_highest (o, next_o);
27961 // compute the efficiency ratio of the card table
27964 generation_skip_ratio = ((n_eph > 400)? (int)(((float)n_gen / (float)n_eph) * 100) : 100);
27965 dprintf (3, ("Msoh: cross: %Id, useful: %Id, cards set: %Id, cards cleared: %Id, ratio: %d",
27966 n_eph, n_gen , n_card_set, total_cards_cleared, generation_skip_ratio));
27970 dprintf (3, ("R: Msoh: cross: %Id, useful: %Id, cards set: %Id, cards cleared: %Id, ratio: %d",
27971 n_gen, n_eph, n_card_set, total_cards_cleared, generation_skip_ratio));
27975 #ifdef SEG_REUSE_STATS
27976 size_t gc_heap::dump_buckets (size_t* ordered_indices, int count, size_t* total_size)
27978 size_t total_items = 0;
27980 for (int i = 0; i < count; i++)
27982 total_items += ordered_indices[i];
27983 *total_size += ordered_indices[i] << (MIN_INDEX_POWER2 + i);
27984 dprintf (SEG_REUSE_LOG_0, ("[%d]%4d 2^%2d", heap_number, ordered_indices[i], (MIN_INDEX_POWER2 + i)));
27986 dprintf (SEG_REUSE_LOG_0, ("[%d]Total %d items, total size is 0x%Ix", heap_number, total_items, *total_size));
27987 return total_items;
27989 #endif // SEG_REUSE_STATS
27991 void gc_heap::count_plug (size_t last_plug_size, uint8_t*& last_plug)
27993 // detect pinned plugs
27994 if (!pinned_plug_que_empty_p() && (last_plug == pinned_plug (oldest_pin())))
27996 deque_pinned_plug();
27997 update_oldest_pinned_plug();
27998 dprintf (3, ("dequed pin,now oldest pin is %Ix", pinned_plug (oldest_pin())));
28002 size_t plug_size = last_plug_size + Align(min_obj_size);
28003 BOOL is_padded = FALSE;
28006 plug_size += Align (min_obj_size);
28008 #endif //SHORT_PLUGS
28010 #ifdef RESPECT_LARGE_ALIGNMENT
28011 plug_size += switch_alignment_size (is_padded);
28012 #endif //RESPECT_LARGE_ALIGNMENT
28014 total_ephemeral_plugs += plug_size;
28015 size_t plug_size_power2 = round_up_power2 (plug_size);
28016 ordered_plug_indices[relative_index_power2_plug (plug_size_power2)]++;
28017 dprintf (SEG_REUSE_LOG_1, ("[%d]count_plug: adding 0x%Ix - %Id (2^%d) to ordered plug array",
28021 (relative_index_power2_plug (plug_size_power2) + MIN_INDEX_POWER2)));
28025 void gc_heap::count_plugs_in_brick (uint8_t* tree, uint8_t*& last_plug)
28027 assert ((tree != NULL));
28028 if (node_left_child (tree))
28030 count_plugs_in_brick (tree + node_left_child (tree), last_plug);
28033 if (last_plug != 0)
28035 uint8_t* plug = tree;
28036 size_t gap_size = node_gap_size (plug);
28037 uint8_t* gap = (plug - gap_size);
28038 uint8_t* last_plug_end = gap;
28039 size_t last_plug_size = (last_plug_end - last_plug);
28040 dprintf (3, ("tree: %Ix, last plug: %Ix, gap size: %Ix, gap: %Ix, last plug size: %Ix",
28041 tree, last_plug, gap_size, gap, last_plug_size));
28043 if (tree == oldest_pinned_plug)
28045 dprintf (3, ("tree %Ix is pinned, last plug is %Ix, size is %Ix",
28046 tree, last_plug, last_plug_size));
28047 mark* m = oldest_pin();
28048 if (m->has_pre_plug_info())
28050 last_plug_size += sizeof (gap_reloc_pair);
28051 dprintf (3, ("pin %Ix has pre plug, adjusting plug size to %Ix", tree, last_plug_size));
28054 // Can't assert here - if it's a pinned plug it can be less.
28055 //assert (last_plug_size >= Align (min_obj_size));
28057 count_plug (last_plug_size, last_plug);
28062 if (node_right_child (tree))
28064 count_plugs_in_brick (tree + node_right_child (tree), last_plug);
28068 void gc_heap::build_ordered_plug_indices ()
28070 memset (ordered_plug_indices, 0, sizeof(ordered_plug_indices));
28071 memset (saved_ordered_plug_indices, 0, sizeof(saved_ordered_plug_indices));
28073 uint8_t* start_address = generation_limit (max_generation);
28074 uint8_t* end_address = heap_segment_allocated (ephemeral_heap_segment);
28075 size_t current_brick = brick_of (start_address);
28076 size_t end_brick = brick_of (end_address - 1);
28077 uint8_t* last_plug = 0;
28079 //Look for the right pinned plug to start from.
28080 reset_pinned_queue_bos();
28081 while (!pinned_plug_que_empty_p())
28083 mark* m = oldest_pin();
28084 if ((m->first >= start_address) && (m->first < end_address))
28086 dprintf (3, ("found a pin %Ix between %Ix and %Ix", m->first, start_address, end_address));
28091 deque_pinned_plug();
28094 update_oldest_pinned_plug();
28096 while (current_brick <= end_brick)
28098 int brick_entry = brick_table [ current_brick ];
28099 if (brick_entry >= 0)
28101 count_plugs_in_brick (brick_address (current_brick) + brick_entry -1, last_plug);
28109 count_plug (end_address - last_plug, last_plug);
28112 // we need to make sure that after fitting all the existing plugs, we
28113 // have big enough free space left to guarantee that the next allocation
28115 size_t extra_size = END_SPACE_AFTER_GC + Align (min_obj_size);
28116 total_ephemeral_plugs += extra_size;
28117 dprintf (SEG_REUSE_LOG_0, ("Making sure we can fit a large object after fitting all plugs"));
28118 ordered_plug_indices[relative_index_power2_plug (round_up_power2 (extra_size))]++;
28120 memcpy (saved_ordered_plug_indices, ordered_plug_indices, sizeof(ordered_plug_indices));
28122 #ifdef SEG_REUSE_STATS
28123 dprintf (SEG_REUSE_LOG_0, ("Plugs:"));
28124 size_t total_plug_power2 = 0;
28125 dump_buckets (ordered_plug_indices, MAX_NUM_BUCKETS, &total_plug_power2);
28126 dprintf (SEG_REUSE_LOG_0, ("plugs: 0x%Ix (rounded up to 0x%Ix (%d%%))",
28127 total_ephemeral_plugs,
28129 (total_ephemeral_plugs ?
28130 (total_plug_power2 * 100 / total_ephemeral_plugs) :
28132 dprintf (SEG_REUSE_LOG_0, ("-------------------"));
28133 #endif // SEG_REUSE_STATS
28136 void gc_heap::init_ordered_free_space_indices ()
28138 memset (ordered_free_space_indices, 0, sizeof(ordered_free_space_indices));
28139 memset (saved_ordered_free_space_indices, 0, sizeof(saved_ordered_free_space_indices));
28142 void gc_heap::trim_free_spaces_indices ()
28144 trimmed_free_space_index = -1;
28145 size_t max_count = max_free_space_items - 1;
28148 for (i = (MAX_NUM_BUCKETS - 1); i >= 0; i--)
28150 count += ordered_free_space_indices[i];
28152 if (count >= max_count)
28158 ptrdiff_t extra_free_space_items = count - max_count;
28160 if (extra_free_space_items > 0)
28162 ordered_free_space_indices[i] -= extra_free_space_items;
28163 free_space_items = max_count;
28164 trimmed_free_space_index = i;
28168 free_space_items = count;
28176 free_space_buckets = MAX_NUM_BUCKETS - i;
28178 for (--i; i >= 0; i--)
28180 ordered_free_space_indices[i] = 0;
28183 memcpy (saved_ordered_free_space_indices,
28184 ordered_free_space_indices,
28185 sizeof(ordered_free_space_indices));
28188 // We fit as many plugs as we can and update the number of plugs left and the number
28189 // of free spaces left.
28190 BOOL gc_heap::can_fit_in_spaces_p (size_t* ordered_blocks, int small_index, size_t* ordered_spaces, int big_index)
28192 assert (small_index <= big_index);
28193 assert (big_index < MAX_NUM_BUCKETS);
28195 size_t small_blocks = ordered_blocks[small_index];
28197 if (small_blocks == 0)
28202 size_t big_spaces = ordered_spaces[big_index];
28204 if (big_spaces == 0)
28209 dprintf (SEG_REUSE_LOG_1, ("[%d]Fitting %Id 2^%d plugs into %Id 2^%d free spaces",
28211 small_blocks, (small_index + MIN_INDEX_POWER2),
28212 big_spaces, (big_index + MIN_INDEX_POWER2)));
28214 size_t big_to_small = big_spaces << (big_index - small_index);
28216 ptrdiff_t extra_small_spaces = big_to_small - small_blocks;
28217 dprintf (SEG_REUSE_LOG_1, ("[%d]%d 2^%d spaces can fit %d 2^%d blocks",
28219 big_spaces, (big_index + MIN_INDEX_POWER2), big_to_small, (small_index + MIN_INDEX_POWER2)));
28220 BOOL can_fit = (extra_small_spaces >= 0);
28224 dprintf (SEG_REUSE_LOG_1, ("[%d]Can fit with %d 2^%d extras blocks",
28226 extra_small_spaces, (small_index + MIN_INDEX_POWER2)));
28231 dprintf (SEG_REUSE_LOG_1, ("[%d]Setting # of 2^%d spaces to 0", heap_number, (big_index + MIN_INDEX_POWER2)));
28232 ordered_spaces[big_index] = 0;
28233 if (extra_small_spaces > 0)
28235 dprintf (SEG_REUSE_LOG_1, ("[%d]Setting # of 2^%d blocks to 0", heap_number, (small_index + MIN_INDEX_POWER2)));
28236 ordered_blocks[small_index] = 0;
28237 for (i = small_index; i < big_index; i++)
28239 if (extra_small_spaces & 1)
28241 dprintf (SEG_REUSE_LOG_1, ("[%d]Increasing # of 2^%d spaces from %d to %d",
28243 (i + MIN_INDEX_POWER2), ordered_spaces[i], (ordered_spaces[i] + 1)));
28244 ordered_spaces[i] += 1;
28246 extra_small_spaces >>= 1;
28249 dprintf (SEG_REUSE_LOG_1, ("[%d]Finally increasing # of 2^%d spaces from %d to %d",
28251 (i + MIN_INDEX_POWER2), ordered_spaces[i], (ordered_spaces[i] + extra_small_spaces)));
28252 ordered_spaces[i] += extra_small_spaces;
28256 dprintf (SEG_REUSE_LOG_1, ("[%d]Decreasing # of 2^%d blocks from %d to %d",
28258 (small_index + MIN_INDEX_POWER2),
28259 ordered_blocks[small_index],
28260 (ordered_blocks[small_index] - big_to_small)));
28261 ordered_blocks[small_index] -= big_to_small;
28264 #ifdef SEG_REUSE_STATS
28266 dprintf (SEG_REUSE_LOG_1, ("[%d]Plugs became:", heap_number));
28267 dump_buckets (ordered_blocks, MAX_NUM_BUCKETS, &temp);
28269 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces became:", heap_number));
28270 dump_buckets (ordered_spaces, MAX_NUM_BUCKETS, &temp);
28271 #endif //SEG_REUSE_STATS
28276 // space_index gets updated to the biggest available space index.
28277 BOOL gc_heap::can_fit_blocks_p (size_t* ordered_blocks, int block_index, size_t* ordered_spaces, int* space_index)
28279 assert (*space_index >= block_index);
28281 while (!can_fit_in_spaces_p (ordered_blocks, block_index, ordered_spaces, *space_index))
28284 if (*space_index < block_index)
28293 BOOL gc_heap::can_fit_all_blocks_p (size_t* ordered_blocks, size_t* ordered_spaces, int count)
28295 #ifdef FEATURE_STRUCTALIGN
28296 // BARTOKTODO (4841): reenable when can_fit_in_spaces_p takes alignment requirements into account
28298 #endif // FEATURE_STRUCTALIGN
28299 int space_index = count - 1;
28300 for (int block_index = (count - 1); block_index >= 0; block_index--)
28302 if (!can_fit_blocks_p (ordered_blocks, block_index, ordered_spaces, &space_index))
28311 void gc_heap::build_ordered_free_spaces (heap_segment* seg)
28313 assert (bestfit_seg);
28315 //bestfit_seg->add_buckets (MAX_NUM_BUCKETS - free_space_buckets + MIN_INDEX_POWER2,
28316 // ordered_free_space_indices + (MAX_NUM_BUCKETS - free_space_buckets),
28317 // free_space_buckets,
28318 // free_space_items);
28320 bestfit_seg->add_buckets (MIN_INDEX_POWER2,
28321 ordered_free_space_indices,
28325 assert (settings.condemned_generation == max_generation);
28327 uint8_t* first_address = heap_segment_mem (seg);
28328 uint8_t* end_address = heap_segment_reserved (seg);
28329 //look through the pinned plugs for relevant ones.
28330 //Look for the right pinned plug to start from.
28331 reset_pinned_queue_bos();
28333 // See comment in can_expand_into_p why we need (max_generation + 1).
28334 size_t eph_gen_starts = (Align (min_obj_size)) * (max_generation + 1);
28335 BOOL has_fit_gen_starts = FALSE;
28337 while (!pinned_plug_que_empty_p())
28340 if ((pinned_plug (m) >= first_address) &&
28341 (pinned_plug (m) < end_address) &&
28342 (pinned_len (m) >= eph_gen_starts))
28345 assert ((pinned_plug (m) - pinned_len (m)) == bestfit_first_pin);
28350 deque_pinned_plug();
28354 if (!pinned_plug_que_empty_p())
28356 bestfit_seg->add ((void*)m, TRUE, TRUE);
28357 deque_pinned_plug();
28359 has_fit_gen_starts = TRUE;
28362 while (!pinned_plug_que_empty_p() &&
28363 ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address)))
28365 bestfit_seg->add ((void*)m, TRUE, FALSE);
28366 deque_pinned_plug();
28370 if (commit_end_of_seg)
28372 if (!has_fit_gen_starts)
28374 assert (bestfit_first_pin == heap_segment_plan_allocated (seg));
28376 bestfit_seg->add ((void*)seg, FALSE, (!has_fit_gen_starts));
28380 bestfit_seg->check();
28384 BOOL gc_heap::try_best_fit (BOOL end_of_segment_p)
28386 if (!end_of_segment_p)
28388 trim_free_spaces_indices ();
28391 BOOL can_bestfit = can_fit_all_blocks_p (ordered_plug_indices,
28392 ordered_free_space_indices,
28395 return can_bestfit;
28398 BOOL gc_heap::best_fit (size_t free_space,
28399 size_t largest_free_space,
28400 size_t additional_space,
28401 BOOL* use_additional_space)
28403 dprintf (SEG_REUSE_LOG_0, ("gen%d: trying best fit mechanism", settings.condemned_generation));
28405 assert (!additional_space || (additional_space && use_additional_space));
28406 if (use_additional_space)
28408 *use_additional_space = FALSE;
28411 if (ordered_plug_indices_init == FALSE)
28413 total_ephemeral_plugs = 0;
28414 build_ordered_plug_indices();
28415 ordered_plug_indices_init = TRUE;
28419 memcpy (ordered_plug_indices, saved_ordered_plug_indices, sizeof(ordered_plug_indices));
28422 if (total_ephemeral_plugs == (END_SPACE_AFTER_GC + Align (min_obj_size)))
28424 dprintf (SEG_REUSE_LOG_0, ("No ephemeral plugs to realloc, done"));
28425 size_t empty_eph = (END_SPACE_AFTER_GC + Align (min_obj_size) + (Align (min_obj_size)) * (max_generation + 1));
28426 BOOL can_fit_empty_eph = (largest_free_space >= empty_eph);
28427 if (!can_fit_empty_eph)
28429 can_fit_empty_eph = (additional_space >= empty_eph);
28431 if (can_fit_empty_eph)
28433 *use_additional_space = TRUE;
28437 return can_fit_empty_eph;
28440 if ((total_ephemeral_plugs + approximate_new_allocation()) >= (free_space + additional_space))
28442 dprintf (SEG_REUSE_LOG_0, ("We won't have enough free space left in this segment after fitting, done"));
28446 if ((free_space + additional_space) == 0)
28448 dprintf (SEG_REUSE_LOG_0, ("No free space in this segment, done"));
28452 #ifdef SEG_REUSE_STATS
28453 dprintf (SEG_REUSE_LOG_0, ("Free spaces:"));
28454 size_t total_free_space_power2 = 0;
28455 size_t total_free_space_items =
28456 dump_buckets (ordered_free_space_indices,
28458 &total_free_space_power2);
28459 dprintf (SEG_REUSE_LOG_0, ("currently max free spaces is %Id", max_free_space_items));
28461 dprintf (SEG_REUSE_LOG_0, ("Ephemeral plugs: 0x%Ix, free space: 0x%Ix (rounded down to 0x%Ix (%Id%%)), additional free_space: 0x%Ix",
28462 total_ephemeral_plugs,
28464 total_free_space_power2,
28465 (free_space ? (total_free_space_power2 * 100 / free_space) : 0),
28466 additional_space));
28468 size_t saved_all_free_space_indices[MAX_NUM_BUCKETS];
28469 memcpy (saved_all_free_space_indices,
28470 ordered_free_space_indices,
28471 sizeof(saved_all_free_space_indices));
28473 #endif // SEG_REUSE_STATS
28475 if (total_ephemeral_plugs > (free_space + additional_space))
28480 use_bestfit = try_best_fit(FALSE);
28482 if (!use_bestfit && additional_space)
28484 int relative_free_space_index = relative_index_power2_free_space (round_down_power2 (additional_space));
28486 if (relative_free_space_index != -1)
28488 int relative_plug_index = 0;
28489 size_t plugs_to_fit = 0;
28491 for (relative_plug_index = (MAX_NUM_BUCKETS - 1); relative_plug_index >= 0; relative_plug_index--)
28493 plugs_to_fit = ordered_plug_indices[relative_plug_index];
28494 if (plugs_to_fit != 0)
28500 if ((relative_plug_index > relative_free_space_index) ||
28501 ((relative_plug_index == relative_free_space_index) &&
28502 (plugs_to_fit > 1)))
28504 #ifdef SEG_REUSE_STATS
28505 dprintf (SEG_REUSE_LOG_0, ("additional space is 2^%d but we stopped at %d 2^%d plug(s)",
28506 (relative_free_space_index + MIN_INDEX_POWER2),
28508 (relative_plug_index + MIN_INDEX_POWER2)));
28509 #endif // SEG_REUSE_STATS
28513 dprintf (SEG_REUSE_LOG_0, ("Adding end of segment (2^%d)", (relative_free_space_index + MIN_INDEX_POWER2)));
28514 ordered_free_space_indices[relative_free_space_index]++;
28515 use_bestfit = try_best_fit(TRUE);
28518 free_space_items++;
28519 // Since we might've trimmed away some of the free spaces we had, we should see
28520 // if we really need to use end of seg space - if it's the same or smaller than
28521 // the largest space we trimmed we can just add that one back instead of
28522 // using end of seg.
28523 if (relative_free_space_index > trimmed_free_space_index)
28525 *use_additional_space = TRUE;
28529 // If the addition space is <= than the last trimmed space, we
28530 // should just use that last trimmed space instead.
28531 saved_ordered_free_space_indices[trimmed_free_space_index]++;
28541 dprintf (SEG_REUSE_LOG_0, ("couldn't fit..."));
28543 #ifdef SEG_REUSE_STATS
28544 size_t saved_max = max_free_space_items;
28545 BOOL temp_bestfit = FALSE;
28547 dprintf (SEG_REUSE_LOG_0, ("----Starting experiment process----"));
28548 dprintf (SEG_REUSE_LOG_0, ("----Couldn't fit with max free items %Id", max_free_space_items));
28550 // TODO: need to take the end of segment into consideration.
28551 while (max_free_space_items <= total_free_space_items)
28553 max_free_space_items += max_free_space_items / 2;
28554 dprintf (SEG_REUSE_LOG_0, ("----Temporarily increasing max free spaces to %Id", max_free_space_items));
28555 memcpy (ordered_free_space_indices,
28556 saved_all_free_space_indices,
28557 sizeof(ordered_free_space_indices));
28558 if (try_best_fit(FALSE))
28560 temp_bestfit = TRUE;
28567 dprintf (SEG_REUSE_LOG_0, ("----With %Id max free spaces we could fit", max_free_space_items));
28571 dprintf (SEG_REUSE_LOG_0, ("----Tried all free spaces and still couldn't fit, lost too much space"));
28574 dprintf (SEG_REUSE_LOG_0, ("----Restoring max free spaces to %Id", saved_max));
28575 max_free_space_items = saved_max;
28576 #endif // SEG_REUSE_STATS
28577 if (free_space_items)
28579 max_free_space_items = min (MAX_NUM_FREE_SPACES, free_space_items * 2);
28580 max_free_space_items = max (max_free_space_items, MIN_NUM_FREE_SPACES);
28584 max_free_space_items = MAX_NUM_FREE_SPACES;
28588 dprintf (SEG_REUSE_LOG_0, ("Adjusted number of max free spaces to %Id", max_free_space_items));
28589 dprintf (SEG_REUSE_LOG_0, ("------End of best fitting process------\n"));
28591 return use_bestfit;
28594 BOOL gc_heap::process_free_space (heap_segment* seg,
28596 size_t min_free_size,
28597 size_t min_cont_size,
28598 size_t* total_free_space,
28599 size_t* largest_free_space)
28601 *total_free_space += free_space;
28602 *largest_free_space = max (*largest_free_space, free_space);
28604 #ifdef SIMPLE_DPRINTF
28605 dprintf (SEG_REUSE_LOG_1, ("free space len: %Ix, total free space: %Ix, largest free space: %Ix",
28606 free_space, *total_free_space, *largest_free_space));
28607 #endif //SIMPLE_DPRINTF
28609 if ((*total_free_space >= min_free_size) && (*largest_free_space >= min_cont_size))
28611 #ifdef SIMPLE_DPRINTF
28612 dprintf (SEG_REUSE_LOG_0, ("(gen%d)total free: %Ix(min: %Ix), largest free: %Ix(min: %Ix). Found segment %Ix to reuse without bestfit",
28613 settings.condemned_generation,
28614 *total_free_space, min_free_size, *largest_free_space, min_cont_size,
28617 UNREFERENCED_PARAMETER(seg);
28618 #endif //SIMPLE_DPRINTF
28622 int free_space_index = relative_index_power2_free_space (round_down_power2 (free_space));
28623 if (free_space_index != -1)
28625 ordered_free_space_indices[free_space_index]++;
28630 BOOL gc_heap::expand_reused_seg_p()
28632 BOOL reused_seg = FALSE;
28633 int heap_expand_mechanism = gc_data_per_heap.get_mechanism (gc_heap_expand);
28634 if ((heap_expand_mechanism == expand_reuse_bestfit) ||
28635 (heap_expand_mechanism == expand_reuse_normal))
28643 BOOL gc_heap::can_expand_into_p (heap_segment* seg, size_t min_free_size, size_t min_cont_size,
28644 allocator* gen_allocator)
28646 min_cont_size += END_SPACE_AFTER_GC;
28647 use_bestfit = FALSE;
28648 commit_end_of_seg = FALSE;
28649 bestfit_first_pin = 0;
28650 uint8_t* first_address = heap_segment_mem (seg);
28651 uint8_t* end_address = heap_segment_reserved (seg);
28652 size_t end_extra_space = end_space_after_gc();
28654 if ((heap_segment_reserved (seg) - end_extra_space) <= heap_segment_plan_allocated (seg))
28656 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: can't use segment [%Ix %Ix, has less than %d bytes at the end",
28657 first_address, end_address, end_extra_space));
28661 end_address -= end_extra_space;
28663 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p(gen%d): min free: %Ix, min continuous: %Ix",
28664 settings.condemned_generation, min_free_size, min_cont_size));
28665 size_t eph_gen_starts = eph_gen_starts_size;
28667 if (settings.condemned_generation == max_generation)
28669 size_t free_space = 0;
28670 size_t largest_free_space = free_space;
28671 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: gen2: testing segment [%Ix %Ix", first_address, end_address));
28672 //Look through the pinned plugs for relevant ones and Look for the right pinned plug to start from.
28673 //We are going to allocate the generation starts in the 1st free space,
28674 //so start from the first free space that's big enough for gen starts and a min object size.
28675 // If we see a free space that is >= gen starts but < gen starts + min obj size we just don't use it -
28676 // we could use it by allocating the last generation start a bit bigger but
28677 // the complexity isn't worth the effort (those plugs are from gen2
28678 // already anyway).
28679 reset_pinned_queue_bos();
28681 BOOL has_fit_gen_starts = FALSE;
28683 init_ordered_free_space_indices ();
28684 while (!pinned_plug_que_empty_p())
28687 if ((pinned_plug (m) >= first_address) &&
28688 (pinned_plug (m) < end_address) &&
28689 (pinned_len (m) >= (eph_gen_starts + Align (min_obj_size))))
28695 deque_pinned_plug();
28699 if (!pinned_plug_que_empty_p())
28701 bestfit_first_pin = pinned_plug (m) - pinned_len (m);
28703 if (process_free_space (seg,
28704 pinned_len (m) - eph_gen_starts,
28705 min_free_size, min_cont_size,
28706 &free_space, &largest_free_space))
28711 deque_pinned_plug();
28713 has_fit_gen_starts = TRUE;
28716 dprintf (3, ("first pin is %Ix", pinned_plug (m)));
28718 //tally up free space
28719 while (!pinned_plug_que_empty_p() &&
28720 ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address)))
28722 dprintf (3, ("looking at pin %Ix", pinned_plug (m)));
28723 if (process_free_space (seg,
28725 min_free_size, min_cont_size,
28726 &free_space, &largest_free_space))
28731 deque_pinned_plug();
28735 //try to find space at the end of the segment.
28736 size_t end_space = (end_address - heap_segment_plan_allocated (seg));
28737 size_t additional_space = ((min_free_size > free_space) ? (min_free_size - free_space) : 0);
28738 dprintf (SEG_REUSE_LOG_0, ("end space: %Ix; additional: %Ix", end_space, additional_space));
28739 if (end_space >= additional_space)
28741 BOOL can_fit = TRUE;
28742 commit_end_of_seg = TRUE;
28744 if (largest_free_space < min_cont_size)
28746 if (end_space >= min_cont_size)
28748 additional_space = max (min_cont_size, additional_space);
28749 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse without bestfit, with committing end of seg for eph",
28754 if (settings.concurrent)
28757 commit_end_of_seg = FALSE;
28761 size_t additional_space_bestfit = additional_space;
28762 if (!has_fit_gen_starts)
28764 if (additional_space_bestfit < (eph_gen_starts + Align (min_obj_size)))
28766 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, gen starts not allocated yet and end space is too small: %Id",
28767 additional_space_bestfit));
28771 bestfit_first_pin = heap_segment_plan_allocated (seg);
28772 additional_space_bestfit -= eph_gen_starts;
28775 can_fit = best_fit (free_space,
28776 largest_free_space,
28777 additional_space_bestfit,
28778 &commit_end_of_seg);
28782 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse with bestfit, %s committing end of seg",
28783 seg, (commit_end_of_seg ? "with" : "without")));
28787 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, total free space is %Ix", (free_space + end_space)));
28794 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse without bestfit, with committing end of seg", seg));
28797 assert (additional_space <= end_space);
28798 if (commit_end_of_seg)
28800 if (!grow_heap_segment (seg, heap_segment_plan_allocated (seg) + additional_space))
28802 dprintf (2, ("Couldn't commit end of segment?!"));
28803 use_bestfit = FALSE;
28810 // We increase the index here because growing heap segment could create a discrepency with
28811 // the additional space we used (could be bigger).
28812 size_t free_space_end_of_seg =
28813 heap_segment_committed (seg) - heap_segment_plan_allocated (seg);
28814 int relative_free_space_index = relative_index_power2_free_space (round_down_power2 (free_space_end_of_seg));
28815 saved_ordered_free_space_indices[relative_free_space_index]++;
28821 memcpy (ordered_free_space_indices,
28822 saved_ordered_free_space_indices,
28823 sizeof(ordered_free_space_indices));
28824 max_free_space_items = max (MIN_NUM_FREE_SPACES, free_space_items * 3 / 2);
28825 max_free_space_items = min (MAX_NUM_FREE_SPACES, max_free_space_items);
28826 dprintf (SEG_REUSE_LOG_0, ("could fit! %Id free spaces, %Id max", free_space_items, max_free_space_items));
28832 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, total free space is %Ix", (free_space + end_space)));
28837 assert (settings.condemned_generation == (max_generation-1));
28838 size_t free_space = (end_address - heap_segment_plan_allocated (seg));
28839 size_t largest_free_space = free_space;
28840 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: gen1: testing segment [%Ix %Ix", first_address, end_address));
28841 //find the first free list in range of the current segment
28842 size_t sz_list = gen_allocator->first_bucket_size();
28843 unsigned int a_l_idx = 0;
28844 uint8_t* free_list = 0;
28845 for (; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
28847 if ((eph_gen_starts <= sz_list) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
28849 free_list = gen_allocator->alloc_list_head_of (a_l_idx);
28852 if ((free_list >= first_address) &&
28853 (free_list < end_address) &&
28854 (unused_array_size (free_list) >= eph_gen_starts))
28860 free_list = free_list_slot (free_list);
28868 init_ordered_free_space_indices ();
28869 if (process_free_space (seg,
28870 unused_array_size (free_list) - eph_gen_starts + Align (min_obj_size),
28871 min_free_size, min_cont_size,
28872 &free_space, &largest_free_space))
28877 free_list = free_list_slot (free_list);
28881 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, no free list"));
28885 //tally up free space
28891 if ((free_list >= first_address) && (free_list < end_address) &&
28892 process_free_space (seg,
28893 unused_array_size (free_list),
28894 min_free_size, min_cont_size,
28895 &free_space, &largest_free_space))
28900 free_list = free_list_slot (free_list);
28903 if (a_l_idx < gen_allocator->number_of_buckets())
28905 free_list = gen_allocator->alloc_list_head_of (a_l_idx);
28911 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, total free space is %Ix", free_space));
28915 BOOL can_fit = best_fit (free_space, 0, NULL);
28918 dprintf (SEG_REUSE_LOG_0, ("(gen1)Found segment %Ix to reuse with bestfit", seg));
28922 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, total free space is %Ix", free_space));
28930 void gc_heap::realloc_plug (size_t last_plug_size, uint8_t*& last_plug,
28931 generation* gen, uint8_t* start_address,
28932 unsigned int& active_new_gen_number,
28933 uint8_t*& last_pinned_gap, BOOL& leftp,
28936 , mark* pinned_plug_entry
28937 #endif //SHORT_PLUGS
28940 // detect generation boundaries
28941 // make sure that active_new_gen_number is not the youngest generation.
28942 // because the generation_limit wouldn't return the right thing in this case.
28945 if ((active_new_gen_number > 1) &&
28946 (last_plug >= generation_limit (active_new_gen_number)))
28948 assert (last_plug >= start_address);
28949 active_new_gen_number--;
28950 realloc_plan_generation_start (generation_of (active_new_gen_number), gen);
28951 assert (generation_plan_allocation_start (generation_of (active_new_gen_number)));
28956 // detect pinned plugs
28957 if (!pinned_plug_que_empty_p() && (last_plug == pinned_plug (oldest_pin())))
28959 size_t entry = deque_pinned_plug();
28960 mark* m = pinned_plug_of (entry);
28962 size_t saved_pinned_len = pinned_len(m);
28963 pinned_len(m) = last_plug - last_pinned_gap;
28964 //dprintf (3,("Adjusting pinned gap: [%Ix, %Ix[", (size_t)last_pinned_gap, (size_t)last_plug));
28966 if (m->has_post_plug_info())
28968 last_plug_size += sizeof (gap_reloc_pair);
28969 dprintf (3, ("ra pinned %Ix was shortened, adjusting plug size to %Ix", last_plug, last_plug_size))
28972 last_pinned_gap = last_plug + last_plug_size;
28973 dprintf (3, ("ra found pin %Ix, len: %Ix->%Ix, last_p: %Ix, last_p_size: %Ix",
28974 pinned_plug (m), saved_pinned_len, pinned_len (m), last_plug, last_plug_size));
28977 //we are creating a generation fault. set the cards.
28979 size_t end_card = card_of (align_on_card (last_plug + last_plug_size));
28980 size_t card = card_of (last_plug);
28981 while (card != end_card)
28988 else if (last_plug >= start_address)
28990 #ifdef FEATURE_STRUCTALIGN
28991 int requiredAlignment;
28993 node_aligninfo (last_plug, requiredAlignment, pad);
28995 // from how we previously aligned the plug's destination address,
28996 // compute the actual alignment offset.
28997 uint8_t* reloc_plug = last_plug + node_relocation_distance (last_plug);
28998 ptrdiff_t alignmentOffset = ComputeStructAlignPad(reloc_plug, requiredAlignment, 0);
28999 if (!alignmentOffset)
29001 // allocate_in_expanded_heap doesn't expect alignmentOffset to be zero.
29002 alignmentOffset = requiredAlignment;
29005 //clear the alignment info because we are reallocating
29006 clear_node_aligninfo (last_plug);
29007 #else // FEATURE_STRUCTALIGN
29008 //clear the realignment flag because we are reallocating
29009 clear_node_realigned (last_plug);
29010 #endif // FEATURE_STRUCTALIGN
29011 BOOL adjacentp = FALSE;
29012 BOOL set_padding_on_saved_p = FALSE;
29016 last_plug_size += sizeof (gap_reloc_pair);
29019 assert (pinned_plug_entry != NULL);
29020 if (last_plug_size <= sizeof (plug_and_gap))
29022 set_padding_on_saved_p = TRUE;
29024 #endif //SHORT_PLUGS
29026 dprintf (3, ("ra plug %Ix was shortened, adjusting plug size to %Ix", last_plug, last_plug_size))
29030 clear_padding_in_expand (last_plug, set_padding_on_saved_p, pinned_plug_entry);
29031 #endif //SHORT_PLUGS
29033 uint8_t* new_address = allocate_in_expanded_heap(gen, last_plug_size, adjacentp, last_plug,
29035 set_padding_on_saved_p,
29037 #endif //SHORT_PLUGS
29038 TRUE, active_new_gen_number REQD_ALIGN_AND_OFFSET_ARG);
29040 dprintf (3, ("ra NA: [%Ix, %Ix[: %Ix", new_address, (new_address + last_plug_size), last_plug_size));
29041 assert (new_address);
29042 set_node_relocation_distance (last_plug, new_address - last_plug);
29043 #ifdef FEATURE_STRUCTALIGN
29044 if (leftp && node_alignpad (last_plug) == 0)
29045 #else // FEATURE_STRUCTALIGN
29046 if (leftp && !node_realigned (last_plug))
29047 #endif // FEATURE_STRUCTALIGN
29049 // TODO - temporarily disable L optimization because of a bug in it.
29050 //set_node_left (last_plug);
29052 dprintf (3,(" Re-allocating %Ix->%Ix len %Id", (size_t)last_plug, (size_t)new_address, last_plug_size));
29057 void gc_heap::realloc_in_brick (uint8_t* tree, uint8_t*& last_plug,
29058 uint8_t* start_address,
29060 unsigned int& active_new_gen_number,
29061 uint8_t*& last_pinned_gap, BOOL& leftp)
29063 assert (tree != NULL);
29064 int left_node = node_left_child (tree);
29065 int right_node = node_right_child (tree);
29067 dprintf (3, ("ra: tree: %Ix, last_pin_gap: %Ix, last_p: %Ix, L: %d, R: %d",
29068 tree, last_pinned_gap, last_plug, left_node, right_node));
29072 dprintf (3, ("LN: realloc %Ix(%Ix)", (tree + left_node), last_plug));
29073 realloc_in_brick ((tree + left_node), last_plug, start_address,
29074 gen, active_new_gen_number, last_pinned_gap,
29078 if (last_plug != 0)
29080 uint8_t* plug = tree;
29082 BOOL has_pre_plug_info_p = FALSE;
29083 BOOL has_post_plug_info_p = FALSE;
29084 mark* pinned_plug_entry = get_next_pinned_entry (tree,
29085 &has_pre_plug_info_p,
29086 &has_post_plug_info_p,
29089 // We only care about the pre plug info 'cause that's what decides if the last plug is shortened.
29090 // The pinned plugs are handled in realloc_plug.
29091 size_t gap_size = node_gap_size (plug);
29092 uint8_t* gap = (plug - gap_size);
29093 uint8_t* last_plug_end = gap;
29094 size_t last_plug_size = (last_plug_end - last_plug);
29095 // Cannot assert this - a plug could be less than that due to the shortened ones.
29096 //assert (last_plug_size >= Align (min_obj_size));
29097 dprintf (3, ("ra: plug %Ix, gap size: %Ix, last_pin_gap: %Ix, last_p: %Ix, last_p_end: %Ix, shortened: %d",
29098 plug, gap_size, last_pinned_gap, last_plug, last_plug_end, (has_pre_plug_info_p ? 1 : 0)));
29099 realloc_plug (last_plug_size, last_plug, gen, start_address,
29100 active_new_gen_number, last_pinned_gap,
29101 leftp, has_pre_plug_info_p
29103 , pinned_plug_entry
29104 #endif //SHORT_PLUGS
29112 dprintf (3, ("RN: realloc %Ix(%Ix)", (tree + right_node), last_plug));
29113 realloc_in_brick ((tree + right_node), last_plug, start_address,
29114 gen, active_new_gen_number, last_pinned_gap,
29120 gc_heap::realloc_plugs (generation* consing_gen, heap_segment* seg,
29121 uint8_t* start_address, uint8_t* end_address,
29122 unsigned active_new_gen_number)
29124 dprintf (3, ("--- Reallocing ---"));
29128 //make sure that every generation has a planned allocation start
29129 int gen_number = max_generation - 1;
29130 while (gen_number >= 0)
29132 generation* gen = generation_of (gen_number);
29133 if (0 == generation_plan_allocation_start (gen))
29135 generation_plan_allocation_start (gen) =
29136 bestfit_first_pin + (max_generation - gen_number - 1) * Align (min_obj_size);
29137 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
29138 assert (generation_plan_allocation_start (gen));
29144 uint8_t* first_address = start_address;
29145 //Look for the right pinned plug to start from.
29146 reset_pinned_queue_bos();
29147 uint8_t* planned_ephemeral_seg_end = heap_segment_plan_allocated (seg);
29148 while (!pinned_plug_que_empty_p())
29150 mark* m = oldest_pin();
29151 if ((pinned_plug (m) >= planned_ephemeral_seg_end) && (pinned_plug (m) < end_address))
29153 if (pinned_plug (m) < first_address)
29155 first_address = pinned_plug (m);
29160 deque_pinned_plug();
29163 size_t current_brick = brick_of (first_address);
29164 size_t end_brick = brick_of (end_address-1);
29165 uint8_t* last_plug = 0;
29167 uint8_t* last_pinned_gap = heap_segment_plan_allocated (seg);
29168 BOOL leftp = FALSE;
29170 dprintf (3, ("start addr: %Ix, first addr: %Ix, current oldest pin: %Ix",
29171 start_address, first_address, pinned_plug (oldest_pin())));
29173 while (current_brick <= end_brick)
29175 int brick_entry = brick_table [ current_brick ];
29176 if (brick_entry >= 0)
29178 realloc_in_brick ((brick_address (current_brick) + brick_entry - 1),
29179 last_plug, start_address, consing_gen,
29180 active_new_gen_number, last_pinned_gap,
29186 if (last_plug != 0)
29188 realloc_plug (end_address - last_plug, last_plug, consing_gen,
29190 active_new_gen_number, last_pinned_gap,
29194 #endif //SHORT_PLUGS
29198 //Fix the old segment allocated size
29199 assert (last_pinned_gap >= heap_segment_mem (seg));
29200 assert (last_pinned_gap <= heap_segment_committed (seg));
29201 heap_segment_plan_allocated (seg) = last_pinned_gap;
29204 void gc_heap::verify_no_pins (uint8_t* start, uint8_t* end)
29207 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
29209 BOOL contains_pinned_plugs = FALSE;
29212 while (mi != mark_stack_tos)
29214 m = pinned_plug_of (mi);
29215 if ((pinned_plug (m) >= start) && (pinned_plug (m) < end))
29217 contains_pinned_plugs = TRUE;
29224 if (contains_pinned_plugs)
29229 #endif //VERIFY_HEAP
29232 void gc_heap::set_expand_in_full_gc (int condemned_gen_number)
29234 if (!should_expand_in_full_gc)
29236 if ((condemned_gen_number != max_generation) &&
29237 (settings.pause_mode != pause_low_latency) &&
29238 (settings.pause_mode != pause_sustained_low_latency))
29240 should_expand_in_full_gc = TRUE;
29245 void gc_heap::save_ephemeral_generation_starts()
29247 for (int ephemeral_generation = 0; ephemeral_generation < max_generation; ephemeral_generation++)
29249 saved_ephemeral_plan_start[ephemeral_generation] =
29250 generation_plan_allocation_start (generation_of (ephemeral_generation));
29251 saved_ephemeral_plan_start_size[ephemeral_generation] =
29252 generation_plan_allocation_start_size (generation_of (ephemeral_generation));
29256 generation* gc_heap::expand_heap (int condemned_generation,
29257 generation* consing_gen,
29258 heap_segment* new_heap_segment)
29260 UNREFERENCED_PARAMETER(condemned_generation);
29261 assert (condemned_generation >= (max_generation -1));
29262 unsigned int active_new_gen_number = max_generation; //Set one too high to get generation gap
29263 uint8_t* start_address = generation_limit (max_generation);
29264 uint8_t* end_address = heap_segment_allocated (ephemeral_heap_segment);
29265 BOOL should_promote_ephemeral = FALSE;
29266 ptrdiff_t eph_size = total_ephemeral_size;
29267 #ifdef BACKGROUND_GC
29268 dprintf(2,("%s: ---- Heap Expansion ----", (recursive_gc_sync::background_running_p() ? "FGC" : "NGC")));
29269 #endif //BACKGROUND_GC
29270 settings.heap_expansion = TRUE;
29272 #ifdef BACKGROUND_GC
29273 if (cm_in_progress)
29275 if (!expanded_in_fgc)
29277 expanded_in_fgc = TRUE;
29280 #endif //BACKGROUND_GC
29282 //reset the elevation state for next time.
29283 dprintf (2, ("Elevation: elevation = el_none"));
29284 if (settings.should_lock_elevation && !expand_reused_seg_p())
29285 settings.should_lock_elevation = FALSE;
29287 heap_segment* new_seg = new_heap_segment;
29290 return consing_gen;
29292 //copy the card and brick tables
29293 if (g_gc_card_table!= card_table)
29294 copy_brick_card_table();
29296 BOOL new_segment_p = (heap_segment_next (new_seg) == 0);
29297 dprintf (2, ("new_segment_p %Ix", (size_t)new_segment_p));
29299 assert (generation_plan_allocation_start (generation_of (max_generation-1)));
29300 assert (generation_plan_allocation_start (generation_of (max_generation-1)) >=
29301 heap_segment_mem (ephemeral_heap_segment));
29302 assert (generation_plan_allocation_start (generation_of (max_generation-1)) <=
29303 heap_segment_committed (ephemeral_heap_segment));
29305 assert (generation_plan_allocation_start (youngest_generation));
29306 assert (generation_plan_allocation_start (youngest_generation) <
29307 heap_segment_plan_allocated (ephemeral_heap_segment));
29309 if (settings.pause_mode == pause_no_gc)
29311 // We don't reuse for no gc, so the size used on the new eph seg is eph_size.
29312 if ((size_t)(heap_segment_reserved (new_seg) - heap_segment_mem (new_seg)) < (eph_size + soh_allocation_no_gc))
29313 should_promote_ephemeral = TRUE;
29319 should_promote_ephemeral = dt_low_ephemeral_space_p (tuning_deciding_promote_ephemeral);
29323 if (should_promote_ephemeral)
29325 ephemeral_promotion = TRUE;
29326 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, expand_new_seg_ep);
29327 dprintf (2, ("promoting ephemeral"));
29328 save_ephemeral_generation_starts();
29332 // commit the new ephemeral segment all at once if it is a new one.
29333 if ((eph_size > 0) && new_segment_p)
29335 #ifdef FEATURE_STRUCTALIGN
29336 // The destination may require a larger alignment padding than the source.
29337 // Assume the worst possible alignment padding.
29338 eph_size += ComputeStructAlignPad(heap_segment_mem (new_seg), MAX_STRUCTALIGN, OBJECT_ALIGNMENT_OFFSET);
29339 #endif // FEATURE_STRUCTALIGN
29340 #ifdef RESPECT_LARGE_ALIGNMENT
29341 //Since the generation start can be larger than min_obj_size
29342 //The alignment could be switched.
29343 eph_size += switch_alignment_size(FALSE);
29344 #endif //RESPECT_LARGE_ALIGNMENT
29345 //Since the generation start can be larger than min_obj_size
29346 //Compare the alignment of the first object in gen1
29347 if (grow_heap_segment (new_seg, heap_segment_mem (new_seg) + eph_size) == 0)
29349 fgm_result.set_fgm (fgm_commit_eph_segment, eph_size, FALSE);
29350 return consing_gen;
29352 heap_segment_used (new_seg) = heap_segment_committed (new_seg);
29355 //Fix the end of the old ephemeral heap segment
29356 heap_segment_plan_allocated (ephemeral_heap_segment) =
29357 generation_plan_allocation_start (generation_of (max_generation-1));
29359 dprintf (3, ("Old ephemeral allocated set to %Ix",
29360 (size_t)heap_segment_plan_allocated (ephemeral_heap_segment)));
29365 // TODO - Is this really necessary? We should think about it.
29366 //initialize the first brick
29367 size_t first_brick = brick_of (heap_segment_mem (new_seg));
29368 set_brick (first_brick,
29369 heap_segment_mem (new_seg) - brick_address (first_brick));
29372 //From this point on, we cannot run out of memory
29374 //reset the allocation of the consing generation back to the end of the
29375 //old ephemeral segment
29376 generation_allocation_limit (consing_gen) =
29377 heap_segment_plan_allocated (ephemeral_heap_segment);
29378 generation_allocation_pointer (consing_gen) = generation_allocation_limit (consing_gen);
29379 generation_allocation_segment (consing_gen) = ephemeral_heap_segment;
29381 //clear the generation gap for all of the ephemeral generations
29383 int generation_num = max_generation-1;
29384 while (generation_num >= 0)
29386 generation* gen = generation_of (generation_num);
29387 generation_plan_allocation_start (gen) = 0;
29392 heap_segment* old_seg = ephemeral_heap_segment;
29393 ephemeral_heap_segment = new_seg;
29395 //Note: the ephemeral segment shouldn't be threaded onto the segment chain
29396 //because the relocation and compact phases shouldn't see it
29398 // set the generation members used by allocate_in_expanded_heap
29399 // and switch to ephemeral generation
29400 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
29402 if (!should_promote_ephemeral)
29404 realloc_plugs (consing_gen, old_seg, start_address, end_address,
29405 active_new_gen_number);
29410 repair_allocation_in_expanded_heap (consing_gen);
29413 // assert that the generation gap for all of the ephemeral generations were allocated.
29416 int generation_num = max_generation-1;
29417 while (generation_num >= 0)
29419 generation* gen = generation_of (generation_num);
29420 assert (generation_plan_allocation_start (gen));
29426 if (!new_segment_p)
29428 dprintf (2, ("Demoting ephemeral segment"));
29429 //demote the entire segment.
29430 settings.demotion = TRUE;
29431 get_gc_data_per_heap()->set_mechanism_bit (gc_demotion_bit);
29432 demotion_low = heap_segment_mem (ephemeral_heap_segment);
29433 demotion_high = heap_segment_reserved (ephemeral_heap_segment);
29437 demotion_low = MAX_PTR;
29439 #ifndef MULTIPLE_HEAPS
29440 settings.demotion = FALSE;
29441 get_gc_data_per_heap()->clear_mechanism_bit (gc_demotion_bit);
29442 #endif //!MULTIPLE_HEAPS
29444 ptrdiff_t eph_size1 = total_ephemeral_size;
29445 MAYBE_UNUSED_VAR(eph_size1);
29447 if (!should_promote_ephemeral && new_segment_p)
29449 assert (eph_size1 <= eph_size);
29452 if (heap_segment_mem (old_seg) == heap_segment_plan_allocated (old_seg))
29454 // This is to catch when we accidently delete a segment that has pins.
29455 verify_no_pins (heap_segment_mem (old_seg), heap_segment_reserved (old_seg));
29458 verify_no_pins (heap_segment_plan_allocated (old_seg), heap_segment_reserved(old_seg));
29460 dprintf(2,("---- End of Heap Expansion ----"));
29461 return consing_gen;
29464 void gc_heap::set_static_data()
29466 static_data* pause_mode_sdata = static_data_table[latency_level];
29467 for (int i = 0; i < NUMBERGENERATIONS; i++)
29469 dynamic_data* dd = dynamic_data_of (i);
29470 static_data* sdata = &pause_mode_sdata[i];
29473 dd->min_size = sdata->min_size;
29475 dprintf (GTC_LOG, ("PM: %d - min: %Id, max: %Id, fr_l: %Id, fr_b: %d%%",
29476 settings.pause_mode,
29477 dd->min_size, dd_max_size,
29478 dd->fragmentation_limit, (int)(dd->fragmentation_burden_limit * 100)));
29482 // Initialize the values that are not const.
29483 void gc_heap::init_static_data()
29485 size_t gen0size = GCHeap::GetValidGen0MaxSize(get_valid_segment_size());
29486 size_t gen0_min_size = Align(gen0size / 8 * 5);
29488 size_t gen0_max_size =
29489 #ifdef MULTIPLE_HEAPS
29490 max (6*1024*1024, min ( Align(soh_segment_size/2), 200*1024*1024));
29491 #else //MULTIPLE_HEAPS
29492 (gc_can_use_concurrent ?
29494 max (6*1024*1024, min ( Align(soh_segment_size/2), 200*1024*1024)));
29495 #endif //MULTIPLE_HEAPS
29497 // TODO: gen0_max_size has a 200mb cap; gen1_max_size should also have a cap.
29498 size_t gen1_max_size =
29499 #ifdef MULTIPLE_HEAPS
29500 max (6*1024*1024, Align(soh_segment_size/2));
29501 #else //MULTIPLE_HEAPS
29502 (gc_can_use_concurrent ?
29504 max (6*1024*1024, Align(soh_segment_size/2)));
29505 #endif //MULTIPLE_HEAPS
29507 dprintf (GTC_LOG, ("gen0size: %Id, gen0 min: %Id, max: %Id, gen1 max: %Id",
29508 gen0size, gen0_min_size, gen0_max_size, gen1_max_size));
29510 for (int i = latency_level_first; i <= latency_level_last; i++)
29512 static_data_table[i][0].min_size = gen0_min_size;
29513 static_data_table[i][0].max_size = gen0_max_size;
29514 static_data_table[i][1].max_size = gen1_max_size;
29518 bool gc_heap::init_dynamic_data()
29520 qpf = GCToOSInterface::QueryPerformanceFrequency();
29522 uint32_t now = (uint32_t)GetHighPrecisionTimeStamp();
29526 for (int i = 0; i <= max_generation+1; i++)
29528 dynamic_data* dd = dynamic_data_of (i);
29530 dd->time_clock = now;
29531 dd->current_size = 0;
29532 dd->promoted_size = 0;
29533 dd->collection_count = 0;
29534 dd->new_allocation = dd->min_size;
29535 dd->gc_new_allocation = dd->new_allocation;
29536 dd->desired_allocation = dd->new_allocation;
29537 dd->fragmentation = 0;
29540 #ifdef GC_CONFIG_DRIVEN
29541 if (heap_number == 0)
29543 #endif //GC_CONFIG_DRIVEN
29548 float gc_heap::surv_to_growth (float cst, float limit, float max_limit)
29550 if (cst < ((max_limit - limit ) / (limit * (max_limit-1.0f))))
29551 return ((limit - limit*cst) / (1.0f - (cst * limit)));
29557 //if the allocation budget wasn't exhausted, the new budget may be wrong because the survival may
29558 //not be correct (collection happened too soon). Correct with a linear estimation based on the previous
29559 //value of the budget
29560 static size_t linear_allocation_model (float allocation_fraction, size_t new_allocation,
29561 size_t previous_desired_allocation, size_t collection_count)
29563 if ((allocation_fraction < 0.95) && (allocation_fraction > 0.0))
29565 dprintf (2, ("allocation fraction: %d", (int)(allocation_fraction/100.0)));
29566 new_allocation = (size_t)(allocation_fraction*new_allocation + (1.0-allocation_fraction)*previous_desired_allocation);
29569 size_t smoothing = 3; // exponential smoothing factor
29570 if (smoothing > collection_count)
29571 smoothing = collection_count;
29572 new_allocation = new_allocation / smoothing + ((previous_desired_allocation / smoothing) * (smoothing-1));
29574 UNREFERENCED_PARAMETER(collection_count);
29576 return new_allocation;
29579 size_t gc_heap::desired_new_allocation (dynamic_data* dd,
29580 size_t out, int gen_number,
29583 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
29585 if (dd_begin_data_size (dd) == 0)
29587 size_t new_allocation = dd_min_size (dd);
29588 current_gc_data_per_heap->gen_data[gen_number].new_allocation = new_allocation;
29589 return new_allocation;
29594 size_t previous_desired_allocation = dd_desired_allocation (dd);
29595 size_t current_size = dd_current_size (dd);
29596 float max_limit = dd_max_limit (dd);
29597 float limit = dd_limit (dd);
29598 size_t min_gc_size = dd_min_size (dd);
29600 size_t max_size = dd_max_size (dd);
29601 size_t new_allocation = 0;
29602 float allocation_fraction = (float) (dd_desired_allocation (dd) - dd_gc_new_allocation (dd)) / (float) (dd_desired_allocation (dd));
29603 if (gen_number >= max_generation)
29605 size_t new_size = 0;
29607 cst = min (1.0f, float (out) / float (dd_begin_data_size (dd)));
29609 f = surv_to_growth (cst, limit, max_limit);
29610 size_t max_growth_size = (size_t)(max_size / f);
29611 if (current_size >= max_growth_size)
29613 new_size = max_size;
29617 new_size = (size_t) min (max ( (f * current_size), min_gc_size), max_size);
29620 assert ((new_size >= current_size) || (new_size == max_size));
29622 if (gen_number == max_generation)
29624 new_allocation = max((new_size - current_size), min_gc_size);
29626 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
29627 dd_desired_allocation (dd), dd_collection_count (dd));
29629 if ((dd_fragmentation (dd) > ((size_t)((f-1)*current_size))))
29631 //reducing allocation in case of fragmentation
29632 size_t new_allocation1 = max (min_gc_size,
29634 (size_t)((float)new_allocation * current_size /
29635 ((float)current_size + 2*dd_fragmentation (dd))));
29636 dprintf (2, ("Reducing max_gen allocation due to fragmentation from %Id to %Id",
29637 new_allocation, new_allocation1));
29638 new_allocation = new_allocation1;
29641 else //large object heap
29643 uint32_t memory_load = 0;
29644 uint64_t available_physical = 0;
29645 get_memory_info (&memory_load, &available_physical);
29646 if (heap_number == 0)
29647 settings.exit_memory_load = memory_load;
29648 if (available_physical > 1024*1024)
29649 available_physical -= 1024*1024;
29651 uint64_t available_free = available_physical + (uint64_t)generation_free_list_space (generation_of (gen_number));
29652 if (available_free > (uint64_t)MAX_PTR)
29654 available_free = (uint64_t)MAX_PTR;
29657 //try to avoid OOM during large object allocation
29658 new_allocation = max (min(max((new_size - current_size), dd_desired_allocation (dynamic_data_of (max_generation))),
29659 (size_t)available_free),
29660 max ((current_size/4), min_gc_size));
29662 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
29663 dd_desired_allocation (dd), dd_collection_count (dd));
29669 size_t survivors = out;
29670 cst = float (survivors) / float (dd_begin_data_size (dd));
29671 f = surv_to_growth (cst, limit, max_limit);
29672 new_allocation = (size_t) min (max ((f * (survivors)), min_gc_size), max_size);
29674 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
29675 dd_desired_allocation (dd), dd_collection_count (dd));
29677 if (gen_number == 0)
29682 //printf ("%f, %Id\n", cst, new_allocation);
29683 size_t free_space = generation_free_list_space (generation_of (gen_number));
29684 // DTREVIEW - is min_gc_size really a good choice?
29685 // on 64-bit this will almost always be true.
29686 dprintf (GTC_LOG, ("frag: %Id, min: %Id", free_space, min_gc_size));
29687 if (free_space > min_gc_size)
29689 settings.gen0_reduction_count = 2;
29693 if (settings.gen0_reduction_count > 0)
29694 settings.gen0_reduction_count--;
29697 if (settings.gen0_reduction_count > 0)
29699 dprintf (2, ("Reducing new allocation based on fragmentation"));
29700 new_allocation = min (new_allocation,
29701 max (min_gc_size, (max_size/3)));
29706 size_t new_allocation_ret =
29707 Align (new_allocation, get_alignment_constant (!(gen_number == (max_generation+1))));
29708 int gen_data_index = gen_number;
29709 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_data_index]);
29710 gen_data->new_allocation = new_allocation_ret;
29712 dd_surv (dd) = cst;
29714 #ifdef SIMPLE_DPRINTF
29715 dprintf (1, ("h%d g%d surv: %Id current: %Id alloc: %Id (%d%%) f: %d%% new-size: %Id new-alloc: %Id",
29716 heap_number, gen_number, out, current_size, (dd_desired_allocation (dd) - dd_gc_new_allocation (dd)),
29717 (int)(cst*100), (int)(f*100), current_size + new_allocation, new_allocation));
29719 dprintf (1,("gen: %d in: %Id out: %Id ", gen_number, generation_allocation_size (generation_of (gen_number)), out));
29720 dprintf (1,("current: %Id alloc: %Id ", current_size, (dd_desired_allocation (dd) - dd_gc_new_allocation (dd))));
29721 dprintf (1,(" surv: %d%% f: %d%% new-size: %Id new-alloc: %Id",
29722 (int)(cst*100), (int)(f*100), current_size + new_allocation, new_allocation));
29723 #endif //SIMPLE_DPRINTF
29725 return new_allocation_ret;
29729 //returns the planned size of a generation (including free list element)
29730 size_t gc_heap::generation_plan_size (int gen_number)
29732 if (0 == gen_number)
29733 return max((heap_segment_plan_allocated (ephemeral_heap_segment) -
29734 generation_plan_allocation_start (generation_of (gen_number))),
29735 (int)Align (min_obj_size));
29738 generation* gen = generation_of (gen_number);
29739 if (heap_segment_rw (generation_start_segment (gen)) == ephemeral_heap_segment)
29740 return (generation_plan_allocation_start (generation_of (gen_number - 1)) -
29741 generation_plan_allocation_start (generation_of (gen_number)));
29744 size_t gensize = 0;
29745 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
29747 PREFIX_ASSUME(seg != NULL);
29749 while (seg && (seg != ephemeral_heap_segment))
29751 gensize += heap_segment_plan_allocated (seg) -
29752 heap_segment_mem (seg);
29753 seg = heap_segment_next_rw (seg);
29757 gensize += (generation_plan_allocation_start (generation_of (gen_number - 1)) -
29758 heap_segment_mem (ephemeral_heap_segment));
29766 //returns the size of a generation (including free list element)
29767 size_t gc_heap::generation_size (int gen_number)
29769 if (0 == gen_number)
29770 return max((heap_segment_allocated (ephemeral_heap_segment) -
29771 generation_allocation_start (generation_of (gen_number))),
29772 (int)Align (min_obj_size));
29775 generation* gen = generation_of (gen_number);
29776 if (heap_segment_rw (generation_start_segment (gen)) == ephemeral_heap_segment)
29777 return (generation_allocation_start (generation_of (gen_number - 1)) -
29778 generation_allocation_start (generation_of (gen_number)));
29781 size_t gensize = 0;
29782 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
29784 PREFIX_ASSUME(seg != NULL);
29786 while (seg && (seg != ephemeral_heap_segment))
29788 gensize += heap_segment_allocated (seg) -
29789 heap_segment_mem (seg);
29790 seg = heap_segment_next_rw (seg);
29794 gensize += (generation_allocation_start (generation_of (gen_number - 1)) -
29795 heap_segment_mem (ephemeral_heap_segment));
29804 size_t gc_heap::compute_in (int gen_number)
29806 assert (gen_number != 0);
29807 dynamic_data* dd = dynamic_data_of (gen_number);
29809 size_t in = generation_allocation_size (generation_of (gen_number));
29811 if (gen_number == max_generation && ephemeral_promotion)
29814 for (int i = 0; i <= max_generation; i++)
29816 dynamic_data* dd = dynamic_data_of (i);
29817 in += dd_survived_size (dd);
29818 if (i != max_generation)
29820 generation_condemned_allocated (generation_of (gen_number)) += dd_survived_size (dd);
29825 dd_gc_new_allocation (dd) -= in;
29826 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
29828 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
29829 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_number]);
29832 generation_allocation_size (generation_of (gen_number)) = 0;
29836 void gc_heap::compute_promoted_allocation (int gen_number)
29838 compute_in (gen_number);
29843 size_t gc_heap::trim_youngest_desired (uint32_t memory_load,
29844 size_t total_new_allocation,
29845 size_t total_min_allocation)
29847 if (memory_load < MAX_ALLOWED_MEM_LOAD)
29849 // If the total of memory load and gen0 budget exceeds
29850 // our max memory load limit, trim the gen0 budget so the total
29851 // is the max memory load limit.
29852 size_t remain_memory_load = (MAX_ALLOWED_MEM_LOAD - memory_load) * mem_one_percent;
29853 return min (total_new_allocation, remain_memory_load);
29857 return max (mem_one_percent, total_min_allocation);
29861 size_t gc_heap::joined_youngest_desired (size_t new_allocation)
29863 dprintf (2, ("Entry memory load: %d; gen0 new_alloc: %Id", settings.entry_memory_load, new_allocation));
29865 size_t final_new_allocation = new_allocation;
29866 if (new_allocation > MIN_YOUNGEST_GEN_DESIRED)
29868 uint32_t num_heaps = 1;
29870 #ifdef MULTIPLE_HEAPS
29871 num_heaps = gc_heap::n_heaps;
29872 #endif //MULTIPLE_HEAPS
29874 size_t total_new_allocation = new_allocation * num_heaps;
29875 size_t total_min_allocation = MIN_YOUNGEST_GEN_DESIRED * num_heaps;
29877 if ((settings.entry_memory_load >= MAX_ALLOWED_MEM_LOAD) ||
29878 (total_new_allocation > max (youngest_gen_desired_th, total_min_allocation)))
29880 uint32_t memory_load = 0;
29881 get_memory_info (&memory_load);
29882 settings.exit_memory_load = memory_load;
29883 dprintf (2, ("Current emory load: %d", memory_load));
29885 size_t final_total =
29886 trim_youngest_desired (memory_load, total_new_allocation, total_min_allocation);
29887 size_t max_new_allocation =
29888 #ifdef MULTIPLE_HEAPS
29889 dd_max_size (g_heaps[0]->dynamic_data_of (0));
29890 #else //MULTIPLE_HEAPS
29891 dd_max_size (dynamic_data_of (0));
29892 #endif //MULTIPLE_HEAPS
29894 final_new_allocation = min (Align ((final_total / num_heaps), get_alignment_constant (TRUE)), max_new_allocation);
29898 if (final_new_allocation < new_allocation)
29900 settings.gen0_reduction_count = 2;
29903 return final_new_allocation;
29908 gc_history_per_heap* gc_heap::get_gc_data_per_heap()
29910 #ifdef BACKGROUND_GC
29911 return (settings.concurrent ? &bgc_data_per_heap : &gc_data_per_heap);
29913 return &gc_data_per_heap;
29914 #endif //BACKGROUND_GC
29917 void gc_heap::compute_new_dynamic_data (int gen_number)
29919 PREFIX_ASSUME(gen_number >= 0);
29920 PREFIX_ASSUME(gen_number <= max_generation);
29922 dynamic_data* dd = dynamic_data_of (gen_number);
29923 generation* gen = generation_of (gen_number);
29924 size_t in = (gen_number==0) ? 0 : compute_in (gen_number);
29926 size_t total_gen_size = generation_size (gen_number);
29927 //keep track of fragmentation
29928 dd_fragmentation (dd) = generation_free_list_space (gen) + generation_free_obj_space (gen);
29929 dd_current_size (dd) = total_gen_size - dd_fragmentation (dd);
29931 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
29933 size_t out = dd_survived_size (dd);
29935 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_number]);
29936 gen_data->size_after = total_gen_size;
29937 gen_data->free_list_space_after = generation_free_list_space (gen);
29938 gen_data->free_obj_space_after = generation_free_obj_space (gen);
29940 if ((settings.pause_mode == pause_low_latency) && (gen_number <= 1))
29942 // When we are in the low latency mode, we can still be
29943 // condemning more than gen1's 'cause of induced GCs.
29944 dd_desired_allocation (dd) = low_latency_alloc;
29948 if (gen_number == 0)
29950 //compensate for dead finalizable objects promotion.
29951 //they shoudn't be counted for growth.
29952 size_t final_promoted = 0;
29953 final_promoted = min (promoted_bytes (heap_number), out);
29954 // Prefast: this is clear from above but prefast needs to be told explicitly
29955 PREFIX_ASSUME(final_promoted <= out);
29957 dprintf (2, ("gen: %d final promoted: %Id", gen_number, final_promoted));
29958 dd_freach_previous_promotion (dd) = final_promoted;
29959 size_t lower_bound = desired_new_allocation (dd, out-final_promoted, gen_number, 0);
29961 if (settings.condemned_generation == 0)
29963 //there is no noise.
29964 dd_desired_allocation (dd) = lower_bound;
29968 size_t higher_bound = desired_new_allocation (dd, out, gen_number, 1);
29970 // <TODO>This assert was causing AppDomains\unload\test1n\test1nrun.bat to fail</TODO>
29971 //assert ( lower_bound <= higher_bound);
29973 //discount the noise. Change the desired allocation
29974 //only if the previous value is outside of the range.
29975 if (dd_desired_allocation (dd) < lower_bound)
29977 dd_desired_allocation (dd) = lower_bound;
29979 else if (dd_desired_allocation (dd) > higher_bound)
29981 dd_desired_allocation (dd) = higher_bound;
29983 #if defined (BIT64) && !defined (MULTIPLE_HEAPS)
29984 dd_desired_allocation (dd) = joined_youngest_desired (dd_desired_allocation (dd));
29985 #endif // BIT64 && !MULTIPLE_HEAPS
29986 trim_youngest_desired_low_memory();
29987 dprintf (2, ("final gen0 new_alloc: %Id", dd_desired_allocation (dd)));
29992 dd_desired_allocation (dd) = desired_new_allocation (dd, out, gen_number, 0);
29996 gen_data->pinned_surv = dd_pinned_survived_size (dd);
29997 gen_data->npinned_surv = dd_survived_size (dd) - dd_pinned_survived_size (dd);
29999 dd_gc_new_allocation (dd) = dd_desired_allocation (dd);
30000 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
30003 dd_promoted_size (dd) = out;
30004 if (gen_number == max_generation)
30006 dd = dynamic_data_of (max_generation+1);
30007 total_gen_size = generation_size (max_generation + 1);
30008 dd_fragmentation (dd) = generation_free_list_space (large_object_generation) +
30009 generation_free_obj_space (large_object_generation);
30010 dd_current_size (dd) = total_gen_size - dd_fragmentation (dd);
30011 dd_survived_size (dd) = dd_current_size (dd);
30013 out = dd_current_size (dd);
30014 dd_desired_allocation (dd) = desired_new_allocation (dd, out, max_generation+1, 0);
30015 dd_gc_new_allocation (dd) = Align (dd_desired_allocation (dd),
30016 get_alignment_constant (FALSE));
30017 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
30019 gen_data = &(current_gc_data_per_heap->gen_data[max_generation+1]);
30020 gen_data->size_after = total_gen_size;
30021 gen_data->free_list_space_after = generation_free_list_space (large_object_generation);
30022 gen_data->free_obj_space_after = generation_free_obj_space (large_object_generation);
30023 gen_data->npinned_surv = out;
30024 #ifdef BACKGROUND_GC
30025 end_loh_size = total_gen_size;
30026 #endif //BACKGROUND_GC
30028 dd_promoted_size (dd) = out;
30032 void gc_heap::trim_youngest_desired_low_memory()
30034 if (g_low_memory_status)
30036 size_t committed_mem = 0;
30037 heap_segment* seg = generation_start_segment (generation_of (max_generation));
30040 committed_mem += heap_segment_committed (seg) - heap_segment_mem (seg);
30041 seg = heap_segment_next (seg);
30043 seg = generation_start_segment (generation_of (max_generation + 1));
30046 committed_mem += heap_segment_committed (seg) - heap_segment_mem (seg);
30047 seg = heap_segment_next (seg);
30050 dynamic_data* dd = dynamic_data_of (0);
30051 size_t current = dd_desired_allocation (dd);
30052 size_t candidate = max (Align ((committed_mem / 10), get_alignment_constant(FALSE)), dd_min_size (dd));
30054 dd_desired_allocation (dd) = min (current, candidate);
30058 void gc_heap::decommit_ephemeral_segment_pages()
30060 if (settings.concurrent)
30065 size_t slack_space = heap_segment_committed (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment);
30066 dynamic_data* dd = dynamic_data_of (0);
30068 #ifndef MULTIPLE_HEAPS
30069 size_t extra_space = (g_low_memory_status ? 0 : (512 * 1024));
30070 size_t decommit_timeout = (g_low_memory_status ? 0 : GC_EPHEMERAL_DECOMMIT_TIMEOUT);
30071 size_t ephemeral_elapsed = dd_time_clock(dd) - gc_last_ephemeral_decommit_time;
30073 if (dd_desired_allocation (dd) > gc_gen0_desired_high)
30075 gc_gen0_desired_high = dd_desired_allocation (dd) + extra_space;
30078 if (ephemeral_elapsed >= decommit_timeout)
30080 slack_space = min (slack_space, gc_gen0_desired_high);
30082 gc_last_ephemeral_decommit_time = dd_time_clock(dd);
30083 gc_gen0_desired_high = 0;
30085 #endif //!MULTIPLE_HEAPS
30087 if (settings.condemned_generation >= (max_generation-1))
30089 size_t new_slack_space =
30091 max(min(min(soh_segment_size/32, dd_max_size(dd)), (generation_size (max_generation) / 10)), dd_desired_allocation(dd));
30093 #ifdef FEATURE_CORECLR
30094 dd_desired_allocation (dd);
30097 #endif //FEATURE_CORECLR
30100 slack_space = min (slack_space, new_slack_space);
30103 decommit_heap_segment_pages (ephemeral_heap_segment, slack_space);
30105 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
30106 current_gc_data_per_heap->extra_gen0_committed = heap_segment_committed (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment);
30109 size_t gc_heap::new_allocation_limit (size_t size, size_t free_size, int gen_number)
30111 dynamic_data* dd = dynamic_data_of (gen_number);
30112 ptrdiff_t new_alloc = dd_new_allocation (dd);
30113 assert (new_alloc == (ptrdiff_t)Align (new_alloc,
30114 get_alignment_constant (!(gen_number == (max_generation+1)))));
30115 size_t limit = min (max (new_alloc, (ptrdiff_t)size), (ptrdiff_t)free_size);
30116 assert (limit == Align (limit, get_alignment_constant (!(gen_number == (max_generation+1)))));
30117 dd_new_allocation (dd) = (new_alloc - limit );
30121 //This is meant to be called by decide_on_compacting.
30123 size_t gc_heap::generation_fragmentation (generation* gen,
30124 generation* consing_gen,
30128 uint8_t* alloc = generation_allocation_pointer (consing_gen);
30129 // If the allocation pointer has reached the ephemeral segment
30130 // fine, otherwise the whole ephemeral segment is considered
30132 if (in_range_for_segment (alloc, ephemeral_heap_segment))
30134 if (alloc <= heap_segment_allocated(ephemeral_heap_segment))
30135 frag = end - alloc;
30138 // case when no survivors, allocated set to beginning
30141 dprintf (3, ("ephemeral frag: %Id", frag));
30144 frag = (heap_segment_allocated (ephemeral_heap_segment) -
30145 heap_segment_mem (ephemeral_heap_segment));
30146 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
30148 PREFIX_ASSUME(seg != NULL);
30150 while (seg != ephemeral_heap_segment)
30152 frag += (heap_segment_allocated (seg) -
30153 heap_segment_plan_allocated (seg));
30154 dprintf (3, ("seg: %Ix, frag: %Id", (size_t)seg,
30155 (heap_segment_allocated (seg) -
30156 heap_segment_plan_allocated (seg))));
30158 seg = heap_segment_next_rw (seg);
30161 dprintf (3, ("frag: %Id discounting pinned plugs", frag));
30162 //add the length of the dequeued plug free space
30164 while (bos < mark_stack_bos)
30166 frag += (pinned_len (pinned_plug_of (bos)));
30173 // for SOH this returns the total sizes of the generation and its
30174 // younger generation(s).
30175 // for LOH this returns just LOH size.
30176 size_t gc_heap::generation_sizes (generation* gen)
30179 if (generation_start_segment (gen ) == ephemeral_heap_segment)
30180 result = (heap_segment_allocated (ephemeral_heap_segment) -
30181 generation_allocation_start (gen));
30184 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
30186 PREFIX_ASSUME(seg != NULL);
30190 result += (heap_segment_allocated (seg) -
30191 heap_segment_mem (seg));
30192 seg = heap_segment_next_in_range (seg);
30199 BOOL gc_heap::decide_on_compacting (int condemned_gen_number,
30200 size_t fragmentation,
30201 BOOL& should_expand)
30203 BOOL should_compact = FALSE;
30204 should_expand = FALSE;
30205 generation* gen = generation_of (condemned_gen_number);
30206 dynamic_data* dd = dynamic_data_of (condemned_gen_number);
30207 size_t gen_sizes = generation_sizes(gen);
30208 float fragmentation_burden = ( ((0 == fragmentation) || (0 == gen_sizes)) ? (0.0f) :
30209 (float (fragmentation) / gen_sizes) );
30211 dprintf (GTC_LOG, ("fragmentation: %Id (%d%%)", fragmentation, (int)(fragmentation_burden * 100.0)));
30214 // for pure GC stress runs we need compaction, for GC stress "mix"
30215 // we need to ensure a better mix of compacting and sweeping collections
30216 if (GCStress<cfg_any>::IsEnabled() && !settings.concurrent
30217 && !g_pConfig->IsGCStressMix())
30218 should_compact = TRUE;
30221 // in GC stress "mix" mode, for stress induced collections make sure we
30222 // keep sweeps and compactions relatively balanced. do not (yet) force sweeps
30223 // against the GC's determination, as it may lead to premature OOMs.
30224 if (g_pConfig->IsGCStressMix() && settings.stress_induced)
30226 int compactions = g_GCStatistics.cntCompactFGC+g_GCStatistics.cntCompactNGC;
30227 int sweeps = g_GCStatistics.cntFGC + g_GCStatistics.cntNGC - compactions;
30228 if (compactions < sweeps / 10)
30230 should_compact = TRUE;
30234 #endif //STRESS_HEAP
30236 if (GCConfig::GetForceCompact())
30237 should_compact = TRUE;
30239 if ((condemned_gen_number == max_generation) && last_gc_before_oom)
30241 should_compact = TRUE;
30242 last_gc_before_oom = FALSE;
30243 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_last_gc);
30246 if (settings.reason == reason_induced_compacting)
30248 dprintf (2, ("induced compacting GC"));
30249 should_compact = TRUE;
30250 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_induced_compacting);
30253 dprintf (2, ("Fragmentation: %d Fragmentation burden %d%%",
30254 fragmentation, (int) (100*fragmentation_burden)));
30256 if (!should_compact)
30258 if (dt_low_ephemeral_space_p (tuning_deciding_compaction))
30260 dprintf(GTC_LOG, ("compacting due to low ephemeral"));
30261 should_compact = TRUE;
30262 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_low_ephemeral);
30266 if (should_compact)
30268 if ((condemned_gen_number >= (max_generation - 1)))
30270 if (dt_low_ephemeral_space_p (tuning_deciding_expansion))
30272 dprintf (GTC_LOG,("Not enough space for all ephemeral generations with compaction"));
30273 should_expand = TRUE;
30279 BOOL high_memory = FALSE;
30282 if (!should_compact)
30284 // We are not putting this in dt_high_frag_p because it's not exactly
30285 // high fragmentation - it's just enough planned fragmentation for us to
30286 // want to compact. Also the "fragmentation" we are talking about here
30287 // is different from anywhere else.
30288 BOOL frag_exceeded = ((fragmentation >= dd_fragmentation_limit (dd)) &&
30289 (fragmentation_burden >= dd_fragmentation_burden_limit (dd)));
30293 #ifdef BACKGROUND_GC
30294 // do not force compaction if this was a stress-induced GC
30295 IN_STRESS_HEAP(if (!settings.stress_induced))
30297 #endif // BACKGROUND_GC
30298 assert (settings.concurrent == FALSE);
30299 should_compact = TRUE;
30300 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_high_frag);
30301 #ifdef BACKGROUND_GC
30303 #endif // BACKGROUND_GC
30307 // check for high memory situation
30308 if(!should_compact)
30310 uint32_t num_heaps = 1;
30311 #ifdef MULTIPLE_HEAPS
30312 num_heaps = gc_heap::n_heaps;
30313 #endif // MULTIPLE_HEAPS
30315 ptrdiff_t reclaim_space = generation_size(max_generation) - generation_plan_size(max_generation);
30316 if((settings.entry_memory_load >= high_memory_load_th) && (settings.entry_memory_load < v_high_memory_load_th))
30318 if(reclaim_space > (int64_t)(min_high_fragmentation_threshold (entry_available_physical_mem, num_heaps)))
30320 dprintf(GTC_LOG,("compacting due to fragmentation in high memory"));
30321 should_compact = TRUE;
30322 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_high_mem_frag);
30324 high_memory = TRUE;
30326 else if(settings.entry_memory_load >= v_high_memory_load_th)
30328 if(reclaim_space > (ptrdiff_t)(min_reclaim_fragmentation_threshold (num_heaps)))
30330 dprintf(GTC_LOG,("compacting due to fragmentation in very high memory"));
30331 should_compact = TRUE;
30332 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_vhigh_mem_frag);
30334 high_memory = TRUE;
30340 // The purpose of calling ensure_gap_allocation here is to make sure
30341 // that we actually are able to commit the memory to allocate generation
30343 if ((should_compact == FALSE) &&
30344 (ensure_gap_allocation (condemned_gen_number) == FALSE))
30346 should_compact = TRUE;
30347 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_no_gaps);
30350 if (settings.condemned_generation == max_generation)
30352 //check the progress
30355 (high_memory && !should_compact) ||
30357 (generation_plan_allocation_start (generation_of (max_generation - 1)) >=
30358 generation_allocation_start (generation_of (max_generation - 1))))
30360 dprintf (2, (" Elevation: gen2 size: %d, gen2 plan size: %d, no progress, elevation = locked",
30361 generation_size (max_generation),
30362 generation_plan_size (max_generation)));
30363 //no progress -> lock
30364 settings.should_lock_elevation = TRUE;
30368 if (settings.pause_mode == pause_no_gc)
30370 should_compact = TRUE;
30371 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_plan_allocated (ephemeral_heap_segment))
30372 < soh_allocation_no_gc)
30374 should_expand = TRUE;
30378 dprintf (2, ("will %s", (should_compact ? "compact" : "sweep")));
30379 return should_compact;
30382 size_t align_lower_good_size_allocation (size_t size)
30384 return (size/64)*64;
30387 size_t gc_heap::approximate_new_allocation()
30389 dynamic_data* dd0 = dynamic_data_of (0);
30390 return max (2*dd_min_size (dd0), ((dd_desired_allocation (dd0)*2)/3));
30393 // After we did a GC we expect to have at least this
30394 // much space at the end of the segment to satisfy
30395 // a reasonable amount of allocation requests.
30396 size_t gc_heap::end_space_after_gc()
30398 return max ((dd_min_size (dynamic_data_of (0))/2), (END_SPACE_AFTER_GC + Align (min_obj_size)));
30401 BOOL gc_heap::ephemeral_gen_fit_p (gc_tuning_point tp)
30403 uint8_t* start = 0;
30405 if ((tp == tuning_deciding_condemned_gen) ||
30406 (tp == tuning_deciding_compaction))
30408 start = (settings.concurrent ? alloc_allocated : heap_segment_allocated (ephemeral_heap_segment));
30409 if (settings.concurrent)
30411 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment (alloc_allocated)",
30412 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated)));
30416 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment (allocated)",
30417 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment))));
30420 else if (tp == tuning_deciding_expansion)
30422 start = heap_segment_plan_allocated (ephemeral_heap_segment);
30423 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment based on plan",
30424 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - start)));
30428 assert (tp == tuning_deciding_full_gc);
30429 dprintf (GTC_LOG, ("FGC: %Id left at the end of ephemeral segment (alloc_allocated)",
30430 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated)));
30431 start = alloc_allocated;
30434 if (start == 0) // empty ephemeral generations
30436 assert (tp == tuning_deciding_expansion);
30437 // if there are no survivors in the ephemeral segment,
30438 // this should be the beginning of ephemeral segment.
30439 start = generation_allocation_pointer (generation_of (max_generation));
30440 assert (start == heap_segment_mem (ephemeral_heap_segment));
30443 if (tp == tuning_deciding_expansion)
30445 assert (settings.condemned_generation >= (max_generation-1));
30446 size_t gen0size = approximate_new_allocation();
30447 size_t eph_size = gen0size;
30449 for (int j = 1; j <= max_generation-1; j++)
30451 eph_size += 2*dd_min_size (dynamic_data_of(j));
30454 // We must find room for one large object and enough room for gen0size
30455 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - start) > eph_size)
30457 dprintf (3, ("Enough room before end of segment"));
30462 size_t room = align_lower_good_size_allocation
30463 (heap_segment_reserved (ephemeral_heap_segment) - start);
30464 size_t end_seg = room;
30466 //look at the plug free space
30467 size_t largest_alloc = END_SPACE_AFTER_GC + Align (min_obj_size);
30468 bool large_chunk_found = FALSE;
30470 uint8_t* gen0start = generation_plan_allocation_start (youngest_generation);
30471 dprintf (3, ("ephemeral_gen_fit_p: gen0 plan start: %Ix", (size_t)gen0start));
30472 if (gen0start == 0)
30474 dprintf (3, ("ephemeral_gen_fit_p: room before free list search %Id, needed: %Id",
30476 while ((bos < mark_stack_bos) &&
30477 !((room >= gen0size) && large_chunk_found))
30479 uint8_t* plug = pinned_plug (pinned_plug_of (bos));
30480 if (in_range_for_segment (plug, ephemeral_heap_segment))
30482 if (plug >= gen0start)
30484 size_t chunk = align_lower_good_size_allocation (pinned_len (pinned_plug_of (bos)));
30486 if (!large_chunk_found)
30488 large_chunk_found = (chunk >= largest_alloc);
30490 dprintf (3, ("ephemeral_gen_fit_p: room now %Id, large chunk: %Id",
30491 room, large_chunk_found));
30497 if (room >= gen0size)
30499 if (large_chunk_found)
30501 dprintf (3, ("Enough room"));
30506 // now we need to find largest_alloc at the end of the segment.
30507 if (end_seg >= end_space_after_gc())
30509 dprintf (3, ("Enough room (may need end of seg)"));
30515 dprintf (3, ("Not enough room"));
30521 size_t end_space = 0;
30522 dynamic_data* dd = dynamic_data_of (0);
30523 if ((tp == tuning_deciding_condemned_gen) ||
30524 (tp == tuning_deciding_full_gc))
30526 end_space = 2*dd_min_size (dd);
30530 assert (tp == tuning_deciding_compaction);
30531 end_space = approximate_new_allocation();
30534 if (!((size_t)(heap_segment_reserved (ephemeral_heap_segment) - start) > end_space))
30536 dprintf (GTC_LOG, ("ephemeral_gen_fit_p: does not fit without compaction"));
30538 return ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - start) > end_space);
30542 CObjectHeader* gc_heap::allocate_large_object (size_t jsize, int64_t& alloc_bytes)
30544 //create a new alloc context because gen3context is shared.
30545 alloc_context acontext;
30546 acontext.alloc_ptr = 0;
30547 acontext.alloc_limit = 0;
30548 acontext.alloc_bytes = 0;
30549 #ifdef MULTIPLE_HEAPS
30550 acontext.set_alloc_heap(vm_heap);
30551 #endif //MULTIPLE_HEAPS
30554 uint8_t* current_lowest_address = lowest_address;
30555 uint8_t* current_highest_address = highest_address;
30556 #ifdef BACKGROUND_GC
30557 if (recursive_gc_sync::background_running_p())
30559 current_lowest_address = background_saved_lowest_address;
30560 current_highest_address = background_saved_highest_address;
30562 #endif //BACKGROUND_GC
30563 #endif // MARK_ARRAY
30566 size_t maxObjectSize = (INT64_MAX - 7 - Align(min_obj_size));
30568 size_t maxObjectSize = (INT32_MAX - 7 - Align(min_obj_size));
30571 if (jsize >= maxObjectSize)
30573 if (GCConfig::GetBreakOnOOM())
30575 GCToOSInterface::DebugBreak();
30580 size_t size = AlignQword (jsize);
30581 int align_const = get_alignment_constant (FALSE);
30582 #ifdef FEATURE_LOH_COMPACTION
30583 size_t pad = Align (loh_padding_obj_size, align_const);
30586 #endif //FEATURE_LOH_COMPACTION
30588 assert (size >= Align (min_obj_size, align_const));
30590 #pragma inline_depth(0)
30592 if (! allocate_more_space (&acontext, (size + pad), max_generation+1))
30598 #pragma inline_depth(20)
30601 #ifdef FEATURE_LOH_COMPACTION
30602 // The GC allocator made a free object already in this alloc context and
30603 // adjusted the alloc_ptr accordingly.
30604 #endif //FEATURE_LOH_COMPACTION
30606 uint8_t* result = acontext.alloc_ptr;
30608 assert ((size_t)(acontext.alloc_limit - acontext.alloc_ptr) == size);
30609 alloc_bytes += size;
30611 CObjectHeader* obj = (CObjectHeader*)result;
30614 if (recursive_gc_sync::background_running_p())
30616 if ((result < current_highest_address) && (result >= current_lowest_address))
30618 dprintf (3, ("Clearing mark bit at address %Ix",
30619 (size_t)(&mark_array [mark_word_of (result)])));
30621 mark_array_clear_marked (result);
30623 #ifdef BACKGROUND_GC
30624 //the object has to cover one full mark uint32_t
30625 assert (size > mark_word_size);
30626 if (current_c_gc_state == c_gc_state_marking)
30628 dprintf (3, ("Concurrent allocation of a large object %Ix",
30630 //mark the new block specially so we know it is a new object
30631 if ((result < current_highest_address) && (result >= current_lowest_address))
30633 dprintf (3, ("Setting mark bit at address %Ix",
30634 (size_t)(&mark_array [mark_word_of (result)])));
30636 mark_array_set_marked (result);
30639 #endif //BACKGROUND_GC
30641 #endif //MARK_ARRAY
30644 assert ((size_t)obj == Align ((size_t)obj, align_const));
30649 void reset_memory (uint8_t* o, size_t sizeo)
30651 if (sizeo > 128 * 1024)
30653 // We cannot reset the memory for the useful part of a free object.
30654 size_t size_to_skip = min_free_list - plug_skew;
30656 size_t page_start = align_on_page ((size_t)(o + size_to_skip));
30657 size_t size = align_lower_page ((size_t)o + sizeo - size_to_skip - plug_skew) - page_start;
30658 // Note we need to compensate for an OS bug here. This bug would cause the MEM_RESET to fail
30659 // on write watched memory.
30662 #ifdef MULTIPLE_HEAPS
30663 bool unlock_p = true;
30665 // We don't do unlock because there could be many processes using workstation GC and it's
30666 // bad perf to have many threads doing unlock at the same time.
30667 bool unlock_p = false;
30668 #endif // MULTIPLE_HEAPS
30670 reset_mm_p = GCToOSInterface::VirtualReset((void*)page_start, size, unlock_p);
30675 void gc_heap::reset_large_object (uint8_t* o)
30677 // If it's a large object, allow the O/S to discard the backing store for these pages.
30678 reset_memory (o, size(o));
30681 BOOL gc_heap::large_object_marked (uint8_t* o, BOOL clearp)
30684 // It shouldn't be necessary to do these comparisons because this is only used for blocking
30685 // GCs and LOH segments cannot be out of range.
30686 if ((o >= lowest_address) && (o < highest_address))
30706 void gc_heap::walk_survivors_relocation (void* profiling_context, record_surv_fn fn)
30708 // Now walk the portion of memory that is actually being relocated.
30709 walk_relocation (profiling_context, fn);
30711 #ifdef FEATURE_LOH_COMPACTION
30712 if (loh_compacted_p)
30714 walk_relocation_for_loh (profiling_context, fn);
30716 #endif //FEATURE_LOH_COMPACTION
30719 void gc_heap::walk_survivors_for_loh (void* profiling_context, record_surv_fn fn)
30721 generation* gen = large_object_generation;
30722 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));;
30724 PREFIX_ASSUME(seg != NULL);
30726 uint8_t* o = generation_allocation_start (gen);
30727 uint8_t* plug_end = o;
30728 uint8_t* plug_start = o;
30732 if (o >= heap_segment_allocated (seg))
30734 seg = heap_segment_next (seg);
30738 o = heap_segment_mem (seg);
30740 if (large_object_marked(o, FALSE))
30747 o = o + AlignQword (size (o));
30748 if (o >= heap_segment_allocated (seg))
30752 m = large_object_marked (o, FALSE);
30757 fn (plug_start, plug_end, 0, profiling_context, false, false);
30761 while (o < heap_segment_allocated (seg) && !large_object_marked(o, FALSE))
30763 o = o + AlignQword (size (o));
30769 #ifdef BACKGROUND_GC
30771 BOOL gc_heap::background_object_marked (uint8_t* o, BOOL clearp)
30774 if ((o >= background_saved_lowest_address) && (o < background_saved_highest_address))
30776 if (mark_array_marked (o))
30780 mark_array_clear_marked (o);
30781 //dprintf (3, ("mark array bit for object %Ix is cleared", o));
30782 dprintf (3, ("CM: %Ix", o));
30792 dprintf (3, ("o %Ix(%d) %s", o, size(o), (m ? "was bm" : "was NOT bm")));
30796 uint8_t* gc_heap::background_next_end (heap_segment* seg, BOOL large_objects_p)
30799 (large_objects_p ? heap_segment_allocated (seg) : heap_segment_background_allocated (seg));
30802 void gc_heap::set_mem_verify (uint8_t* start, uint8_t* end, uint8_t b)
30807 if ((GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC) &&
30808 !(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_NO_MEM_FILL))
30810 dprintf (3, ("setting mem to %c [%Ix, [%Ix", b, start, end));
30811 memset (start, b, (end - start));
30814 #endif //VERIFY_HEAP
30817 void gc_heap::generation_delete_heap_segment (generation* gen,
30819 heap_segment* prev_seg,
30820 heap_segment* next_seg)
30822 dprintf (3, ("bgc sweep: deleting seg %Ix", seg));
30823 if (gen == large_object_generation)
30825 heap_segment_next (prev_seg) = next_seg;
30827 dprintf (3, ("Preparing empty large segment %Ix for deletion", (size_t)seg));
30829 heap_segment_next (seg) = freeable_large_heap_segment;
30830 freeable_large_heap_segment = seg;
30834 if (seg == ephemeral_heap_segment)
30839 heap_segment_next (next_seg) = prev_seg;
30841 dprintf (3, ("Preparing empty small segment %Ix for deletion", (size_t)seg));
30842 heap_segment_next (seg) = freeable_small_heap_segment;
30843 freeable_small_heap_segment = seg;
30846 decommit_heap_segment (seg);
30847 seg->flags |= heap_segment_flags_decommitted;
30849 set_mem_verify (heap_segment_allocated (seg) - plug_skew, heap_segment_used (seg), 0xbb);
30852 void gc_heap::process_background_segment_end (heap_segment* seg,
30854 uint8_t* last_plug_end,
30855 heap_segment* start_seg,
30859 uint8_t* allocated = heap_segment_allocated (seg);
30860 uint8_t* background_allocated = heap_segment_background_allocated (seg);
30862 dprintf (3, ("Processing end of background segment [%Ix, %Ix[(%Ix[)",
30863 (size_t)heap_segment_mem (seg), background_allocated, allocated));
30866 if (allocated != background_allocated)
30868 if (gen == large_object_generation)
30873 dprintf (3, ("Make a free object before newly promoted objects [%Ix, %Ix[",
30874 (size_t)last_plug_end, background_allocated));
30875 thread_gap (last_plug_end, background_allocated - last_plug_end, generation_of (max_generation));
30877 fix_brick_to_highest (last_plug_end, background_allocated);
30879 // When we allowed fgc's during going through gaps, we could have erased the brick
30880 // that corresponds to bgc_allocated 'cause we had to update the brick there,
30881 // recover it here.
30882 fix_brick_to_highest (background_allocated, background_allocated);
30886 // by default, if allocated == background_allocated, it can't
30887 // be the ephemeral segment.
30888 if (seg == ephemeral_heap_segment)
30893 if (allocated == heap_segment_mem (seg))
30895 // this can happen with LOH segments when multiple threads
30896 // allocate new segments and not all of them were needed to
30897 // satisfy allocation requests.
30898 assert (gen == large_object_generation);
30901 if (last_plug_end == heap_segment_mem (seg))
30903 dprintf (3, ("Segment allocated is %Ix (beginning of this seg) - %s be deleted",
30904 (size_t)allocated, (*delete_p ? "should" : "should not")));
30906 if (seg != start_seg)
30913 dprintf (3, ("Trimming seg to %Ix[", (size_t)last_plug_end));
30914 heap_segment_allocated (seg) = last_plug_end;
30915 set_mem_verify (heap_segment_allocated (seg) - plug_skew, heap_segment_used (seg), 0xbb);
30917 decommit_heap_segment_pages (seg, 0);
30921 dprintf (3, ("verifying seg %Ix's mark array was completely cleared", seg));
30922 bgc_verify_mark_array_cleared (seg);
30925 void gc_heap::process_n_background_segments (heap_segment* seg,
30926 heap_segment* prev_seg,
30929 assert (gen != large_object_generation);
30933 dprintf (2, ("processing seg %Ix (not seen by bgc mark)", seg));
30934 heap_segment* next_seg = heap_segment_next (seg);
30936 if (heap_segment_read_only_p (seg))
30942 if (heap_segment_allocated (seg) == heap_segment_mem (seg))
30944 // This can happen - if we have a LOH segment where nothing survived
30945 // or a SOH segment allocated by a gen1 GC when BGC was going where
30946 // nothing survived last time we did a gen1 GC.
30947 generation_delete_heap_segment (gen, seg, prev_seg, next_seg);
30955 verify_soh_segment_list();
30961 BOOL gc_heap::fgc_should_consider_object (uint8_t* o,
30963 BOOL consider_bgc_mark_p,
30964 BOOL check_current_sweep_p,
30965 BOOL check_saved_sweep_p)
30967 // the logic for this function must be kept in sync with the analogous function
30968 // in ToolBox\SOS\Strike\gc.cpp
30970 // TRUE means we don't need to check the bgc mark bit
30971 // FALSE means we do.
30972 BOOL no_bgc_mark_p = FALSE;
30974 if (consider_bgc_mark_p)
30976 if (check_current_sweep_p && (o < current_sweep_pos))
30978 dprintf (3, ("no bgc mark - o: %Ix < cs: %Ix", o, current_sweep_pos));
30979 no_bgc_mark_p = TRUE;
30982 if (!no_bgc_mark_p)
30984 if(check_saved_sweep_p && (o >= saved_sweep_ephemeral_start))
30986 dprintf (3, ("no bgc mark - o: %Ix >= ss: %Ix", o, saved_sweep_ephemeral_start));
30987 no_bgc_mark_p = TRUE;
30990 if (!check_saved_sweep_p)
30992 uint8_t* background_allocated = heap_segment_background_allocated (seg);
30993 // if this was the saved ephemeral segment, check_saved_sweep_p
30994 // would've been true.
30995 assert (heap_segment_background_allocated (seg) != saved_sweep_ephemeral_start);
30996 // background_allocated could be 0 for the new segments acquired during bgc
30997 // sweep and we still want no_bgc_mark_p to be true.
30998 if (o >= background_allocated)
31000 dprintf (3, ("no bgc mark - o: %Ix >= ba: %Ix", o, background_allocated));
31001 no_bgc_mark_p = TRUE;
31008 no_bgc_mark_p = TRUE;
31011 dprintf (3, ("bgc mark %Ix: %s (bm: %s)", o, (no_bgc_mark_p ? "no" : "yes"), (background_object_marked (o, FALSE) ? "yes" : "no")));
31012 return (no_bgc_mark_p ? TRUE : background_object_marked (o, FALSE));
31015 // consider_bgc_mark_p tells you if you need to care about the bgc mark bit at all
31016 // if it's TRUE, check_current_sweep_p tells you if you should consider the
31017 // current sweep position or not.
31018 void gc_heap::should_check_bgc_mark (heap_segment* seg,
31019 BOOL* consider_bgc_mark_p,
31020 BOOL* check_current_sweep_p,
31021 BOOL* check_saved_sweep_p)
31023 // the logic for this function must be kept in sync with the analogous function
31024 // in ToolBox\SOS\Strike\gc.cpp
31025 *consider_bgc_mark_p = FALSE;
31026 *check_current_sweep_p = FALSE;
31027 *check_saved_sweep_p = FALSE;
31029 if (current_c_gc_state == c_gc_state_planning)
31031 // We are doing the current_sweep_pos comparison here because we have yet to
31032 // turn on the swept flag for the segment but in_range_for_segment will return
31033 // FALSE if the address is the same as reserved.
31034 if ((seg->flags & heap_segment_flags_swept) || (current_sweep_pos == heap_segment_reserved (seg)))
31036 dprintf (3, ("seg %Ix is already swept by bgc", seg));
31040 *consider_bgc_mark_p = TRUE;
31042 dprintf (3, ("seg %Ix hasn't been swept by bgc", seg));
31044 if (seg == saved_sweep_ephemeral_seg)
31046 dprintf (3, ("seg %Ix is the saved ephemeral seg", seg));
31047 *check_saved_sweep_p = TRUE;
31050 if (in_range_for_segment (current_sweep_pos, seg))
31052 dprintf (3, ("current sweep pos is %Ix and within seg %Ix",
31053 current_sweep_pos, seg));
31054 *check_current_sweep_p = TRUE;
31060 void gc_heap::background_ephemeral_sweep()
31062 dprintf (3, ("bgc ephemeral sweep"));
31064 int align_const = get_alignment_constant (TRUE);
31066 saved_sweep_ephemeral_seg = ephemeral_heap_segment;
31067 saved_sweep_ephemeral_start = generation_allocation_start (generation_of (max_generation - 1));
31069 // Since we don't want to interfere with gen0 allocation while we are threading gen0 free list,
31070 // we thread onto a list first then publish it when we are done.
31071 allocator youngest_free_list;
31072 size_t youngest_free_list_space = 0;
31073 size_t youngest_free_obj_space = 0;
31075 youngest_free_list.clear();
31077 for (int i = 0; i <= (max_generation - 1); i++)
31079 generation* gen_to_reset = generation_of (i);
31080 assert (generation_free_list_space (gen_to_reset) == 0);
31081 // Can only assert free_list_space is 0, not free_obj_space as the allocator could have added
31082 // something there.
31085 for (int i = (max_generation - 1); i >= 0; i--)
31087 generation* current_gen = generation_of (i);
31088 uint8_t* o = generation_allocation_start (current_gen);
31089 //Skip the generation gap object
31090 o = o + Align(size (o), align_const);
31091 uint8_t* end = ((i > 0) ?
31092 generation_allocation_start (generation_of (i - 1)) :
31093 heap_segment_allocated (ephemeral_heap_segment));
31095 uint8_t* plug_end = o;
31096 uint8_t* plug_start = o;
31097 BOOL marked_p = FALSE;
31101 marked_p = background_object_marked (o, TRUE);
31105 size_t plug_size = plug_start - plug_end;
31109 thread_gap (plug_end, plug_size, current_gen);
31115 make_unused_array (plug_end, plug_size);
31116 if (plug_size >= min_free_list)
31118 youngest_free_list_space += plug_size;
31119 youngest_free_list.thread_item (plug_end, plug_size);
31123 youngest_free_obj_space += plug_size;
31128 fix_brick_to_highest (plug_end, plug_start);
31129 fix_brick_to_highest (plug_start, plug_start);
31134 o = o + Align (size (o), align_const);
31140 m = background_object_marked (o, TRUE);
31143 dprintf (3, ("bgs: plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
31147 while ((o < end) && !background_object_marked (o, FALSE))
31149 o = o + Align (size (o), align_const);
31154 if (plug_end != end)
31158 thread_gap (plug_end, end - plug_end, current_gen);
31159 fix_brick_to_highest (plug_end, end);
31163 heap_segment_allocated (ephemeral_heap_segment) = plug_end;
31164 // the following line is temporary.
31165 heap_segment_saved_bg_allocated (ephemeral_heap_segment) = plug_end;
31167 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
31169 make_unused_array (plug_end, (end - plug_end));
31171 #endif //VERIFY_HEAP
31175 dd_fragmentation (dynamic_data_of (i)) =
31176 generation_free_list_space (current_gen) + generation_free_obj_space (current_gen);
31179 generation* youngest_gen = generation_of (0);
31180 generation_free_list_space (youngest_gen) = youngest_free_list_space;
31181 generation_free_obj_space (youngest_gen) = youngest_free_obj_space;
31182 dd_fragmentation (dynamic_data_of (0)) = youngest_free_list_space + youngest_free_obj_space;
31183 generation_allocator (youngest_gen)->copy_with_no_repair (&youngest_free_list);
31186 void gc_heap::background_sweep()
31188 generation* gen = generation_of (max_generation);
31189 dynamic_data* dd = dynamic_data_of (max_generation);
31190 // For SOH segments we go backwards.
31191 heap_segment* start_seg = ephemeral_heap_segment;
31192 PREFIX_ASSUME(start_seg != NULL);
31193 heap_segment* fseg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
31194 heap_segment* seg = start_seg;
31195 uint8_t* o = heap_segment_mem (seg);
31197 heap_segment* prev_seg = heap_segment_next (seg);
31198 int align_const = get_alignment_constant (TRUE);
31201 assert (o == generation_allocation_start (generation_of (max_generation)));
31202 o = o + Align(size (o), align_const);
31205 uint8_t* plug_end = o;
31206 uint8_t* plug_start = o;
31207 next_sweep_obj = o;
31208 current_sweep_pos = o;
31210 //uint8_t* end = background_next_end (seg, (gen == large_object_generation));
31211 uint8_t* end = heap_segment_background_allocated (seg);
31212 BOOL delete_p = FALSE;
31214 //concurrent_print_time_delta ("finished with mark and start with sweep");
31215 concurrent_print_time_delta ("Sw");
31216 dprintf (2, ("---- (GC%d)Background Sweep Phase ----", VolatileLoad(&settings.gc_index)));
31218 //block concurrent allocation for large objects
31219 dprintf (3, ("lh state: planning"));
31220 if (gc_lh_block_event.IsValid())
31222 gc_lh_block_event.Reset();
31225 for (int i = 0; i <= (max_generation + 1); i++)
31227 generation* gen_to_reset = generation_of (i);
31228 generation_allocator (gen_to_reset)->clear();
31229 generation_free_list_space (gen_to_reset) = 0;
31230 generation_free_obj_space (gen_to_reset) = 0;
31231 generation_free_list_allocated (gen_to_reset) = 0;
31232 generation_end_seg_allocated (gen_to_reset) = 0;
31233 generation_condemned_allocated (gen_to_reset) = 0;
31234 //reset the allocation so foreground gc can allocate into older generation
31235 generation_allocation_pointer (gen_to_reset)= 0;
31236 generation_allocation_limit (gen_to_reset) = 0;
31237 generation_allocation_segment (gen_to_reset) = heap_segment_rw (generation_start_segment (gen_to_reset));
31240 FIRE_EVENT(BGC2ndNonConEnd);
31242 current_bgc_state = bgc_sweep_soh;
31243 verify_soh_segment_list();
31245 #ifdef FEATURE_BASICFREEZE
31246 if ((generation_start_segment (gen) != ephemeral_heap_segment) &&
31247 ro_segments_in_range)
31249 sweep_ro_segments (generation_start_segment (gen));
31251 #endif // FEATURE_BASICFREEZE
31253 //TODO BACKGROUND_GC: can we move this to where we switch to the LOH?
31254 if (current_c_gc_state != c_gc_state_planning)
31256 current_c_gc_state = c_gc_state_planning;
31259 concurrent_print_time_delta ("Swe");
31261 heap_segment* loh_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation + 1)));
31262 PREFIX_ASSUME(loh_seg != NULL);
31265 loh_seg->flags &= ~heap_segment_flags_swept;
31266 heap_segment_background_allocated (loh_seg) = heap_segment_allocated (loh_seg);
31267 loh_seg = heap_segment_next_rw (loh_seg);
31270 #ifdef MULTIPLE_HEAPS
31271 bgc_t_join.join(this, gc_join_restart_ee);
31272 if (bgc_t_join.joined())
31273 #endif //MULTIPLE_HEAPS
31275 #ifdef MULTIPLE_HEAPS
31276 dprintf(2, ("Starting BGC threads for resuming EE"));
31277 bgc_t_join.restart();
31278 #endif //MULTIPLE_HEAPS
31281 if (heap_number == 0)
31286 FIRE_EVENT(BGC2ndConBegin);
31288 background_ephemeral_sweep();
31290 #ifdef MULTIPLE_HEAPS
31291 bgc_t_join.join(this, gc_join_after_ephemeral_sweep);
31292 if (bgc_t_join.joined())
31293 #endif //MULTIPLE_HEAPS
31295 #ifdef FEATURE_EVENT_TRACE
31296 bgc_heap_walk_for_etw_p = GCEventStatus::IsEnabled(GCEventProvider_Default,
31297 GCEventKeyword_GCHeapSurvivalAndMovement,
31298 GCEventLevel_Information);
31299 #endif //FEATURE_EVENT_TRACE
31301 leave_spin_lock (&gc_lock);
31303 #ifdef MULTIPLE_HEAPS
31304 dprintf(2, ("Starting BGC threads for BGC sweeping"));
31305 bgc_t_join.restart();
31306 #endif //MULTIPLE_HEAPS
31309 disable_preemptive (true);
31311 dprintf (2, ("bgs: sweeping gen2 objects"));
31312 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
31313 (size_t)heap_segment_mem (seg),
31314 (size_t)heap_segment_allocated (seg),
31315 (size_t)heap_segment_background_allocated (seg)));
31317 int num_objs = 256;
31318 int current_num_objs = 0;
31319 heap_segment* next_seg = 0;
31325 if (gen == large_object_generation)
31327 next_seg = heap_segment_next (seg);
31331 next_seg = heap_segment_prev (fseg, seg);
31336 if (!heap_segment_read_only_p (seg))
31338 if (gen == large_object_generation)
31340 // we can treat all LOH segments as in the bgc domain
31341 // regardless of whether we saw in bgc mark or not
31342 // because we don't allow LOH allocations during bgc
31343 // sweep anyway - the LOH segments can't change.
31344 process_background_segment_end (seg, gen, plug_end,
31345 start_seg, &delete_p);
31349 assert (heap_segment_background_allocated (seg) != 0);
31350 process_background_segment_end (seg, gen, plug_end,
31351 start_seg, &delete_p);
31353 assert (next_seg || !delete_p);
31359 generation_delete_heap_segment (gen, seg, prev_seg, next_seg);
31364 dprintf (2, ("seg %Ix has been swept", seg));
31365 seg->flags |= heap_segment_flags_swept;
31368 verify_soh_segment_list();
31372 dprintf (GTC_LOG, ("seg: %Ix, next_seg: %Ix, prev_seg: %Ix", seg, next_seg, prev_seg));
31376 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
31378 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
31380 if (gen != large_object_generation)
31382 dprintf (2, ("bgs: sweeping gen3 objects"));
31383 current_bgc_state = bgc_sweep_loh;
31384 gen = generation_of (max_generation+1);
31385 start_seg = heap_segment_rw (generation_start_segment (gen));
31387 PREFIX_ASSUME(start_seg != NULL);
31391 o = generation_allocation_start (gen);
31392 assert (method_table (o) == g_gc_pFreeObjectMethodTable);
31393 align_const = get_alignment_constant (FALSE);
31394 o = o + Align(size (o), align_const);
31396 end = heap_segment_allocated (seg);
31397 dprintf (2, ("sweeping gen3 objects"));
31398 generation_free_obj_space (gen) = 0;
31399 generation_allocator (gen)->clear();
31400 generation_free_list_space (gen) = 0;
31402 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
31403 (size_t)heap_segment_mem (seg),
31404 (size_t)heap_segment_allocated (seg),
31405 (size_t)heap_segment_background_allocated (seg)));
31412 o = heap_segment_mem (seg);
31415 assert (gen != large_object_generation);
31416 assert (o == generation_allocation_start (generation_of (max_generation)));
31417 align_const = get_alignment_constant (TRUE);
31418 o = o + Align(size (o), align_const);
31422 current_sweep_pos = o;
31423 next_sweep_obj = o;
31426 end = background_next_end (seg, (gen == large_object_generation));
31427 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
31428 (size_t)heap_segment_mem (seg),
31429 (size_t)heap_segment_allocated (seg),
31430 (size_t)heap_segment_background_allocated (seg)));
31434 if ((o < end) && background_object_marked (o, TRUE))
31437 if (gen == large_object_generation)
31439 dprintf (2, ("loh fr: [%Ix-%Ix[(%Id)", plug_end, plug_start, plug_start-plug_end));
31442 thread_gap (plug_end, plug_start-plug_end, gen);
31443 if (gen != large_object_generation)
31445 add_gen_free (max_generation, plug_start-plug_end);
31446 fix_brick_to_highest (plug_end, plug_start);
31447 // we need to fix the brick for the next plug here 'cause an FGC can
31448 // happen and can't read a stale brick.
31449 fix_brick_to_highest (plug_start, plug_start);
31456 next_sweep_obj = o + Align(size (o), align_const);
31457 current_num_objs++;
31458 if (current_num_objs >= num_objs)
31460 current_sweep_pos = next_sweep_obj;
31463 current_num_objs = 0;
31466 o = next_sweep_obj;
31472 m = background_object_marked (o, TRUE);
31475 if (gen != large_object_generation)
31477 add_gen_plug (max_generation, plug_end-plug_start);
31478 dd_survived_size (dd) += (plug_end - plug_start);
31480 dprintf (3, ("bgs: plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
31484 while ((o < end) && !background_object_marked (o, FALSE))
31486 next_sweep_obj = o + Align(size (o), align_const);;
31487 current_num_objs++;
31488 if (current_num_objs >= num_objs)
31490 current_sweep_pos = plug_end;
31491 dprintf (1234, ("f: swept till %Ix", current_sweep_pos));
31493 current_num_objs = 0;
31496 o = next_sweep_obj;
31501 size_t total_loh_size = generation_size (max_generation + 1);
31502 size_t total_soh_size = generation_sizes (generation_of (max_generation));
31504 dprintf (GTC_LOG, ("h%d: S: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
31506 dprintf (GTC_LOG, ("end of bgc sweep: gen2 FL: %Id, FO: %Id",
31507 generation_free_list_space (generation_of (max_generation)),
31508 generation_free_obj_space (generation_of (max_generation))));
31509 dprintf (GTC_LOG, ("h%d: end of bgc sweep: gen3 FL: %Id, FO: %Id",
31511 generation_free_list_space (generation_of (max_generation + 1)),
31512 generation_free_obj_space (generation_of (max_generation + 1))));
31514 FIRE_EVENT(BGC2ndConEnd);
31515 concurrent_print_time_delta ("background sweep");
31517 heap_segment* reset_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
31518 PREFIX_ASSUME(reset_seg != NULL);
31522 heap_segment_saved_bg_allocated (reset_seg) = heap_segment_background_allocated (reset_seg);
31523 heap_segment_background_allocated (reset_seg) = 0;
31524 reset_seg = heap_segment_next_rw (reset_seg);
31527 // We calculate dynamic data here because if we wait till we signal the lh event,
31528 // the allocation thread can change the fragmentation and we may read an intermediate
31529 // value (which can be greater than the generation size). Plus by that time it won't
31531 compute_new_dynamic_data (max_generation);
31533 enable_preemptive ();
31535 #ifdef MULTIPLE_HEAPS
31536 bgc_t_join.join(this, gc_join_set_state_free);
31537 if (bgc_t_join.joined())
31538 #endif //MULTIPLE_HEAPS
31540 // TODO: We are using this join just to set the state. Should
31541 // look into eliminating it - check to make sure things that use
31542 // this state can live with per heap state like should_check_bgc_mark.
31543 current_c_gc_state = c_gc_state_free;
31545 #ifdef MULTIPLE_HEAPS
31546 dprintf(2, ("Starting BGC threads after background sweep phase"));
31547 bgc_t_join.restart();
31548 #endif //MULTIPLE_HEAPS
31551 disable_preemptive (true);
31553 if (gc_lh_block_event.IsValid())
31555 gc_lh_block_event.Set();
31558 //dprintf (GTC_LOG, ("---- (GC%d)End Background Sweep Phase ----", VolatileLoad(&settings.gc_index)));
31559 dprintf (GTC_LOG, ("---- (GC%d)ESw ----", VolatileLoad(&settings.gc_index)));
31561 #endif //BACKGROUND_GC
31563 void gc_heap::sweep_large_objects ()
31565 //this min value is for the sake of the dynamic tuning.
31566 //so we know that we are not starting even if we have no
31568 generation* gen = large_object_generation;
31569 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
31571 PREFIX_ASSUME(start_seg != NULL);
31573 heap_segment* seg = start_seg;
31574 heap_segment* prev_seg = 0;
31575 uint8_t* o = generation_allocation_start (gen);
31576 int align_const = get_alignment_constant (FALSE);
31578 //Skip the generation gap object
31579 o = o + Align(size (o), align_const);
31581 uint8_t* plug_end = o;
31582 uint8_t* plug_start = o;
31584 generation_allocator (gen)->clear();
31585 generation_free_list_space (gen) = 0;
31586 generation_free_obj_space (gen) = 0;
31589 dprintf (3, ("sweeping large objects"));
31590 dprintf (3, ("seg: %Ix, [%Ix, %Ix[, starting from %Ix",
31592 (size_t)heap_segment_mem (seg),
31593 (size_t)heap_segment_allocated (seg),
31598 if (o >= heap_segment_allocated (seg))
31600 heap_segment* next_seg = heap_segment_next (seg);
31601 //delete the empty segment if not the only one
31602 if ((plug_end == heap_segment_mem (seg)) &&
31603 (seg != start_seg) && !heap_segment_read_only_p (seg))
31605 //prepare for deletion
31606 dprintf (3, ("Preparing empty large segment %Ix", (size_t)seg));
31608 heap_segment_next (prev_seg) = next_seg;
31609 heap_segment_next (seg) = freeable_large_heap_segment;
31610 freeable_large_heap_segment = seg;
31614 if (!heap_segment_read_only_p (seg))
31616 dprintf (3, ("Trimming seg to %Ix[", (size_t)plug_end));
31617 heap_segment_allocated (seg) = plug_end;
31618 decommit_heap_segment_pages (seg, 0);
31627 o = heap_segment_mem (seg);
31629 dprintf (3, ("seg: %Ix, [%Ix, %Ix[", (size_t)seg,
31630 (size_t)heap_segment_mem (seg),
31631 (size_t)heap_segment_allocated (seg)));
31634 if (large_object_marked(o, TRUE))
31637 //everything between plug_end and plug_start is free
31638 thread_gap (plug_end, plug_start-plug_end, gen);
31643 o = o + AlignQword (size (o));
31644 if (o >= heap_segment_allocated (seg))
31648 m = large_object_marked (o, TRUE);
31651 dprintf (3, ("plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
31655 while (o < heap_segment_allocated (seg) && !large_object_marked(o, FALSE))
31657 o = o + AlignQword (size (o));
31662 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
31664 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
31667 void gc_heap::relocate_in_large_objects ()
31669 relocate_args args;
31671 args.high = gc_high;
31672 args.last_plug = 0;
31674 generation* gen = large_object_generation;
31676 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
31678 PREFIX_ASSUME(seg != NULL);
31680 uint8_t* o = generation_allocation_start (gen);
31684 if (o >= heap_segment_allocated (seg))
31686 seg = heap_segment_next_rw (seg);
31691 o = heap_segment_mem (seg);
31694 while (o < heap_segment_allocated (seg))
31696 check_class_object_demotion (o);
31697 if (contain_pointers (o))
31699 dprintf(3, ("Relocating through large object %Ix", (size_t)o));
31700 go_through_object_nostart (method_table (o), o, size(o), pval,
31702 reloc_survivor_helper (pval);
31705 o = o + AlignQword (size (o));
31710 void gc_heap::mark_through_cards_for_large_objects (card_fn fn,
31713 uint8_t* low = gc_low;
31714 size_t end_card = 0;
31715 generation* oldest_gen = generation_of (max_generation+1);
31716 heap_segment* seg = heap_segment_rw (generation_start_segment (oldest_gen));
31718 PREFIX_ASSUME(seg != NULL);
31720 uint8_t* beg = generation_allocation_start (oldest_gen);
31721 uint8_t* end = heap_segment_allocated (seg);
31723 size_t cg_pointers_found = 0;
31725 size_t card_word_end = (card_of (align_on_card_word (end)) /
31730 size_t n_card_set = 0;
31731 uint8_t* next_boundary = (relocating ?
31732 generation_plan_allocation_start (generation_of (max_generation -1)) :
31735 uint8_t* nhigh = (relocating ?
31736 heap_segment_plan_allocated (ephemeral_heap_segment) :
31739 BOOL foundp = FALSE;
31740 uint8_t* start_address = 0;
31741 uint8_t* limit = 0;
31742 size_t card = card_of (beg);
31744 #ifdef BACKGROUND_GC
31745 BOOL consider_bgc_mark_p = FALSE;
31746 BOOL check_current_sweep_p = FALSE;
31747 BOOL check_saved_sweep_p = FALSE;
31748 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
31749 #endif //BACKGROUND_GC
31751 size_t total_cards_cleared = 0;
31753 //dprintf(3,( "scanning large objects from %Ix to %Ix", (size_t)beg, (size_t)end));
31754 dprintf(3, ("CMl: %Ix->%Ix", (size_t)beg, (size_t)end));
31757 if ((o < end) && (card_of(o) > card))
31759 dprintf (3, ("Found %Id cg pointers", cg_pointers_found));
31760 if (cg_pointers_found == 0)
31762 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card), (size_t)o));
31763 clear_cards (card, card_of((uint8_t*)o));
31764 total_cards_cleared += (card_of((uint8_t*)o) - card);
31766 n_eph +=cg_pointers_found;
31767 cg_pointers_found = 0;
31768 card = card_of ((uint8_t*)o);
31770 if ((o < end) &&(card >= end_card))
31772 foundp = find_card (card_table, card, card_word_end, end_card);
31775 n_card_set+= end_card - card;
31776 start_address = max (beg, card_address (card));
31778 limit = min (end, card_address (end_card));
31780 if ((!foundp) || (o >= end) || (card_address (card) >= end))
31782 if ((foundp) && (cg_pointers_found == 0))
31784 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card),
31785 (size_t)card_address(card+1)));
31786 clear_cards (card, card+1);
31787 total_cards_cleared += 1;
31789 n_eph +=cg_pointers_found;
31790 cg_pointers_found = 0;
31791 if ((seg = heap_segment_next_rw (seg)) != 0)
31793 #ifdef BACKGROUND_GC
31794 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
31795 #endif //BACKGROUND_GC
31796 beg = heap_segment_mem (seg);
31797 end = compute_next_end (seg, low);
31798 card_word_end = card_of (align_on_card_word (end)) / card_word_width;
31799 card = card_of (beg);
31810 assert (card_set_p (card));
31812 dprintf(3,("card %Ix: o: %Ix, l: %Ix[ ",
31813 card, (size_t)o, (size_t)limit));
31815 assert (Align (size (o)) >= Align (min_obj_size));
31816 size_t s = size (o);
31817 uint8_t* next_o = o + AlignQword (s);
31823 assert (Align (s) >= Align (min_obj_size));
31824 next_o = o + AlignQword (s);
31827 dprintf (4, ("|%Ix|", (size_t)o));
31828 if (next_o < start_address)
31833 #ifdef BACKGROUND_GC
31834 if (!fgc_should_consider_object (o, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p))
31838 #endif //BACKGROUND_GC
31840 #ifdef COLLECTIBLE_CLASS
31841 if (is_collectible(o))
31843 BOOL passed_end_card_p = FALSE;
31845 if (card_of (o) > card)
31847 passed_end_card_p = card_transition (o, end, card_word_end,
31851 foundp, start_address,
31852 limit, total_cards_cleared);
31855 if ((!passed_end_card_p || foundp) && (card_of (o) == card))
31857 // card is valid and it covers the head of the object
31858 if (fn == &gc_heap::relocate_address)
31860 keep_card_live (o, n_gen, cg_pointers_found);
31864 uint8_t* class_obj = get_class_object (o);
31865 mark_through_cards_helper (&class_obj, n_gen,
31866 cg_pointers_found, fn,
31867 nhigh, next_boundary);
31871 if (passed_end_card_p)
31873 if (foundp && (card_address (card) < next_o))
31875 goto go_through_refs;
31885 #endif //COLLECTIBLE_CLASS
31887 if (contain_pointers (o))
31889 dprintf(3,("Going through %Ix", (size_t)o));
31891 go_through_object (method_table(o), o, s, poo,
31892 start_address, use_start, (o + s),
31894 if (card_of ((uint8_t*)poo) > card)
31896 BOOL passed_end_card_p = card_transition ((uint8_t*)poo, end,
31901 foundp, start_address,
31902 limit, total_cards_cleared);
31904 if (passed_end_card_p)
31906 if (foundp && (card_address (card) < next_o))
31910 if (ppstop <= (uint8_t**)start_address)
31912 else if (poo < (uint8_t**)start_address)
31913 {poo = (uint8_t**)start_address;}
31923 mark_through_cards_helper (poo, n_gen,
31924 cg_pointers_found, fn,
31925 nhigh, next_boundary);
31937 // compute the efficiency ratio of the card table
31940 generation_skip_ratio = min (((n_eph > 800) ?
31941 (int)(((float)n_gen / (float)n_eph) * 100) : 100),
31942 generation_skip_ratio);
31944 dprintf (3, ("Mloh: cross: %Id, useful: %Id, cards cleared: %Id, cards set: %Id, ratio: %d",
31945 n_eph, n_gen, total_cards_cleared, n_card_set, generation_skip_ratio));
31949 dprintf (3, ("R: Mloh: cross: %Id, useful: %Id, cards set: %Id, ratio: %d",
31950 n_eph, n_gen, n_card_set, generation_skip_ratio));
31954 void gc_heap::descr_segment (heap_segment* seg )
31957 uint8_t* x = heap_segment_mem (seg);
31958 while (x < heap_segment_allocated (seg))
31960 dprintf(2, ( "%Ix: %d ", (size_t)x, size (x)));
31961 x = x + Align(size (x));
31964 UNREFERENCED_PARAMETER(seg);
31968 void gc_heap::descr_generations_to_profiler (gen_walk_fn fn, void *context)
31970 #ifdef MULTIPLE_HEAPS
31971 int n_heaps = g_theGCHeap->GetNumberOfHeaps ();
31972 for (int i = 0; i < n_heaps; i++)
31974 gc_heap* hp = GCHeap::GetHeap(i)->pGenGCHeap;
31975 #else //MULTIPLE_HEAPS
31977 gc_heap* hp = NULL;
31979 // prefix complains about us dereferencing hp in wks build even though we only access static members
31980 // this way. not sure how to shut it up except for this ugly workaround:
31981 PREFIX_ASSUME(hp != NULL);
31982 #endif // _PREFAST_
31983 #endif //MULTIPLE_HEAPS
31985 int curr_gen_number0 = max_generation+1;
31986 while (curr_gen_number0 >= 0)
31988 generation* gen = hp->generation_of (curr_gen_number0);
31989 heap_segment* seg = generation_start_segment (gen);
31990 while (seg && (seg != hp->ephemeral_heap_segment))
31992 assert (curr_gen_number0 > 0);
31994 // report bounds from heap_segment_mem (seg) to
31995 // heap_segment_allocated (seg);
31996 // for generation # curr_gen_number0
31997 // for heap # heap_no
31999 fn(context, curr_gen_number0, heap_segment_mem (seg),
32000 heap_segment_allocated (seg),
32001 curr_gen_number0 == max_generation+1 ? heap_segment_reserved (seg) : heap_segment_allocated (seg));
32003 seg = heap_segment_next (seg);
32007 assert (seg == hp->ephemeral_heap_segment);
32008 assert (curr_gen_number0 <= max_generation);
32010 if (curr_gen_number0 == max_generation)
32012 if (heap_segment_mem (seg) < generation_allocation_start (hp->generation_of (max_generation-1)))
32014 // report bounds from heap_segment_mem (seg) to
32015 // generation_allocation_start (generation_of (max_generation-1))
32016 // for heap # heap_number
32018 fn(context, curr_gen_number0, heap_segment_mem (seg),
32019 generation_allocation_start (hp->generation_of (max_generation-1)),
32020 generation_allocation_start (hp->generation_of (max_generation-1)) );
32023 else if (curr_gen_number0 != 0)
32025 //report bounds from generation_allocation_start (generation_of (curr_gen_number0))
32026 // to generation_allocation_start (generation_of (curr_gen_number0-1))
32027 // for heap # heap_number
32029 fn(context, curr_gen_number0, generation_allocation_start (hp->generation_of (curr_gen_number0)),
32030 generation_allocation_start (hp->generation_of (curr_gen_number0-1)),
32031 generation_allocation_start (hp->generation_of (curr_gen_number0-1)));
32035 //report bounds from generation_allocation_start (generation_of (curr_gen_number0))
32036 // to heap_segment_allocated (ephemeral_heap_segment);
32037 // for heap # heap_number
32039 fn(context, curr_gen_number0, generation_allocation_start (hp->generation_of (curr_gen_number0)),
32040 heap_segment_allocated (hp->ephemeral_heap_segment),
32041 heap_segment_reserved (hp->ephemeral_heap_segment) );
32044 curr_gen_number0--;
32050 // Note that when logging is on it can take a long time to go through the free items.
32051 void gc_heap::print_free_list (int gen, heap_segment* seg)
32053 UNREFERENCED_PARAMETER(gen);
32054 UNREFERENCED_PARAMETER(seg);
32056 if (settings.concurrent == FALSE)
32058 uint8_t* seg_start = heap_segment_mem (seg);
32059 uint8_t* seg_end = heap_segment_allocated (seg);
32061 dprintf (3, ("Free list in seg %Ix:", seg_start));
32063 size_t total_free_item = 0;
32065 allocator* gen_allocator = generation_allocator (generation_of (gen));
32066 for (unsigned int b = 0; b < gen_allocator->number_of_buckets(); b++)
32068 uint8_t* fo = gen_allocator->alloc_list_head_of (b);
32071 if (fo >= seg_start && fo < seg_end)
32075 size_t free_item_len = size(fo);
32077 dprintf (3, ("[%Ix, %Ix[:%Id",
32079 (size_t)(fo + free_item_len),
32083 fo = free_list_slot (fo);
32087 dprintf (3, ("total %Id free items", total_free_item));
32093 void gc_heap::descr_generations (BOOL begin_gc_p)
32095 UNREFERENCED_PARAMETER(begin_gc_p);
32097 if (StressLog::StressLogOn(LF_GC, LL_INFO10))
32100 #ifdef MULTIPLE_HEAPS
32102 #endif //MULTIPLE_HEAPS
32104 STRESS_LOG1(LF_GC, LL_INFO10, "GC Heap %p\n", hp);
32105 for (int n = max_generation; n >= 0; --n)
32107 STRESS_LOG4(LF_GC, LL_INFO10, " Generation %d [%p, %p] cur = %p\n",
32109 generation_allocation_start(generation_of(n)),
32110 generation_allocation_limit(generation_of(n)),
32111 generation_allocation_pointer(generation_of(n)));
32113 heap_segment* seg = generation_start_segment(generation_of(n));
32116 STRESS_LOG4(LF_GC, LL_INFO10, " Segment mem %p alloc = %p used %p committed %p\n",
32117 heap_segment_mem(seg),
32118 heap_segment_allocated(seg),
32119 heap_segment_used(seg),
32120 heap_segment_committed(seg));
32121 seg = heap_segment_next(seg);
32125 #endif // STRESS_LOG
32128 dprintf (2, ("lowest_address: %Ix highest_address: %Ix",
32129 (size_t) lowest_address, (size_t) highest_address));
32130 #ifdef BACKGROUND_GC
32131 dprintf (2, ("bgc lowest_address: %Ix bgc highest_address: %Ix",
32132 (size_t) background_saved_lowest_address, (size_t) background_saved_highest_address));
32133 #endif //BACKGROUND_GC
32135 if (heap_number == 0)
32137 dprintf (1, ("total heap size: %Id, commit size: %Id", get_total_heap_size(), get_total_committed_size()));
32140 int curr_gen_number = max_generation+1;
32141 while (curr_gen_number >= 0)
32143 size_t total_gen_size = generation_size (curr_gen_number);
32144 #ifdef SIMPLE_DPRINTF
32145 dprintf (GTC_LOG, ("[%s][g%d]gen %d:, size: %Id, frag: %Id(L: %Id, O: %Id), f: %d%% %s %s %s",
32146 (begin_gc_p ? "BEG" : "END"),
32147 settings.condemned_generation,
32150 dd_fragmentation (dynamic_data_of (curr_gen_number)),
32151 generation_free_list_space (generation_of (curr_gen_number)),
32152 generation_free_obj_space (generation_of (curr_gen_number)),
32154 (int)(((double)dd_fragmentation (dynamic_data_of (curr_gen_number)) / (double)total_gen_size) * 100) :
32156 (begin_gc_p ? ("") : (settings.compaction ? "(compact)" : "(sweep)")),
32157 (settings.heap_expansion ? "(EX)" : " "),
32158 (settings.promotion ? "Promotion" : "NoPromotion")));
32160 dprintf (2, ( "Generation %d: gap size: %d, generation size: %Id, fragmentation: %Id",
32162 size (generation_allocation_start (generation_of (curr_gen_number))),
32164 dd_fragmentation (dynamic_data_of (curr_gen_number))));
32165 #endif //SIMPLE_DPRINTF
32167 generation* gen = generation_of (curr_gen_number);
32168 heap_segment* seg = generation_start_segment (gen);
32169 while (seg && (seg != ephemeral_heap_segment))
32171 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[-%Ix[ (%Id) (%Id)",
32173 (size_t)heap_segment_mem (seg),
32174 (size_t)heap_segment_allocated (seg),
32175 (size_t)heap_segment_committed (seg),
32176 (size_t)(heap_segment_allocated (seg) - heap_segment_mem (seg)),
32177 (size_t)(heap_segment_committed (seg) - heap_segment_allocated (seg))));
32178 print_free_list (curr_gen_number, seg);
32179 seg = heap_segment_next (seg);
32181 if (seg && (seg != generation_start_segment (gen)))
32183 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[",
32185 (size_t)heap_segment_mem (seg),
32186 (size_t)generation_allocation_start (generation_of (curr_gen_number-1))));
32187 print_free_list (curr_gen_number, seg);
32192 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[",
32194 (size_t)generation_allocation_start (generation_of (curr_gen_number)),
32195 (size_t)(((curr_gen_number == 0)) ?
32196 (heap_segment_allocated
32197 (generation_start_segment
32198 (generation_of (curr_gen_number)))) :
32199 (generation_allocation_start
32200 (generation_of (curr_gen_number - 1))))
32202 print_free_list (curr_gen_number, seg);
32214 //-----------------------------------------------------------------------------
32216 // VM Specific support
32218 //-----------------------------------------------------------------------------
32223 unsigned int PromotedObjectCount = 0;
32224 unsigned int CreatedObjectCount = 0;
32225 unsigned int AllocDuration = 0;
32226 unsigned int AllocCount = 0;
32227 unsigned int AllocBigCount = 0;
32228 unsigned int AllocSmallCount = 0;
32229 unsigned int AllocStart = 0;
32232 //Static member variables.
32233 VOLATILE(BOOL) GCHeap::GcInProgress = FALSE;
32235 //CMCSafeLock* GCHeap::fGcLock;
32236 GCEvent *GCHeap::WaitForGCEvent = NULL;
32239 unsigned int GCHeap::GcDuration;
32241 unsigned GCHeap::GcCondemnedGeneration = 0;
32242 size_t GCHeap::totalSurvivedSize = 0;
32243 #ifdef FEATURE_PREMORTEM_FINALIZATION
32244 CFinalize* GCHeap::m_Finalize = 0;
32245 BOOL GCHeap::GcCollectClasses = FALSE;
32246 VOLATILE(int32_t) GCHeap::m_GCFLock = 0;
32248 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
32250 #ifdef BACKGROUND_GC
32251 int GCHeap::gc_stress_fgcs_in_bgc = 0;
32252 #endif // BACKGROUND_GC
32253 #ifndef MULTIPLE_HEAPS
32254 OBJECTHANDLE GCHeap::m_StressObjs[NUM_HEAP_STRESS_OBJS];
32255 int GCHeap::m_CurStressObj = 0;
32256 #endif // !MULTIPLE_HEAPS
32257 #endif // STRESS_HEAP
32258 #endif // FEATURE_REDHAWK
32260 #endif //FEATURE_PREMORTEM_FINALIZATION
32262 class NoGCRegionLockHolder
32265 NoGCRegionLockHolder()
32267 enter_spin_lock_noinstru(&g_no_gc_lock);
32270 ~NoGCRegionLockHolder()
32272 leave_spin_lock_noinstru(&g_no_gc_lock);
32276 // An explanation of locking for finalization:
32278 // Multiple threads allocate objects. During the allocation, they are serialized by
32279 // the AllocLock above. But they release that lock before they register the object
32280 // for finalization. That's because there is much contention for the alloc lock, but
32281 // finalization is presumed to be a rare case.
32283 // So registering an object for finalization must be protected by the FinalizeLock.
32285 // There is another logical queue that involves finalization. When objects registered
32286 // for finalization become unreachable, they are moved from the "registered" queue to
32287 // the "unreachable" queue. Note that this only happens inside a GC, so no other
32288 // threads can be manipulating either queue at that time. Once the GC is over and
32289 // threads are resumed, the Finalizer thread will dequeue objects from the "unreachable"
32290 // queue and call their finalizers. This dequeue operation is also protected with
32291 // the finalize lock.
32293 // At first, this seems unnecessary. Only one thread is ever enqueuing or dequeuing
32294 // on the unreachable queue (either the GC thread during a GC or the finalizer thread
32295 // when a GC is not in progress). The reason we share a lock with threads enqueuing
32296 // on the "registered" queue is that the "registered" and "unreachable" queues are
32299 // They are actually two regions of a longer list, which can only grow at one end.
32300 // So to enqueue an object to the "registered" list, you actually rotate an unreachable
32301 // object at the boundary between the logical queues, out to the other end of the
32302 // unreachable queue -- where all growing takes place. Then you move the boundary
32303 // pointer so that the gap we created at the boundary is now on the "registered"
32304 // side rather than the "unreachable" side. Now the object can be placed into the
32305 // "registered" side at that point. This is much more efficient than doing moves
32306 // of arbitrarily long regions, but it causes the two queues to require a shared lock.
32308 // Notice that Enter/LeaveFinalizeLock is not a GC-aware spin lock. Instead, it relies
32309 // on the fact that the lock will only be taken for a brief period and that it will
32310 // never provoke or allow a GC while the lock is held. This is critical. If the
32311 // FinalizeLock used enter_spin_lock (and thus sometimes enters preemptive mode to
32312 // allow a GC), then the Alloc client would have to GC protect a finalizable object
32313 // to protect against that eventuality. That is too slow!
32317 BOOL IsValidObject99(uint8_t *pObject)
32320 if (!((CObjectHeader*)pObject)->IsFree())
32321 ((CObjectHeader *) pObject)->Validate();
32322 #endif //VERIFY_HEAP
32326 #ifdef BACKGROUND_GC
32327 BOOL gc_heap::bgc_mark_array_range (heap_segment* seg,
32329 uint8_t** range_beg,
32330 uint8_t** range_end)
32332 uint8_t* seg_start = heap_segment_mem (seg);
32333 uint8_t* seg_end = (whole_seg_p ? heap_segment_reserved (seg) : align_on_mark_word (heap_segment_allocated (seg)));
32335 if ((seg_start < background_saved_highest_address) &&
32336 (seg_end > background_saved_lowest_address))
32338 *range_beg = max (seg_start, background_saved_lowest_address);
32339 *range_end = min (seg_end, background_saved_highest_address);
32348 void gc_heap::bgc_verify_mark_array_cleared (heap_segment* seg)
32350 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32351 if (recursive_gc_sync::background_running_p() && GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
32353 uint8_t* range_beg = 0;
32354 uint8_t* range_end = 0;
32356 if (bgc_mark_array_range (seg, TRUE, &range_beg, &range_end))
32358 size_t markw = mark_word_of (range_beg);
32359 size_t markw_end = mark_word_of (range_end);
32360 while (markw < markw_end)
32362 if (mark_array [markw])
32364 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
32365 markw, mark_array [markw], mark_word_address (markw)));
32370 uint8_t* p = mark_word_address (markw_end);
32371 while (p < range_end)
32373 assert (!(mark_array_marked (p)));
32378 #endif //VERIFY_HEAP && MARK_ARRAY
32381 void gc_heap::verify_mark_bits_cleared (uint8_t* obj, size_t s)
32383 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32384 size_t start_mark_bit = mark_bit_of (obj) + 1;
32385 size_t end_mark_bit = mark_bit_of (obj + s);
32386 unsigned int startbit = mark_bit_bit (start_mark_bit);
32387 unsigned int endbit = mark_bit_bit (end_mark_bit);
32388 size_t startwrd = mark_bit_word (start_mark_bit);
32389 size_t endwrd = mark_bit_word (end_mark_bit);
32390 unsigned int result = 0;
32392 unsigned int firstwrd = ~(lowbits (~0, startbit));
32393 unsigned int lastwrd = ~(highbits (~0, endbit));
32395 if (startwrd == endwrd)
32397 unsigned int wrd = firstwrd & lastwrd;
32398 result = mark_array[startwrd] & wrd;
32406 // verify the first mark word is cleared.
32409 result = mark_array[startwrd] & firstwrd;
32417 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
32419 result = mark_array[wrdtmp];
32426 // set the last mark word.
32429 result = mark_array[endwrd] & lastwrd;
32435 #endif //VERIFY_HEAP && MARK_ARRAY
32438 void gc_heap::clear_all_mark_array()
32441 //size_t num_dwords_written = 0;
32442 //size_t begin_time = GetHighPrecisionTimeStamp();
32444 generation* gen = generation_of (max_generation);
32445 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32451 if (gen != large_object_generation)
32453 gen = generation_of (max_generation+1);
32454 seg = heap_segment_rw (generation_start_segment (gen));
32462 uint8_t* range_beg = 0;
32463 uint8_t* range_end = 0;
32465 if (bgc_mark_array_range (seg, (seg == ephemeral_heap_segment), &range_beg, &range_end))
32467 size_t markw = mark_word_of (range_beg);
32468 size_t markw_end = mark_word_of (range_end);
32469 size_t size_total = (markw_end - markw) * sizeof (uint32_t);
32470 //num_dwords_written = markw_end - markw;
32472 size_t size_left = 0;
32474 assert (((size_t)&mark_array[markw] & (sizeof(PTR_PTR)-1)) == 0);
32476 if ((size_total & (sizeof(PTR_PTR) - 1)) != 0)
32478 size = (size_total & ~(sizeof(PTR_PTR) - 1));
32479 size_left = size_total - size;
32480 assert ((size_left & (sizeof (uint32_t) - 1)) == 0);
32487 memclr ((uint8_t*)&mark_array[markw], size);
32489 if (size_left != 0)
32491 uint32_t* markw_to_clear = &mark_array[markw + size / sizeof (uint32_t)];
32492 for (size_t i = 0; i < (size_left / sizeof (uint32_t)); i++)
32494 *markw_to_clear = 0;
32500 seg = heap_segment_next_rw (seg);
32503 //size_t end_time = GetHighPrecisionTimeStamp() - begin_time;
32505 //printf ("took %Id ms to clear %Id bytes\n", end_time, num_dwords_written*sizeof(uint32_t));
32507 #endif //MARK_ARRAY
32510 #endif //BACKGROUND_GC
32512 void gc_heap::verify_mark_array_cleared (heap_segment* seg)
32514 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32515 assert (card_table == g_gc_card_table);
32516 size_t markw = mark_word_of (heap_segment_mem (seg));
32517 size_t markw_end = mark_word_of (heap_segment_reserved (seg));
32519 while (markw < markw_end)
32521 if (mark_array [markw])
32523 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
32524 markw, mark_array [markw], mark_word_address (markw)));
32529 #endif //VERIFY_HEAP && MARK_ARRAY
32532 void gc_heap::verify_mark_array_cleared ()
32534 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32535 if (recursive_gc_sync::background_running_p() && GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
32537 generation* gen = generation_of (max_generation);
32538 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32544 if (gen != large_object_generation)
32546 gen = generation_of (max_generation+1);
32547 seg = heap_segment_rw (generation_start_segment (gen));
32555 bgc_verify_mark_array_cleared (seg);
32556 seg = heap_segment_next_rw (seg);
32559 #endif //VERIFY_HEAP && MARK_ARRAY
32562 void gc_heap::verify_seg_end_mark_array_cleared()
32564 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32565 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
32567 generation* gen = generation_of (max_generation);
32568 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32574 if (gen != large_object_generation)
32576 gen = generation_of (max_generation+1);
32577 seg = heap_segment_rw (generation_start_segment (gen));
32585 // We already cleared all mark array bits for ephemeral generations
32586 // at the beginning of bgc sweep
32587 uint8_t* from = ((seg == ephemeral_heap_segment) ?
32588 generation_allocation_start (generation_of (max_generation - 1)) :
32589 heap_segment_allocated (seg));
32590 size_t markw = mark_word_of (align_on_mark_word (from));
32591 size_t markw_end = mark_word_of (heap_segment_reserved (seg));
32593 while (from < mark_word_address (markw))
32595 if (is_mark_bit_set (from))
32597 dprintf (3, ("mark bit for %Ix was not cleared", from));
32601 from += mark_bit_pitch;
32604 while (markw < markw_end)
32606 if (mark_array [markw])
32608 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
32609 markw, mark_array [markw], mark_word_address (markw)));
32614 seg = heap_segment_next_rw (seg);
32617 #endif //VERIFY_HEAP && MARK_ARRAY
32620 // This function is called to make sure we don't mess up the segment list
32621 // in SOH. It's called by:
32622 // 1) begin and end of ephemeral GCs
32623 // 2) during bgc sweep when we switch segments.
32624 void gc_heap::verify_soh_segment_list()
32627 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
32629 generation* gen = generation_of (max_generation);
32630 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32631 heap_segment* last_seg = 0;
32635 seg = heap_segment_next_rw (seg);
32637 if (last_seg != ephemeral_heap_segment)
32642 #endif //VERIFY_HEAP
32645 // This function can be called at any foreground GCs or blocking GCs. For background GCs,
32646 // it can be called at the end of the final marking; and at any point during background
32648 // NOTE - to be able to call this function during background sweep, we need to temporarily
32649 // NOT clear the mark array bits as we go.
32650 void gc_heap::verify_partial ()
32652 #ifdef BACKGROUND_GC
32653 //printf ("GC#%d: Verifying loh during sweep\n", settings.gc_index);
32654 //generation* gen = large_object_generation;
32655 generation* gen = generation_of (max_generation);
32656 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32657 int align_const = get_alignment_constant (gen != large_object_generation);
32663 // Different ways to fail.
32664 BOOL mark_missed_p = FALSE;
32665 BOOL bad_ref_p = FALSE;
32666 BOOL free_ref_p = FALSE;
32672 if (gen != large_object_generation)
32675 gen = large_object_generation;
32676 align_const = get_alignment_constant (gen != large_object_generation);
32677 seg = heap_segment_rw (generation_start_segment (gen));
32686 o = heap_segment_mem (seg);
32687 end = heap_segment_allocated (seg);
32688 //printf ("validating [%Ix-[%Ix\n", o, end);
32693 BOOL marked_p = background_object_marked (o, FALSE);
32697 go_through_object_cl (method_table (o), o, s, oo,
32701 //dprintf (3, ("VOM: verifying member %Ix in obj %Ix", (size_t)*oo, o));
32702 MethodTable *pMT = method_table (*oo);
32704 if (pMT == g_gc_pFreeObjectMethodTable)
32710 if (!pMT->SanityCheck())
32713 dprintf (3, ("Bad member of %Ix %Ix",
32714 (size_t)oo, (size_t)*oo));
32718 if (current_bgc_state == bgc_final_marking)
32720 if (marked_p && !background_object_marked (*oo, FALSE))
32722 mark_missed_p = TRUE;
32731 o = o + Align(s, align_const);
32733 seg = heap_segment_next_rw (seg);
32736 //printf ("didn't find any large object large enough...\n");
32737 //printf ("finished verifying loh\n");
32738 #endif //BACKGROUND_GC
32744 gc_heap::verify_free_lists ()
32746 for (int gen_num = 0; gen_num <= max_generation+1; gen_num++)
32748 dprintf (3, ("Verifying free list for gen:%d", gen_num));
32749 allocator* gen_alloc = generation_allocator (generation_of (gen_num));
32750 size_t sz = gen_alloc->first_bucket_size();
32751 bool verify_undo_slot = (gen_num != 0) && (gen_num != max_generation+1) && !gen_alloc->discard_if_no_fit_p();
32753 for (unsigned int a_l_number = 0; a_l_number < gen_alloc->number_of_buckets(); a_l_number++)
32755 uint8_t* free_list = gen_alloc->alloc_list_head_of (a_l_number);
32759 if (!((CObjectHeader*)free_list)->IsFree())
32761 dprintf (3, ("Verifiying Heap: curr free list item %Ix isn't a free object)",
32762 (size_t)free_list));
32765 if (((a_l_number < (gen_alloc->number_of_buckets()-1))&& (unused_array_size (free_list) >= sz))
32766 || ((a_l_number != 0) && (unused_array_size (free_list) < sz/2)))
32768 dprintf (3, ("Verifiying Heap: curr free list item %Ix isn't in the right bucket",
32769 (size_t)free_list));
32772 if (verify_undo_slot && (free_list_undo (free_list) != UNDO_EMPTY))
32774 dprintf (3, ("Verifiying Heap: curr free list item %Ix has non empty undo slot",
32775 (size_t)free_list));
32778 if ((gen_num != max_generation+1)&&(object_gennum (free_list)!= gen_num))
32780 dprintf (3, ("Verifiying Heap: curr free list item %Ix is in the wrong generation free list",
32781 (size_t)free_list));
32786 free_list = free_list_slot (free_list);
32788 //verify the sanity of the tail
32789 uint8_t* tail = gen_alloc->alloc_list_tail_of (a_l_number);
32790 if (!((tail == 0) || (tail == prev)))
32792 dprintf (3, ("Verifying Heap: tail of free list is not correct"));
32797 uint8_t* head = gen_alloc->alloc_list_head_of (a_l_number);
32798 if ((head != 0) && (free_list_slot (head) != 0))
32800 dprintf (3, ("Verifying Heap: tail of free list is not correct"));
32811 gc_heap::verify_heap (BOOL begin_gc_p)
32813 int heap_verify_level = static_cast<int>(GCConfig::GetHeapVerifyLevel());
32814 size_t last_valid_brick = 0;
32815 BOOL bCurrentBrickInvalid = FALSE;
32816 BOOL large_brick_p = TRUE;
32817 size_t curr_brick = 0;
32818 size_t prev_brick = (size_t)-1;
32819 int curr_gen_num = max_generation+1;
32820 heap_segment* seg = heap_segment_in_range (generation_start_segment (generation_of (curr_gen_num ) ));
32822 PREFIX_ASSUME(seg != NULL);
32824 uint8_t* curr_object = heap_segment_mem (seg);
32825 uint8_t* prev_object = 0;
32826 uint8_t* begin_youngest = generation_allocation_start(generation_of(0));
32827 uint8_t* end_youngest = heap_segment_allocated (ephemeral_heap_segment);
32828 uint8_t* next_boundary = generation_allocation_start (generation_of (max_generation - 1));
32829 int align_const = get_alignment_constant (FALSE);
32830 size_t total_objects_verified = 0;
32831 size_t total_objects_verified_deep = 0;
32833 #ifdef BACKGROUND_GC
32834 BOOL consider_bgc_mark_p = FALSE;
32835 BOOL check_current_sweep_p = FALSE;
32836 BOOL check_saved_sweep_p = FALSE;
32837 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
32838 #endif //BACKGROUND_GC
32840 #ifdef MULTIPLE_HEAPS
32841 t_join* current_join = &gc_t_join;
32842 #ifdef BACKGROUND_GC
32843 if (settings.concurrent && (bgc_thread_id.IsCurrentThread()))
32845 // We always call verify_heap on entry of GC on the SVR GC threads.
32846 current_join = &bgc_t_join;
32848 #endif //BACKGROUND_GC
32849 #endif //MULTIPLE_HEAPS
32851 UNREFERENCED_PARAMETER(begin_gc_p);
32852 #ifdef BACKGROUND_GC
32853 dprintf (2,("[%s]GC#%d(%s): Verifying heap - begin",
32854 (begin_gc_p ? "BEG" : "END"),
32855 VolatileLoad(&settings.gc_index),
32856 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC"))));
32858 dprintf (2,("[%s]GC#%d: Verifying heap - begin",
32859 (begin_gc_p ? "BEG" : "END"), VolatileLoad(&settings.gc_index)));
32860 #endif //BACKGROUND_GC
32862 #ifndef MULTIPLE_HEAPS
32863 if ((ephemeral_low != generation_allocation_start (generation_of (max_generation - 1))) ||
32864 (ephemeral_high != heap_segment_reserved (ephemeral_heap_segment)))
32868 #endif //MULTIPLE_HEAPS
32870 #ifdef BACKGROUND_GC
32871 //don't touch the memory because the program is allocating from it.
32872 if (!settings.concurrent)
32873 #endif //BACKGROUND_GC
32875 if (!(heap_verify_level & GCConfig::HEAPVERIFY_NO_MEM_FILL))
32877 //uninit the unused portions of segments.
32878 generation* gen1 = large_object_generation;
32879 heap_segment* seg1 = heap_segment_rw (generation_start_segment (gen1));
32880 PREFIX_ASSUME(seg1 != NULL);
32886 uint8_t* clear_start = heap_segment_allocated (seg1) - plug_skew;
32887 if (heap_segment_used (seg1) > clear_start)
32889 dprintf (3, ("setting end of seg %Ix: [%Ix-[%Ix to 0xaa",
32890 heap_segment_mem (seg1),
32892 heap_segment_used (seg1)));
32893 memset (heap_segment_allocated (seg1) - plug_skew, 0xaa,
32894 (heap_segment_used (seg1) - clear_start));
32896 seg1 = heap_segment_next_rw (seg1);
32900 if (gen1 == large_object_generation)
32902 gen1 = generation_of (max_generation);
32903 seg1 = heap_segment_rw (generation_start_segment (gen1));
32904 PREFIX_ASSUME(seg1 != NULL);
32915 #ifdef MULTIPLE_HEAPS
32916 current_join->join(this, gc_join_verify_copy_table);
32917 if (current_join->joined())
32919 // in concurrent GC, new segment could be allocated when GC is working so the card brick table might not be updated at this point
32920 for (int i = 0; i < n_heaps; i++)
32922 //copy the card and brick tables
32923 if (g_gc_card_table != g_heaps[i]->card_table)
32925 g_heaps[i]->copy_brick_card_table();
32929 current_join->restart();
32932 if (g_gc_card_table != card_table)
32933 copy_brick_card_table();
32934 #endif //MULTIPLE_HEAPS
32936 //verify that the generation structures makes sense
32938 generation* gen = generation_of (max_generation);
32940 assert (generation_allocation_start (gen) ==
32941 heap_segment_mem (heap_segment_rw (generation_start_segment (gen))));
32942 int gen_num = max_generation-1;
32943 generation* prev_gen = gen;
32944 while (gen_num >= 0)
32946 gen = generation_of (gen_num);
32947 assert (generation_allocation_segment (gen) == ephemeral_heap_segment);
32948 assert (generation_allocation_start (gen) >= heap_segment_mem (ephemeral_heap_segment));
32949 assert (generation_allocation_start (gen) < heap_segment_allocated (ephemeral_heap_segment));
32951 if (generation_start_segment (prev_gen ) ==
32952 generation_start_segment (gen))
32954 assert (generation_allocation_start (prev_gen) <
32955 generation_allocation_start (gen));
32964 // Handle segment transitions
32965 if (curr_object >= heap_segment_allocated (seg))
32967 if (curr_object > heap_segment_allocated(seg))
32969 dprintf (3, ("Verifiying Heap: curr_object: %Ix > heap_segment_allocated (seg: %Ix)",
32970 (size_t)curr_object, (size_t)seg));
32973 seg = heap_segment_next_in_range (seg);
32976 #ifdef BACKGROUND_GC
32977 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
32978 #endif //BACKGROUND_GC
32979 curr_object = heap_segment_mem(seg);
32985 if (curr_gen_num == (max_generation+1))
32988 seg = heap_segment_in_range (generation_start_segment (generation_of (curr_gen_num)));
32990 PREFIX_ASSUME(seg != NULL);
32992 #ifdef BACKGROUND_GC
32993 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
32994 #endif //BACKGROUND_GC
32995 curr_object = heap_segment_mem (seg);
32997 large_brick_p = FALSE;
32998 align_const = get_alignment_constant (TRUE);
33001 break; // Done Verifying Heap -- no more segments
33005 // Are we at the end of the youngest_generation?
33006 if (seg == ephemeral_heap_segment)
33008 if (curr_object >= end_youngest)
33010 // prev_object length is too long if we hit this int3
33011 if (curr_object > end_youngest)
33013 dprintf (3, ("Verifiying Heap: curr_object: %Ix > end_youngest: %Ix",
33014 (size_t)curr_object, (size_t)end_youngest));
33020 if ((curr_object >= next_boundary) && (curr_gen_num > 0))
33023 if (curr_gen_num > 0)
33025 next_boundary = generation_allocation_start (generation_of (curr_gen_num - 1));
33030 //if (is_mark_set (curr_object))
33032 // printf ("curr_object: %Ix is marked!",(size_t)curr_object);
33033 // FATAL_GC_ERROR();
33036 size_t s = size (curr_object);
33037 dprintf (3, ("o: %Ix, s: %d", (size_t)curr_object, s));
33040 dprintf (3, ("Verifying Heap: size of current object %Ix == 0", curr_object));
33044 // If object is not in the youngest generation, then lets
33045 // verify that the brick table is correct....
33046 if (((seg != ephemeral_heap_segment) ||
33047 (brick_of(curr_object) < brick_of(begin_youngest))))
33049 curr_brick = brick_of(curr_object);
33051 // Brick Table Verification...
33053 // On brick transition
33054 // if brick is negative
33055 // verify that brick indirects to previous valid brick
33057 // set current brick invalid flag to be flipped if we
33058 // encounter an object at the correct place
33060 if (curr_brick != prev_brick)
33062 // If the last brick we were examining had positive
33063 // entry but we never found the matching object, then
33064 // we have a problem
33065 // If prev_brick was the last one of the segment
33066 // it's ok for it to be invalid because it is never looked at
33067 if (bCurrentBrickInvalid &&
33068 (curr_brick != brick_of (heap_segment_mem (seg))) &&
33069 !heap_segment_read_only_p (seg))
33071 dprintf (3, ("curr brick %Ix invalid", curr_brick));
33077 //large objects verify the table only if they are in
33079 if ((heap_segment_reserved (seg) <= highest_address) &&
33080 (heap_segment_mem (seg) >= lowest_address) &&
33081 brick_table [curr_brick] != 0)
33083 dprintf (3, ("curr_brick %Ix for large object %Ix not set to -32768",
33084 curr_brick, (size_t)curr_object));
33089 bCurrentBrickInvalid = FALSE;
33094 // If the current brick contains a negative value make sure
33095 // that the indirection terminates at the last valid brick
33096 if (brick_table [curr_brick] <= 0)
33098 if (brick_table [curr_brick] == 0)
33100 dprintf(3, ("curr_brick %Ix for object %Ix set to 0",
33101 curr_brick, (size_t)curr_object));
33104 ptrdiff_t i = curr_brick;
33105 while ((i >= ((ptrdiff_t) brick_of (heap_segment_mem (seg)))) &&
33106 (brick_table[i] < 0))
33108 i = i + brick_table[i];
33110 if (i < ((ptrdiff_t)(brick_of (heap_segment_mem (seg))) - 1))
33112 dprintf (3, ("ptrdiff i: %Ix < brick_of (heap_segment_mem (seg)):%Ix - 1. curr_brick: %Ix",
33113 i, brick_of (heap_segment_mem (seg)),
33117 // if (i != last_valid_brick)
33118 // FATAL_GC_ERROR();
33119 bCurrentBrickInvalid = FALSE;
33121 else if (!heap_segment_read_only_p (seg))
33123 bCurrentBrickInvalid = TRUE;
33128 if (bCurrentBrickInvalid)
33130 if (curr_object == (brick_address(curr_brick) + brick_table[curr_brick] - 1))
33132 bCurrentBrickInvalid = FALSE;
33133 last_valid_brick = curr_brick;
33138 if (*((uint8_t**)curr_object) != (uint8_t *) g_gc_pFreeObjectMethodTable)
33140 #ifdef FEATURE_LOH_COMPACTION
33141 if ((curr_gen_num == (max_generation+1)) && (prev_object != 0))
33143 assert (method_table (prev_object) == g_gc_pFreeObjectMethodTable);
33145 #endif //FEATURE_LOH_COMPACTION
33147 total_objects_verified++;
33149 BOOL can_verify_deep = TRUE;
33150 #ifdef BACKGROUND_GC
33151 can_verify_deep = fgc_should_consider_object (curr_object, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p);
33152 #endif //BACKGROUND_GC
33154 BOOL deep_verify_obj = can_verify_deep;
33155 if ((heap_verify_level & GCConfig::HEAPVERIFY_DEEP_ON_COMPACT) && !settings.compaction)
33156 deep_verify_obj = FALSE;
33158 ((CObjectHeader*)curr_object)->ValidateHeap((Object*)curr_object, deep_verify_obj);
33160 if (can_verify_deep)
33162 if (curr_gen_num > 0)
33164 BOOL need_card_p = FALSE;
33165 if (contain_pointers_or_collectible (curr_object))
33167 dprintf (4, ("curr_object: %Ix", (size_t)curr_object));
33168 size_t crd = card_of (curr_object);
33169 BOOL found_card_p = card_set_p (crd);
33171 #ifdef COLLECTIBLE_CLASS
33172 if (is_collectible(curr_object))
33174 uint8_t* class_obj = get_class_object (curr_object);
33175 if ((class_obj < ephemeral_high) && (class_obj >= next_boundary))
33179 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix pointing to class object %Ix",
33180 card_of (curr_object), (size_t)curr_object, class_obj));
33186 #endif //COLLECTIBLE_CLASS
33188 if (contain_pointers(curr_object))
33190 go_through_object_nostart
33191 (method_table(curr_object), curr_object, s, oo,
33193 if ((crd != card_of ((uint8_t*)oo)) && !found_card_p)
33195 crd = card_of ((uint8_t*)oo);
33196 found_card_p = card_set_p (crd);
33197 need_card_p = FALSE;
33199 if ((*oo < ephemeral_high) && (*oo >= next_boundary))
33201 need_card_p = TRUE;
33204 if (need_card_p && !found_card_p)
33207 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix, %Ix:%Ix[",
33208 card_of (curr_object), (size_t)curr_object,
33209 card_of (curr_object+Align(s, align_const)), (size_t)curr_object+Align(s, align_const)));
33215 if (need_card_p && !found_card_p)
33217 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix, %Ix:%Ix[",
33218 card_of (curr_object), (size_t)curr_object,
33219 card_of (curr_object+Align(s, align_const)), (size_t)curr_object+Align(s, align_const)));
33224 total_objects_verified_deep++;
33228 prev_object = curr_object;
33229 prev_brick = curr_brick;
33230 curr_object = curr_object + Align(s, align_const);
33231 if (curr_object < prev_object)
33233 dprintf (3, ("overflow because of a bad object size: %Ix size %Ix", prev_object, s));
33238 #ifdef BACKGROUND_GC
33239 dprintf (2, ("(%s)(%s)(%s) total_objects_verified is %Id, total_objects_verified_deep is %Id",
33240 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p () ? "FGC" : "NGC")),
33241 (begin_gc_p ? "BEG" : "END"),
33242 ((current_c_gc_state == c_gc_state_planning) ? "in plan" : "not in plan"),
33243 total_objects_verified, total_objects_verified_deep));
33244 if (current_c_gc_state != c_gc_state_planning)
33246 assert (total_objects_verified == total_objects_verified_deep);
33248 #endif //BACKGROUND_GC
33250 verify_free_lists();
33252 #ifdef FEATURE_PREMORTEM_FINALIZATION
33253 finalize_queue->CheckFinalizerObjects();
33254 #endif // FEATURE_PREMORTEM_FINALIZATION
33257 // to be consistent with handle table APIs pass a ScanContext*
33258 // to provide the heap number. the SC isn't complete though so
33259 // limit its scope to handle table verification.
33261 sc.thread_number = heap_number;
33262 GCScan::VerifyHandleTable(max_generation, max_generation, &sc);
33265 #ifdef MULTIPLE_HEAPS
33266 current_join->join(this, gc_join_verify_objects_done);
33267 if (current_join->joined())
33268 #endif //MULTIPLE_HEAPS
33270 SyncBlockCache::GetSyncBlockCache()->VerifySyncTableEntry();
33271 #ifdef MULTIPLE_HEAPS
33272 current_join->restart();
33273 #endif //MULTIPLE_HEAPS
33276 #ifdef BACKGROUND_GC
33277 if (!settings.concurrent)
33279 if (current_c_gc_state == c_gc_state_planning)
33281 // temporarily commenting this out 'cause an FGC
33282 // could be triggered before we sweep ephemeral.
33283 //verify_seg_end_mark_array_cleared();
33287 if (settings.concurrent)
33289 verify_mark_array_cleared();
33291 dprintf (2,("GC%d(%s): Verifying heap - end",
33292 VolatileLoad(&settings.gc_index),
33293 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC"))));
33295 dprintf (2,("GC#d: Verifying heap - end", VolatileLoad(&settings.gc_index)));
33296 #endif //BACKGROUND_GC
33299 #endif //VERIFY_HEAP
33302 void GCHeap::ValidateObjectMember (Object* obj)
33305 size_t s = size (obj);
33306 uint8_t* o = (uint8_t*)obj;
33308 go_through_object_cl (method_table (obj), o, s, oo,
33310 uint8_t* child_o = *oo;
33313 dprintf (3, ("VOM: m: %Ix obj %Ix", (size_t)child_o, o));
33314 MethodTable *pMT = method_table (child_o);
33316 if (!pMT->SanityCheck()) {
33317 dprintf (3, ("Bad member of %Ix %Ix",
33318 (size_t)oo, (size_t)child_o));
33323 #endif // VERIFY_HEAP
33326 void DestructObject (CObjectHeader* hdr)
33328 UNREFERENCED_PARAMETER(hdr); // compiler bug? -- this *is*, indeed, referenced
33329 hdr->~CObjectHeader();
33332 HRESULT GCHeap::Shutdown ()
33336 GCScan::GcRuntimeStructuresValid (FALSE);
33338 // Cannot assert this, since we use SuspendEE as the mechanism to quiesce all
33339 // threads except the one performing the shutdown.
33340 // ASSERT( !GcInProgress );
33342 // Guard against any more GC occurring and against any threads blocking
33343 // for GC to complete when the GC heap is gone. This fixes a race condition
33344 // where a thread in GC is destroyed as part of process destruction and
33345 // the remaining threads block for GC complete.
33348 //EnterAllocLock();
33350 //EnterFinalizeLock();
33353 // during shutdown lot of threads are suspended
33354 // on this even, we don't want to wake them up just yet
33355 //CloseHandle (WaitForGCEvent);
33357 //find out if the global card table hasn't been used yet
33358 uint32_t* ct = &g_gc_card_table[card_word (gcard_of (g_gc_lowest_address))];
33359 if (card_table_refcount (ct) == 0)
33361 destroy_card_table (ct);
33362 g_gc_card_table = nullptr;
33364 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
33365 g_gc_card_bundle_table = nullptr;
33367 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
33368 SoftwareWriteWatch::StaticClose();
33369 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
33372 //destroy all segments on the standby list
33373 while(gc_heap::segment_standby_list != 0)
33375 heap_segment* next_seg = heap_segment_next (gc_heap::segment_standby_list);
33376 #ifdef MULTIPLE_HEAPS
33377 (gc_heap::g_heaps[0])->delete_heap_segment (gc_heap::segment_standby_list, FALSE);
33378 #else //MULTIPLE_HEAPS
33379 pGenGCHeap->delete_heap_segment (gc_heap::segment_standby_list, FALSE);
33380 #endif //MULTIPLE_HEAPS
33381 gc_heap::segment_standby_list = next_seg;
33385 #ifdef MULTIPLE_HEAPS
33387 for (int i = 0; i < gc_heap::n_heaps; i ++)
33389 delete gc_heap::g_heaps[i]->vm_heap;
33390 //destroy pure GC stuff
33391 gc_heap::destroy_gc_heap (gc_heap::g_heaps[i]);
33394 gc_heap::destroy_gc_heap (pGenGCHeap);
33396 #endif //MULTIPLE_HEAPS
33397 gc_heap::shutdown_gc();
33402 // Wait until a garbage collection is complete
33403 // returns NOERROR if wait was OK, other error code if failure.
33404 // WARNING: This will not undo the must complete state. If you are
33405 // in a must complete when you call this, you'd better know what you're
33408 #ifdef FEATURE_PREMORTEM_FINALIZATION
33410 HRESULT AllocateCFinalize(CFinalize **pCFinalize)
33412 *pCFinalize = new (nothrow) CFinalize();
33413 if (*pCFinalize == NULL || !(*pCFinalize)->Initialize())
33414 return E_OUTOFMEMORY;
33418 #endif // FEATURE_PREMORTEM_FINALIZATION
33420 // init the instance heap
33421 HRESULT GCHeap::Init(size_t hn)
33423 HRESULT hres = S_OK;
33425 #ifdef MULTIPLE_HEAPS
33426 if ((pGenGCHeap = gc_heap::make_gc_heap(this, (int)hn)) == 0)
33427 hres = E_OUTOFMEMORY;
33429 UNREFERENCED_PARAMETER(hn);
33430 if (!gc_heap::make_gc_heap())
33431 hres = E_OUTOFMEMORY;
33432 #endif //MULTIPLE_HEAPS
33438 //System wide initialization
33439 HRESULT GCHeap::Initialize ()
33443 g_gc_pFreeObjectMethodTable = GCToEEInterface::GetFreeObjectMethodTable();
33444 g_num_processors = GCToOSInterface::GetTotalProcessorCount();
33445 assert(g_num_processors != 0);
33447 //Initialize the static members.
33450 CreatedObjectCount = 0;
33453 size_t seg_size = get_valid_segment_size();
33454 gc_heap::soh_segment_size = seg_size;
33455 size_t large_seg_size = get_valid_segment_size(TRUE);
33456 gc_heap::min_loh_segment_size = large_seg_size;
33457 gc_heap::min_segment_size = min (seg_size, large_seg_size);
33458 #ifdef SEG_MAPPING_TABLE
33459 gc_heap::min_segment_size_shr = index_of_highest_set_bit (gc_heap::min_segment_size);
33460 #endif //SEG_MAPPING_TABLE
33462 #ifdef MULTIPLE_HEAPS
33463 if (GCConfig::GetNoAffinitize())
33464 gc_heap::gc_thread_no_affinitize_p = true;
33466 uint32_t nhp_from_config = static_cast<uint32_t>(GCConfig::GetHeapCount());
33468 uint32_t nhp_from_process = GCToOSInterface::GetCurrentProcessCpuCount();
33470 uint32_t nhp = ((nhp_from_config == 0) ? nhp_from_process :
33471 (min (nhp_from_config, nhp_from_process)));
33473 nhp = min (nhp, MAX_SUPPORTED_CPUS);
33475 hr = gc_heap::initialize_gc (seg_size, large_seg_size /*LHEAP_ALLOC*/, nhp);
33477 hr = gc_heap::initialize_gc (seg_size, large_seg_size /*LHEAP_ALLOC*/);
33478 #endif //MULTIPLE_HEAPS
33483 gc_heap::total_physical_mem = GCToOSInterface::GetPhysicalMemoryLimit();
33485 gc_heap::mem_one_percent = gc_heap::total_physical_mem / 100;
33486 #ifndef MULTIPLE_HEAPS
33487 gc_heap::mem_one_percent /= g_num_processors;
33488 #endif //!MULTIPLE_HEAPS
33490 // We should only use this if we are in the "many process" mode which really is only applicable
33491 // to very powerful machines - before that's implemented, temporarily I am only enabling this for 80GB+ memory.
33492 // For now I am using an estimate to calculate these numbers but this should really be obtained
33493 // programmatically going forward.
33494 // I am assuming 47 processes using WKS GC and 3 using SVR GC.
33495 // I am assuming 3 in part due to the "very high memory load" is 97%.
33496 int available_mem_th = 10;
33497 if (gc_heap::total_physical_mem >= ((uint64_t)80 * 1024 * 1024 * 1024))
33499 int adjusted_available_mem_th = 3 + (int)((float)47 / (float)(g_num_processors));
33500 available_mem_th = min (available_mem_th, adjusted_available_mem_th);
33503 gc_heap::high_memory_load_th = 100 - available_mem_th;
33506 gc_heap::youngest_gen_desired_th = gc_heap::mem_one_percent;
33509 WaitForGCEvent = new (nothrow) GCEvent;
33511 if (!WaitForGCEvent)
33513 return E_OUTOFMEMORY;
33516 if (!WaitForGCEvent->CreateManualEventNoThrow(TRUE))
33521 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
33522 #if defined (STRESS_HEAP) && !defined (MULTIPLE_HEAPS)
33523 if (GCStress<cfg_any>::IsEnabled()) {
33524 for(int i = 0; i < GCHeap::NUM_HEAP_STRESS_OBJS; i++)
33525 m_StressObjs[i] = CreateGlobalHandle(0);
33526 m_CurStressObj = 0;
33528 #endif //STRESS_HEAP && !MULTIPLE_HEAPS
33529 #endif // FEATURE_REDHAWK
33531 initGCShadow(); // If we are debugging write barriers, initialize heap shadow
33533 #ifdef MULTIPLE_HEAPS
33535 for (unsigned i = 0; i < nhp; i++)
33537 GCHeap* Hp = new (nothrow) GCHeap();
33539 return E_OUTOFMEMORY;
33541 if ((hr = Hp->Init (i))!= S_OK)
33546 // initialize numa node to heap map
33547 heap_select::init_numa_node_to_heap_map(nhp);
33550 #endif //MULTIPLE_HEAPS
33554 GCScan::GcRuntimeStructuresValid (TRUE);
33556 GCToEEInterface::DiagUpdateGenerationBounds();
33563 // GC callback functions
33564 bool GCHeap::IsPromoted(Object* object)
33567 ((CObjectHeader*)object)->Validate();
33570 uint8_t* o = (uint8_t*)object;
33572 if (gc_heap::settings.condemned_generation == max_generation)
33574 #ifdef MULTIPLE_HEAPS
33575 gc_heap* hp = gc_heap::g_heaps[0];
33577 gc_heap* hp = pGenGCHeap;
33578 #endif //MULTIPLE_HEAPS
33580 #ifdef BACKGROUND_GC
33581 if (gc_heap::settings.concurrent)
33583 bool is_marked = (!((o < hp->background_saved_highest_address) && (o >= hp->background_saved_lowest_address))||
33584 hp->background_marked (o));
33588 #endif //BACKGROUND_GC
33590 return (!((o < hp->highest_address) && (o >= hp->lowest_address))
33591 || hp->is_mark_set (o));
33596 gc_heap* hp = gc_heap::heap_of (o);
33597 return (!((o < hp->gc_high) && (o >= hp->gc_low))
33598 || hp->is_mark_set (o));
33602 size_t GCHeap::GetPromotedBytes(int heap_index)
33604 #ifdef BACKGROUND_GC
33605 if (gc_heap::settings.concurrent)
33607 return gc_heap::bpromoted_bytes (heap_index);
33610 #endif //BACKGROUND_GC
33612 return gc_heap::promoted_bytes (heap_index);
33616 unsigned int GCHeap::WhichGeneration (Object* object)
33618 gc_heap* hp = gc_heap::heap_of ((uint8_t*)object);
33619 unsigned int g = hp->object_gennum ((uint8_t*)object);
33620 dprintf (3, ("%Ix is in gen %d", (size_t)object, g));
33624 bool GCHeap::IsEphemeral (Object* object)
33626 uint8_t* o = (uint8_t*)object;
33627 gc_heap* hp = gc_heap::heap_of (o);
33628 return !!hp->ephemeral_pointer_p (o);
33631 // Return NULL if can't find next object. When EE is not suspended,
33632 // the result is not accurate: if the input arg is in gen0, the function could
33633 // return zeroed out memory as next object
33634 Object * GCHeap::NextObj (Object * object)
33637 uint8_t* o = (uint8_t*)object;
33639 #ifndef FEATURE_BASICFREEZE
33640 if (!((o < g_gc_highest_address) && (o >= g_gc_lowest_address)))
33644 #endif //!FEATURE_BASICFREEZE
33646 heap_segment * hs = gc_heap::find_segment (o, FALSE);
33652 BOOL large_object_p = heap_segment_loh_p (hs);
33653 if (large_object_p)
33654 return NULL; //could be racing with another core allocating.
33655 #ifdef MULTIPLE_HEAPS
33656 gc_heap* hp = heap_segment_heap (hs);
33657 #else //MULTIPLE_HEAPS
33659 #endif //MULTIPLE_HEAPS
33660 unsigned int g = hp->object_gennum ((uint8_t*)object);
33661 if ((g == 0) && hp->settings.demotion)
33662 return NULL;//could be racing with another core allocating.
33663 int align_const = get_alignment_constant (!large_object_p);
33664 uint8_t* nextobj = o + Align (size (o), align_const);
33665 if (nextobj <= o) // either overflow or 0 sized object.
33670 if ((nextobj < heap_segment_mem(hs)) ||
33671 (nextobj >= heap_segment_allocated(hs) && hs != hp->ephemeral_heap_segment) ||
33672 (nextobj >= hp->alloc_allocated))
33677 return (Object *)nextobj;
33680 #endif // VERIFY_HEAP
33685 #ifdef FEATURE_BASICFREEZE
33686 BOOL GCHeap::IsInFrozenSegment (Object * object)
33688 uint8_t* o = (uint8_t*)object;
33689 heap_segment * hs = gc_heap::find_segment (o, FALSE);
33690 //We create a frozen object for each frozen segment before the segment is inserted
33691 //to segment list; during ngen, we could also create frozen objects in segments which
33692 //don't belong to current GC heap.
33693 //So we return true if hs is NULL. It might create a hole about detecting invalidate
33694 //object. But given all other checks present, the hole should be very small
33695 return !hs || heap_segment_read_only_p (hs);
33697 #endif //FEATURE_BASICFREEZE
33699 #endif //VERIFY_HEAP
33701 // returns TRUE if the pointer is in one of the GC heaps.
33702 bool GCHeap::IsHeapPointer (void* vpObject, bool small_heap_only)
33704 STATIC_CONTRACT_SO_TOLERANT;
33706 // removed STATIC_CONTRACT_CAN_TAKE_LOCK here because find_segment
33707 // no longer calls GCEvent::Wait which eventually takes a lock.
33709 uint8_t* object = (uint8_t*) vpObject;
33710 #ifndef FEATURE_BASICFREEZE
33711 if (!((object < g_gc_highest_address) && (object >= g_gc_lowest_address)))
33713 #endif //!FEATURE_BASICFREEZE
33715 heap_segment * hs = gc_heap::find_segment (object, small_heap_only);
33719 #ifdef STRESS_PINNING
33720 static n_promote = 0;
33721 #endif //STRESS_PINNING
33722 // promote an object
33723 void GCHeap::Promote(Object** ppObject, ScanContext* sc, uint32_t flags)
33725 THREAD_NUMBER_FROM_CONTEXT;
33726 #ifndef MULTIPLE_HEAPS
33727 const int thread = 0;
33728 #endif //!MULTIPLE_HEAPS
33730 uint8_t* o = (uint8_t*)*ppObject;
33735 #ifdef DEBUG_DestroyedHandleValue
33736 // we can race with destroy handle during concurrent scan
33737 if (o == (uint8_t*)DEBUG_DestroyedHandleValue)
33739 #endif //DEBUG_DestroyedHandleValue
33743 gc_heap* hp = gc_heap::heap_of (o);
33745 dprintf (3, ("Promote %Ix", (size_t)o));
33747 #ifdef INTERIOR_POINTERS
33748 if (flags & GC_CALL_INTERIOR)
33750 if ((o < hp->gc_low) || (o >= hp->gc_high))
33754 if ( (o = hp->find_object (o, hp->gc_low)) == 0)
33760 #endif //INTERIOR_POINTERS
33762 #ifdef FEATURE_CONSERVATIVE_GC
33763 // For conservative GC, a value on stack may point to middle of a free object.
33764 // In this case, we don't need to promote the pointer.
33765 if (GCConfig::GetConservativeGC()
33766 && ((CObjectHeader*)o)->IsFree())
33773 ((CObjectHeader*)o)->ValidatePromote(sc, flags);
33775 UNREFERENCED_PARAMETER(sc);
33778 if (flags & GC_CALL_PINNED)
33779 hp->pin_object (o, (uint8_t**) ppObject, hp->gc_low, hp->gc_high);
33781 #ifdef STRESS_PINNING
33782 if ((++n_promote % 20) == 1)
33783 hp->pin_object (o, (uint8_t**) ppObject, hp->gc_low, hp->gc_high);
33784 #endif //STRESS_PINNING
33786 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
33787 size_t promoted_size_begin = hp->promoted_bytes (thread);
33788 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
33790 if ((o >= hp->gc_low) && (o < hp->gc_high))
33792 hpt->mark_object_simple (&o THREAD_NUMBER_ARG);
33795 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
33796 size_t promoted_size_end = hp->promoted_bytes (thread);
33799 if (sc->pCurrentDomain)
33801 sc->pCurrentDomain->RecordSurvivedBytes ((promoted_size_end - promoted_size_begin), thread);
33804 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
33806 STRESS_LOG_ROOT_PROMOTE(ppObject, o, o ? header(o)->GetMethodTable() : NULL);
33809 void GCHeap::Relocate (Object** ppObject, ScanContext* sc,
33812 UNREFERENCED_PARAMETER(sc);
33814 uint8_t* object = (uint8_t*)(Object*)(*ppObject);
33816 THREAD_NUMBER_FROM_CONTEXT;
33818 //dprintf (3, ("Relocate location %Ix\n", (size_t)ppObject));
33819 dprintf (3, ("R: %Ix", (size_t)ppObject));
33824 gc_heap* hp = gc_heap::heap_of (object);
33827 if (!(flags & GC_CALL_INTERIOR))
33829 // We cannot validate this object if it's in the condemned gen because it could
33830 // be one of the objects that were overwritten by an artificial gap due to a pinned plug.
33831 if (!((object >= hp->gc_low) && (object < hp->gc_high)))
33833 ((CObjectHeader*)object)->Validate(FALSE);
33838 dprintf (3, ("Relocate %Ix\n", (size_t)object));
33842 if ((flags & GC_CALL_INTERIOR) && gc_heap::settings.loh_compaction)
33844 if (!((object >= hp->gc_low) && (object < hp->gc_high)))
33849 if (gc_heap::loh_object_p (object))
33851 pheader = hp->find_object (object, 0);
33857 ptrdiff_t ref_offset = object - pheader;
33858 hp->relocate_address(&pheader THREAD_NUMBER_ARG);
33859 *ppObject = (Object*)(pheader + ref_offset);
33866 hp->relocate_address(&pheader THREAD_NUMBER_ARG);
33867 *ppObject = (Object*)pheader;
33870 STRESS_LOG_ROOT_RELOCATE(ppObject, object, pheader, ((!(flags & GC_CALL_INTERIOR)) ? ((Object*)object)->GetGCSafeMethodTable() : 0));
33873 /*static*/ bool GCHeap::IsObjectInFixedHeap(Object *pObj)
33875 // For now we simply look at the size of the object to determine if it in the
33876 // fixed heap or not. If the bit indicating this gets set at some point
33877 // we should key off that instead.
33878 return size( pObj ) >= LARGE_OBJECT_SIZE;
33881 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
33884 void StressHeapDummy ();
33886 static int32_t GCStressStartCount = -1;
33887 static int32_t GCStressCurCount = 0;
33888 static int32_t GCStressStartAtJit = -1;
33890 // the maximum number of foreground GCs we'll induce during one BGC
33891 // (this number does not include "naturally" occuring GCs).
33892 static int32_t GCStressMaxFGCsPerBGC = -1;
33894 // CLRRandom implementation can produce FPU exceptions if
33895 // the test/application run by CLR is enabling any FPU exceptions.
33896 // We want to avoid any unexpected exception coming from stress
33897 // infrastructure, so CLRRandom is not an option.
33898 // The code below is a replicate of CRT rand() implementation.
33899 // Using CRT rand() is not an option because we will interfere with the user application
33900 // that may also use it.
33901 int StressRNG(int iMaxValue)
33903 static BOOL bisRandInit = FALSE;
33904 static int lHoldrand = 1L;
33908 lHoldrand = (int)time(NULL);
33909 bisRandInit = TRUE;
33911 int randValue = (((lHoldrand = lHoldrand * 214013L + 2531011L) >> 16) & 0x7fff);
33912 return randValue % iMaxValue;
33914 #endif // STRESS_HEAP
33915 #endif // !FEATURE_REDHAWK
33917 // free up object so that things will move and then do a GC
33918 //return TRUE if GC actually happens, otherwise FALSE
33919 bool GCHeap::StressHeap(gc_alloc_context * context)
33921 #if defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
33922 alloc_context* acontext = static_cast<alloc_context*>(context);
33923 assert(context != nullptr);
33925 // if GC stress was dynamically disabled during this run we return FALSE
33926 if (!GCStressPolicy::IsEnabled())
33930 if (g_pConfig->FastGCStressLevel() && !GCToEEInterface::GetThread()->StressHeapIsEnabled()) {
33936 if ((g_pConfig->GetGCStressLevel() & EEConfig::GCSTRESS_UNIQUE)
33938 || g_pConfig->FastGCStressLevel() > 1
33941 if (!Thread::UniqueStack(&acontext)) {
33946 #ifdef BACKGROUND_GC
33947 // don't trigger a GC from the GC threads but still trigger GCs from user threads.
33948 if (GCToEEInterface::WasCurrentThreadCreatedByGC())
33952 #endif //BACKGROUND_GC
33954 if (GCStressStartAtJit == -1 || GCStressStartCount == -1)
33956 GCStressStartCount = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_GCStressStart);
33957 GCStressStartAtJit = CLRConfig::GetConfigValue(CLRConfig::INTERNAL_GCStressStartAtJit);
33960 if (GCStressMaxFGCsPerBGC == -1)
33962 GCStressMaxFGCsPerBGC = CLRConfig::GetConfigValue(CLRConfig::INTERNAL_GCStressMaxFGCsPerBGC);
33963 if (g_pConfig->IsGCStressMix() && GCStressMaxFGCsPerBGC == -1)
33964 GCStressMaxFGCsPerBGC = 6;
33968 if (g_JitCount < GCStressStartAtJit)
33972 // Allow programmer to skip the first N Stress GCs so that you can
33973 // get to the interesting ones faster.
33974 Interlocked::Increment(&GCStressCurCount);
33975 if (GCStressCurCount < GCStressStartCount)
33978 // throttle the number of stress-induced GCs by a factor given by GCStressStep
33979 if ((GCStressCurCount % g_pConfig->GetGCStressStep()) != 0)
33984 #ifdef BACKGROUND_GC
33985 if (IsConcurrentGCEnabled() && IsConcurrentGCInProgress())
33987 // allow a maximum number of stress induced FGCs during one BGC
33988 if (gc_stress_fgcs_in_bgc >= GCStressMaxFGCsPerBGC)
33990 ++gc_stress_fgcs_in_bgc;
33992 #endif // BACKGROUND_GC
33994 if (g_pStringClass == 0)
33996 // If the String class has not been loaded, dont do any stressing. This should
33997 // be kept to a minimum to get as complete coverage as possible.
33998 _ASSERTE(g_fEEInit);
34002 #ifndef MULTIPLE_HEAPS
34003 static int32_t OneAtATime = -1;
34005 // Only bother with this if the stress level is big enough and if nobody else is
34006 // doing it right now. Note that some callers are inside the AllocLock and are
34007 // guaranteed synchronized. But others are using AllocationContexts and have no
34008 // particular synchronization.
34010 // For this latter case, we want a very high-speed way of limiting this to one
34011 // at a time. A secondary advantage is that we release part of our StressObjs
34012 // buffer sparingly but just as effectively.
34014 if (Interlocked::Increment(&OneAtATime) == 0 &&
34015 !TrackAllocations()) // Messing with object sizes can confuse the profiler (see ICorProfilerInfo::GetObjectSize)
34019 // If the current string is used up
34020 if (HndFetchHandle(m_StressObjs[m_CurStressObj]) == 0)
34022 // Populate handles with strings
34023 int i = m_CurStressObj;
34024 while(HndFetchHandle(m_StressObjs[i]) == 0)
34026 _ASSERTE(m_StressObjs[i] != 0);
34027 unsigned strLen = (LARGE_OBJECT_SIZE - 32) / sizeof(WCHAR);
34028 unsigned strSize = PtrAlign(StringObject::GetSize(strLen));
34030 // update the cached type handle before allocating
34031 SetTypeHandleOnThreadForAlloc(TypeHandle(g_pStringClass));
34032 str = (StringObject*) pGenGCHeap->allocate (strSize, acontext);
34035 str->SetMethodTable (g_pStringClass);
34036 str->SetStringLength (strLen);
34038 HndAssignHandle(m_StressObjs[i], ObjectToOBJECTREF(str));
34040 i = (i + 1) % NUM_HEAP_STRESS_OBJS;
34041 if (i == m_CurStressObj) break;
34044 // advance the current handle to the next string
34045 m_CurStressObj = (m_CurStressObj + 1) % NUM_HEAP_STRESS_OBJS;
34048 // Get the current string
34049 str = (StringObject*) OBJECTREFToObject(HndFetchHandle(m_StressObjs[m_CurStressObj]));
34052 // Chop off the end of the string and form a new object out of it.
34053 // This will 'free' an object at the begining of the heap, which will
34054 // force data movement. Note that we can only do this so many times.
34055 // before we have to move on to the next string.
34056 unsigned sizeOfNewObj = (unsigned)Align(min_obj_size * 31);
34057 if (str->GetStringLength() > sizeOfNewObj / sizeof(WCHAR))
34059 unsigned sizeToNextObj = (unsigned)Align(size(str));
34060 uint8_t* freeObj = ((uint8_t*) str) + sizeToNextObj - sizeOfNewObj;
34061 pGenGCHeap->make_unused_array (freeObj, sizeOfNewObj);
34062 str->SetStringLength(str->GetStringLength() - (sizeOfNewObj / sizeof(WCHAR)));
34066 // Let the string itself become garbage.
34067 // will be realloced next time around
34068 HndAssignHandle(m_StressObjs[m_CurStressObj], 0);
34072 Interlocked::Decrement(&OneAtATime);
34073 #endif // !MULTIPLE_HEAPS
34074 if (IsConcurrentGCEnabled())
34076 int rgen = StressRNG(10);
34078 // gen0:gen1:gen2 distribution: 40:40:20
34081 else if (rgen >= 4)
34086 GarbageCollectTry (rgen, FALSE, collection_gcstress);
34090 GarbageCollect(max_generation, FALSE, collection_gcstress);
34095 UNREFERENCED_PARAMETER(context);
34097 #endif // defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
34101 #ifdef FEATURE_PREMORTEM_FINALIZATION
34102 #define REGISTER_FOR_FINALIZATION(_object, _size) \
34103 hp->finalize_queue->RegisterForFinalization (0, (_object), (_size))
34104 #else // FEATURE_PREMORTEM_FINALIZATION
34105 #define REGISTER_FOR_FINALIZATION(_object, _size) true
34106 #endif // FEATURE_PREMORTEM_FINALIZATION
34108 #define CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(_object, _size, _register) do { \
34109 if ((_object) == NULL || ((_register) && !REGISTER_FOR_FINALIZATION(_object, _size))) \
34111 STRESS_LOG_OOM_STACK(_size); \
34117 // Small Object Allocator
34120 // Allocate small object with an alignment requirement of 8-bytes.
34122 GCHeap::AllocAlign8(gc_alloc_context* ctx, size_t size, uint32_t flags )
34124 #ifdef FEATURE_64BIT_ALIGNMENT
34130 alloc_context* acontext = static_cast<alloc_context*>(ctx);
34132 #ifdef MULTIPLE_HEAPS
34133 if (acontext->get_alloc_heap() == 0)
34135 AssignHeap (acontext);
34136 assert (acontext->get_alloc_heap());
34139 gc_heap* hp = acontext->get_alloc_heap()->pGenGCHeap;
34141 gc_heap* hp = pGenGCHeap;
34142 #endif //MULTIPLE_HEAPS
34144 return AllocAlign8Common(hp, acontext, size, flags);
34146 UNREFERENCED_PARAMETER(ctx);
34147 UNREFERENCED_PARAMETER(size);
34148 UNREFERENCED_PARAMETER(flags);
34149 assert(!"should not call GCHeap::AllocAlign8 without FEATURE_64BIT_ALIGNMENT defined!");
34151 #endif //FEATURE_64BIT_ALIGNMENT
34154 // Common code used by both variants of AllocAlign8 above.
34156 GCHeap::AllocAlign8Common(void* _hp, alloc_context* acontext, size_t size, uint32_t flags)
34158 #ifdef FEATURE_64BIT_ALIGNMENT
34164 gc_heap* hp = (gc_heap*)_hp;
34168 Object* newAlloc = NULL;
34171 #ifdef COUNT_CYCLES
34172 AllocStart = GetCycleCount32();
34174 #elif defined(ENABLE_INSTRUMENTATION)
34175 unsigned AllocStart = GetInstLogTime();
34177 #endif //COUNT_CYCLES
34180 if (size < LARGE_OBJECT_SIZE)
34186 // Depending on where in the object the payload requiring 8-byte alignment resides we might have to
34187 // align the object header on an 8-byte boundary or midway between two such boundaries. The unaligned
34188 // case is indicated to the GC via the GC_ALLOC_ALIGN8_BIAS flag.
34189 size_t desiredAlignment = (flags & GC_ALLOC_ALIGN8_BIAS) ? 4 : 0;
34191 // Retrieve the address of the next allocation from the context (note that we're inside the alloc
34192 // lock at this point).
34193 uint8_t* result = acontext->alloc_ptr;
34195 // Will an allocation at this point yield the correct alignment and fit into the remainder of the
34197 if ((((size_t)result & 7) == desiredAlignment) && ((result + size) <= acontext->alloc_limit))
34199 // Yes, we can just go ahead and make the allocation.
34200 newAlloc = (Object*) hp->allocate (size, acontext);
34201 ASSERT(((size_t)newAlloc & 7) == desiredAlignment);
34205 // No, either the next available address is not aligned in the way we require it or there's
34206 // not enough space to allocate an object of the required size. In both cases we allocate a
34207 // padding object (marked as a free object). This object's size is such that it will reverse
34208 // the alignment of the next header (asserted below).
34210 // We allocate both together then decide based on the result whether we'll format the space as
34211 // free object + real object or real object + free object.
34212 ASSERT((Align(min_obj_size) & 7) == 4);
34213 CObjectHeader *freeobj = (CObjectHeader*) hp->allocate (Align(size) + Align(min_obj_size), acontext);
34216 if (((size_t)freeobj & 7) == desiredAlignment)
34218 // New allocation has desired alignment, return this one and place the free object at the
34219 // end of the allocated space.
34220 newAlloc = (Object*)freeobj;
34221 freeobj = (CObjectHeader*)((uint8_t*)freeobj + Align(size));
34225 // New allocation is still mis-aligned, format the initial space as a free object and the
34226 // rest of the space should be correctly aligned for the real object.
34227 newAlloc = (Object*)((uint8_t*)freeobj + Align(min_obj_size));
34228 ASSERT(((size_t)newAlloc & 7) == desiredAlignment);
34230 freeobj->SetFree(min_obj_size);
34236 // The LOH always guarantees at least 8-byte alignment, regardless of platform. Moreover it doesn't
34237 // support mis-aligned object headers so we can't support biased headers as above. Luckily for us
34238 // we've managed to arrange things so the only case where we see a bias is for boxed value types and
34239 // these can never get large enough to be allocated on the LOH.
34240 ASSERT(65536 < LARGE_OBJECT_SIZE);
34241 ASSERT((flags & GC_ALLOC_ALIGN8_BIAS) == 0);
34243 alloc_context* acontext = generation_alloc_context (hp->generation_of (max_generation+1));
34245 newAlloc = (Object*) hp->allocate_large_object (size, acontext->alloc_bytes_loh);
34246 ASSERT(((size_t)newAlloc & 7) == 0);
34249 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
34252 #ifdef COUNT_CYCLES
34253 finish = GetCycleCount32();
34254 #elif defined(ENABLE_INSTRUMENTATION)
34255 finish = GetInstLogTime();
34256 #endif //COUNT_CYCLES
34257 AllocDuration += finish - AllocStart;
34262 UNREFERENCED_PARAMETER(_hp);
34263 UNREFERENCED_PARAMETER(acontext);
34264 UNREFERENCED_PARAMETER(size);
34265 UNREFERENCED_PARAMETER(flags);
34266 assert(!"Should not call GCHeap::AllocAlign8Common without FEATURE_64BIT_ALIGNMENT defined!");
34268 #endif // FEATURE_64BIT_ALIGNMENT
34272 GCHeap::AllocLHeap( size_t size, uint32_t flags REQD_ALIGN_DCL)
34281 Object* newAlloc = NULL;
34284 #ifdef COUNT_CYCLES
34285 AllocStart = GetCycleCount32();
34287 #elif defined(ENABLE_INSTRUMENTATION)
34288 unsigned AllocStart = GetInstLogTime();
34290 #endif //COUNT_CYCLES
34293 #ifdef MULTIPLE_HEAPS
34294 //take the first heap....
34295 gc_heap* hp = gc_heap::g_heaps[0];
34297 gc_heap* hp = pGenGCHeap;
34299 // prefix complains about us dereferencing hp in wks build even though we only access static members
34300 // this way. not sure how to shut it up except for this ugly workaround:
34301 PREFIX_ASSUME(hp != NULL);
34303 #endif //MULTIPLE_HEAPS
34305 alloc_context* acontext = generation_alloc_context (hp->generation_of (max_generation+1));
34307 newAlloc = (Object*) hp->allocate_large_object (size + ComputeMaxStructAlignPadLarge(requiredAlignment), acontext->alloc_bytes_loh);
34308 #ifdef FEATURE_STRUCTALIGN
34309 newAlloc = (Object*) hp->pad_for_alignment_large ((uint8_t*) newAlloc, requiredAlignment, size);
34310 #endif // FEATURE_STRUCTALIGN
34311 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
34314 #ifdef COUNT_CYCLES
34315 finish = GetCycleCount32();
34316 #elif defined(ENABLE_INSTRUMENTATION)
34317 finish = GetInstLogTime();
34318 #endif //COUNT_CYCLES
34319 AllocDuration += finish - AllocStart;
34326 GCHeap::Alloc(gc_alloc_context* context, size_t size, uint32_t flags REQD_ALIGN_DCL)
34335 Object* newAlloc = NULL;
34336 alloc_context* acontext = static_cast<alloc_context*>(context);
34339 #ifdef COUNT_CYCLES
34340 AllocStart = GetCycleCount32();
34342 #elif defined(ENABLE_INSTRUMENTATION)
34343 unsigned AllocStart = GetInstLogTime();
34345 #endif //COUNT_CYCLES
34348 #ifdef MULTIPLE_HEAPS
34349 if (acontext->get_alloc_heap() == 0)
34351 AssignHeap (acontext);
34352 assert (acontext->get_alloc_heap());
34354 #endif //MULTIPLE_HEAPS
34356 #ifdef MULTIPLE_HEAPS
34357 gc_heap* hp = acontext->get_alloc_heap()->pGenGCHeap;
34359 gc_heap* hp = pGenGCHeap;
34361 // prefix complains about us dereferencing hp in wks build even though we only access static members
34362 // this way. not sure how to shut it up except for this ugly workaround:
34363 PREFIX_ASSUME(hp != NULL);
34365 #endif //MULTIPLE_HEAPS
34367 if (size < LARGE_OBJECT_SIZE)
34373 newAlloc = (Object*) hp->allocate (size + ComputeMaxStructAlignPad(requiredAlignment), acontext);
34374 #ifdef FEATURE_STRUCTALIGN
34375 newAlloc = (Object*) hp->pad_for_alignment ((uint8_t*) newAlloc, requiredAlignment, size, acontext);
34376 #endif // FEATURE_STRUCTALIGN
34377 // ASSERT (newAlloc);
34381 newAlloc = (Object*) hp->allocate_large_object (size + ComputeMaxStructAlignPadLarge(requiredAlignment), acontext->alloc_bytes_loh);
34382 #ifdef FEATURE_STRUCTALIGN
34383 newAlloc = (Object*) hp->pad_for_alignment_large ((uint8_t*) newAlloc, requiredAlignment, size);
34384 #endif // FEATURE_STRUCTALIGN
34387 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
34390 #ifdef COUNT_CYCLES
34391 finish = GetCycleCount32();
34392 #elif defined(ENABLE_INSTRUMENTATION)
34393 finish = GetInstLogTime();
34394 #endif //COUNT_CYCLES
34395 AllocDuration += finish - AllocStart;
34402 GCHeap::FixAllocContext (gc_alloc_context* context, bool lockp, void* arg, void *heap)
34404 alloc_context* acontext = static_cast<alloc_context*>(context);
34405 #ifdef MULTIPLE_HEAPS
34408 acontext->alloc_count = 0;
34410 uint8_t * alloc_ptr = acontext->alloc_ptr;
34415 // The acontext->alloc_heap can be out of sync with the ptrs because
34416 // of heap re-assignment in allocate
34417 gc_heap* hp = gc_heap::heap_of (alloc_ptr);
34419 gc_heap* hp = pGenGCHeap;
34420 #endif //MULTIPLE_HEAPS
34422 if (heap == NULL || heap == hp)
34426 enter_spin_lock (&hp->more_space_lock);
34428 hp->fix_allocation_context (acontext, ((arg != 0)? TRUE : FALSE),
34429 get_alignment_constant(TRUE));
34432 leave_spin_lock (&hp->more_space_lock);
34438 GCHeap::GetContainingObject (void *pInteriorPtr, bool fCollectedGenOnly)
34440 uint8_t *o = (uint8_t*)pInteriorPtr;
34442 gc_heap* hp = gc_heap::heap_of (o);
34444 uint8_t* lowest = (fCollectedGenOnly ? hp->gc_low : hp->lowest_address);
34445 uint8_t* highest = (fCollectedGenOnly ? hp->gc_high : hp->highest_address);
34447 if (o >= lowest && o < highest)
34449 o = hp->find_object (o, lowest);
34456 return (Object *)o;
34459 BOOL should_collect_optimized (dynamic_data* dd, BOOL low_memory_p)
34461 if (dd_new_allocation (dd) < 0)
34466 if (((float)(dd_new_allocation (dd)) / (float)dd_desired_allocation (dd)) < (low_memory_p ? 0.7 : 0.3))
34474 //----------------------------------------------------------------------------
34475 // #GarbageCollector
34477 // API to ensure that a complete new garbage collection takes place
34480 GCHeap::GarbageCollect (int generation, bool low_memory_p, int mode)
34485 size_t total_allocated = 0;
34486 size_t total_desired = 0;
34487 #ifdef MULTIPLE_HEAPS
34489 for (hn = 0; hn < gc_heap::n_heaps; hn++)
34491 gc_heap* hp = gc_heap::g_heaps [hn];
34492 total_desired += dd_desired_allocation (hp->dynamic_data_of (0));
34493 total_allocated += dd_desired_allocation (hp->dynamic_data_of (0))-
34494 dd_new_allocation (hp->dynamic_data_of (0));
34497 gc_heap* hp = pGenGCHeap;
34498 total_desired = dd_desired_allocation (hp->dynamic_data_of (0));
34499 total_allocated = dd_desired_allocation (hp->dynamic_data_of (0))-
34500 dd_new_allocation (hp->dynamic_data_of (0));
34501 #endif //MULTIPLE_HEAPS
34503 if ((total_desired > gc_heap::mem_one_percent) && (total_allocated < gc_heap::mem_one_percent))
34505 dprintf (2, ("Async low mem but we've only allocated %d (< 10%% of physical mem) out of %d, returning",
34506 total_allocated, total_desired));
34513 #ifdef MULTIPLE_HEAPS
34514 gc_heap* hpt = gc_heap::g_heaps[0];
34517 #endif //MULTIPLE_HEAPS
34519 generation = (generation < 0) ? max_generation : min (generation, max_generation);
34520 dynamic_data* dd = hpt->dynamic_data_of (generation);
34522 #ifdef BACKGROUND_GC
34523 if (recursive_gc_sync::background_running_p())
34525 if ((mode == collection_optimized) || (mode & collection_non_blocking))
34529 if (mode & collection_blocking)
34531 pGenGCHeap->background_gc_wait();
34532 if (mode & collection_optimized)
34538 #endif //BACKGROUND_GC
34540 if (mode & collection_optimized)
34542 if (pGenGCHeap->gc_started)
34548 BOOL should_collect = FALSE;
34549 BOOL should_check_loh = (generation == max_generation);
34550 #ifdef MULTIPLE_HEAPS
34551 for (int i = 0; i < gc_heap::n_heaps; i++)
34553 dynamic_data* dd1 = gc_heap::g_heaps [i]->dynamic_data_of (generation);
34554 dynamic_data* dd2 = (should_check_loh ?
34555 (gc_heap::g_heaps [i]->dynamic_data_of (max_generation + 1)) :
34558 if (should_collect_optimized (dd1, low_memory_p))
34560 should_collect = TRUE;
34563 if (dd2 && should_collect_optimized (dd2, low_memory_p))
34565 should_collect = TRUE;
34570 should_collect = should_collect_optimized (dd, low_memory_p);
34571 if (!should_collect && should_check_loh)
34574 should_collect_optimized (hpt->dynamic_data_of (max_generation + 1), low_memory_p);
34576 #endif //MULTIPLE_HEAPS
34577 if (!should_collect)
34584 size_t CollectionCountAtEntry = dd_collection_count (dd);
34585 size_t BlockingCollectionCountAtEntry = gc_heap::full_gc_counts[gc_type_blocking];
34586 size_t CurrentCollectionCount = 0;
34590 CurrentCollectionCount = GarbageCollectTry(generation, low_memory_p, mode);
34592 if ((mode & collection_blocking) &&
34593 (generation == max_generation) &&
34594 (gc_heap::full_gc_counts[gc_type_blocking] == BlockingCollectionCountAtEntry))
34596 #ifdef BACKGROUND_GC
34597 if (recursive_gc_sync::background_running_p())
34599 pGenGCHeap->background_gc_wait();
34601 #endif //BACKGROUND_GC
34606 if (CollectionCountAtEntry == CurrentCollectionCount)
34615 GCHeap::GarbageCollectTry (int generation, BOOL low_memory_p, int mode)
34617 int gen = (generation < 0) ?
34618 max_generation : min (generation, max_generation);
34620 gc_reason reason = reason_empty;
34624 if (mode & collection_blocking)
34625 reason = reason_lowmemory_blocking;
34627 reason = reason_lowmemory;
34630 reason = reason_induced;
34632 if (reason == reason_induced)
34634 if (mode & collection_compacting)
34636 reason = reason_induced_compacting;
34638 else if (mode & collection_non_blocking)
34640 reason = reason_induced_noforce;
34643 else if (mode & collection_gcstress)
34645 reason = reason_gcstress;
34650 return GarbageCollectGeneration (gen, reason);
34653 void gc_heap::do_pre_gc()
34655 STRESS_LOG_GC_STACK;
34658 STRESS_LOG_GC_START(VolatileLoad(&settings.gc_index),
34659 (uint32_t)settings.condemned_generation,
34660 (uint32_t)settings.reason);
34661 #endif // STRESS_LOG
34663 #ifdef MULTIPLE_HEAPS
34664 gc_heap* hp = g_heaps[0];
34667 #endif //MULTIPLE_HEAPS
34669 #ifdef BACKGROUND_GC
34670 settings.b_state = hp->current_bgc_state;
34671 #endif //BACKGROUND_GC
34673 #ifdef BACKGROUND_GC
34674 dprintf (1, ("*GC* %d(gen0:%d)(%d)(%s)(%d)",
34675 VolatileLoad(&settings.gc_index),
34676 dd_collection_count (hp->dynamic_data_of (0)),
34677 settings.condemned_generation,
34678 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC")),
34679 settings.b_state));
34681 dprintf (1, ("*GC* %d(gen0:%d)(%d)",
34682 VolatileLoad(&settings.gc_index),
34683 dd_collection_count(hp->dynamic_data_of(0)),
34684 settings.condemned_generation));
34685 #endif //BACKGROUND_GC
34687 // TODO: this can happen...it's because of the way we are calling
34688 // do_pre_gc, will fix later.
34689 //if (last_gc_index > VolatileLoad(&settings.gc_index))
34691 // FATAL_GC_ERROR();
34694 last_gc_index = VolatileLoad(&settings.gc_index);
34695 GCHeap::UpdatePreGCCounters();
34697 if (settings.concurrent)
34699 #ifdef BACKGROUND_GC
34700 full_gc_counts[gc_type_background]++;
34701 #if defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
34702 GCHeap::gc_stress_fgcs_in_bgc = 0;
34703 #endif // STRESS_HEAP && !FEATURE_REDHAWK
34704 #endif // BACKGROUND_GC
34708 if (settings.condemned_generation == max_generation)
34710 full_gc_counts[gc_type_blocking]++;
34714 #ifdef BACKGROUND_GC
34715 if (settings.background_p)
34717 ephemeral_fgc_counts[settings.condemned_generation]++;
34719 #endif //BACKGROUND_GC
34723 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
34726 SystemDomain::ResetADSurvivedBytes();
34728 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
34731 #ifdef GC_CONFIG_DRIVEN
34732 void gc_heap::record_interesting_info_per_heap()
34734 // datapoints are always from the last blocking GC so don't record again
34736 if (!(settings.concurrent))
34738 for (int i = 0; i < max_idp_count; i++)
34740 interesting_data_per_heap[i] += interesting_data_per_gc[i];
34744 int compact_reason = get_gc_data_per_heap()->get_mechanism (gc_heap_compact);
34745 if (compact_reason >= 0)
34746 (compact_reasons_per_heap[compact_reason])++;
34747 int expand_mechanism = get_gc_data_per_heap()->get_mechanism (gc_heap_expand);
34748 if (expand_mechanism >= 0)
34749 (expand_mechanisms_per_heap[expand_mechanism])++;
34751 for (int i = 0; i < max_gc_mechanism_bits_count; i++)
34753 if (get_gc_data_per_heap()->is_mechanism_bit_set ((gc_mechanism_bit_per_heap)i))
34754 (interesting_mechanism_bits_per_heap[i])++;
34757 // h# | GC | gen | C | EX | NF | BF | ML | DM || PreS | PostS | Merge | Conv | Pre | Post | PrPo | PreP | PostP |
34758 cprintf (("%2d | %6d | %1d | %1s | %2s | %2s | %2s | %2s | %2s || %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id |",
34760 (size_t)settings.gc_index,
34761 settings.condemned_generation,
34762 // TEMP - I am just doing this for wks GC 'cuase I wanna see the pattern of doing C/S GCs.
34763 (settings.compaction ? (((compact_reason >= 0) && gc_heap_compact_reason_mandatory_p[compact_reason]) ? "M" : "W") : ""), // compaction
34764 ((expand_mechanism >= 0)? "X" : ""), // EX
34765 ((expand_mechanism == expand_reuse_normal) ? "X" : ""), // NF
34766 ((expand_mechanism == expand_reuse_bestfit) ? "X" : ""), // BF
34767 (get_gc_data_per_heap()->is_mechanism_bit_set (gc_mark_list_bit) ? "X" : ""), // ML
34768 (get_gc_data_per_heap()->is_mechanism_bit_set (gc_demotion_bit) ? "X" : ""), // DM
34769 interesting_data_per_gc[idp_pre_short],
34770 interesting_data_per_gc[idp_post_short],
34771 interesting_data_per_gc[idp_merged_pin],
34772 interesting_data_per_gc[idp_converted_pin],
34773 interesting_data_per_gc[idp_pre_pin],
34774 interesting_data_per_gc[idp_post_pin],
34775 interesting_data_per_gc[idp_pre_and_post_pin],
34776 interesting_data_per_gc[idp_pre_short_padded],
34777 interesting_data_per_gc[idp_post_short_padded]));
34780 void gc_heap::record_global_mechanisms()
34782 for (int i = 0; i < max_global_mechanisms_count; i++)
34784 if (gc_data_global.get_mechanism_p ((gc_global_mechanism_p)i))
34786 ::record_global_mechanism (i);
34791 BOOL gc_heap::should_do_sweeping_gc (BOOL compact_p)
34793 if (!compact_ratio)
34794 return (!compact_p);
34796 size_t compact_count = compact_or_sweep_gcs[0];
34797 size_t sweep_count = compact_or_sweep_gcs[1];
34799 size_t total_count = compact_count + sweep_count;
34800 BOOL should_compact = compact_p;
34801 if (total_count > 3)
34805 int temp_ratio = (int)((compact_count + 1) * 100 / (total_count + 1));
34806 if (temp_ratio > compact_ratio)
34808 // cprintf (("compact would be: %d, total_count: %d, ratio would be %d%% > target\n",
34809 // (compact_count + 1), (total_count + 1), temp_ratio));
34810 should_compact = FALSE;
34815 int temp_ratio = (int)((sweep_count + 1) * 100 / (total_count + 1));
34816 if (temp_ratio > (100 - compact_ratio))
34818 // cprintf (("sweep would be: %d, total_count: %d, ratio would be %d%% > target\n",
34819 // (sweep_count + 1), (total_count + 1), temp_ratio));
34820 should_compact = TRUE;
34825 return !should_compact;
34827 #endif //GC_CONFIG_DRIVEN
34829 void gc_heap::do_post_gc()
34831 if (!settings.concurrent)
34837 #ifdef COUNT_CYCLES
34838 AllocStart = GetCycleCount32();
34840 AllocStart = clock();
34841 #endif //COUNT_CYCLES
34844 #ifdef MULTIPLE_HEAPS
34845 gc_heap* hp = g_heaps[0];
34848 #endif //MULTIPLE_HEAPS
34850 GCToEEInterface::GcDone(settings.condemned_generation);
34852 GCToEEInterface::DiagGCEnd(VolatileLoad(&settings.gc_index),
34853 (uint32_t)settings.condemned_generation,
34854 (uint32_t)settings.reason,
34855 !!settings.concurrent);
34857 //dprintf (1, (" ****end of Garbage Collection**** %d(gen0:%d)(%d)",
34858 dprintf (1, ("*EGC* %d(gen0:%d)(%d)(%s)",
34859 VolatileLoad(&settings.gc_index),
34860 dd_collection_count(hp->dynamic_data_of(0)),
34861 settings.condemned_generation,
34862 (settings.concurrent ? "BGC" : "GC")));
34864 if (settings.exit_memory_load != 0)
34865 last_gc_memory_load = settings.exit_memory_load;
34866 else if (settings.entry_memory_load != 0)
34867 last_gc_memory_load = settings.entry_memory_load;
34869 last_gc_heap_size = get_total_heap_size();
34870 last_gc_fragmentation = get_total_fragmentation();
34872 GCHeap::UpdatePostGCCounters();
34873 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
34874 //if (g_fEnableARM)
34876 // SystemDomain::GetADSurvivedBytes();
34878 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
34881 STRESS_LOG_GC_END(VolatileLoad(&settings.gc_index),
34882 (uint32_t)settings.condemned_generation,
34883 (uint32_t)settings.reason);
34884 #endif // STRESS_LOG
34886 #ifdef GC_CONFIG_DRIVEN
34887 if (!settings.concurrent)
34889 if (settings.compaction)
34890 (compact_or_sweep_gcs[0])++;
34892 (compact_or_sweep_gcs[1])++;
34895 #ifdef MULTIPLE_HEAPS
34896 for (int i = 0; i < n_heaps; i++)
34897 g_heaps[i]->record_interesting_info_per_heap();
34899 record_interesting_info_per_heap();
34900 #endif //MULTIPLE_HEAPS
34901 record_global_mechanisms();
34902 #endif //GC_CONFIG_DRIVEN
34905 unsigned GCHeap::GetGcCount()
34907 return (unsigned int)VolatileLoad(&pGenGCHeap->settings.gc_index);
34911 GCHeap::GarbageCollectGeneration (unsigned int gen, gc_reason reason)
34913 dprintf (2, ("triggered a GC!"));
34915 #ifdef MULTIPLE_HEAPS
34916 gc_heap* hpt = gc_heap::g_heaps[0];
34919 #endif //MULTIPLE_HEAPS
34920 bool cooperative_mode = true;
34921 dynamic_data* dd = hpt->dynamic_data_of (gen);
34922 size_t localCount = dd_collection_count (dd);
34924 enter_spin_lock (&gc_heap::gc_lock);
34925 dprintf (SPINLOCK_LOG, ("GC Egc"));
34926 ASSERT_HOLDING_SPIN_LOCK(&gc_heap::gc_lock);
34928 //don't trigger another GC if one was already in progress
34929 //while waiting for the lock
34931 size_t col_count = dd_collection_count (dd);
34933 if (localCount != col_count)
34935 #ifdef SYNCHRONIZATION_STATS
34936 gc_lock_contended++;
34937 #endif //SYNCHRONIZATION_STATS
34938 dprintf (SPINLOCK_LOG, ("no need GC Lgc"));
34939 leave_spin_lock (&gc_heap::gc_lock);
34941 // We don't need to release msl here 'cause this means a GC
34942 // has happened and would have release all msl's.
34947 #ifdef COUNT_CYCLES
34948 int gc_start = GetCycleCount32();
34949 #endif //COUNT_CYCLES
34952 #ifdef COUNT_CYCLES
34953 AllocDuration += GetCycleCount32() - AllocStart;
34955 AllocDuration += clock() - AllocStart;
34956 #endif //COUNT_CYCLES
34959 gc_heap::g_low_memory_status = (reason == reason_lowmemory) ||
34960 (reason == reason_lowmemory_blocking) ||
34961 (gc_heap::latency_level == latency_level_memory_footprint);
34963 gc_trigger_reason = reason;
34965 #ifdef MULTIPLE_HEAPS
34966 for (int i = 0; i < gc_heap::n_heaps; i++)
34968 gc_heap::g_heaps[i]->reset_gc_done();
34971 gc_heap::reset_gc_done();
34972 #endif //MULTIPLE_HEAPS
34974 gc_heap::gc_started = TRUE;
34977 init_sync_log_stats();
34979 #ifndef MULTIPLE_HEAPS
34980 cooperative_mode = gc_heap::enable_preemptive ();
34982 dprintf (2, ("Suspending EE"));
34983 BEGIN_TIMING(suspend_ee_during_log);
34984 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC);
34985 END_TIMING(suspend_ee_during_log);
34986 gc_heap::proceed_with_gc_p = gc_heap::should_proceed_with_gc();
34987 gc_heap::disable_preemptive (cooperative_mode);
34988 if (gc_heap::proceed_with_gc_p)
34989 pGenGCHeap->settings.init_mechanisms();
34991 gc_heap::update_collection_counts_for_no_gc();
34993 #endif //!MULTIPLE_HEAPS
34996 // MAP_EVENT_MONITORS(EE_MONITOR_GARBAGE_COLLECTIONS, NotifyEvent(EE_EVENT_TYPE_GC_STARTED, 0));
34999 #ifdef COUNT_CYCLES
35002 start = GetCycleCount32();
35007 #endif //COUNT_CYCLES
35008 PromotedObjectCount = 0;
35011 unsigned int condemned_generation_number = gen;
35013 // We want to get a stack from the user thread that triggered the GC
35014 // instead of on the GC thread which is the case for Server GC.
35015 // But we are doing it for Workstation GC as well to be uniform.
35016 FIRE_EVENT(GCTriggered, static_cast<uint32_t>(reason));
35018 #ifdef MULTIPLE_HEAPS
35019 GcCondemnedGeneration = condemned_generation_number;
35021 cooperative_mode = gc_heap::enable_preemptive ();
35023 BEGIN_TIMING(gc_during_log);
35024 gc_heap::ee_suspend_event.Set();
35025 gc_heap::wait_for_gc_done();
35026 END_TIMING(gc_during_log);
35028 gc_heap::disable_preemptive (cooperative_mode);
35030 condemned_generation_number = GcCondemnedGeneration;
35032 if (gc_heap::proceed_with_gc_p)
35034 BEGIN_TIMING(gc_during_log);
35035 pGenGCHeap->garbage_collect (condemned_generation_number);
35036 END_TIMING(gc_during_log);
35038 #endif //MULTIPLE_HEAPS
35041 #ifdef COUNT_CYCLES
35042 finish = GetCycleCount32();
35045 #endif //COUNT_CYCLES
35046 GcDuration += finish - start;
35048 ("<GC# %d> Condemned: %d, Duration: %d, total: %d Alloc Avg: %d, Small Objects:%d Large Objects:%d",
35049 VolatileLoad(&pGenGCHeap->settings.gc_index), condemned_generation_number,
35050 finish - start, GcDuration,
35051 AllocCount ? (AllocDuration / AllocCount) : 0,
35052 AllocSmallCount, AllocBigCount));
35057 #ifdef BACKGROUND_GC
35058 // We are deciding whether we should fire the alloc wait end event here
35059 // because in begin_foreground we could be calling end_foreground
35060 // if we need to retry.
35061 if (gc_heap::alloc_wait_event_p)
35063 hpt->fire_alloc_wait_event_end (awr_fgc_wait_for_bgc);
35064 gc_heap::alloc_wait_event_p = FALSE;
35066 #endif //BACKGROUND_GC
35068 #ifndef MULTIPLE_HEAPS
35069 #ifdef BACKGROUND_GC
35070 if (!gc_heap::dont_restart_ee_p)
35072 #endif //BACKGROUND_GC
35073 BEGIN_TIMING(restart_ee_during_log);
35074 GCToEEInterface::RestartEE(TRUE);
35075 END_TIMING(restart_ee_during_log);
35076 #ifdef BACKGROUND_GC
35078 #endif //BACKGROUND_GC
35079 #endif //!MULTIPLE_HEAPS
35081 #ifdef COUNT_CYCLES
35082 printf ("GC: %d Time: %d\n", GcCondemnedGeneration,
35083 GetCycleCount32() - gc_start);
35084 #endif //COUNT_CYCLES
35086 #ifndef MULTIPLE_HEAPS
35087 process_sync_log_stats();
35088 gc_heap::gc_started = FALSE;
35089 gc_heap::set_gc_done();
35090 dprintf (SPINLOCK_LOG, ("GC Lgc"));
35091 leave_spin_lock (&gc_heap::gc_lock);
35092 #endif //!MULTIPLE_HEAPS
35094 #ifdef FEATURE_PREMORTEM_FINALIZATION
35095 GCToEEInterface::EnableFinalization(!pGenGCHeap->settings.concurrent && pGenGCHeap->settings.found_finalizers);
35096 #endif // FEATURE_PREMORTEM_FINALIZATION
35098 return dd_collection_count (dd);
35101 size_t GCHeap::GetTotalBytesInUse ()
35103 #ifdef MULTIPLE_HEAPS
35104 //enumarate all the heaps and get their size.
35105 size_t tot_size = 0;
35106 for (int i = 0; i < gc_heap::n_heaps; i++)
35108 GCHeap* Hp = gc_heap::g_heaps [i]->vm_heap;
35109 tot_size += Hp->ApproxTotalBytesInUse (FALSE);
35113 return ApproxTotalBytesInUse ();
35114 #endif //MULTIPLE_HEAPS
35117 int GCHeap::CollectionCount (int generation, int get_bgc_fgc_count)
35119 if (get_bgc_fgc_count != 0)
35121 #ifdef BACKGROUND_GC
35122 if (generation == max_generation)
35124 return (int)(gc_heap::full_gc_counts[gc_type_background]);
35128 return (int)(gc_heap::ephemeral_fgc_counts[generation]);
35132 #endif //BACKGROUND_GC
35135 #ifdef MULTIPLE_HEAPS
35136 gc_heap* hp = gc_heap::g_heaps [0];
35137 #else //MULTIPLE_HEAPS
35138 gc_heap* hp = pGenGCHeap;
35139 #endif //MULTIPLE_HEAPS
35140 if (generation > max_generation)
35143 return (int)dd_collection_count (hp->dynamic_data_of (generation));
35146 size_t GCHeap::ApproxTotalBytesInUse(BOOL small_heap_only)
35148 size_t totsize = 0;
35150 //ASSERT(InMustComplete());
35151 enter_spin_lock (&pGenGCHeap->gc_lock);
35153 heap_segment* eph_seg = generation_allocation_segment (pGenGCHeap->generation_of (0));
35154 // Get small block heap size info
35155 totsize = (pGenGCHeap->alloc_allocated - heap_segment_mem (eph_seg));
35156 heap_segment* seg1 = generation_start_segment (pGenGCHeap->generation_of (max_generation));
35157 while (seg1 != eph_seg)
35159 totsize += heap_segment_allocated (seg1) -
35160 heap_segment_mem (seg1);
35161 seg1 = heap_segment_next (seg1);
35164 //discount the fragmentation
35165 for (int i = 0; i <= max_generation; i++)
35167 generation* gen = pGenGCHeap->generation_of (i);
35168 totsize -= (generation_free_list_space (gen) + generation_free_obj_space (gen));
35171 if (!small_heap_only)
35173 heap_segment* seg2 = generation_start_segment (pGenGCHeap->generation_of (max_generation+1));
35177 totsize += heap_segment_allocated (seg2) -
35178 heap_segment_mem (seg2);
35179 seg2 = heap_segment_next (seg2);
35182 //discount the fragmentation
35183 generation* loh_gen = pGenGCHeap->generation_of (max_generation+1);
35184 size_t frag = generation_free_list_space (loh_gen) + generation_free_obj_space (loh_gen);
35187 leave_spin_lock (&pGenGCHeap->gc_lock);
35191 #ifdef MULTIPLE_HEAPS
35192 void GCHeap::AssignHeap (alloc_context* acontext)
35194 // Assign heap based on processor
35195 acontext->set_alloc_heap(GetHeap(heap_select::select_heap(acontext, 0)));
35196 acontext->set_home_heap(acontext->get_alloc_heap());
35198 GCHeap* GCHeap::GetHeap (int n)
35200 assert (n < gc_heap::n_heaps);
35201 return gc_heap::g_heaps [n]->vm_heap;
35203 #endif //MULTIPLE_HEAPS
35205 bool GCHeap::IsThreadUsingAllocationContextHeap(gc_alloc_context* context, int thread_number)
35207 alloc_context* acontext = static_cast<alloc_context*>(context);
35208 #ifdef MULTIPLE_HEAPS
35209 return ((acontext->get_home_heap() == GetHeap(thread_number)) ||
35210 ((acontext->get_home_heap() == 0) && (thread_number == 0)));
35212 UNREFERENCED_PARAMETER(acontext);
35213 UNREFERENCED_PARAMETER(thread_number);
35215 #endif //MULTIPLE_HEAPS
35218 // Returns the number of processors required to trigger the use of thread based allocation contexts
35219 int GCHeap::GetNumberOfHeaps ()
35221 #ifdef MULTIPLE_HEAPS
35222 return gc_heap::n_heaps;
35225 #endif //MULTIPLE_HEAPS
35229 in this way we spend extra time cycling through all the heaps while create the handle
35230 it ought to be changed by keeping alloc_context.home_heap as number (equals heap_number)
35232 int GCHeap::GetHomeHeapNumber ()
35234 #ifdef MULTIPLE_HEAPS
35235 Thread *pThread = GCToEEInterface::GetThread();
35236 for (int i = 0; i < gc_heap::n_heaps; i++)
35240 gc_alloc_context* ctx = GCToEEInterface::GetAllocContext();
35241 GCHeap *hp = static_cast<alloc_context*>(ctx)->get_home_heap();
35242 if (hp == gc_heap::g_heaps[i]->vm_heap) return i;
35248 #endif //MULTIPLE_HEAPS
35251 unsigned int GCHeap::GetCondemnedGeneration()
35253 return gc_heap::settings.condemned_generation;
35256 void GCHeap::GetMemoryInfo(uint32_t* highMemLoadThreshold,
35257 uint64_t* totalPhysicalMem,
35258 uint32_t* lastRecordedMemLoad,
35259 size_t* lastRecordedHeapSize,
35260 size_t* lastRecordedFragmentation)
35262 *highMemLoadThreshold = gc_heap::high_memory_load_th;
35263 *totalPhysicalMem = gc_heap::total_physical_mem;
35264 *lastRecordedMemLoad = gc_heap::last_gc_memory_load;
35265 *lastRecordedHeapSize = gc_heap::last_gc_heap_size;
35266 *lastRecordedFragmentation = gc_heap::last_gc_fragmentation;
35269 int GCHeap::GetGcLatencyMode()
35271 return (int)(pGenGCHeap->settings.pause_mode);
35274 int GCHeap::SetGcLatencyMode (int newLatencyMode)
35276 if (gc_heap::settings.pause_mode == pause_no_gc)
35277 return (int)set_pause_mode_no_gc;
35279 gc_pause_mode new_mode = (gc_pause_mode)newLatencyMode;
35281 if (new_mode == pause_low_latency)
35283 #ifndef MULTIPLE_HEAPS
35284 pGenGCHeap->settings.pause_mode = new_mode;
35285 #endif //!MULTIPLE_HEAPS
35287 else if (new_mode == pause_sustained_low_latency)
35289 #ifdef BACKGROUND_GC
35290 if (gc_heap::gc_can_use_concurrent)
35292 pGenGCHeap->settings.pause_mode = new_mode;
35294 #endif //BACKGROUND_GC
35298 pGenGCHeap->settings.pause_mode = new_mode;
35301 #ifdef BACKGROUND_GC
35302 if (recursive_gc_sync::background_running_p())
35304 // If we get here, it means we are doing an FGC. If the pause
35305 // mode was altered we will need to save it in the BGC settings.
35306 if (gc_heap::saved_bgc_settings.pause_mode != new_mode)
35308 gc_heap::saved_bgc_settings.pause_mode = new_mode;
35311 #endif //BACKGROUND_GC
35313 return (int)set_pause_mode_success;
35316 int GCHeap::GetLOHCompactionMode()
35318 return pGenGCHeap->loh_compaction_mode;
35321 void GCHeap::SetLOHCompactionMode (int newLOHCompactionyMode)
35323 #ifdef FEATURE_LOH_COMPACTION
35324 pGenGCHeap->loh_compaction_mode = (gc_loh_compaction_mode)newLOHCompactionyMode;
35325 #endif //FEATURE_LOH_COMPACTION
35328 bool GCHeap::RegisterForFullGCNotification(uint32_t gen2Percentage,
35329 uint32_t lohPercentage)
35331 #ifdef MULTIPLE_HEAPS
35332 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35334 gc_heap* hp = gc_heap::g_heaps [hn];
35335 hp->fgn_last_alloc = dd_new_allocation (hp->dynamic_data_of (0));
35337 #else //MULTIPLE_HEAPS
35338 pGenGCHeap->fgn_last_alloc = dd_new_allocation (pGenGCHeap->dynamic_data_of (0));
35339 #endif //MULTIPLE_HEAPS
35341 pGenGCHeap->full_gc_approach_event.Reset();
35342 pGenGCHeap->full_gc_end_event.Reset();
35343 pGenGCHeap->full_gc_approach_event_set = false;
35345 pGenGCHeap->fgn_maxgen_percent = gen2Percentage;
35346 pGenGCHeap->fgn_loh_percent = lohPercentage;
35351 bool GCHeap::CancelFullGCNotification()
35353 pGenGCHeap->fgn_maxgen_percent = 0;
35354 pGenGCHeap->fgn_loh_percent = 0;
35356 pGenGCHeap->full_gc_approach_event.Set();
35357 pGenGCHeap->full_gc_end_event.Set();
35362 int GCHeap::WaitForFullGCApproach(int millisecondsTimeout)
35364 dprintf (2, ("WFGA: Begin wait"));
35365 int result = gc_heap::full_gc_wait (&(pGenGCHeap->full_gc_approach_event), millisecondsTimeout);
35366 dprintf (2, ("WFGA: End wait"));
35370 int GCHeap::WaitForFullGCComplete(int millisecondsTimeout)
35372 dprintf (2, ("WFGE: Begin wait"));
35373 int result = gc_heap::full_gc_wait (&(pGenGCHeap->full_gc_end_event), millisecondsTimeout);
35374 dprintf (2, ("WFGE: End wait"));
35378 int GCHeap::StartNoGCRegion(uint64_t totalSize, bool lohSizeKnown, uint64_t lohSize, bool disallowFullBlockingGC)
35380 NoGCRegionLockHolder lh;
35382 dprintf (1, ("begin no gc called"));
35383 start_no_gc_region_status status = gc_heap::prepare_for_no_gc_region (totalSize, lohSizeKnown, lohSize, disallowFullBlockingGC);
35384 if (status == start_no_gc_success)
35386 GarbageCollect (max_generation);
35387 status = gc_heap::get_start_no_gc_region_status();
35390 if (status != start_no_gc_success)
35391 gc_heap::handle_failure_for_no_gc();
35393 return (int)status;
35396 int GCHeap::EndNoGCRegion()
35398 NoGCRegionLockHolder lh;
35399 return (int)gc_heap::end_no_gc_region();
35402 void GCHeap::PublishObject (uint8_t* Obj)
35404 #ifdef BACKGROUND_GC
35405 gc_heap* hp = gc_heap::heap_of (Obj);
35406 hp->bgc_alloc_lock->loh_alloc_done (Obj);
35407 #endif //BACKGROUND_GC
35410 // The spec for this one isn't clear. This function
35411 // returns the size that can be allocated without
35412 // triggering a GC of any kind.
35413 size_t GCHeap::ApproxFreeBytes()
35416 //ASSERT(InMustComplete());
35417 enter_spin_lock (&pGenGCHeap->gc_lock);
35419 generation* gen = pGenGCHeap->generation_of (0);
35420 size_t res = generation_allocation_limit (gen) - generation_allocation_pointer (gen);
35422 leave_spin_lock (&pGenGCHeap->gc_lock);
35427 HRESULT GCHeap::GetGcCounters(int gen, gc_counters* counters)
35429 if ((gen < 0) || (gen > max_generation))
35431 #ifdef MULTIPLE_HEAPS
35432 counters->current_size = 0;
35433 counters->promoted_size = 0;
35434 counters->collection_count = 0;
35436 //enumarate all the heaps and get their counters.
35437 for (int i = 0; i < gc_heap::n_heaps; i++)
35439 dynamic_data* dd = gc_heap::g_heaps [i]->dynamic_data_of (gen);
35441 counters->current_size += dd_current_size (dd);
35442 counters->promoted_size += dd_promoted_size (dd);
35444 counters->collection_count += dd_collection_count (dd);
35447 dynamic_data* dd = pGenGCHeap->dynamic_data_of (gen);
35448 counters->current_size = dd_current_size (dd);
35449 counters->promoted_size = dd_promoted_size (dd);
35450 counters->collection_count = dd_collection_count (dd);
35451 #endif //MULTIPLE_HEAPS
35455 // Get the segment size to use, making sure it conforms.
35456 size_t GCHeap::GetValidSegmentSize(bool large_seg)
35458 return get_valid_segment_size (large_seg);
35461 // Get the max gen0 heap size, making sure it conforms.
35462 size_t GCHeap::GetValidGen0MaxSize(size_t seg_size)
35464 size_t gen0size = static_cast<size_t>(GCConfig::GetGen0Size());
35466 if ((gen0size == 0) || !g_theGCHeap->IsValidGen0MaxSize(gen0size))
35469 // performance data seems to indicate halving the size results
35470 // in optimal perf. Ask for adjusted gen0 size.
35471 gen0size = max(GCToOSInterface::GetCacheSizePerLogicalCpu(FALSE),(256*1024));
35473 // if gen0 size is too large given the available memory, reduce it.
35474 // Get true cache size, as we don't want to reduce below this.
35475 size_t trueSize = max(GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE),(256*1024));
35476 dprintf (2, ("cache: %Id-%Id, cpu: %Id",
35477 GCToOSInterface::GetCacheSizePerLogicalCpu(FALSE),
35478 GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE)));
35480 int n_heaps = gc_heap::n_heaps;
35482 size_t trueSize = GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE);
35483 gen0size = max((4*trueSize/5),(256*1024));
35484 trueSize = max(trueSize, (256*1024));
35488 // if the total min GC across heaps will exceed 1/6th of available memory,
35489 // then reduce the min GC size until it either fits or has been reduced to cache size.
35490 while ((gen0size * n_heaps) > GCToOSInterface::GetPhysicalMemoryLimit() / 6)
35492 gen0size = gen0size / 2;
35493 if (gen0size <= trueSize)
35495 gen0size = trueSize;
35501 // Generation 0 must never be more than 1/2 the segment size.
35502 if (gen0size >= (seg_size / 2))
35503 gen0size = seg_size / 2;
35508 void GCHeap::SetReservedVMLimit (size_t vmlimit)
35510 gc_heap::reserved_memory_limit = vmlimit;
35514 //versions of same method on each heap
35516 #ifdef FEATURE_PREMORTEM_FINALIZATION
35518 Object* GCHeap::GetNextFinalizableObject()
35521 #ifdef MULTIPLE_HEAPS
35523 //return the first non critical one in the first queue.
35524 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35526 gc_heap* hp = gc_heap::g_heaps [hn];
35527 Object* O = hp->finalize_queue->GetNextFinalizableObject(TRUE);
35531 //return the first non crtitical/critical one in the first queue.
35532 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35534 gc_heap* hp = gc_heap::g_heaps [hn];
35535 Object* O = hp->finalize_queue->GetNextFinalizableObject(FALSE);
35542 #else //MULTIPLE_HEAPS
35543 return pGenGCHeap->finalize_queue->GetNextFinalizableObject();
35544 #endif //MULTIPLE_HEAPS
35548 size_t GCHeap::GetNumberFinalizableObjects()
35550 #ifdef MULTIPLE_HEAPS
35552 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35554 gc_heap* hp = gc_heap::g_heaps [hn];
35555 cnt += hp->finalize_queue->GetNumberFinalizableObjects();
35560 #else //MULTIPLE_HEAPS
35561 return pGenGCHeap->finalize_queue->GetNumberFinalizableObjects();
35562 #endif //MULTIPLE_HEAPS
35565 size_t GCHeap::GetFinalizablePromotedCount()
35567 #ifdef MULTIPLE_HEAPS
35570 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35572 gc_heap* hp = gc_heap::g_heaps [hn];
35573 cnt += hp->finalize_queue->GetPromotedCount();
35577 #else //MULTIPLE_HEAPS
35578 return pGenGCHeap->finalize_queue->GetPromotedCount();
35579 #endif //MULTIPLE_HEAPS
35582 bool GCHeap::FinalizeAppDomain(void *pDomain, bool fRunFinalizers)
35584 #ifdef MULTIPLE_HEAPS
35585 bool foundp = false;
35586 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35588 gc_heap* hp = gc_heap::g_heaps [hn];
35589 if (hp->finalize_queue->FinalizeAppDomain (pDomain, fRunFinalizers))
35594 #else //MULTIPLE_HEAPS
35595 return pGenGCHeap->finalize_queue->FinalizeAppDomain (pDomain, fRunFinalizers);
35596 #endif //MULTIPLE_HEAPS
35599 bool GCHeap::ShouldRestartFinalizerWatchDog()
35601 // This condition was historically used as part of the condition to detect finalizer thread timeouts
35602 return gc_heap::gc_lock.lock != -1;
35605 void GCHeap::SetFinalizeQueueForShutdown(bool fHasLock)
35607 #ifdef MULTIPLE_HEAPS
35608 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35610 gc_heap* hp = gc_heap::g_heaps [hn];
35611 hp->finalize_queue->SetSegForShutDown(fHasLock);
35614 #else //MULTIPLE_HEAPS
35615 pGenGCHeap->finalize_queue->SetSegForShutDown(fHasLock);
35616 #endif //MULTIPLE_HEAPS
35619 //---------------------------------------------------------------------------
35620 // Finalized class tracking
35621 //---------------------------------------------------------------------------
35623 bool GCHeap::RegisterForFinalization (int gen, Object* obj)
35627 if (((((CObjectHeader*)obj)->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN))
35629 //just reset the bit
35630 ((CObjectHeader*)obj)->GetHeader()->ClrBit(BIT_SBLK_FINALIZER_RUN);
35635 gc_heap* hp = gc_heap::heap_of ((uint8_t*)obj);
35636 return hp->finalize_queue->RegisterForFinalization (gen, obj);
35640 void GCHeap::SetFinalizationRun (Object* obj)
35642 ((CObjectHeader*)obj)->GetHeader()->SetBit(BIT_SBLK_FINALIZER_RUN);
35646 //--------------------------------------------------------------------
35648 // Support for finalization
35650 //--------------------------------------------------------------------
35653 unsigned int gen_segment (int gen)
35655 assert (((signed)NUMBERGENERATIONS - gen - 1)>=0);
35656 return (NUMBERGENERATIONS - gen - 1);
35659 bool CFinalize::Initialize()
35666 m_Array = new (nothrow)(Object*[100]);
35671 STRESS_LOG_OOM_STACK(sizeof(Object*[100]));
35672 if (GCConfig::GetBreakOnOOM())
35674 GCToOSInterface::DebugBreak();
35678 m_EndArray = &m_Array[100];
35680 for (int i =0; i < FreeList; i++)
35682 SegQueueLimit (i) = m_Array;
35684 m_PromotedCount = 0;
35687 lockowner_threadid.Clear();
35693 CFinalize::~CFinalize()
35698 size_t CFinalize::GetPromotedCount ()
35700 return m_PromotedCount;
35704 void CFinalize::EnterFinalizeLock()
35706 _ASSERTE(dbgOnly_IsSpecialEEThread() ||
35707 GCToEEInterface::GetThread() == 0 ||
35708 GCToEEInterface::IsPreemptiveGCDisabled());
35711 if (Interlocked::CompareExchange(&lock, 0, -1) >= 0)
35713 unsigned int i = 0;
35716 YieldProcessor(); // indicate to the processor that we are spining
35718 GCToOSInterface::YieldThread (0);
35720 GCToOSInterface::Sleep (5);
35726 lockowner_threadid.SetToCurrentThread();
35731 void CFinalize::LeaveFinalizeLock()
35733 _ASSERTE(dbgOnly_IsSpecialEEThread() ||
35734 GCToEEInterface::GetThread() == 0 ||
35735 GCToEEInterface::IsPreemptiveGCDisabled());
35738 lockowner_threadid.Clear();
35744 CFinalize::RegisterForFinalization (int gen, Object* obj, size_t size)
35751 EnterFinalizeLock();
35753 unsigned int dest = 0;
35755 if (g_fFinalizerRunOnShutDown)
35757 //no method table available yet,
35758 //put it in the finalizer queue and sort out when
35760 dest = FinalizerListSeg;
35764 dest = gen_segment (gen);
35766 // Adjust boundary for segments so that GC will keep objects alive.
35767 Object*** s_i = &SegQueue (FreeList);
35768 if ((*s_i) == m_EndArray)
35772 LeaveFinalizeLock();
35773 if (method_table(obj) == NULL)
35775 // If the object is uninitialized, a valid size should have been passed.
35776 assert (size >= Align (min_obj_size));
35777 dprintf (3, ("Making unused array [%Ix, %Ix[", (size_t)obj, (size_t)(obj+size)));
35778 ((CObjectHeader*)obj)->SetFree(size);
35780 STRESS_LOG_OOM_STACK(0);
35781 if (GCConfig::GetBreakOnOOM())
35783 GCToOSInterface::DebugBreak();
35788 Object*** end_si = &SegQueueLimit (dest);
35791 //is the segment empty?
35792 if (!(*s_i == *(s_i-1)))
35794 //no, swap the end elements.
35795 *(*s_i) = *(*(s_i-1));
35797 //increment the fill pointer
35799 //go to the next segment.
35801 } while (s_i > end_si);
35803 // We have reached the destination segment
35804 // store the object
35806 // increment the fill pointer
35809 LeaveFinalizeLock();
35815 CFinalize::GetNextFinalizableObject (BOOL only_non_critical)
35819 EnterFinalizeLock();
35822 if (!IsSegEmpty(FinalizerListSeg))
35824 if (g_fFinalizerRunOnShutDown)
35826 obj = *(SegQueueLimit (FinalizerListSeg)-1);
35827 if (method_table(obj)->HasCriticalFinalizer())
35829 MoveItem ((SegQueueLimit (FinalizerListSeg)-1),
35830 FinalizerListSeg, CriticalFinalizerListSeg);
35834 --SegQueueLimit (FinalizerListSeg);
35837 obj = *(--SegQueueLimit (FinalizerListSeg));
35840 else if (!only_non_critical && !IsSegEmpty(CriticalFinalizerListSeg))
35842 //the FinalizerList is empty, we can adjust both
35843 // limit instead of moving the object to the free list
35844 obj = *(--SegQueueLimit (CriticalFinalizerListSeg));
35845 --SegQueueLimit (FinalizerListSeg);
35849 dprintf (3, ("running finalizer for %Ix (mt: %Ix)", obj, method_table (obj)));
35851 LeaveFinalizeLock();
35856 CFinalize::SetSegForShutDown(BOOL fHasLock)
35861 EnterFinalizeLock();
35862 for (i = 0; i <= max_generation; i++)
35864 unsigned int seg = gen_segment (i);
35865 Object** startIndex = SegQueueLimit (seg)-1;
35866 Object** stopIndex = SegQueue (seg);
35867 for (Object** po = startIndex; po >= stopIndex; po--)
35870 if (method_table(obj)->HasCriticalFinalizer())
35872 MoveItem (po, seg, CriticalFinalizerListSeg);
35876 MoveItem (po, seg, FinalizerListSeg);
35881 LeaveFinalizeLock();
35885 CFinalize::DiscardNonCriticalObjects()
35887 //empty the finalization queue
35888 Object** startIndex = SegQueueLimit (FinalizerListSeg)-1;
35889 Object** stopIndex = SegQueue (FinalizerListSeg);
35890 for (Object** po = startIndex; po >= stopIndex; po--)
35892 MoveItem (po, FinalizerListSeg, FreeList);
35897 CFinalize::GetNumberFinalizableObjects()
35899 return SegQueueLimit (FinalizerListSeg) -
35900 (g_fFinalizerRunOnShutDown ? m_Array : SegQueue(FinalizerListSeg));
35904 CFinalize::FinalizeSegForAppDomain (void *pDomain,
35905 BOOL fRunFinalizers,
35908 BOOL finalizedFound = FALSE;
35909 Object** endIndex = SegQueue (Seg);
35910 for (Object** i = SegQueueLimit (Seg)-1; i >= endIndex ;i--)
35912 CObjectHeader* obj = (CObjectHeader*)*i;
35914 // Objects are put into the finalization queue before they are complete (ie their methodtable
35915 // may be null) so we must check that the object we found has a method table before checking
35916 // if it has the index we are looking for. If the methodtable is null, it can't be from the
35917 // unloading domain, so skip it.
35918 if (method_table(obj) == NULL)
35923 // does the EE actually want us to finalize this object?
35924 if (!GCToEEInterface::ShouldFinalizeObjectForUnload(pDomain, obj))
35929 if (!fRunFinalizers || (obj->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN)
35931 //remove the object because we don't want to
35932 //run the finalizer
35933 MoveItem (i, Seg, FreeList);
35934 //Reset the bit so it will be put back on the queue
35935 //if resurrected and re-registered.
35936 obj->GetHeader()->ClrBit (BIT_SBLK_FINALIZER_RUN);
35940 if (method_table(obj)->HasCriticalFinalizer())
35942 finalizedFound = TRUE;
35943 MoveItem (i, Seg, CriticalFinalizerListSeg);
35947 if (GCToEEInterface::AppDomainIsRudeUnload(pDomain))
35949 MoveItem (i, Seg, FreeList);
35953 finalizedFound = TRUE;
35954 MoveItem (i, Seg, FinalizerListSeg);
35960 return finalizedFound;
35964 CFinalize::FinalizeAppDomain (void *pDomain, bool fRunFinalizers)
35966 bool finalizedFound = false;
35968 unsigned int startSeg = gen_segment (max_generation);
35970 EnterFinalizeLock();
35972 for (unsigned int Seg = startSeg; Seg <= gen_segment (0); Seg++)
35974 if (FinalizeSegForAppDomain (pDomain, fRunFinalizers, Seg))
35976 finalizedFound = true;
35980 LeaveFinalizeLock();
35982 return finalizedFound;
35986 CFinalize::MoveItem (Object** fromIndex,
35987 unsigned int fromSeg,
35988 unsigned int toSeg)
35992 ASSERT (fromSeg != toSeg);
35993 if (fromSeg > toSeg)
35997 // Place the element at the boundary closest to dest
35998 Object** srcIndex = fromIndex;
35999 for (unsigned int i = fromSeg; i != toSeg; i+= step)
36001 Object**& destFill = m_FillPointers[i+(step - 1 )/2];
36002 Object** destIndex = destFill - (step + 1)/2;
36003 if (srcIndex != destIndex)
36005 Object* tmp = *srcIndex;
36006 *srcIndex = *destIndex;
36010 srcIndex = destIndex;
36015 CFinalize::GcScanRoots (promote_func* fn, int hn, ScanContext *pSC)
36021 pSC->thread_number = hn;
36023 //scan the finalization queue
36024 Object** startIndex = SegQueue (CriticalFinalizerListSeg);
36025 Object** stopIndex = SegQueueLimit (FinalizerListSeg);
36027 for (Object** po = startIndex; po < stopIndex; po++)
36030 //dprintf (3, ("scan freacheable %Ix", (size_t)o));
36031 dprintf (3, ("scan f %Ix", (size_t)o));
36032 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
36035 pSC->pCurrentDomain = SystemDomain::GetAppDomainAtIndex(o->GetAppDomainIndex());
36037 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
36043 void CFinalize::WalkFReachableObjects (fq_walk_fn fn)
36045 Object** startIndex = SegQueue (CriticalFinalizerListSeg);
36046 Object** stopCriticalIndex = SegQueueLimit (CriticalFinalizerListSeg);
36047 Object** stopIndex = SegQueueLimit (FinalizerListSeg);
36048 for (Object** po = startIndex; po < stopIndex; po++)
36051 fn(po < stopCriticalIndex, *po);
36056 CFinalize::ScanForFinalization (promote_func* pfn, int gen, BOOL mark_only_p,
36060 sc.promotion = TRUE;
36061 #ifdef MULTIPLE_HEAPS
36062 sc.thread_number = hp->heap_number;
36064 UNREFERENCED_PARAMETER(hp);
36065 #endif //MULTIPLE_HEAPS
36067 BOOL finalizedFound = FALSE;
36069 //start with gen and explore all the younger generations.
36070 unsigned int startSeg = gen_segment (gen);
36072 m_PromotedCount = 0;
36073 for (unsigned int Seg = startSeg; Seg <= gen_segment(0); Seg++)
36075 Object** endIndex = SegQueue (Seg);
36076 for (Object** i = SegQueueLimit (Seg)-1; i >= endIndex ;i--)
36078 CObjectHeader* obj = (CObjectHeader*)*i;
36079 dprintf (3, ("scanning: %Ix", (size_t)obj));
36080 if (!g_theGCHeap->IsPromoted (obj))
36082 dprintf (3, ("freacheable: %Ix", (size_t)obj));
36084 assert (method_table(obj)->HasFinalizer());
36086 if (GCToEEInterface::EagerFinalized(obj))
36088 MoveItem (i, Seg, FreeList);
36090 else if ((obj->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN)
36092 //remove the object because we don't want to
36093 //run the finalizer
36095 MoveItem (i, Seg, FreeList);
36097 //Reset the bit so it will be put back on the queue
36098 //if resurrected and re-registered.
36099 obj->GetHeader()->ClrBit (BIT_SBLK_FINALIZER_RUN);
36106 if (method_table(obj)->HasCriticalFinalizer())
36108 MoveItem (i, Seg, CriticalFinalizerListSeg);
36112 MoveItem (i, Seg, FinalizerListSeg);
36116 #ifdef BACKGROUND_GC
36119 if ((gen == max_generation) && (recursive_gc_sync::background_running_p()))
36121 // TODO - fix the following line.
36122 //assert (gc_heap::background_object_marked ((uint8_t*)obj, FALSE));
36123 dprintf (3, ("%Ix is marked", (size_t)obj));
36126 #endif //BACKGROUND_GC
36130 finalizedFound = !IsSegEmpty(FinalizerListSeg) ||
36131 !IsSegEmpty(CriticalFinalizerListSeg);
36133 if (finalizedFound)
36135 //Promote the f-reachable objects
36137 #ifdef MULTIPLE_HEAPS
36141 #endif //MULTIPLE_HEAPS
36144 hp->settings.found_finalizers = TRUE;
36146 #ifdef BACKGROUND_GC
36147 if (hp->settings.concurrent)
36149 hp->settings.found_finalizers = !(IsSegEmpty(FinalizerListSeg) && IsSegEmpty(CriticalFinalizerListSeg));
36151 #endif //BACKGROUND_GC
36152 if (hp->settings.concurrent && hp->settings.found_finalizers)
36155 GCToEEInterface::EnableFinalization(true);
36159 return finalizedFound;
36162 //Relocates all of the objects in the finalization array
36164 CFinalize::RelocateFinalizationData (int gen, gc_heap* hp)
36167 sc.promotion = FALSE;
36168 #ifdef MULTIPLE_HEAPS
36169 sc.thread_number = hp->heap_number;
36171 UNREFERENCED_PARAMETER(hp);
36172 #endif //MULTIPLE_HEAPS
36174 unsigned int Seg = gen_segment (gen);
36176 Object** startIndex = SegQueue (Seg);
36177 for (Object** po = startIndex; po < SegQueue (FreeList);po++)
36179 GCHeap::Relocate (po, &sc);
36184 CFinalize::UpdatePromotedGenerations (int gen, BOOL gen_0_empty_p)
36186 // update the generation fill pointers.
36187 // if gen_0_empty is FALSE, test each object to find out if
36188 // it was promoted or not
36191 for (int i = min (gen+1, max_generation); i > 0; i--)
36193 m_FillPointers [gen_segment(i)] = m_FillPointers [gen_segment(i-1)];
36198 //Look for demoted or promoted plugs
36200 for (int i = gen; i >= 0; i--)
36202 unsigned int Seg = gen_segment (i);
36203 Object** startIndex = SegQueue (Seg);
36205 for (Object** po = startIndex;
36206 po < SegQueueLimit (gen_segment(i)); po++)
36208 int new_gen = g_theGCHeap->WhichGeneration (*po);
36214 MoveItem (po, gen_segment (i), gen_segment (new_gen));
36219 MoveItem (po, gen_segment (i), gen_segment (new_gen));
36220 //back down in order to see all objects.
36231 CFinalize::GrowArray()
36233 size_t oldArraySize = (m_EndArray - m_Array);
36234 size_t newArraySize = (size_t)(((float)oldArraySize / 10) * 12);
36236 Object** newArray = new (nothrow) Object*[newArraySize];
36239 // It's not safe to throw here, because of the FinalizeLock. Tell our caller
36240 // to throw for us.
36241 // ASSERT (newArray);
36244 memcpy (newArray, m_Array, oldArraySize*sizeof(Object*));
36246 //adjust the fill pointers
36247 for (int i = 0; i < FreeList; i++)
36249 m_FillPointers [i] += (newArray - m_Array);
36252 m_Array = newArray;
36253 m_EndArray = &m_Array [newArraySize];
36259 void CFinalize::CheckFinalizerObjects()
36261 for (int i = 0; i <= max_generation; i++)
36263 Object **startIndex = SegQueue (gen_segment (i));
36264 Object **stopIndex = SegQueueLimit (gen_segment (i));
36266 for (Object **po = startIndex; po < stopIndex; po++)
36268 if ((int)g_theGCHeap->WhichGeneration (*po) < i)
36270 ((CObjectHeader*)*po)->Validate();
36274 #endif //VERIFY_HEAP
36276 #endif // FEATURE_PREMORTEM_FINALIZATION
36279 //------------------------------------------------------------------------------
36281 // End of VM specific support
36283 //------------------------------------------------------------------------------
36284 void gc_heap::walk_heap_per_heap (walk_fn fn, void* context, int gen_number, BOOL walk_large_object_heap_p)
36286 generation* gen = gc_heap::generation_of (gen_number);
36287 heap_segment* seg = generation_start_segment (gen);
36288 uint8_t* x = ((gen_number == max_generation) ? heap_segment_mem (seg) :
36289 generation_allocation_start (gen));
36291 uint8_t* end = heap_segment_allocated (seg);
36292 BOOL small_object_segments = TRUE;
36293 int align_const = get_alignment_constant (small_object_segments);
36300 if ((seg = heap_segment_next (seg)) != 0)
36302 x = heap_segment_mem (seg);
36303 end = heap_segment_allocated (seg);
36308 if (small_object_segments && walk_large_object_heap_p)
36311 small_object_segments = FALSE;
36312 align_const = get_alignment_constant (small_object_segments);
36313 seg = generation_start_segment (large_object_generation);
36314 x = heap_segment_mem (seg);
36315 end = heap_segment_allocated (seg);
36325 size_t s = size (x);
36326 CObjectHeader* o = (CObjectHeader*)x;
36331 _ASSERTE(((size_t)o & 0x3) == 0); // Last two bits should never be set at this point
36333 if (!fn (o->GetObjectBase(), context))
36336 x = x + Align (s, align_const);
36340 void gc_heap::walk_finalize_queue (fq_walk_fn fn)
36342 #ifdef FEATURE_PREMORTEM_FINALIZATION
36343 finalize_queue->WalkFReachableObjects (fn);
36344 #endif //FEATURE_PREMORTEM_FINALIZATION
36347 void gc_heap::walk_heap (walk_fn fn, void* context, int gen_number, BOOL walk_large_object_heap_p)
36349 #ifdef MULTIPLE_HEAPS
36350 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36352 gc_heap* hp = gc_heap::g_heaps [hn];
36354 hp->walk_heap_per_heap (fn, context, gen_number, walk_large_object_heap_p);
36357 walk_heap_per_heap(fn, context, gen_number, walk_large_object_heap_p);
36358 #endif //MULTIPLE_HEAPS
36361 void GCHeap::DiagWalkObject (Object* obj, walk_fn fn, void* context)
36363 uint8_t* o = (uint8_t*)obj;
36366 go_through_object_cl (method_table (o), o, size(o), oo,
36370 Object *oh = (Object*)*oo;
36371 if (!fn (oh, context))
36379 void GCHeap::DiagWalkSurvivorsWithType (void* gc_context, record_surv_fn fn, void* diag_context, walk_surv_type type)
36381 gc_heap* hp = (gc_heap*)gc_context;
36382 hp->walk_survivors (fn, diag_context, type);
36385 void GCHeap::DiagWalkHeap (walk_fn fn, void* context, int gen_number, bool walk_large_object_heap_p)
36387 gc_heap::walk_heap (fn, context, gen_number, walk_large_object_heap_p);
36390 void GCHeap::DiagWalkFinalizeQueue (void* gc_context, fq_walk_fn fn)
36392 gc_heap* hp = (gc_heap*)gc_context;
36393 hp->walk_finalize_queue (fn);
36396 void GCHeap::DiagScanFinalizeQueue (fq_scan_fn fn, ScanContext* sc)
36398 #ifdef MULTIPLE_HEAPS
36399 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36401 gc_heap* hp = gc_heap::g_heaps [hn];
36402 hp->finalize_queue->GcScanRoots(fn, hn, sc);
36405 pGenGCHeap->finalize_queue->GcScanRoots(fn, 0, sc);
36406 #endif //MULTIPLE_HEAPS
36409 void GCHeap::DiagScanHandles (handle_scan_fn fn, int gen_number, ScanContext* context)
36411 UNREFERENCED_PARAMETER(gen_number);
36412 GCScan::GcScanHandlesForProfilerAndETW (max_generation, context, fn);
36415 void GCHeap::DiagScanDependentHandles (handle_scan_fn fn, int gen_number, ScanContext* context)
36417 UNREFERENCED_PARAMETER(gen_number);
36418 GCScan::GcScanDependentHandlesForProfilerAndETW (max_generation, context, fn);
36421 // Go through and touch (read) each page straddled by a memory block.
36422 void TouchPages(void * pStart, size_t cb)
36424 const uint32_t pagesize = OS_PAGE_SIZE;
36425 _ASSERTE(0 == (pagesize & (pagesize-1))); // Must be a power of 2.
36428 VOLATILE(char)* pEnd = (VOLATILE(char)*)(cb + (char*)pStart);
36429 VOLATILE(char)* p = (VOLATILE(char)*)(((char*)pStart) - (((size_t)pStart) & (pagesize-1)));
36433 a = VolatileLoad(p);
36434 //printf("Touching page %lxh\n", (uint32_t)p);
36440 #if defined(WRITE_BARRIER_CHECK) && !defined (SERVER_GC)
36441 // This code is designed to catch the failure to update the write barrier
36442 // The way it works is to copy the whole heap right after every GC. The write
36443 // barrier code has been modified so that it updates the shadow as well as the
36444 // real GC heap. Before doing the next GC, we walk the heap, looking for pointers
36445 // that were updated in the real heap, but not the shadow. A mismatch indicates
36446 // an error. The offending code can be found by breaking after the correct GC,
36447 // and then placing a data breakpoint on the Heap location that was updated without
36448 // going through the write barrier.
36450 // Called at process shutdown
36451 void deleteGCShadow()
36453 if (g_GCShadow != 0)
36454 GCToOSInterface::VirtualRelease (g_GCShadow, g_GCShadowEnd - g_GCShadow);
36459 // Called at startup and right after a GC, get a snapshot of the GC Heap
36460 void initGCShadow()
36462 if (!(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_BARRIERCHECK))
36465 size_t len = g_gc_highest_address - g_gc_lowest_address;
36466 if (len > (size_t)(g_GCShadowEnd - g_GCShadow))
36469 g_GCShadowEnd = g_GCShadow = (uint8_t *)GCToOSInterface::VirtualReserve(len, 0, VirtualReserveFlags::None);
36470 if (g_GCShadow == NULL || !GCToOSInterface::VirtualCommit(g_GCShadow, len))
36472 _ASSERTE(!"Not enough memory to run HeapVerify level 2");
36473 // If after the assert we decide to allow the program to continue
36474 // running we need to be in a state that will not trigger any
36475 // additional AVs while we fail to allocate a shadow segment, i.e.
36476 // ensure calls to updateGCShadow() checkGCWriteBarrier() don't AV
36481 g_GCShadowEnd += len;
36484 // save the value of g_gc_lowest_address at this time. If this value changes before
36485 // the next call to checkGCWriteBarrier() it means we extended the heap (with a
36486 // large object segment most probably), and the whole shadow segment is inconsistent.
36487 g_shadow_lowest_address = g_gc_lowest_address;
36489 //****** Copy the whole GC heap ******
36491 // NOTE: This is the one situation where the combination of heap_segment_rw(gen_start_segment())
36492 // can produce a NULL result. This is because the initialization has not completed.
36494 generation* gen = gc_heap::generation_of (max_generation);
36495 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
36497 ptrdiff_t delta = g_GCShadow - g_gc_lowest_address;
36498 BOOL small_object_segments = TRUE;
36503 if (small_object_segments)
36505 small_object_segments = FALSE;
36506 seg = heap_segment_rw (generation_start_segment (gc_heap::generation_of (max_generation+1)));
36512 // Copy the segment
36513 uint8_t* start = heap_segment_mem(seg);
36514 uint8_t* end = heap_segment_allocated (seg);
36515 memcpy(start + delta, start, end - start);
36516 seg = heap_segment_next_rw (seg);
36520 #define INVALIDGCVALUE (void*)((size_t)0xcccccccd)
36522 // test to see if 'ptr' was only updated via the write barrier.
36523 inline void testGCShadow(Object** ptr)
36525 Object** shadow = (Object**) &g_GCShadow[((uint8_t*) ptr - g_gc_lowest_address)];
36526 if (*ptr != 0 && (uint8_t*) shadow < g_GCShadowEnd && *ptr != *shadow)
36529 // If you get this assertion, someone updated a GC pointer in the heap without
36530 // using the write barrier. To find out who, check the value of
36531 // dd_collection_count (dynamic_data_of (0)). Also
36532 // note the value of 'ptr'. Rerun the App that the previous GC just occurred.
36533 // Then put a data breakpoint for the value of 'ptr' Then check every write
36534 // to pointer between the two GCs. The last one is not using the write barrier.
36536 // If the memory of interest does not exist at system startup,
36537 // you need to set the data breakpoint right after the memory gets committed
36538 // Set a breakpoint at the end of grow_heap_segment, and put the value of 'ptr'
36539 // in the memory window. run until the memory gets mapped. Then you can set
36542 // Note a recent change, we've identified race conditions when updating the gc shadow.
36543 // Throughout the runtime, code will update an address in the gc heap, then erect the
36544 // write barrier, which calls updateGCShadow. With an app that pounds one heap location
36545 // from multiple threads, you can hit this assert even though all involved are using the
36546 // write barrier properly. Thusly, we detect the race and set this location to INVALIDGCVALUE.
36547 // TODO: the code in jithelp.asm doesn't call updateGCShadow, and hasn't been
36548 // TODO: fixed to detect the race. We've only seen this race from VolatileWritePtr,
36549 // TODO: so elect not to fix jithelp.asm at this time. It should be done if we start hitting
36550 // TODO: erroneous asserts in here.
36552 if(*shadow!=INVALIDGCVALUE)
36554 #ifdef FEATURE_BASICFREEZE
36555 // Write barriers for stores of references to frozen objects may be optimized away.
36556 if (!gc_heap::frozen_object_p(*ptr))
36557 #endif // FEATURE_BASICFREEZE
36559 _ASSERTE(!"Pointer updated without using write barrier");
36565 printf("saw a INVALIDGCVALUE. (just to let you know)\n");
36571 void testGCShadowHelper (uint8_t* x)
36573 size_t s = size (x);
36574 if (contain_pointers (x))
36576 go_through_object_nostart (method_table(x), x, s, oo,
36577 { testGCShadow((Object**) oo); });
36581 // Walk the whole heap, looking for pointers that were not updated with the write barrier.
36582 void checkGCWriteBarrier()
36584 // g_shadow_lowest_address != g_gc_lowest_address means the GC heap was extended by a segment
36585 // and the GC shadow segment did not track that change!
36586 if (g_GCShadowEnd <= g_GCShadow || g_shadow_lowest_address != g_gc_lowest_address)
36588 // No shadow stack, nothing to check.
36593 generation* gen = gc_heap::generation_of (max_generation);
36594 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
36596 PREFIX_ASSUME(seg != NULL);
36600 uint8_t* x = heap_segment_mem(seg);
36601 while (x < heap_segment_allocated (seg))
36603 size_t s = size (x);
36604 testGCShadowHelper (x);
36607 seg = heap_segment_next_rw (seg);
36612 // go through large object heap
36613 int alignment = get_alignment_constant(FALSE);
36614 generation* gen = gc_heap::generation_of (max_generation+1);
36615 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
36617 PREFIX_ASSUME(seg != NULL);
36621 uint8_t* x = heap_segment_mem(seg);
36622 while (x < heap_segment_allocated (seg))
36624 size_t s = size (x);
36625 testGCShadowHelper (x);
36626 x = x + Align (s, alignment);
36628 seg = heap_segment_next_rw (seg);
36632 #endif //WRITE_BARRIER_CHECK && !SERVER_GC
36634 #endif // !DACCESS_COMPILE
36636 #ifdef FEATURE_BASICFREEZE
36637 void gc_heap::walk_read_only_segment(heap_segment *seg, void *pvContext, object_callback_func pfnMethodTable, object_callback_func pfnObjRef)
36639 #ifdef DACCESS_COMPILE
36640 UNREFERENCED_PARAMETER(seg);
36641 UNREFERENCED_PARAMETER(pvContext);
36642 UNREFERENCED_PARAMETER(pfnMethodTable);
36643 UNREFERENCED_PARAMETER(pfnObjRef);
36645 uint8_t *o = heap_segment_mem(seg);
36647 // small heap alignment constant
36648 int alignment = get_alignment_constant(TRUE);
36650 while (o < heap_segment_allocated(seg))
36652 pfnMethodTable(pvContext, o);
36654 if (contain_pointers (o))
36656 go_through_object_nostart (method_table (o), o, size(o), oo,
36659 pfnObjRef(pvContext, oo);
36664 o += Align(size(o), alignment);
36666 #endif //!DACCESS_COMPILE
36668 #endif // FEATURE_BASICFREEZE
36670 #ifndef DACCESS_COMPILE
36671 HRESULT GCHeap::WaitUntilConcurrentGCCompleteAsync(int millisecondsTimeout)
36673 #ifdef BACKGROUND_GC
36674 if (recursive_gc_sync::background_running_p())
36676 uint32_t dwRet = pGenGCHeap->background_gc_wait(awr_ignored, millisecondsTimeout);
36677 if (dwRet == WAIT_OBJECT_0)
36679 else if (dwRet == WAIT_TIMEOUT)
36680 return HRESULT_FROM_WIN32(ERROR_TIMEOUT);
36682 return E_FAIL; // It is not clear if what the last error would be if the wait failed,
36683 // as there are too many layers in between. The best we can do is to return E_FAIL;
36689 #endif // !DACCESS_COMPILE
36691 void GCHeap::TemporaryEnableConcurrentGC()
36693 #ifdef BACKGROUND_GC
36694 gc_heap::temp_disable_concurrent_p = false;
36695 #endif //BACKGROUND_GC
36698 void GCHeap::TemporaryDisableConcurrentGC()
36700 #ifdef BACKGROUND_GC
36701 gc_heap::temp_disable_concurrent_p = true;
36702 #endif //BACKGROUND_GC
36705 bool GCHeap::IsConcurrentGCEnabled()
36707 #ifdef BACKGROUND_GC
36708 return (gc_heap::gc_can_use_concurrent && !(gc_heap::temp_disable_concurrent_p));
36711 #endif //BACKGROUND_GC
36714 void GCHeap::SetFinalizeRunOnShutdown(bool value)
36716 g_fFinalizerRunOnShutDown = value;
36719 void PopulateDacVars(GcDacVars *gcDacVars)
36721 #ifndef DACCESS_COMPILE
36722 assert(gcDacVars != nullptr);
36724 gcDacVars->major_version_number = 1;
36725 gcDacVars->minor_version_number = 0;
36726 gcDacVars->built_with_svr = &g_built_with_svr_gc;
36727 gcDacVars->build_variant = &g_build_variant;
36728 gcDacVars->gc_structures_invalid_cnt = const_cast<int32_t*>(&GCScan::m_GcStructuresInvalidCnt);
36729 gcDacVars->generation_size = sizeof(generation);
36730 gcDacVars->max_gen = &g_max_generation;
36731 #ifndef MULTIPLE_HEAPS
36732 gcDacVars->mark_array = &gc_heap::mark_array;
36733 gcDacVars->ephemeral_heap_segment = reinterpret_cast<dac_heap_segment**>(&gc_heap::ephemeral_heap_segment);
36734 gcDacVars->current_c_gc_state = const_cast<c_gc_state*>(&gc_heap::current_c_gc_state);
36735 gcDacVars->saved_sweep_ephemeral_seg = reinterpret_cast<dac_heap_segment**>(&gc_heap::saved_sweep_ephemeral_seg);
36736 gcDacVars->saved_sweep_ephemeral_start = &gc_heap::saved_sweep_ephemeral_start;
36737 gcDacVars->background_saved_lowest_address = &gc_heap::background_saved_lowest_address;
36738 gcDacVars->background_saved_highest_address = &gc_heap::background_saved_highest_address;
36739 gcDacVars->alloc_allocated = &gc_heap::alloc_allocated;
36740 gcDacVars->next_sweep_obj = &gc_heap::next_sweep_obj;
36741 gcDacVars->oom_info = &gc_heap::oom_info;
36742 gcDacVars->finalize_queue = reinterpret_cast<dac_finalize_queue**>(&gc_heap::finalize_queue);
36743 gcDacVars->generation_table = reinterpret_cast<dac_generation**>(&gc_heap::generation_table);
36744 #ifdef GC_CONFIG_DRIVEN
36745 gcDacVars->gc_global_mechanisms = reinterpret_cast<size_t**>(&gc_global_mechanisms);
36746 gcDacVars->interesting_data_per_heap = reinterpret_cast<size_t**>(&gc_heap::interesting_data_per_heap);
36747 gcDacVars->compact_reasons_per_heap = reinterpret_cast<size_t**>(&gc_heap::compact_reasons_per_heap);
36748 gcDacVars->expand_mechanisms_per_heap = reinterpret_cast<size_t**>(&gc_heap::expand_mechanisms_per_heap);
36749 gcDacVars->interesting_mechanism_bits_per_heap = reinterpret_cast<size_t**>(&gc_heap::interesting_mechanism_bits_per_heap);
36750 #endif // GC_CONFIG_DRIVEN
36751 #ifdef HEAP_ANALYZE
36752 gcDacVars->internal_root_array = &gc_heap::internal_root_array;
36753 gcDacVars->internal_root_array_index = &gc_heap::internal_root_array_index;
36754 gcDacVars->heap_analyze_success = &gc_heap::heap_analyze_success;
36755 #endif // HEAP_ANALYZE
36757 gcDacVars->n_heaps = &gc_heap::n_heaps;
36758 gcDacVars->g_heaps = reinterpret_cast<dac_gc_heap***>(&gc_heap::g_heaps);
36759 #endif // MULTIPLE_HEAPS
36760 #endif // DACCESS_COMPILE