1 // Licensed to the .NET Foundation under one or more agreements.
2 // The .NET Foundation licenses this file to you under the MIT license.
3 // See the LICENSE file in the project root for more information.
9 // GC automatically manages memory allocated by managed code.
10 // The design doc for GC can be found at Documentation/botr/garbage-collection.md
12 // This file includes both the code for GC and the allocator. The most common
13 // case for a GC to be triggered is from the allocator code. See
14 // code:#try_allocate_more_space where it calls GarbageCollectGeneration.
16 // Entry points for the allocator are GCHeap::Alloc* which are called by the
17 // allocation helpers in gcscan.cpp
21 #include "softwarewritewatch.h"
25 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
26 BOOL bgc_heap_walk_for_etw_p = FALSE;
27 #endif //BACKGROUND_GC && FEATURE_EVENT_TRACE
29 #if defined(FEATURE_REDHAWK)
30 #define MAYBE_UNUSED_VAR(v) v = v
32 #define MAYBE_UNUSED_VAR(v)
33 #endif // FEATURE_REDHAWK
35 #define MAX_PTR ((uint8_t*)(~(ptrdiff_t)0))
38 #define partial_size_th 100
39 #define num_partial_refs 64
41 #define partial_size_th 100
42 #define num_partial_refs 32
45 #define demotion_plug_len_th (6*1024*1024)
48 #define MARK_STACK_INITIAL_LENGTH 1024
50 #define MARK_STACK_INITIAL_LENGTH 128
53 #define LOH_PIN_QUEUE_LENGTH 100
54 #define LOH_PIN_DECAY 10
57 // Right now we support maximum 1024 procs - meaning that we will create at most
58 // that many GC threads and GC heaps.
59 #define MAX_SUPPORTED_CPUS 1024
61 #define MAX_SUPPORTED_CPUS 64
64 #ifdef GC_CONFIG_DRIVEN
65 int compact_ratio = 0;
66 #endif //GC_CONFIG_DRIVEN
68 // See comments in reset_memory.
69 BOOL reset_mm_p = TRUE;
71 bool g_fFinalizerRunOnShutDown = false;
74 bool g_built_with_svr_gc = true;
76 bool g_built_with_svr_gc = false;
77 #endif // FEATURE_SVR_GC
79 #if defined(BUILDENV_DEBUG)
80 uint8_t g_build_variant = 0;
81 #elif defined(BUILDENV_CHECKED)
82 uint8_t g_build_variant = 1;
84 uint8_t g_build_variant = 2;
85 #endif // defined(BUILDENV_DEBUG)
87 VOLATILE(int32_t) g_no_gc_lock = -1;
89 #if defined (TRACE_GC) && !defined (DACCESS_COMPILE)
90 const char * const allocation_state_str[] = {
96 "try_fit_new_seg_after_cg",
100 "try_free_full_seg_in_bgc",
101 "try_free_after_bgc",
104 "acquire_seg_after_cg",
105 "acquire_seg_after_bgc",
106 "check_and_wait_for_bgc",
107 "trigger_full_compact_gc",
108 "trigger_ephemeral_gc",
109 "trigger_2nd_ephemeral_gc",
112 #endif //TRACE_GC && !DACCESS_COMPILE
114 // Keep this in sync with the definition of gc_reason
115 #if (defined(DT_LOG) || defined(TRACE_GC)) && !defined (DACCESS_COMPILE)
116 static const char* const str_gc_reasons[] =
131 static const char* const str_gc_pause_modes[] =
136 "sustained_low_latency",
139 #endif // defined(DT_LOG) || defined(TRACE_GC)
142 BOOL is_induced (gc_reason reason)
144 return ((reason == reason_induced) ||
145 (reason == reason_induced_noforce) ||
146 (reason == reason_lowmemory) ||
147 (reason == reason_lowmemory_blocking) ||
148 (reason == reason_induced_compacting));
152 BOOL is_induced_blocking (gc_reason reason)
154 return ((reason == reason_induced) ||
155 (reason == reason_lowmemory_blocking) ||
156 (reason == reason_induced_compacting));
159 #ifndef DACCESS_COMPILE
162 size_t GetHighPrecisionTimeStamp()
164 int64_t ts = GCToOSInterface::QueryPerformanceCounter();
166 return (size_t)(ts / (qpf / 1000));
172 // There is a current and a prior copy of the statistics. This allows us to display deltas per reporting
173 // interval, as well as running totals. The 'min' and 'max' values require special treatment. They are
174 // Reset (zeroed) in the current statistics when we begin a new interval and they are updated via a
175 // comparison with the global min/max.
176 GCStatistics g_GCStatistics;
177 GCStatistics g_LastGCStatistics;
179 char* GCStatistics::logFileName = NULL;
180 FILE* GCStatistics::logFile = NULL;
182 void GCStatistics::AddGCStats(const gc_mechanisms& settings, size_t timeInMSec)
185 if (settings.concurrent)
187 bgc.Accumulate((uint32_t)timeInMSec*1000);
190 else if (settings.background_p)
192 fgc.Accumulate((uint32_t)timeInMSec*1000);
194 if (settings.compaction)
196 assert(settings.condemned_generation < max_generation);
197 cntFGCGen[settings.condemned_generation]++;
200 #endif // BACKGROUND_GC
202 ngc.Accumulate((uint32_t)timeInMSec*1000);
204 if (settings.compaction)
206 cntNGCGen[settings.condemned_generation]++;
209 if (is_induced (settings.reason))
210 cntReasons[(int)reason_induced]++;
212 else if (settings.stress_induced)
213 cntReasons[(int)reason_gcstress]++;
214 #endif // STRESS_HEAP
216 cntReasons[(int)settings.reason]++;
219 if (settings.concurrent || !settings.background_p)
221 #endif // BACKGROUND_GC
225 #endif // BACKGROUND_GC
228 void GCStatistics::Initialize()
230 LIMITED_METHOD_CONTRACT;
231 // for efficiency sake we're taking a dependency on the layout of a C++ object
232 // with a vtable. protect against violations of our premise:
233 static_assert(offsetof(GCStatistics, cntDisplay) == sizeof(void*),
234 "The first field of GCStatistics follows the pointer sized vtable");
236 int podOffs = offsetof(GCStatistics, cntDisplay); // offset of the first POD field
237 memset((uint8_t*)(&g_GCStatistics)+podOffs, 0, sizeof(g_GCStatistics)-podOffs);
238 memset((uint8_t*)(&g_LastGCStatistics)+podOffs, 0, sizeof(g_LastGCStatistics)-podOffs);
241 void GCStatistics::DisplayAndUpdate()
243 LIMITED_METHOD_CONTRACT;
245 if (logFileName == NULL || logFile == NULL)
250 fprintf(logFile, "\nGCMix **** Initialize *****\n\n");
252 fprintf(logFile, "GCMix **** Summary ***** %d\n", cntDisplay);
254 // NGC summary (total, timing info)
255 ngc.DisplayAndUpdate(logFile, "NGC ", &g_LastGCStatistics.ngc, cntNGC, g_LastGCStatistics.cntNGC, msec);
257 // FGC summary (total, timing info)
258 fgc.DisplayAndUpdate(logFile, "FGC ", &g_LastGCStatistics.fgc, cntFGC, g_LastGCStatistics.cntFGC, msec);
261 bgc.DisplayAndUpdate(logFile, "BGC ", &g_LastGCStatistics.bgc, cntBGC, g_LastGCStatistics.cntBGC, msec);
263 // NGC/FGC break out by generation & compacting vs. sweeping
264 fprintf(logFile, "NGC ");
265 for (int i = max_generation; i >= 0; --i)
266 fprintf(logFile, "gen%d %d (%d). ", i, cntNGCGen[i]-g_LastGCStatistics.cntNGCGen[i], cntNGCGen[i]);
267 fprintf(logFile, "\n");
269 fprintf(logFile, "FGC ");
270 for (int i = max_generation-1; i >= 0; --i)
271 fprintf(logFile, "gen%d %d (%d). ", i, cntFGCGen[i]-g_LastGCStatistics.cntFGCGen[i], cntFGCGen[i]);
272 fprintf(logFile, "\n");
274 // Compacting vs. Sweeping break out
275 int _cntSweep = cntNGC-cntCompactNGC;
276 int _cntLastSweep = g_LastGCStatistics.cntNGC-g_LastGCStatistics.cntCompactNGC;
277 fprintf(logFile, "NGC Sweeping %d (%d) Compacting %d (%d)\n",
278 _cntSweep - _cntLastSweep, _cntSweep,
279 cntCompactNGC - g_LastGCStatistics.cntCompactNGC, cntCompactNGC);
281 _cntSweep = cntFGC-cntCompactFGC;
282 _cntLastSweep = g_LastGCStatistics.cntFGC-g_LastGCStatistics.cntCompactFGC;
283 fprintf(logFile, "FGC Sweeping %d (%d) Compacting %d (%d)\n",
284 _cntSweep - _cntLastSweep, _cntSweep,
285 cntCompactFGC - g_LastGCStatistics.cntCompactFGC, cntCompactFGC);
289 for (int reason=(int)reason_alloc_soh; reason <= (int)reason_gcstress; ++reason)
291 if (cntReasons[reason] != 0)
292 fprintf(logFile, "%s %d (%d). ", str_gc_reasons[reason],
293 cntReasons[reason]-g_LastGCStatistics.cntReasons[reason], cntReasons[reason]);
296 fprintf(logFile, "\n\n");
298 // flush the log file...
302 g_LastGCStatistics = *this;
312 #define TOTAL_TIMES_TO_SHIFT 6
314 #define TOTAL_TIMES_TO_SHIFT 5
317 size_t round_up_power2 (size_t size)
319 unsigned short shift = 1;
323 for (unsigned short i = 0; i < TOTAL_TIMES_TO_SHIFT; i++)
325 shifted = size | (size >> shift);
340 size_t round_down_power2 (size_t size)
342 size_t power2 = round_up_power2 (size);
352 // the index starts from 0.
353 int index_of_set_bit (size_t power2)
356 int high = sizeof (size_t) * 8 - 1;
360 mid = ((low + high)/2);
361 size_t temp = (size_t)1 << mid;
366 else if (power2 < temp)
380 int relative_index_power2_plug (size_t power2)
382 int index = index_of_set_bit (power2);
383 assert (index <= MAX_INDEX_POWER2);
385 return ((index < MIN_INDEX_POWER2) ? 0 : (index - MIN_INDEX_POWER2));
389 int relative_index_power2_free_space (size_t power2)
391 int index = index_of_set_bit (power2);
392 assert (index <= MAX_INDEX_POWER2);
394 return ((index < MIN_INDEX_POWER2) ? -1 : (index - MIN_INDEX_POWER2));
398 uint32_t bgc_alloc_spin_count = 140;
399 uint32_t bgc_alloc_spin_count_loh = 16;
400 uint32_t bgc_alloc_spin = 2;
404 void c_write (uint32_t& place, uint32_t value)
406 Interlocked::Exchange (&place, value);
410 #ifndef DACCESS_COMPILE
411 // If every heap's gen2 or gen3 size is less than this threshold we will do a blocking GC.
412 const size_t bgc_min_per_heap = 4*1024*1024;
414 int gc_heap::gchist_index = 0;
415 gc_mechanisms_store gc_heap::gchist[max_history_count];
417 #ifndef MULTIPLE_HEAPS
418 size_t gc_heap::total_promoted_bytes = 0;
419 VOLATILE(bgc_state) gc_heap::current_bgc_state = bgc_not_in_process;
420 int gc_heap::gchist_index_per_heap = 0;
421 gc_heap::gc_history gc_heap::gchist_per_heap[max_history_count];
422 #endif //MULTIPLE_HEAPS
424 void gc_heap::add_to_history_per_heap()
427 gc_history* current_hist = &gchist_per_heap[gchist_index_per_heap];
428 current_hist->gc_index = settings.gc_index;
429 current_hist->current_bgc_state = current_bgc_state;
430 size_t elapsed = dd_gc_elapsed_time (dynamic_data_of (0));
431 current_hist->gc_time_ms = (uint32_t)elapsed;
432 current_hist->gc_efficiency = (elapsed ? (total_promoted_bytes / elapsed) : total_promoted_bytes);
433 current_hist->eph_low = generation_allocation_start (generation_of (max_generation-1));
434 current_hist->gen0_start = generation_allocation_start (generation_of (0));
435 current_hist->eph_high = heap_segment_allocated (ephemeral_heap_segment);
437 current_hist->bgc_lowest = background_saved_lowest_address;
438 current_hist->bgc_highest = background_saved_highest_address;
439 #endif //BACKGROUND_GC
440 current_hist->fgc_lowest = lowest_address;
441 current_hist->fgc_highest = highest_address;
442 current_hist->g_lowest = g_gc_lowest_address;
443 current_hist->g_highest = g_gc_highest_address;
445 gchist_index_per_heap++;
446 if (gchist_index_per_heap == max_history_count)
448 gchist_index_per_heap = 0;
453 void gc_heap::add_to_history()
456 gc_mechanisms_store* current_settings = &gchist[gchist_index];
457 current_settings->store (&settings);
460 if (gchist_index == max_history_count)
467 #endif //DACCESS_COMPILE
468 #endif //BACKGROUND_GC
470 #if defined(TRACE_GC) && !defined(DACCESS_COMPILE)
471 BOOL gc_log_on = TRUE;
473 size_t gc_log_file_size = 0;
475 size_t gc_buffer_index = 0;
476 size_t max_gc_buffers = 0;
478 static CLRCriticalSection gc_log_lock;
480 // we keep this much in a buffer and only flush when the buffer is full
481 #define gc_log_buffer_size (1024*1024)
482 uint8_t* gc_log_buffer = 0;
483 size_t gc_log_buffer_offset = 0;
485 void log_va_msg(const char *fmt, va_list args)
489 const int BUFFERSIZE = 512;
490 static char rgchBuffer[BUFFERSIZE];
491 char * pBuffer = &rgchBuffer[0];
494 int buffer_start = 1;
495 int pid_len = sprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, "[%5d]", (uint32_t)GCToOSInterface::GetCurrentThreadIdForLogging());
496 buffer_start += pid_len;
497 memset(&pBuffer[buffer_start], '-', BUFFERSIZE - buffer_start);
498 int msg_len = _vsnprintf_s(&pBuffer[buffer_start], BUFFERSIZE - buffer_start, _TRUNCATE, fmt, args );
501 msg_len = BUFFERSIZE - buffer_start;
504 msg_len += buffer_start;
506 if ((gc_log_buffer_offset + msg_len) > (gc_log_buffer_size - 12))
509 memset (index_str, '-', 8);
510 sprintf_s (index_str, _countof(index_str), "%d", (int)gc_buffer_index);
511 gc_log_buffer[gc_log_buffer_offset] = '\n';
512 memcpy (gc_log_buffer + (gc_log_buffer_offset + 1), index_str, 8);
515 if (gc_buffer_index > max_gc_buffers)
517 fseek (gc_log, 0, SEEK_SET);
520 fwrite(gc_log_buffer, gc_log_buffer_size, 1, gc_log);
522 memset (gc_log_buffer, '*', gc_log_buffer_size);
523 gc_log_buffer_offset = 0;
526 memcpy (gc_log_buffer + gc_log_buffer_offset, pBuffer, msg_len);
527 gc_log_buffer_offset += msg_len;
532 void GCLog (const char *fmt, ... )
534 if (gc_log_on && (gc_log != NULL))
538 log_va_msg (fmt, args);
542 #endif // TRACE_GC && !DACCESS_COMPILE
544 #if defined(GC_CONFIG_DRIVEN) && !defined(DACCESS_COMPILE)
546 BOOL gc_config_log_on = FALSE;
547 FILE* gc_config_log = NULL;
549 // we keep this much in a buffer and only flush when the buffer is full
550 #define gc_config_log_buffer_size (1*1024) // TEMP
551 uint8_t* gc_config_log_buffer = 0;
552 size_t gc_config_log_buffer_offset = 0;
554 // For config since we log so little we keep the whole history. Also it's only
555 // ever logged by one thread so no need to synchronize.
556 void log_va_msg_config(const char *fmt, va_list args)
558 const int BUFFERSIZE = 256;
559 static char rgchBuffer[BUFFERSIZE];
560 char * pBuffer = &rgchBuffer[0];
563 int buffer_start = 1;
564 int msg_len = _vsnprintf_s (&pBuffer[buffer_start], BUFFERSIZE - buffer_start, _TRUNCATE, fmt, args );
565 assert (msg_len != -1);
566 msg_len += buffer_start;
568 if ((gc_config_log_buffer_offset + msg_len) > gc_config_log_buffer_size)
570 fwrite(gc_config_log_buffer, gc_config_log_buffer_offset, 1, gc_config_log);
571 fflush(gc_config_log);
572 gc_config_log_buffer_offset = 0;
575 memcpy (gc_config_log_buffer + gc_config_log_buffer_offset, pBuffer, msg_len);
576 gc_config_log_buffer_offset += msg_len;
579 void GCLogConfig (const char *fmt, ... )
581 if (gc_config_log_on && (gc_config_log != NULL))
584 va_start( args, fmt );
585 log_va_msg_config (fmt, args);
588 #endif // GC_CONFIG_DRIVEN && !DACCESS_COMPILE
590 #ifdef SYNCHRONIZATION_STATS
592 // Number of GCs have we done since we last logged.
593 static unsigned int gc_count_during_log;
594 // In ms. This is how often we print out stats.
595 static const unsigned int log_interval = 5000;
596 // Time (in ms) when we start a new log interval.
597 static unsigned int log_start_tick;
598 static unsigned int gc_lock_contended;
599 static int64_t log_start_hires;
600 // Cycles accumulated in SuspendEE during log_interval.
601 static uint64_t suspend_ee_during_log;
602 // Cycles accumulated in RestartEE during log_interval.
603 static uint64_t restart_ee_during_log;
604 static uint64_t gc_during_log;
606 #endif //SYNCHRONIZATION_STATS
609 init_sync_log_stats()
611 #ifdef SYNCHRONIZATION_STATS
612 if (gc_count_during_log == 0)
614 gc_heap::init_sync_stats();
615 suspend_ee_during_log = 0;
616 restart_ee_during_log = 0;
618 gc_lock_contended = 0;
620 log_start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
621 log_start_hires = GCToOSInterface::QueryPerformanceCounter();
623 gc_count_during_log++;
624 #endif //SYNCHRONIZATION_STATS
628 process_sync_log_stats()
630 #ifdef SYNCHRONIZATION_STATS
632 unsigned int log_elapsed = GCToOSInterface::GetLowPrecisionTimeStamp() - log_start_tick;
634 if (log_elapsed > log_interval)
636 uint64_t total = GCToOSInterface::QueryPerformanceCounter() - log_start_hires;
637 // Print out the cycles we spent on average in each suspend and restart.
638 printf("\n_________________________________________________________________________________\n"
639 "Past %d(s): #%3d GCs; Total gc_lock contended: %8u; GC: %12u\n"
640 "SuspendEE: %8u; RestartEE: %8u GC %.3f%%\n",
644 (unsigned int)(gc_during_log / gc_count_during_log),
645 (unsigned int)(suspend_ee_during_log / gc_count_during_log),
646 (unsigned int)(restart_ee_during_log / gc_count_during_log),
647 (double)(100.0f * gc_during_log / total));
648 gc_heap::print_sync_stats(gc_count_during_log);
650 gc_count_during_log = 0;
652 #endif //SYNCHRONIZATION_STATS
655 #ifdef MULTIPLE_HEAPS
659 gc_join_init_cpu_mapping = 0,
661 gc_join_generation_determined = 2,
662 gc_join_begin_mark_phase = 3,
663 gc_join_scan_dependent_handles = 4,
664 gc_join_rescan_dependent_handles = 5,
665 gc_join_scan_sizedref_done = 6,
666 gc_join_null_dead_short_weak = 7,
667 gc_join_scan_finalization = 8,
668 gc_join_null_dead_long_weak = 9,
669 gc_join_null_dead_syncblk = 10,
670 gc_join_decide_on_compaction = 11,
671 gc_join_rearrange_segs_compaction = 12,
672 gc_join_adjust_handle_age_compact = 13,
673 gc_join_adjust_handle_age_sweep = 14,
674 gc_join_begin_relocate_phase = 15,
675 gc_join_relocate_phase_done = 16,
676 gc_join_verify_objects_done = 17,
677 gc_join_start_bgc = 18,
678 gc_join_restart_ee = 19,
679 gc_join_concurrent_overflow = 20,
680 gc_join_suspend_ee = 21,
681 gc_join_bgc_after_ephemeral = 22,
682 gc_join_allow_fgc = 23,
683 gc_join_bgc_sweep = 24,
684 gc_join_suspend_ee_verify = 25,
685 gc_join_restart_ee_verify = 26,
686 gc_join_set_state_free = 27,
687 gc_r_join_update_card_bundle = 28,
688 gc_join_after_absorb = 29,
689 gc_join_verify_copy_table = 30,
690 gc_join_after_reset = 31,
691 gc_join_after_ephemeral_sweep = 32,
692 gc_join_after_profiler_heap_walk = 33,
693 gc_join_minimal_gc = 34,
694 gc_join_after_commit_soh_no_gc = 35,
695 gc_join_expand_loh_no_gc = 36,
696 gc_join_final_no_gc = 37,
697 gc_join_disable_software_write_watch = 38,
703 join_flavor_server_gc = 0,
707 #define first_thread_arrived 2
708 struct DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE) join_structure
710 // Shared non volatile keep on separate line to prevent eviction
713 // Keep polling/wait structures on separate line write once per join
714 DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE)
715 GCEvent joined_event[3]; // the last event in the array is only used for first_thread_arrived.
716 Volatile<int> lock_color;
717 VOLATILE(BOOL) wait_done;
718 VOLATILE(BOOL) joined_p;
720 // Keep volatile counted locks on separate cache line write many per join
721 DECLSPEC_ALIGN(HS_CACHE_LINE_SIZE)
722 VOLATILE(int32_t) join_lock;
723 VOLATILE(int32_t) r_join_lock;
732 type_first_r_join = 3,
744 join_heap_restart = 100,
745 join_heap_r_restart = 200
757 join_structure join_struct;
760 gc_join_flavor flavor;
763 uint64_t start[MAX_SUPPORTED_CPUS], end[MAX_SUPPORTED_CPUS], start_seq;
764 // remember join id and last thread to arrive so restart can use these
766 // we want to print statistics every 10 seconds - this is to remember the start of the 10 sec interval
768 // counters for joins, in 1000's of clock cycles
769 uint64_t elapsed_total[gc_join_max], wake_total[gc_join_max], seq_loss_total[gc_join_max], par_loss_total[gc_join_max], in_join_total[gc_join_max];
773 BOOL init (int n_th, gc_join_flavor f)
775 dprintf (JOIN_LOG, ("Initializing join structure"));
776 join_struct.n_threads = n_th;
777 join_struct.lock_color = 0;
778 for (int i = 0; i < 3; i++)
780 if (!join_struct.joined_event[i].IsValid())
782 join_struct.joined_p = FALSE;
783 dprintf (JOIN_LOG, ("Creating join event %d", i));
784 // TODO - changing this to a non OS event
785 // because this is also used by BGC threads which are
786 // managed threads and WaitEx does not allow you to wait
787 // for an OS event on a managed thread.
788 // But we are not sure if this plays well in the hosting
790 //join_struct.joined_event[i].CreateOSManualEventNoThrow(FALSE);
791 if (!join_struct.joined_event[i].CreateManualEventNoThrow(FALSE))
795 join_struct.join_lock = join_struct.n_threads;
796 join_struct.r_join_lock = join_struct.n_threads;
797 join_struct.wait_done = FALSE;
801 start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
809 dprintf (JOIN_LOG, ("Destroying join structure"));
810 for (int i = 0; i < 3; i++)
812 if (join_struct.joined_event[i].IsValid())
813 join_struct.joined_event[i].CloseEvent();
817 inline void fire_event (int heap, join_time time, join_type type, int join_id)
819 FIRE_EVENT(GCJoin_V2, heap, time, type, join_id);
822 void join (gc_heap* gch, int join_id)
825 // parallel execution ends here
826 end[gch->heap_number] = get_ts();
829 assert (!join_struct.joined_p);
830 int color = join_struct.lock_color.LoadWithoutBarrier();
832 if (Interlocked::Decrement(&join_struct.join_lock) != 0)
834 dprintf (JOIN_LOG, ("join%d(%d): Join() Waiting...join_lock is now %d",
835 flavor, join_id, (int32_t)(join_struct.join_lock)));
837 fire_event (gch->heap_number, time_start, type_join, join_id);
839 //busy wait around the color
840 if (color == join_struct.lock_color.LoadWithoutBarrier())
843 int spin_count = 4096 * (gc_heap::n_heaps - 1);
844 for (int j = 0; j < spin_count; j++)
846 if (color != join_struct.lock_color.LoadWithoutBarrier())
850 YieldProcessor(); // indicate to the processor that we are spinning
853 // we've spun, and if color still hasn't changed, fall into hard wait
854 if (color == join_struct.lock_color.LoadWithoutBarrier())
856 dprintf (JOIN_LOG, ("join%d(%d): Join() hard wait on reset event %d, join_lock is now %d",
857 flavor, join_id, color, (int32_t)(join_struct.join_lock)));
859 //Thread* current_thread = GCToEEInterface::GetThread();
860 //BOOL cooperative_mode = gc_heap::enable_preemptive (current_thread);
861 uint32_t dwJoinWait = join_struct.joined_event[color].Wait(INFINITE, FALSE);
862 //gc_heap::disable_preemptive (current_thread, cooperative_mode);
864 if (dwJoinWait != WAIT_OBJECT_0)
866 STRESS_LOG1 (LF_GC, LL_FATALERROR, "joined event wait failed with code: %Ix", dwJoinWait);
871 // avoid race due to the thread about to reset the event (occasionally) being preempted before ResetEvent()
872 if (color == join_struct.lock_color.LoadWithoutBarrier())
877 dprintf (JOIN_LOG, ("join%d(%d): Join() done, join_lock is %d",
878 flavor, join_id, (int32_t)(join_struct.join_lock)));
881 fire_event (gch->heap_number, time_end, type_join, join_id);
884 // parallel execution starts here
885 start[gch->heap_number] = get_ts();
886 Interlocked::ExchangeAdd(&in_join_total[join_id], (start[gch->heap_number] - end[gch->heap_number]));
891 fire_event (gch->heap_number, time_start, type_last_join, join_id);
893 join_struct.joined_p = TRUE;
894 dprintf (JOIN_LOG, ("join%d(%d): Last thread to complete the join, setting id", flavor, join_id));
895 join_struct.joined_event[!color].Reset();
897 // this one is alone so it can proceed
899 // remember the join id, the last thread arriving, the start of the sequential phase,
900 // and keep track of the cycles spent waiting in the join
901 thd = gch->heap_number;
902 start_seq = get_ts();
903 Interlocked::ExchangeAdd(&in_join_total[join_id], (start_seq - end[gch->heap_number]));
908 // Reverse join - first thread gets here does the work; other threads will only proceed
909 // after the work is done.
910 // Note that you cannot call this twice in a row on the same thread. Plus there's no
911 // need to call it twice in row - you should just merge the work.
912 BOOL r_join (gc_heap* gch, int join_id)
915 if (join_struct.n_threads == 1)
920 if (Interlocked::CompareExchange(&join_struct.r_join_lock, 0, join_struct.n_threads) == 0)
922 if (!join_struct.wait_done)
924 dprintf (JOIN_LOG, ("r_join() Waiting..."));
926 fire_event (gch->heap_number, time_start, type_join, join_id);
928 //busy wait around the color
929 if (!join_struct.wait_done)
932 int spin_count = 2 * 4096 * (gc_heap::n_heaps - 1);
933 for (int j = 0; j < spin_count; j++)
935 if (join_struct.wait_done)
939 YieldProcessor(); // indicate to the processor that we are spinning
942 // we've spun, and if color still hasn't changed, fall into hard wait
943 if (!join_struct.wait_done)
945 dprintf (JOIN_LOG, ("Join() hard wait on reset event %d", first_thread_arrived));
946 uint32_t dwJoinWait = join_struct.joined_event[first_thread_arrived].Wait(INFINITE, FALSE);
947 if (dwJoinWait != WAIT_OBJECT_0)
949 STRESS_LOG1 (LF_GC, LL_FATALERROR, "joined event wait failed with code: %Ix", dwJoinWait);
954 // avoid race due to the thread about to reset the event (occasionally) being preempted before ResetEvent()
955 if (!join_struct.wait_done)
960 dprintf (JOIN_LOG, ("r_join() done"));
963 fire_event (gch->heap_number, time_end, type_join, join_id);
970 fire_event (gch->heap_number, time_start, type_first_r_join, join_id);
978 return GCToOSInterface::QueryPerformanceCounter();
981 void start_ts (gc_heap* gch)
983 // parallel execution ends here
984 start[gch->heap_number] = get_ts();
991 uint64_t elapsed_seq = get_ts() - start_seq;
992 uint64_t max = 0, sum = 0, wake = 0;
993 uint64_t min_ts = start[0];
994 for (int i = 1; i < join_struct.n_threads; i++)
996 if(min_ts > start[i]) min_ts = start[i];
999 for (int i = 0; i < join_struct.n_threads; i++)
1001 uint64_t wake_delay = start[i] - min_ts;
1002 uint64_t elapsed = end[i] - start[i];
1008 uint64_t seq_loss = (join_struct.n_threads - 1)*elapsed_seq;
1009 uint64_t par_loss = join_struct.n_threads*max - sum;
1010 double efficiency = 0.0;
1012 efficiency = sum*100.0/(join_struct.n_threads*max);
1014 const double ts_scale = 1e-6;
1016 // enable this printf to get statistics on each individual join as it occurs
1017 // printf("join #%3d seq_loss = %5g par_loss = %5g efficiency = %3.0f%%\n", join_id, ts_scale*seq_loss, ts_scale*par_loss, efficiency);
1019 elapsed_total[id] += sum;
1020 wake_total[id] += wake;
1021 seq_loss_total[id] += seq_loss;
1022 par_loss_total[id] += par_loss;
1024 // every 10 seconds, print a summary of the time spent in each type of join
1025 if (GCToOSInterface::GetLowPrecisionTimeStamp() - start_tick > 10*1000)
1027 printf("**** summary *****\n");
1028 for (int i = 0; i < 16; i++)
1030 printf("join #%3d elapsed_total = %8g wake_loss = %8g seq_loss = %8g par_loss = %8g in_join_total = %8g\n",
1032 ts_scale*elapsed_total[i],
1033 ts_scale*wake_total[i],
1034 ts_scale*seq_loss_total[i],
1035 ts_scale*par_loss_total[i],
1036 ts_scale*in_join_total[i]);
1037 elapsed_total[i] = wake_total[i] = seq_loss_total[i] = par_loss_total[i] = in_join_total[i] = 0;
1039 start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
1043 fire_event (join_heap_restart, time_start, type_restart, -1);
1044 assert (join_struct.joined_p);
1045 join_struct.joined_p = FALSE;
1046 join_struct.join_lock = join_struct.n_threads;
1047 dprintf (JOIN_LOG, ("join%d(%d): Restarting from join: join_lock is %d", flavor, id, (int32_t)(join_struct.join_lock)));
1048 // printf("restart from join #%d at cycle %u from start of gc\n", join_id, GetCycleCount32() - gc_start);
1049 int color = join_struct.lock_color.LoadWithoutBarrier();
1050 join_struct.lock_color = !color;
1051 join_struct.joined_event[color].Set();
1053 // printf("Set joined_event %d\n", !join_struct.lock_color);
1055 fire_event (join_heap_restart, time_end, type_restart, -1);
1058 start[thd] = get_ts();
1064 dprintf (JOIN_LOG, ("join%d(%d): joined, join_lock is %d", flavor, id, (int32_t)(join_struct.join_lock)));
1065 return join_struct.joined_p;
1070 if (join_struct.n_threads != 1)
1072 fire_event (join_heap_r_restart, time_start, type_restart, -1);
1073 join_struct.wait_done = TRUE;
1074 join_struct.joined_event[first_thread_arrived].Set();
1075 fire_event (join_heap_r_restart, time_end, type_restart, -1);
1081 if (join_struct.n_threads != 1)
1083 join_struct.r_join_lock = join_struct.n_threads;
1084 join_struct.wait_done = FALSE;
1085 join_struct.joined_event[first_thread_arrived].Reset();
1092 #ifdef BACKGROUND_GC
1094 #endif //BACKGROUND_GC
1096 #endif //MULTIPLE_HEAPS
1098 #define spin_and_switch(count_to_spin, expr) \
1100 for (int j = 0; j < count_to_spin; j++) \
1110 GCToOSInterface::YieldThread(0); \
1114 #ifndef DACCESS_COMPILE
1115 #ifdef BACKGROUND_GC
1117 #define max_pending_allocs 64
1119 class exclusive_sync
1121 // TODO - verify that this is the right syntax for Volatile.
1122 VOLATILE(uint8_t*) rwp_object;
1123 VOLATILE(int32_t) needs_checking;
1127 uint8_t cache_separator[HS_CACHE_LINE_SIZE - sizeof (int) - sizeof (int32_t)];
1129 // TODO - perhaps each object should be on its own cache line...
1130 VOLATILE(uint8_t*) alloc_objects[max_pending_allocs];
1132 int find_free_index ()
1134 for (int i = 0; i < max_pending_allocs; i++)
1136 if (alloc_objects [i] == (uint8_t*)0)
1148 spin_count = 32 * (g_num_processors - 1);
1151 for (int i = 0; i < max_pending_allocs; i++)
1153 alloc_objects [i] = (uint8_t*)0;
1159 for (int i = 0; i < max_pending_allocs; i++)
1161 if (alloc_objects [i] != (uint8_t*)0)
1163 GCToOSInterface::DebugBreak();
1168 void bgc_mark_set (uint8_t* obj)
1170 dprintf (3, ("cm: probing %Ix", obj));
1172 if (Interlocked::CompareExchange(&needs_checking, 1, 0) == 0)
1174 // If we spend too much time spending all the allocs,
1175 // consider adding a high water mark and scan up
1176 // to that; we'll need to interlock in done when
1177 // we update the high watermark.
1178 for (int i = 0; i < max_pending_allocs; i++)
1180 if (obj == alloc_objects[i])
1183 dprintf (3, ("cm: will spin"));
1184 spin_and_switch (spin_count, (obj != alloc_objects[i]));
1191 dprintf (3, ("cm: set %Ix", obj));
1196 spin_and_switch (spin_count, (needs_checking == 0));
1201 int loh_alloc_set (uint8_t* obj)
1203 if (!gc_heap::cm_in_progress)
1209 dprintf (3, ("loh alloc: probing %Ix", obj));
1211 if (Interlocked::CompareExchange(&needs_checking, 1, 0) == 0)
1213 if (obj == rwp_object)
1216 spin_and_switch (spin_count, (obj != rwp_object));
1221 int cookie = find_free_index();
1225 alloc_objects[cookie] = obj;
1229 // GCToOSInterface::DebugBreak();
1232 dprintf (3, ("loh alloc: set %Ix at %d", obj, cookie));
1238 dprintf (3, ("loh alloc: setting %Ix will spin to acquire a free index", obj));
1239 spin_and_switch (spin_count, (find_free_index () != -1));
1246 dprintf (3, ("loh alloc: will spin on checking %Ix", obj));
1247 spin_and_switch (spin_count, (needs_checking == 0));
1252 void bgc_mark_done ()
1254 dprintf (3, ("cm: release lock on %Ix", (uint8_t *)rwp_object));
1258 void loh_alloc_done_with_index (int index)
1260 dprintf (3, ("loh alloc: release lock on %Ix based on %d", (uint8_t *)alloc_objects[index], index));
1261 assert ((index >= 0) && (index < max_pending_allocs));
1262 alloc_objects[index] = (uint8_t*)0;
1265 void loh_alloc_done (uint8_t* obj)
1267 #ifdef BACKGROUND_GC
1268 if (!gc_heap::cm_in_progress)
1273 for (int i = 0; i < max_pending_allocs; i++)
1275 if (alloc_objects [i] == obj)
1277 dprintf (3, ("loh alloc: release lock on %Ix at %d", (uint8_t *)alloc_objects[i], i));
1278 alloc_objects[i] = (uint8_t*)0;
1282 #endif //BACKGROUND_GC
1286 // Note that this class was written assuming just synchronization between
1287 // one background GC thread and multiple user threads that might request
1288 // an FGC - it does not take into account what kind of locks the multiple
1289 // user threads might be holding at the time (eg, there could only be one
1290 // user thread requesting an FGC because it needs to take gc_lock first)
1291 // so you'll see checks that may not be necessary if you take those conditions
1292 // into consideration.
1294 // With the introduction of Server Background GC we no longer use this
1295 // class to do synchronization between FGCs and BGC.
1296 class recursive_gc_sync
1298 static VOLATILE(int32_t) foreground_request_count;//initial state 0
1299 static VOLATILE(BOOL) gc_background_running; //initial state FALSE
1300 static VOLATILE(int32_t) foreground_count; // initial state 0;
1301 static VOLATILE(uint32_t) foreground_gate; // initial state FALSE;
1302 static GCEvent foreground_complete;//Auto Reset
1303 static GCEvent foreground_allowed;//Auto Reset
1305 static void begin_background();
1306 static void end_background();
1307 static void begin_foreground();
1308 static void end_foreground();
1309 BOOL allow_foreground ();
1311 static void shutdown();
1312 static BOOL background_running_p() {return gc_background_running;}
1315 VOLATILE(int32_t) recursive_gc_sync::foreground_request_count = 0;//initial state 0
1316 VOLATILE(int32_t) recursive_gc_sync::foreground_count = 0; // initial state 0;
1317 VOLATILE(BOOL) recursive_gc_sync::gc_background_running = FALSE; //initial state FALSE
1318 VOLATILE(uint32_t) recursive_gc_sync::foreground_gate = 0;
1319 GCEvent recursive_gc_sync::foreground_complete;//Auto Reset
1320 GCEvent recursive_gc_sync::foreground_allowed;//Manual Reset
1322 BOOL recursive_gc_sync::init ()
1324 foreground_request_count = 0;
1325 foreground_count = 0;
1326 gc_background_running = FALSE;
1327 foreground_gate = 0;
1329 if (!foreground_complete.CreateOSAutoEventNoThrow(FALSE))
1333 if (!foreground_allowed.CreateManualEventNoThrow(FALSE))
1345 void recursive_gc_sync::shutdown()
1347 if (foreground_complete.IsValid())
1348 foreground_complete.CloseEvent();
1349 if (foreground_allowed.IsValid())
1350 foreground_allowed.CloseEvent();
1353 void recursive_gc_sync::begin_background()
1355 dprintf (2, ("begin background"));
1356 foreground_request_count = 1;
1357 foreground_count = 1;
1358 foreground_allowed.Reset();
1359 gc_background_running = TRUE;
1361 void recursive_gc_sync::end_background()
1363 dprintf (2, ("end background"));
1364 gc_background_running = FALSE;
1365 foreground_gate = 1;
1366 foreground_allowed.Set();
1369 void recursive_gc_sync::begin_foreground()
1371 dprintf (2, ("begin_foreground"));
1373 bool cooperative_mode = false;
1374 if (gc_background_running)
1376 gc_heap::fire_alloc_wait_event_begin (awr_fgc_wait_for_bgc);
1377 gc_heap::alloc_wait_event_p = TRUE;
1381 Interlocked::Increment (&foreground_request_count);
1384 dprintf(2, ("Waiting sync gc point"));
1385 assert (foreground_allowed.IsValid());
1386 assert (foreground_complete.IsValid());
1388 cooperative_mode = gc_heap::enable_preemptive ();
1390 foreground_allowed.Wait(INFINITE, FALSE);
1392 dprintf(2, ("Waiting sync gc point is done"));
1394 gc_heap::disable_preemptive (cooperative_mode);
1396 if (foreground_gate)
1398 Interlocked::Increment (&foreground_count);
1399 dprintf (2, ("foreground_count: %d", (int32_t)foreground_count));
1400 if (foreground_gate)
1402 gc_heap::settings.concurrent = FALSE;
1413 goto try_again_no_inc;
1418 void recursive_gc_sync::end_foreground()
1420 dprintf (2, ("end_foreground"));
1421 if (gc_background_running)
1423 Interlocked::Decrement (&foreground_request_count);
1424 dprintf (2, ("foreground_count before decrement: %d", (int32_t)foreground_count));
1425 if (Interlocked::Decrement (&foreground_count) == 0)
1427 //c_write ((BOOL*)&foreground_gate, 0);
1428 // TODO - couldn't make the syntax work with Volatile<T>
1429 foreground_gate = 0;
1430 if (foreground_count == 0)
1432 foreground_allowed.Reset ();
1433 dprintf(2, ("setting foreground complete event"));
1434 foreground_complete.Set();
1441 BOOL recursive_gc_sync::allow_foreground()
1443 assert (gc_heap::settings.concurrent);
1444 dprintf (100, ("enter allow_foreground, f_req_count: %d, f_count: %d",
1445 (int32_t)foreground_request_count, (int32_t)foreground_count));
1447 BOOL did_fgc = FALSE;
1449 //if we have suspended the EE, just return because
1450 //some thread could be waiting on this to proceed.
1451 if (!GCHeap::GcInProgress)
1453 //TODO BACKGROUND_GC This is to stress the concurrency between
1454 //background and foreground
1455 // gc_heap::disallow_new_allocation (0);
1457 //GCToOSInterface::YieldThread(0);
1460 if (foreground_request_count != 0)
1462 //foreground wants to run
1463 //save the important settings
1464 //TODO BACKGROUND_GC be more selective about the important settings.
1465 gc_mechanisms saved_settings = gc_heap::settings;
1469 //c_write ((BOOL*)&foreground_gate, 1);
1470 // TODO - couldn't make the syntax work with Volatile<T>
1471 foreground_gate = 1;
1472 foreground_allowed.Set ();
1473 foreground_complete.Wait (INFINITE, FALSE);
1474 }while (/*foreground_request_count ||*/ foreground_gate);
1476 assert (!foreground_gate);
1478 //restore the important settings
1479 gc_heap::settings = saved_settings;
1480 GCHeap::GcCondemnedGeneration = gc_heap::settings.condemned_generation;
1481 //the background GC shouldn't be using gc_high and gc_low
1482 //gc_low = lowest_address;
1483 //gc_high = highest_address;
1486 //TODO BACKGROUND_GC This is to stress the concurrency between
1487 //background and foreground
1488 // gc_heap::allow_new_allocation (0);
1492 dprintf (100, ("leave allow_foreground"));
1493 assert (gc_heap::settings.concurrent);
1497 #endif //BACKGROUND_GC
1498 #endif //DACCESS_COMPILE
1501 #if defined(COUNT_CYCLES)
1503 #pragma warning(disable:4035)
1507 unsigned GetCycleCount32() // enough for about 40 seconds
1515 #pragma warning(default:4035)
1517 #endif //COUNT_CYCLES
1520 int mark_time, plan_time, sweep_time, reloc_time, compact_time;
1523 #ifndef MULTIPLE_HEAPS
1525 #endif // MULTIPLE_HEAPS
1527 void reset_memory (uint8_t* o, size_t sizeo);
1531 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1532 static bool virtual_alloc_hardware_write_watch = false;
1533 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1535 static bool hardware_write_watch_capability = false;
1537 #ifndef DACCESS_COMPILE
1539 //check if the write watch APIs are supported.
1541 void hardware_write_watch_api_supported()
1543 if (GCToOSInterface::SupportsWriteWatch())
1545 hardware_write_watch_capability = true;
1546 dprintf (2, ("WriteWatch supported"));
1550 dprintf (2,("WriteWatch not supported"));
1554 #endif //!DACCESS_COMPILE
1556 inline bool can_use_hardware_write_watch()
1558 return hardware_write_watch_capability;
1561 inline bool can_use_write_watch_for_gc_heap()
1563 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1565 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1566 return can_use_hardware_write_watch();
1567 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
1570 inline bool can_use_write_watch_for_card_table()
1572 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
1575 return can_use_hardware_write_watch();
1580 #define mem_reserve (MEM_RESERVE)
1581 #endif //WRITE_WATCH
1583 //check if the low memory notification is supported
1585 #ifndef DACCESS_COMPILE
1587 void WaitLongerNoInstru (int i)
1589 // every 8th attempt:
1590 bool bToggleGC = GCToEEInterface::EnablePreemptiveGC();
1592 // if we're waiting for gc to finish, we should block immediately
1593 if (g_fSuspensionPending == 0)
1595 if (g_num_processors > 1)
1597 YieldProcessor(); // indicate to the processor that we are spining
1599 GCToOSInterface::YieldThread (0);
1601 GCToOSInterface::Sleep (5);
1604 GCToOSInterface::Sleep (5);
1607 // If CLR is hosted, a thread may reach here while it is in preemptive GC mode,
1608 // or it has no Thread object, in order to force a task to yield, or to triger a GC.
1609 // It is important that the thread is going to wait for GC. Otherwise the thread
1610 // is in a tight loop. If the thread has high priority, the perf is going to be very BAD.
1614 // In debug builds, all enter_spin_lock operations go through this code. If a GC has
1615 // started, it is important to block until the GC thread calls set_gc_done (since it is
1616 // guaranteed to have cleared g_TrapReturningThreads by this point). This avoids livelock
1617 // conditions which can otherwise occur if threads are allowed to spin in this function
1618 // (and therefore starve the GC thread) between the point when the GC thread sets the
1619 // WaitForGC event and the point when the GC thread clears g_TrapReturningThreads.
1620 if (gc_heap::gc_started)
1622 gc_heap::wait_for_gc_done();
1625 GCToEEInterface::DisablePreemptiveGC();
1627 else if (g_fSuspensionPending > 0)
1629 g_theGCHeap->WaitUntilGCComplete();
1634 static void safe_switch_to_thread()
1636 bool cooperative_mode = gc_heap::enable_preemptive();
1638 GCToOSInterface::YieldThread(0);
1640 gc_heap::disable_preemptive(cooperative_mode);
1644 // We need the following methods to have volatile arguments, so that they can accept
1645 // raw pointers in addition to the results of the & operator on Volatile<T>.
1648 static void enter_spin_lock_noinstru (RAW_KEYWORD(volatile) int32_t* lock)
1652 if (Interlocked::CompareExchange(lock, 0, -1) >= 0)
1655 while (VolatileLoad(lock) >= 0)
1657 if ((++i & 7) && !IsGCInProgress())
1659 if (g_num_processors > 1)
1661 #ifndef MULTIPLE_HEAPS
1662 int spin_count = 1024 * g_num_processors;
1663 #else //!MULTIPLE_HEAPS
1664 int spin_count = 32 * g_num_processors;
1665 #endif //!MULTIPLE_HEAPS
1666 for (int j = 0; j < spin_count; j++)
1668 if (VolatileLoad(lock) < 0 || IsGCInProgress())
1670 YieldProcessor(); // indicate to the processor that we are spining
1672 if (VolatileLoad(lock) >= 0 && !IsGCInProgress())
1674 safe_switch_to_thread();
1679 safe_switch_to_thread();
1684 WaitLongerNoInstru(i);
1692 static BOOL try_enter_spin_lock_noinstru(RAW_KEYWORD(volatile) int32_t* lock)
1694 return (Interlocked::CompareExchange(&*lock, 0, -1) < 0);
1698 static void leave_spin_lock_noinstru (RAW_KEYWORD(volatile) int32_t* lock)
1700 VolatileStore<int32_t>((int32_t*)lock, -1);
1706 static void enter_spin_lock(GCSpinLock *pSpinLock)
1708 enter_spin_lock_noinstru(&pSpinLock->lock);
1709 assert (pSpinLock->holding_thread == (Thread*)-1);
1710 pSpinLock->holding_thread = GCToEEInterface::GetThread();
1714 static BOOL try_enter_spin_lock(GCSpinLock *pSpinLock)
1716 BOOL ret = try_enter_spin_lock_noinstru(&pSpinLock->lock);
1718 pSpinLock->holding_thread = GCToEEInterface::GetThread();
1723 static void leave_spin_lock(GCSpinLock *pSpinLock)
1725 bool gc_thread_p = GCToEEInterface::WasCurrentThreadCreatedByGC();
1726 // _ASSERTE((pSpinLock->holding_thread == GCToEEInterface::GetThread()) || gc_thread_p || pSpinLock->released_by_gc_p);
1727 pSpinLock->released_by_gc_p = gc_thread_p;
1728 pSpinLock->holding_thread = (Thread*) -1;
1729 if (pSpinLock->lock != -1)
1730 leave_spin_lock_noinstru(&pSpinLock->lock);
1733 #define ASSERT_HOLDING_SPIN_LOCK(pSpinLock) \
1734 _ASSERTE((pSpinLock)->holding_thread == GCToEEInterface::GetThread());
1736 #define ASSERT_NOT_HOLDING_SPIN_LOCK(pSpinLock) \
1737 _ASSERTE((pSpinLock)->holding_thread != GCToEEInterface::GetThread());
1741 //In the concurrent version, the Enable/DisablePreemptiveGC is optional because
1742 //the gc thread call WaitLonger.
1743 void WaitLonger (int i
1744 #ifdef SYNCHRONIZATION_STATS
1745 , GCSpinLock* spin_lock
1746 #endif //SYNCHRONIZATION_STATS
1749 #ifdef SYNCHRONIZATION_STATS
1750 (spin_lock->num_wait_longer)++;
1751 #endif //SYNCHRONIZATION_STATS
1753 // every 8th attempt:
1754 bool bToggleGC = GCToEEInterface::EnablePreemptiveGC();
1757 // if we're waiting for gc to finish, we should block immediately
1758 if (!gc_heap::gc_started)
1760 #ifdef SYNCHRONIZATION_STATS
1761 (spin_lock->num_switch_thread_w)++;
1762 #endif //SYNCHRONIZATION_STATS
1763 if (g_num_processors > 1)
1765 YieldProcessor(); // indicate to the processor that we are spining
1767 GCToOSInterface::YieldThread (0);
1769 GCToOSInterface::Sleep (5);
1772 GCToOSInterface::Sleep (5);
1775 // If CLR is hosted, a thread may reach here while it is in preemptive GC mode,
1776 // or it has no Thread object, in order to force a task to yield, or to triger a GC.
1777 // It is important that the thread is going to wait for GC. Otherwise the thread
1778 // is in a tight loop. If the thread has high priority, the perf is going to be very BAD.
1779 if (gc_heap::gc_started)
1781 gc_heap::wait_for_gc_done();
1786 #ifdef SYNCHRONIZATION_STATS
1787 (spin_lock->num_disable_preemptive_w)++;
1788 #endif //SYNCHRONIZATION_STATS
1789 GCToEEInterface::DisablePreemptiveGC();
1794 static void enter_spin_lock (GCSpinLock* spin_lock)
1798 if (Interlocked::CompareExchange(&spin_lock->lock, 0, -1) >= 0)
1801 while (spin_lock->lock >= 0)
1803 if ((++i & 7) && !gc_heap::gc_started)
1805 if (g_num_processors > 1)
1807 #ifndef MULTIPLE_HEAPS
1808 int spin_count = 1024 * g_num_processors;
1809 #else //!MULTIPLE_HEAPS
1810 int spin_count = 32 * g_num_processors;
1811 #endif //!MULTIPLE_HEAPS
1812 for (int j = 0; j < spin_count; j++)
1814 if (spin_lock->lock < 0 || gc_heap::gc_started)
1816 YieldProcessor(); // indicate to the processor that we are spining
1818 if (spin_lock->lock >= 0 && !gc_heap::gc_started)
1820 #ifdef SYNCHRONIZATION_STATS
1821 (spin_lock->num_switch_thread)++;
1822 #endif //SYNCHRONIZATION_STATS
1823 bool cooperative_mode = gc_heap::enable_preemptive ();
1825 GCToOSInterface::YieldThread(0);
1827 gc_heap::disable_preemptive (cooperative_mode);
1831 GCToOSInterface::YieldThread(0);
1836 #ifdef SYNCHRONIZATION_STATS
1838 #endif //SYNCHRONIZATION_STATS
1846 inline BOOL try_enter_spin_lock(GCSpinLock* spin_lock)
1848 return (Interlocked::CompareExchange(&spin_lock->lock, 0, -1) < 0);
1852 static void leave_spin_lock (GCSpinLock * spin_lock)
1854 spin_lock->lock = -1;
1857 #define ASSERT_HOLDING_SPIN_LOCK(pSpinLock)
1861 bool gc_heap::enable_preemptive ()
1863 return GCToEEInterface::EnablePreemptiveGC();
1866 void gc_heap::disable_preemptive (bool restore_cooperative)
1868 if (restore_cooperative)
1870 GCToEEInterface::DisablePreemptiveGC();
1874 #endif // !DACCESS_COMPILE
1876 typedef void ** PTR_PTR;
1877 //This function clears a piece of memory
1878 // size has to be Dword aligned
1881 void memclr ( uint8_t* mem, size_t size)
1883 dprintf (3, ("MEMCLR: %Ix, %d", mem, size));
1884 assert ((size & (sizeof(PTR_PTR)-1)) == 0);
1885 assert (sizeof(PTR_PTR) == DATA_ALIGNMENT);
1888 // The compiler will recognize this pattern and replace it with memset call. We can as well just call
1889 // memset directly to make it obvious what's going on.
1890 PTR_PTR m = (PTR_PTR) mem;
1891 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
1895 memset (mem, 0, size);
1898 void memcopy (uint8_t* dmem, uint8_t* smem, size_t size)
1900 const size_t sz4ptr = sizeof(PTR_PTR)*4;
1901 const size_t sz2ptr = sizeof(PTR_PTR)*2;
1902 const size_t sz1ptr = sizeof(PTR_PTR)*1;
1904 // size must be a multiple of the pointer size
1905 assert ((size & (sizeof (PTR_PTR)-1)) == 0);
1906 assert (sizeof(PTR_PTR) == DATA_ALIGNMENT);
1908 // copy in groups of four pointer sized things at a time
1913 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1914 ((PTR_PTR)dmem)[1] = ((PTR_PTR)smem)[1];
1915 ((PTR_PTR)dmem)[2] = ((PTR_PTR)smem)[2];
1916 ((PTR_PTR)dmem)[3] = ((PTR_PTR)smem)[3];
1920 while ((size -= sz4ptr) >= sz4ptr);
1923 // still two pointer sized things or more left to copy?
1926 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1927 ((PTR_PTR)dmem)[1] = ((PTR_PTR)smem)[1];
1932 // still one pointer sized thing left to copy?
1935 ((PTR_PTR)dmem)[0] = ((PTR_PTR)smem)[0];
1943 ptrdiff_t round_down (ptrdiff_t add, int pitch)
1945 return ((add / pitch) * pitch);
1948 #if defined(FEATURE_STRUCTALIGN) && defined(RESPECT_LARGE_ALIGNMENT)
1949 // FEATURE_STRUCTALIGN allows the compiler to dictate the alignment,
1950 // i.e, if a larger alignment matters or is beneficial, the compiler
1951 // generated info tells us so. RESPECT_LARGE_ALIGNMENT is just the
1952 // converse - it's a heuristic for the GC to use a larger alignment.
1953 #error FEATURE_STRUCTALIGN should imply !RESPECT_LARGE_ALIGNMENT
1956 #if defined(FEATURE_STRUCTALIGN) && defined(FEATURE_LOH_COMPACTION)
1957 #error FEATURE_STRUCTALIGN and FEATURE_LOH_COMPACTION are mutually exclusive
1960 #if defined(GROWABLE_SEG_MAPPING_TABLE) && !defined(SEG_MAPPING_TABLE)
1961 #error if GROWABLE_SEG_MAPPING_TABLE is defined, SEG_MAPPING_TABLE must be defined
1965 BOOL same_large_alignment_p (uint8_t* p1, uint8_t* p2)
1967 #ifdef RESPECT_LARGE_ALIGNMENT
1968 return ((((size_t)p1 ^ (size_t)p2) & 7) == 0);
1970 UNREFERENCED_PARAMETER(p1);
1971 UNREFERENCED_PARAMETER(p2);
1973 #endif //RESPECT_LARGE_ALIGNMENT
1977 size_t switch_alignment_size (BOOL already_padded_p)
1979 if (already_padded_p)
1980 return DATA_ALIGNMENT;
1982 return (Align (min_obj_size) +((Align (min_obj_size)&DATA_ALIGNMENT)^DATA_ALIGNMENT));
1986 #ifdef FEATURE_STRUCTALIGN
1987 void set_node_aligninfo (uint8_t *node, int requiredAlignment, ptrdiff_t pad);
1988 void clear_node_aligninfo (uint8_t *node);
1989 #else // FEATURE_STRUCTALIGN
1990 #define node_realigned(node) (((plug_and_reloc*)(node))[-1].reloc & 1)
1991 void set_node_realigned (uint8_t* node);
1992 void clear_node_realigned(uint8_t* node);
1993 #endif // FEATURE_STRUCTALIGN
1996 size_t AlignQword (size_t nbytes)
1998 #ifdef FEATURE_STRUCTALIGN
1999 // This function is used to align everything on the large object
2000 // heap to an 8-byte boundary, to reduce the number of unaligned
2001 // accesses to (say) arrays of doubles. With FEATURE_STRUCTALIGN,
2002 // the compiler dictates the optimal alignment instead of having
2003 // a heuristic in the GC.
2004 return Align (nbytes);
2005 #else // FEATURE_STRUCTALIGN
2006 return (nbytes + 7) & ~7;
2007 #endif // FEATURE_STRUCTALIGN
2011 BOOL Aligned (size_t n)
2013 return (n & ALIGNCONST) == 0;
2016 #define OBJECT_ALIGNMENT_OFFSET (sizeof(MethodTable *))
2018 #ifdef FEATURE_STRUCTALIGN
2019 #define MAX_STRUCTALIGN OS_PAGE_SIZE
2020 #else // FEATURE_STRUCTALIGN
2021 #define MAX_STRUCTALIGN 0
2022 #endif // FEATURE_STRUCTALIGN
2024 #ifdef FEATURE_STRUCTALIGN
2026 ptrdiff_t AdjustmentForMinPadSize(ptrdiff_t pad, int requiredAlignment)
2028 // The resulting alignpad must be either 0 or at least min_obj_size.
2029 // Note that by computing the following difference on unsigned types,
2030 // we can do the range check 0 < alignpad < min_obj_size with a
2031 // single conditional branch.
2032 if ((size_t)(pad - DATA_ALIGNMENT) < Align (min_obj_size) - DATA_ALIGNMENT)
2034 return requiredAlignment;
2040 uint8_t* StructAlign (uint8_t* origPtr, int requiredAlignment, ptrdiff_t alignmentOffset=OBJECT_ALIGNMENT_OFFSET)
2042 // required alignment must be a power of two
2043 _ASSERTE(((size_t)origPtr & ALIGNCONST) == 0);
2044 _ASSERTE(((requiredAlignment - 1) & requiredAlignment) == 0);
2045 _ASSERTE(requiredAlignment >= sizeof(void *));
2046 _ASSERTE(requiredAlignment <= MAX_STRUCTALIGN);
2048 // When this method is invoked for individual objects (i.e., alignmentOffset
2049 // is just the size of the PostHeader), what needs to be aligned when
2050 // we're done is the pointer to the payload of the object (which means
2051 // the actual resulting object pointer is typically not aligned).
2053 uint8_t* result = (uint8_t*)Align ((size_t)origPtr + alignmentOffset, requiredAlignment-1) - alignmentOffset;
2054 ptrdiff_t alignpad = result - origPtr;
2056 return result + AdjustmentForMinPadSize (alignpad, requiredAlignment);
2060 ptrdiff_t ComputeStructAlignPad (uint8_t* plug, int requiredAlignment, size_t alignmentOffset=OBJECT_ALIGNMENT_OFFSET)
2062 return StructAlign (plug, requiredAlignment, alignmentOffset) - plug;
2065 BOOL IsStructAligned (uint8_t *ptr, int requiredAlignment)
2067 return StructAlign (ptr, requiredAlignment) == ptr;
2071 ptrdiff_t ComputeMaxStructAlignPad (int requiredAlignment)
2073 if (requiredAlignment == DATA_ALIGNMENT)
2075 // Since a non-zero alignment padding cannot be less than min_obj_size (so we can fit the
2076 // alignment padding object), the worst-case alignment padding is correspondingly larger
2077 // than the required alignment.
2078 return requiredAlignment + Align (min_obj_size) - DATA_ALIGNMENT;
2082 ptrdiff_t ComputeMaxStructAlignPadLarge (int requiredAlignment)
2084 if (requiredAlignment <= get_alignment_constant (TRUE)+1)
2086 // This is the same as ComputeMaxStructAlignPad, except that in addition to leaving space
2087 // for padding before the actual object, it also leaves space for filling a gap after the
2088 // actual object. This is needed on the large object heap, as the outer allocation functions
2089 // don't operate on an allocation context (which would have left space for the final gap).
2090 return requiredAlignment + Align (min_obj_size) * 2 - DATA_ALIGNMENT;
2093 uint8_t* gc_heap::pad_for_alignment (uint8_t* newAlloc, int requiredAlignment, size_t size, alloc_context* acontext)
2095 uint8_t* alignedPtr = StructAlign (newAlloc, requiredAlignment);
2096 if (alignedPtr != newAlloc) {
2097 make_unused_array (newAlloc, alignedPtr - newAlloc);
2099 acontext->alloc_ptr = alignedPtr + Align (size);
2103 uint8_t* gc_heap::pad_for_alignment_large (uint8_t* newAlloc, int requiredAlignment, size_t size)
2105 uint8_t* alignedPtr = StructAlign (newAlloc, requiredAlignment);
2106 if (alignedPtr != newAlloc) {
2107 make_unused_array (newAlloc, alignedPtr - newAlloc);
2109 if (alignedPtr < newAlloc + ComputeMaxStructAlignPadLarge (requiredAlignment)) {
2110 make_unused_array (alignedPtr + AlignQword (size), newAlloc + ComputeMaxStructAlignPadLarge (requiredAlignment) - alignedPtr);
2114 #else // FEATURE_STRUCTALIGN
2115 #define ComputeMaxStructAlignPad(requiredAlignment) 0
2116 #define ComputeMaxStructAlignPadLarge(requiredAlignment) 0
2117 #endif // FEATURE_STRUCTALIGN
2119 //CLR_SIZE is the max amount of bytes from gen0 that is set to 0 in one chunk
2121 #define CLR_SIZE ((size_t)(8*1024))
2123 #define CLR_SIZE ((size_t)(8*1024))
2126 #define END_SPACE_AFTER_GC (LARGE_OBJECT_SIZE + MAX_STRUCTALIGN)
2128 #ifdef BACKGROUND_GC
2129 #define SEGMENT_INITIAL_COMMIT (2*OS_PAGE_SIZE)
2131 #define SEGMENT_INITIAL_COMMIT (OS_PAGE_SIZE)
2132 #endif //BACKGROUND_GC
2138 #define INITIAL_ALLOC ((size_t)((size_t)4*1024*1024*1024))
2139 #define LHEAP_ALLOC ((size_t)(1024*1024*256))
2143 #define INITIAL_ALLOC ((size_t)(1024*1024*64))
2144 #define LHEAP_ALLOC ((size_t)(1024*1024*32))
2152 #define INITIAL_ALLOC ((size_t)(1024*1024*256))
2153 #define LHEAP_ALLOC ((size_t)(1024*1024*128))
2157 #define INITIAL_ALLOC ((size_t)(1024*1024*16))
2158 #define LHEAP_ALLOC ((size_t)(1024*1024*16))
2164 //amount in bytes of the etw allocation tick
2165 const size_t etw_allocation_tick = 100*1024;
2167 const size_t low_latency_alloc = 256*1024;
2169 const size_t fgn_check_quantum = 2*1024*1024;
2172 const int max_snoop_level = 128;
2177 //threshold of heap size to turn on card bundles.
2178 #define SH_TH_CARD_BUNDLE (40*1024*1024)
2179 #define MH_TH_CARD_BUNDLE (180*1024*1024)
2180 #endif //CARD_BUNDLE
2182 #define GC_EPHEMERAL_DECOMMIT_TIMEOUT 5000
2185 size_t align_on_page (size_t add)
2187 return ((add + OS_PAGE_SIZE - 1) & ~((size_t)OS_PAGE_SIZE - 1));
2191 uint8_t* align_on_page (uint8_t* add)
2193 return (uint8_t*)align_on_page ((size_t) add);
2197 size_t align_lower_page (size_t add)
2199 return (add & ~((size_t)OS_PAGE_SIZE - 1));
2203 uint8_t* align_lower_page (uint8_t* add)
2205 return (uint8_t*)align_lower_page ((size_t)add);
2209 size_t align_write_watch_lower_page (size_t add)
2211 return (add & ~(WRITE_WATCH_UNIT_SIZE - 1));
2215 uint8_t* align_write_watch_lower_page (uint8_t* add)
2217 return (uint8_t*)align_lower_page ((size_t)add);
2222 BOOL power_of_two_p (size_t integer)
2224 return !(integer & (integer-1));
2228 BOOL oddp (size_t integer)
2230 return (integer & 1) != 0;
2233 // we only ever use this for WORDs.
2234 size_t logcount (size_t word)
2236 //counts the number of high bits in a 16 bit word.
2237 assert (word < 0x10000);
2239 count = (word & 0x5555) + ( (word >> 1 ) & 0x5555);
2240 count = (count & 0x3333) + ( (count >> 2) & 0x3333);
2241 count = (count & 0x0F0F) + ( (count >> 4) & 0x0F0F);
2242 count = (count & 0x00FF) + ( (count >> 8) & 0x00FF);
2246 #ifndef DACCESS_COMPILE
2248 void stomp_write_barrier_resize(bool is_runtime_suspended, bool requires_upper_bounds_check)
2250 WriteBarrierParameters args = {};
2251 args.operation = WriteBarrierOp::StompResize;
2252 args.is_runtime_suspended = is_runtime_suspended;
2253 args.requires_upper_bounds_check = requires_upper_bounds_check;
2255 args.card_table = g_gc_card_table;
2256 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
2257 args.card_bundle_table = g_gc_card_bundle_table;
2260 args.lowest_address = g_gc_lowest_address;
2261 args.highest_address = g_gc_highest_address;
2263 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
2264 if (SoftwareWriteWatch::IsEnabledForGCHeap())
2266 args.write_watch_table = g_gc_sw_ww_table;
2268 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
2270 GCToEEInterface::StompWriteBarrier(&args);
2273 void stomp_write_barrier_ephemeral(uint8_t* ephemeral_low, uint8_t* ephemeral_high)
2275 WriteBarrierParameters args = {};
2276 args.operation = WriteBarrierOp::StompEphemeral;
2277 args.is_runtime_suspended = true;
2278 args.ephemeral_low = ephemeral_low;
2279 args.ephemeral_high = ephemeral_high;
2280 GCToEEInterface::StompWriteBarrier(&args);
2283 void stomp_write_barrier_initialize(uint8_t* ephemeral_low, uint8_t* ephemeral_high)
2285 WriteBarrierParameters args = {};
2286 args.operation = WriteBarrierOp::Initialize;
2287 args.is_runtime_suspended = true;
2288 args.requires_upper_bounds_check = false;
2289 args.card_table = g_gc_card_table;
2291 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
2292 args.card_bundle_table = g_gc_card_bundle_table;
2295 args.lowest_address = g_gc_lowest_address;
2296 args.highest_address = g_gc_highest_address;
2297 args.ephemeral_low = ephemeral_low;
2298 args.ephemeral_high = ephemeral_high;
2299 GCToEEInterface::StompWriteBarrier(&args);
2302 #endif // DACCESS_COMPILE
2304 //extract the low bits [0,low[ of a uint32_t
2305 #define lowbits(wrd, bits) ((wrd) & ((1 << (bits))-1))
2306 //extract the high bits [high, 32] of a uint32_t
2307 #define highbits(wrd, bits) ((wrd) & ~((1 << (bits))-1))
2309 // Things we need to manually initialize:
2310 // gen0 min_size - based on cache
2311 // gen0/1 max_size - based on segment size
2312 static static_data static_data_table[latency_level_last - latency_level_first + 1][NUMBERGENERATIONS] =
2314 // latency_level_memory_footprint
2317 {0, 0, 40000, 0.5f, 9.0f, 20.0f, 1000, 1},
2319 {163840, 0, 80000, 0.5f, 2.0f, 7.0f, 10000, 10},
2321 {256*1024, SSIZE_T_MAX, 200000, 0.25f, 1.2f, 1.8f, 100000, 100},
2323 {3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0}
2326 // latency_level_balanced
2330 #ifdef MULTIPLE_HEAPS
2334 #endif //MULTIPLE_HEAPS
2337 {9*32*1024, 0, 80000, 0.5f, 2.0f, 7.0f, 10000, 10},
2339 {256*1024, SSIZE_T_MAX, 200000, 0.25f, 1.2f, 1.8f, 100000, 100},
2341 {3*1024*1024, SSIZE_T_MAX, 0, 0.0f, 1.25f, 4.5f, 0, 0}
2348 class CObjectHeader;
2352 class c_synchronize;
2354 #ifdef FEATURE_PREMORTEM_FINALIZATION
2355 #ifndef DACCESS_COMPILE
2357 HRESULT AllocateCFinalize(CFinalize **pCFinalize);
2358 #endif //!DACCESS_COMPILE
2359 #endif // FEATURE_PREMORTEM_FINALIZATION
2361 uint8_t* tree_search (uint8_t* tree, uint8_t* old_address);
2364 #ifdef USE_INTROSORT
2365 #define _sort introsort::sort
2366 #else //USE_INTROSORT
2367 #define _sort qsort1
2368 void qsort1(uint8_t** low, uint8_t** high, unsigned int depth);
2369 #endif //USE_INTROSORT
2371 void* virtual_alloc (size_t size);
2372 void virtual_free (void* add, size_t size);
2374 /* per heap static initialization */
2376 #ifndef MULTIPLE_HEAPS
2377 uint32_t* gc_heap::mark_array;
2378 #endif //MULTIPLE_HEAPS
2382 uint8_t** gc_heap::g_mark_list;
2384 #ifdef PARALLEL_MARK_LIST_SORT
2385 uint8_t** gc_heap::g_mark_list_copy;
2386 #endif //PARALLEL_MARK_LIST_SORT
2388 size_t gc_heap::mark_list_size;
2391 #ifdef SEG_MAPPING_TABLE
2392 seg_mapping* seg_mapping_table;
2393 #endif //SEG_MAPPING_TABLE
2395 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
2396 sorted_table* gc_heap::seg_table;
2397 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
2399 #ifdef MULTIPLE_HEAPS
2400 GCEvent gc_heap::ee_suspend_event;
2401 size_t gc_heap::min_balance_threshold = 0;
2402 #endif //MULTIPLE_HEAPS
2404 VOLATILE(BOOL) gc_heap::gc_started;
2406 #ifdef MULTIPLE_HEAPS
2408 GCEvent gc_heap::gc_start_event;
2410 bool gc_heap::gc_thread_no_affinitize_p = false;
2412 int gc_heap::n_heaps;
2414 gc_heap** gc_heap::g_heaps;
2416 size_t* gc_heap::g_promoted;
2419 int* gc_heap::g_mark_stack_busy;
2423 #ifdef BACKGROUND_GC
2424 size_t* gc_heap::g_bpromoted;
2425 #endif //BACKGROUND_GC
2427 #else //MULTIPLE_HEAPS
2429 size_t gc_heap::g_promoted;
2431 #ifdef BACKGROUND_GC
2432 size_t gc_heap::g_bpromoted;
2433 #endif //BACKGROUND_GC
2435 #endif //MULTIPLE_HEAPS
2437 size_t gc_heap::reserved_memory = 0;
2438 size_t gc_heap::reserved_memory_limit = 0;
2439 BOOL gc_heap::g_low_memory_status;
2441 #ifndef DACCESS_COMPILE
2442 static gc_reason gc_trigger_reason = reason_empty;
2443 #endif //DACCESS_COMPILE
2445 gc_latency_level gc_heap::latency_level = latency_level_default;
2447 gc_mechanisms gc_heap::settings;
2449 gc_history_global gc_heap::gc_data_global;
2451 size_t gc_heap::gc_last_ephemeral_decommit_time = 0;
2453 size_t gc_heap::gc_gen0_desired_high;
2456 double gc_heap::short_plugs_pad_ratio = 0;
2457 #endif //SHORT_PLUGS
2460 #define MAX_ALLOWED_MEM_LOAD 85
2462 // consider putting this in dynamic data -
2463 // we may want different values for workstation
2465 #define MIN_YOUNGEST_GEN_DESIRED (16*1024*1024)
2467 size_t gc_heap::youngest_gen_desired_th;
2470 uint32_t gc_heap::last_gc_memory_load = 0;
2472 size_t gc_heap::last_gc_heap_size = 0;
2474 size_t gc_heap::last_gc_fragmentation = 0;
2476 uint64_t gc_heap::mem_one_percent = 0;
2478 uint32_t gc_heap::high_memory_load_th = 0;
2480 uint64_t gc_heap::total_physical_mem = 0;
2482 uint64_t gc_heap::entry_available_physical_mem = 0;
2484 #ifdef BACKGROUND_GC
2485 GCEvent gc_heap::bgc_start_event;
2487 gc_mechanisms gc_heap::saved_bgc_settings;
2489 GCEvent gc_heap::background_gc_done_event;
2491 GCEvent gc_heap::ee_proceed_event;
2493 bool gc_heap::gc_can_use_concurrent = false;
2495 bool gc_heap::temp_disable_concurrent_p = false;
2497 uint32_t gc_heap::cm_in_progress = FALSE;
2499 BOOL gc_heap::dont_restart_ee_p = FALSE;
2501 BOOL gc_heap::keep_bgc_threads_p = FALSE;
2503 GCEvent gc_heap::bgc_threads_sync_event;
2505 BOOL gc_heap::do_ephemeral_gc_p = FALSE;
2507 BOOL gc_heap::do_concurrent_p = FALSE;
2509 size_t gc_heap::ephemeral_fgc_counts[max_generation];
2511 BOOL gc_heap::alloc_wait_event_p = FALSE;
2513 VOLATILE(c_gc_state) gc_heap::current_c_gc_state = c_gc_state_free;
2515 #endif //BACKGROUND_GC
2517 #ifndef MULTIPLE_HEAPS
2518 #ifdef SPINLOCK_HISTORY
2519 int gc_heap::spinlock_info_index = 0;
2520 spinlock_info gc_heap::last_spinlock_info[max_saved_spinlock_info + 8];
2521 #endif //SPINLOCK_HISTORY
2523 size_t gc_heap::fgn_last_alloc = 0;
2525 int gc_heap::generation_skip_ratio = 100;
2527 uint64_t gc_heap::loh_alloc_since_cg = 0;
2529 BOOL gc_heap::elevation_requested = FALSE;
2531 BOOL gc_heap::last_gc_before_oom = FALSE;
2533 #ifdef BACKGROUND_GC
2534 uint8_t* gc_heap::background_saved_lowest_address = 0;
2535 uint8_t* gc_heap::background_saved_highest_address = 0;
2536 uint8_t* gc_heap::next_sweep_obj = 0;
2537 uint8_t* gc_heap::current_sweep_pos = 0;
2538 exclusive_sync* gc_heap::bgc_alloc_lock;
2539 #endif //BACKGROUND_GC
2541 oom_history gc_heap::oom_info;
2543 fgm_history gc_heap::fgm_result;
2545 BOOL gc_heap::ro_segments_in_range;
2547 size_t gc_heap::gen0_big_free_spaces = 0;
2549 uint8_t* gc_heap::ephemeral_low;
2551 uint8_t* gc_heap::ephemeral_high;
2553 uint8_t* gc_heap::lowest_address;
2555 uint8_t* gc_heap::highest_address;
2557 BOOL gc_heap::ephemeral_promotion;
2559 uint8_t* gc_heap::saved_ephemeral_plan_start[NUMBERGENERATIONS-1];
2560 size_t gc_heap::saved_ephemeral_plan_start_size[NUMBERGENERATIONS-1];
2562 short* gc_heap::brick_table;
2564 uint32_t* gc_heap::card_table;
2567 uint32_t* gc_heap::card_bundle_table;
2568 #endif //CARD_BUNDLE
2570 uint8_t* gc_heap::gc_low;
2572 uint8_t* gc_heap::gc_high;
2574 uint8_t* gc_heap::demotion_low;
2576 uint8_t* gc_heap::demotion_high;
2578 BOOL gc_heap::demote_gen1_p = TRUE;
2580 uint8_t* gc_heap::last_gen1_pin_end;
2582 gen_to_condemn_tuning gc_heap::gen_to_condemn_reasons;
2584 size_t gc_heap::etw_allocation_running_amount[2];
2586 int gc_heap::gc_policy = 0;
2588 size_t gc_heap::allocation_running_time;
2590 size_t gc_heap::allocation_running_amount;
2592 heap_segment* gc_heap::ephemeral_heap_segment = 0;
2594 BOOL gc_heap::blocking_collection = FALSE;
2596 heap_segment* gc_heap::freeable_large_heap_segment = 0;
2598 size_t gc_heap::time_bgc_last = 0;
2600 size_t gc_heap::mark_stack_tos = 0;
2602 size_t gc_heap::mark_stack_bos = 0;
2604 size_t gc_heap::mark_stack_array_length = 0;
2606 mark* gc_heap::mark_stack_array = 0;
2608 BOOL gc_heap::verify_pinned_queue_p = FALSE;
2610 uint8_t* gc_heap::oldest_pinned_plug = 0;
2612 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
2613 size_t gc_heap::num_pinned_objects = 0;
2614 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
2616 #ifdef FEATURE_LOH_COMPACTION
2617 size_t gc_heap::loh_pinned_queue_tos = 0;
2619 size_t gc_heap::loh_pinned_queue_bos = 0;
2621 size_t gc_heap::loh_pinned_queue_length = 0;
2623 mark* gc_heap::loh_pinned_queue = 0;
2625 BOOL gc_heap::loh_compacted_p = FALSE;
2626 #endif //FEATURE_LOH_COMPACTION
2628 #ifdef BACKGROUND_GC
2630 EEThreadId gc_heap::bgc_thread_id;
2632 uint8_t* gc_heap::background_written_addresses [array_size+2];
2634 heap_segment* gc_heap::freeable_small_heap_segment = 0;
2636 size_t gc_heap::bgc_overflow_count = 0;
2638 size_t gc_heap::bgc_begin_loh_size = 0;
2639 size_t gc_heap::end_loh_size = 0;
2641 uint32_t gc_heap::bgc_alloc_spin_loh = 0;
2643 size_t gc_heap::bgc_loh_size_increased = 0;
2645 size_t gc_heap::bgc_loh_allocated_in_free = 0;
2647 size_t gc_heap::background_soh_alloc_count = 0;
2649 size_t gc_heap::background_loh_alloc_count = 0;
2651 uint8_t** gc_heap::background_mark_stack_tos = 0;
2653 uint8_t** gc_heap::background_mark_stack_array = 0;
2655 size_t gc_heap::background_mark_stack_array_length = 0;
2657 uint8_t* gc_heap::background_min_overflow_address =0;
2659 uint8_t* gc_heap::background_max_overflow_address =0;
2661 BOOL gc_heap::processed_soh_overflow_p = FALSE;
2663 uint8_t* gc_heap::background_min_soh_overflow_address =0;
2665 uint8_t* gc_heap::background_max_soh_overflow_address =0;
2667 heap_segment* gc_heap::saved_sweep_ephemeral_seg = 0;
2669 uint8_t* gc_heap::saved_sweep_ephemeral_start = 0;
2671 heap_segment* gc_heap::saved_overflow_ephemeral_seg = 0;
2673 Thread* gc_heap::bgc_thread = 0;
2675 BOOL gc_heap::expanded_in_fgc = FALSE;
2677 uint8_t** gc_heap::c_mark_list = 0;
2679 size_t gc_heap::c_mark_list_length = 0;
2681 size_t gc_heap::c_mark_list_index = 0;
2683 gc_history_per_heap gc_heap::bgc_data_per_heap;
2685 BOOL gc_heap::bgc_thread_running;
2687 CLRCriticalSection gc_heap::bgc_threads_timeout_cs;
2689 GCEvent gc_heap::gc_lh_block_event;
2691 #endif //BACKGROUND_GC
2694 uint8_t** gc_heap::mark_list;
2695 uint8_t** gc_heap::mark_list_index;
2696 uint8_t** gc_heap::mark_list_end;
2700 snoop_stats_data gc_heap::snoop_stat;
2701 #endif //SNOOP_STATS
2703 uint8_t* gc_heap::min_overflow_address = MAX_PTR;
2705 uint8_t* gc_heap::max_overflow_address = 0;
2707 uint8_t* gc_heap::shigh = 0;
2709 uint8_t* gc_heap::slow = MAX_PTR;
2711 size_t gc_heap::ordered_free_space_indices[MAX_NUM_BUCKETS];
2713 size_t gc_heap::saved_ordered_free_space_indices[MAX_NUM_BUCKETS];
2715 size_t gc_heap::ordered_plug_indices[MAX_NUM_BUCKETS];
2717 size_t gc_heap::saved_ordered_plug_indices[MAX_NUM_BUCKETS];
2719 BOOL gc_heap::ordered_plug_indices_init = FALSE;
2721 BOOL gc_heap::use_bestfit = FALSE;
2723 uint8_t* gc_heap::bestfit_first_pin = 0;
2725 BOOL gc_heap::commit_end_of_seg = FALSE;
2727 size_t gc_heap::max_free_space_items = 0;
2729 size_t gc_heap::free_space_buckets = 0;
2731 size_t gc_heap::free_space_items = 0;
2733 int gc_heap::trimmed_free_space_index = 0;
2735 size_t gc_heap::total_ephemeral_plugs = 0;
2737 seg_free_spaces* gc_heap::bestfit_seg = 0;
2739 size_t gc_heap::total_ephemeral_size = 0;
2743 size_t gc_heap::internal_root_array_length = initial_internal_roots;
2745 uint8_t** gc_heap::internal_root_array = 0;
2747 size_t gc_heap::internal_root_array_index = 0;
2749 BOOL gc_heap::heap_analyze_success = TRUE;
2751 uint8_t* gc_heap::current_obj = 0;
2752 size_t gc_heap::current_obj_size = 0;
2754 #endif //HEAP_ANALYZE
2756 #ifdef GC_CONFIG_DRIVEN
2757 size_t gc_heap::interesting_data_per_gc[max_idp_count];
2758 //size_t gc_heap::interesting_data_per_heap[max_idp_count];
2759 //size_t gc_heap::interesting_mechanisms_per_heap[max_im_count];
2760 #endif //GC_CONFIG_DRIVEN
2761 #endif //MULTIPLE_HEAPS
2763 no_gc_region_info gc_heap::current_no_gc_region_info;
2764 BOOL gc_heap::proceed_with_gc_p = FALSE;
2765 GCSpinLock gc_heap::gc_lock;
2767 size_t gc_heap::eph_gen_starts_size = 0;
2768 heap_segment* gc_heap::segment_standby_list;
2769 size_t gc_heap::last_gc_index = 0;
2770 #ifdef SEG_MAPPING_TABLE
2771 size_t gc_heap::min_segment_size = 0;
2772 size_t gc_heap::min_segment_size_shr = 0;
2773 #endif //SEG_MAPPING_TABLE
2774 size_t gc_heap::soh_segment_size = 0;
2775 size_t gc_heap::min_loh_segment_size = 0;
2776 size_t gc_heap::segment_info_size = 0;
2778 #ifdef GC_CONFIG_DRIVEN
2779 size_t gc_heap::time_init = 0;
2780 size_t gc_heap::time_since_init = 0;
2781 size_t gc_heap::compact_or_sweep_gcs[2];
2782 #endif //GC_CONFIG_DRIVEN
2784 #ifdef FEATURE_LOH_COMPACTION
2785 BOOL gc_heap::loh_compaction_always_p = FALSE;
2786 gc_loh_compaction_mode gc_heap::loh_compaction_mode = loh_compaction_default;
2787 int gc_heap::loh_pinned_queue_decay = LOH_PIN_DECAY;
2789 #endif //FEATURE_LOH_COMPACTION
2791 GCEvent gc_heap::full_gc_approach_event;
2793 GCEvent gc_heap::full_gc_end_event;
2795 uint32_t gc_heap::fgn_maxgen_percent = 0;
2797 uint32_t gc_heap::fgn_loh_percent = 0;
2799 #ifdef BACKGROUND_GC
2800 BOOL gc_heap::fgn_last_gc_was_concurrent = FALSE;
2801 #endif //BACKGROUND_GC
2803 VOLATILE(bool) gc_heap::full_gc_approach_event_set;
2805 size_t gc_heap::full_gc_counts[gc_type_max];
2807 BOOL gc_heap::should_expand_in_full_gc = FALSE;
2810 BOOL gc_heap::heap_analyze_enabled = FALSE;
2811 #endif //HEAP_ANALYZE
2813 #ifndef MULTIPLE_HEAPS
2815 alloc_list gc_heap::loh_alloc_list [NUM_LOH_ALIST-1];
2816 alloc_list gc_heap::gen2_alloc_list[NUM_GEN2_ALIST-1];
2818 dynamic_data gc_heap::dynamic_data_table [NUMBERGENERATIONS+1];
2819 gc_history_per_heap gc_heap::gc_data_per_heap;
2820 size_t gc_heap::maxgen_pinned_compact_before_advance = 0;
2822 uint8_t* gc_heap::alloc_allocated = 0;
2824 size_t gc_heap::allocation_quantum = CLR_SIZE;
2826 GCSpinLock gc_heap::more_space_lock;
2828 #ifdef SYNCHRONIZATION_STATS
2829 unsigned int gc_heap::good_suspension = 0;
2830 unsigned int gc_heap::bad_suspension = 0;
2831 uint64_t gc_heap::total_msl_acquire = 0;
2832 unsigned int gc_heap::num_msl_acquired = 0;
2833 unsigned int gc_heap::num_high_msl_acquire = 0;
2834 unsigned int gc_heap::num_low_msl_acquire = 0;
2835 #endif //SYNCHRONIZATION_STATS
2837 size_t gc_heap::alloc_contexts_used = 0;
2838 size_t gc_heap::soh_allocation_no_gc = 0;
2839 size_t gc_heap::loh_allocation_no_gc = 0;
2840 bool gc_heap::no_gc_oom_p = false;
2841 heap_segment* gc_heap::saved_loh_segment_no_gc = 0;
2843 #endif //MULTIPLE_HEAPS
2845 #ifndef MULTIPLE_HEAPS
2847 BOOL gc_heap::gen0_bricks_cleared = FALSE;
2850 int gc_heap::gen0_must_clear_bricks = 0;
2851 #endif //FFIND_OBJECT
2853 #ifdef FEATURE_PREMORTEM_FINALIZATION
2854 CFinalize* gc_heap::finalize_queue = 0;
2855 #endif // FEATURE_PREMORTEM_FINALIZATION
2857 generation gc_heap::generation_table [NUMBERGENERATIONS + 1];
2859 size_t gc_heap::interesting_data_per_heap[max_idp_count];
2861 size_t gc_heap::compact_reasons_per_heap[max_compact_reasons_count];
2863 size_t gc_heap::expand_mechanisms_per_heap[max_expand_mechanisms_count];
2865 size_t gc_heap::interesting_mechanism_bits_per_heap[max_gc_mechanism_bits_count];
2867 #endif // MULTIPLE_HEAPS
2869 /* end of per heap static initialization */
2871 /* end of static initialization */
2873 #ifndef DACCESS_COMPILE
2875 void gen_to_condemn_tuning::print (int heap_num)
2878 dprintf (DT_LOG_0, ("condemned reasons (%d %d)", condemn_reasons_gen, condemn_reasons_condition));
2879 dprintf (DT_LOG_0, ("%s", record_condemn_reasons_gen_header));
2880 gc_condemn_reason_gen r_gen;
2881 for (int i = 0; i < gcrg_max; i++)
2883 r_gen = (gc_condemn_reason_gen)(i);
2884 str_reasons_gen[i * 2] = get_gen_char (get_gen (r_gen));
2886 dprintf (DT_LOG_0, ("[%2d]%s", heap_num, str_reasons_gen));
2888 dprintf (DT_LOG_0, ("%s", record_condemn_reasons_condition_header));
2889 gc_condemn_reason_condition r_condition;
2890 for (int i = 0; i < gcrc_max; i++)
2892 r_condition = (gc_condemn_reason_condition)(i);
2893 str_reasons_condition[i * 2] = get_condition_char (get_condition (r_condition));
2896 dprintf (DT_LOG_0, ("[%2d]%s", heap_num, str_reasons_condition));
2898 UNREFERENCED_PARAMETER(heap_num);
2902 void gc_generation_data::print (int heap_num, int gen_num)
2904 #if defined(SIMPLE_DPRINTF) && defined(DT_LOG)
2905 dprintf (DT_LOG_0, ("[%2d]gen%d beg %Id fl %Id fo %Id end %Id fl %Id fo %Id in %Id p %Id np %Id alloc %Id",
2908 free_list_space_before, free_obj_space_before,
2910 free_list_space_after, free_obj_space_after,
2911 in, pinned_surv, npinned_surv,
2914 UNREFERENCED_PARAMETER(heap_num);
2915 UNREFERENCED_PARAMETER(gen_num);
2916 #endif //SIMPLE_DPRINTF && DT_LOG
2919 void gc_history_per_heap::set_mechanism (gc_mechanism_per_heap mechanism_per_heap, uint32_t value)
2921 uint32_t* mechanism = &mechanisms[mechanism_per_heap];
2923 *mechanism |= mechanism_mask;
2924 *mechanism |= (1 << value);
2927 gc_mechanism_descr* descr = &gc_mechanisms_descr[mechanism_per_heap];
2928 dprintf (DT_LOG_0, ("setting %s: %s",
2930 (descr->descr)[value]));
2934 void gc_history_per_heap::print()
2936 #if defined(SIMPLE_DPRINTF) && defined(DT_LOG)
2937 for (int i = 0; i < (sizeof (gen_data)/sizeof (gc_generation_data)); i++)
2939 gen_data[i].print (heap_index, i);
2942 dprintf (DT_LOG_0, ("fla %Id flr %Id esa %Id ca %Id pa %Id paa %Id, rfle %d, ec %Id",
2943 maxgen_size_info.free_list_allocated,
2944 maxgen_size_info.free_list_rejected,
2945 maxgen_size_info.end_seg_allocated,
2946 maxgen_size_info.condemned_allocated,
2947 maxgen_size_info.pinned_allocated,
2948 maxgen_size_info.pinned_allocated_advance,
2949 maxgen_size_info.running_free_list_efficiency,
2950 extra_gen0_committed));
2953 gc_mechanism_descr* descr = 0;
2955 for (int i = 0; i < max_mechanism_per_heap; i++)
2957 mechanism = get_mechanism ((gc_mechanism_per_heap)i);
2961 descr = &gc_mechanisms_descr[(gc_mechanism_per_heap)i];
2962 dprintf (DT_LOG_0, ("[%2d]%s%s",
2965 (descr->descr)[mechanism]));
2968 #endif //SIMPLE_DPRINTF && DT_LOG
2971 void gc_history_global::print()
2974 char str_settings[64];
2975 memset (str_settings, '|', sizeof (char) * 64);
2976 str_settings[max_global_mechanisms_count*2] = 0;
2978 for (int i = 0; i < max_global_mechanisms_count; i++)
2980 str_settings[i * 2] = (get_mechanism_p ((gc_global_mechanism_p)i) ? 'Y' : 'N');
2983 dprintf (DT_LOG_0, ("[hp]|c|p|o|d|b|e|"));
2985 dprintf (DT_LOG_0, ("%4d|%s", num_heaps, str_settings));
2986 dprintf (DT_LOG_0, ("Condemned gen%d(reason: %s; mode: %s), youngest budget %Id(%d), memload %d",
2987 condemned_generation,
2988 str_gc_reasons[reason],
2989 str_gc_pause_modes[pause_mode],
2990 final_youngest_desired,
2991 gen0_reduction_count,
2996 void gc_heap::fire_per_heap_hist_event (gc_history_per_heap* current_gc_data_per_heap, int heap_num)
2998 maxgen_size_increase* maxgen_size_info = &(current_gc_data_per_heap->maxgen_size_info);
2999 FIRE_EVENT(GCPerHeapHistory_V3,
3000 (void *)(maxgen_size_info->free_list_allocated),
3001 (void *)(maxgen_size_info->free_list_rejected),
3002 (void *)(maxgen_size_info->end_seg_allocated),
3003 (void *)(maxgen_size_info->condemned_allocated),
3004 (void *)(maxgen_size_info->pinned_allocated),
3005 (void *)(maxgen_size_info->pinned_allocated_advance),
3006 maxgen_size_info->running_free_list_efficiency,
3007 current_gc_data_per_heap->gen_to_condemn_reasons.get_reasons0(),
3008 current_gc_data_per_heap->gen_to_condemn_reasons.get_reasons1(),
3009 current_gc_data_per_heap->mechanisms[gc_heap_compact],
3010 current_gc_data_per_heap->mechanisms[gc_heap_expand],
3011 current_gc_data_per_heap->heap_index,
3012 (void *)(current_gc_data_per_heap->extra_gen0_committed),
3013 (max_generation + 2),
3014 (uint32_t)(sizeof (gc_generation_data)),
3015 (void *)&(current_gc_data_per_heap->gen_data[0]));
3017 current_gc_data_per_heap->print();
3018 current_gc_data_per_heap->gen_to_condemn_reasons.print (heap_num);
3021 void gc_heap::fire_pevents()
3024 settings.record (&gc_data_global);
3025 gc_data_global.print();
3027 FIRE_EVENT(GCGlobalHeapHistory_V2, gc_data_global.final_youngest_desired,
3028 gc_data_global.num_heaps,
3029 gc_data_global.condemned_generation,
3030 gc_data_global.gen0_reduction_count,
3031 gc_data_global.reason,
3032 gc_data_global.global_mechanims_p,
3033 gc_data_global.pause_mode,
3034 gc_data_global.mem_pressure);
3036 #ifdef MULTIPLE_HEAPS
3037 for (int i = 0; i < gc_heap::n_heaps; i++)
3039 gc_heap* hp = gc_heap::g_heaps[i];
3040 gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap();
3041 fire_per_heap_hist_event (current_gc_data_per_heap, hp->heap_number);
3044 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
3045 fire_per_heap_hist_event (current_gc_data_per_heap, heap_number);
3051 gc_heap::dt_low_ephemeral_space_p (gc_tuning_point tp)
3057 case tuning_deciding_condemned_gen:
3058 case tuning_deciding_compaction:
3059 case tuning_deciding_expansion:
3060 case tuning_deciding_full_gc:
3062 ret = (!ephemeral_gen_fit_p (tp));
3065 case tuning_deciding_promote_ephemeral:
3067 size_t new_gen0size = approximate_new_allocation();
3068 ptrdiff_t plan_ephemeral_size = total_ephemeral_size;
3070 dprintf (GTC_LOG, ("h%d: plan eph size is %Id, new gen0 is %Id",
3071 heap_number, plan_ephemeral_size, new_gen0size));
3073 // If we were in no_gc_region we could have allocated a larger than normal segment,
3074 // and the next seg we allocate will be a normal sized seg so if we can't fit the new
3075 // ephemeral generations there, do an ephemeral promotion.
3076 ret = ((soh_segment_size - segment_info_size) < (plan_ephemeral_size + new_gen0size));
3088 gc_heap::dt_high_frag_p (gc_tuning_point tp,
3096 case tuning_deciding_condemned_gen:
3098 dynamic_data* dd = dynamic_data_of (gen_number);
3099 float fragmentation_burden = 0;
3103 ret = (dd_fragmentation (dynamic_data_of (max_generation)) >= dd_max_size(dd));
3104 dprintf (GTC_LOG, ("h%d: frag is %Id, max size is %Id",
3105 heap_number, dd_fragmentation (dd), dd_max_size(dd)));
3109 #ifndef MULTIPLE_HEAPS
3110 if (gen_number == max_generation)
3112 float frag_ratio = (float)(dd_fragmentation (dynamic_data_of (max_generation))) / (float)generation_size (max_generation);
3113 if (frag_ratio > 0.65)
3115 dprintf (GTC_LOG, ("g2 FR: %d%%", (int)(frag_ratio*100)));
3119 #endif //!MULTIPLE_HEAPS
3120 size_t fr = generation_unusable_fragmentation (generation_of (gen_number));
3121 ret = (fr > dd_fragmentation_limit(dd));
3124 fragmentation_burden = (float)fr / generation_size (gen_number);
3125 ret = (fragmentation_burden > dd_v_fragmentation_burden_limit (dd));
3127 dprintf (GTC_LOG, ("h%d: gen%d, frag is %Id, alloc effi: %d%%, unusable frag is %Id, ratio is %d",
3128 heap_number, gen_number, dd_fragmentation (dd),
3129 (int)(100*generation_allocator_efficiency (generation_of (gen_number))),
3130 fr, (int)(fragmentation_burden*100)));
3142 gc_heap::dt_estimate_reclaim_space_p (gc_tuning_point tp, int gen_number)
3148 case tuning_deciding_condemned_gen:
3150 if (gen_number == max_generation)
3152 dynamic_data* dd = dynamic_data_of (gen_number);
3153 size_t maxgen_allocated = (dd_desired_allocation (dd) - dd_new_allocation (dd));
3154 size_t maxgen_total_size = maxgen_allocated + dd_current_size (dd);
3155 size_t est_maxgen_surv = (size_t)((float) (maxgen_total_size) * dd_surv (dd));
3156 size_t est_maxgen_free = maxgen_total_size - est_maxgen_surv + dd_fragmentation (dd);
3158 dprintf (GTC_LOG, ("h%d: Total gen2 size: %Id, est gen2 dead space: %Id (s: %d, allocated: %Id), frag: %Id",
3162 (int)(dd_surv (dd) * 100),
3164 dd_fragmentation (dd)));
3166 uint32_t num_heaps = 1;
3168 #ifdef MULTIPLE_HEAPS
3169 num_heaps = gc_heap::n_heaps;
3170 #endif //MULTIPLE_HEAPS
3172 size_t min_frag_th = min_reclaim_fragmentation_threshold (num_heaps);
3173 dprintf (GTC_LOG, ("h%d, min frag is %Id", heap_number, min_frag_th));
3174 ret = (est_maxgen_free >= min_frag_th);
3190 // DTREVIEW: Right now we only estimate gen2 fragmentation.
3191 // on 64-bit though we should consider gen1 or even gen0 fragmentatioin as
3194 gc_heap::dt_estimate_high_frag_p (gc_tuning_point tp, int gen_number, uint64_t available_mem)
3200 case tuning_deciding_condemned_gen:
3202 if (gen_number == max_generation)
3204 dynamic_data* dd = dynamic_data_of (gen_number);
3205 float est_frag_ratio = 0;
3206 if (dd_current_size (dd) == 0)
3210 else if ((dd_fragmentation (dd) == 0) || (dd_fragmentation (dd) + dd_current_size (dd) == 0))
3216 est_frag_ratio = (float)dd_fragmentation (dd) / (float)(dd_fragmentation (dd) + dd_current_size (dd));
3219 size_t est_frag = (dd_fragmentation (dd) + (size_t)((dd_desired_allocation (dd) - dd_new_allocation (dd)) * est_frag_ratio));
3220 dprintf (GTC_LOG, ("h%d: gen%d: current_size is %Id, frag is %Id, est_frag_ratio is %d%%, estimated frag is %Id",
3223 dd_current_size (dd),
3224 dd_fragmentation (dd),
3225 (int)(est_frag_ratio*100),
3228 uint32_t num_heaps = 1;
3230 #ifdef MULTIPLE_HEAPS
3231 num_heaps = gc_heap::n_heaps;
3232 #endif //MULTIPLE_HEAPS
3233 uint64_t min_frag_th = min_high_fragmentation_threshold(available_mem, num_heaps);
3234 //dprintf (GTC_LOG, ("h%d, min frag is %I64d", heap_number, min_frag_th));
3235 ret = (est_frag >= min_frag_th);
3252 gc_heap::dt_low_card_table_efficiency_p (gc_tuning_point tp)
3258 case tuning_deciding_condemned_gen:
3260 /* promote into max-generation if the card table has too many
3261 * generation faults besides the n -> 0
3263 ret = (generation_skip_ratio < 30);
3275 in_range_for_segment(uint8_t* add, heap_segment* seg)
3277 return ((add >= heap_segment_mem (seg)) && (add < heap_segment_reserved (seg)));
3280 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
3281 // The array we allocate is organized as follows:
3282 // 0th element is the address of the last array we allocated.
3283 // starting from the 1st element are the segment addresses, that's
3284 // what buckets() returns.
3297 bk* buckets() { return (slots + 1); }
3298 uint8_t*& last_slot (bk* arr) { return arr[0].add; }
3301 static sorted_table* make_sorted_table ();
3302 BOOL insert (uint8_t* add, size_t val);;
3303 size_t lookup (uint8_t*& add);
3304 void remove (uint8_t* add);
3306 void delete_sorted_table();
3307 void delete_old_slots();
3308 void enqueue_old_slot(bk* sl);
3309 BOOL ensure_space_for_insert();
3313 sorted_table::make_sorted_table ()
3317 // allocate one more bk to store the older slot address.
3318 sorted_table* res = (sorted_table*)new char [sizeof (sorted_table) + (size + 1) * sizeof (bk)];
3322 res->slots = (bk*)(res + 1);
3329 sorted_table::delete_sorted_table()
3331 if (slots != (bk*)(this+1))
3339 sorted_table::delete_old_slots()
3341 uint8_t* sl = (uint8_t*)old_slots;
3345 sl = last_slot ((bk*)sl);
3351 sorted_table::enqueue_old_slot(bk* sl)
3353 last_slot (sl) = (uint8_t*)old_slots;
3359 sorted_table::lookup (uint8_t*& add)
3361 ptrdiff_t high = (count-1);
3365 bk* buck = buckets();
3368 mid = ((low + high)/2);
3370 if (buck[ti].add > add)
3372 if ((ti > 0) && (buck[ti-1].add <= add))
3374 add = buck[ti-1].add;
3375 return buck[ti - 1].val;
3381 if (buck[ti+1].add > add)
3384 return buck[ti].val;
3394 sorted_table::ensure_space_for_insert()
3398 size = (size * 3)/2;
3399 assert((size * sizeof (bk)) > 0);
3400 bk* res = (bk*)new (nothrow) char [(size + 1) * sizeof (bk)];
3405 last_slot (res) = 0;
3406 memcpy (((bk*)res + 1), buckets(), count * sizeof (bk));
3407 bk* last_old_slots = slots;
3409 if (last_old_slots != (bk*)(this + 1))
3410 enqueue_old_slot (last_old_slots);
3416 sorted_table::insert (uint8_t* add, size_t val)
3418 //grow if no more room
3419 assert (count < size);
3422 ptrdiff_t high = (count-1);
3426 bk* buck = buckets();
3429 mid = ((low + high)/2);
3431 if (buck[ti].add > add)
3433 if ((ti == 0) || (buck[ti-1].add <= add))
3435 // found insertion point
3436 for (ptrdiff_t k = count; k > ti;k--)
3438 buck [k] = buck [k-1];
3449 if (buck[ti+1].add > add)
3451 //found the insertion point
3452 for (ptrdiff_t k = count; k > ti+1;k--)
3454 buck [k] = buck [k-1];
3456 buck[ti+1].add = add;
3457 buck[ti+1].val = val;
3469 sorted_table::remove (uint8_t* add)
3471 ptrdiff_t high = (count-1);
3475 bk* buck = buckets();
3478 mid = ((low + high)/2);
3480 if (buck[ti].add > add)
3482 if (buck[ti-1].add <= add)
3484 // found the guy to remove
3485 for (ptrdiff_t k = ti; k < count; k++)
3486 buck[k-1] = buck[k];
3494 if (buck[ti+1].add > add)
3496 // found the guy to remove
3497 for (ptrdiff_t k = ti+1; k < count; k++)
3498 buck[k-1] = buck[k];
3509 sorted_table::clear()
3512 buckets()[0].add = MAX_PTR;
3514 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
3516 #ifdef SEG_MAPPING_TABLE
3517 #ifdef GROWABLE_SEG_MAPPING_TABLE
3519 uint8_t* align_on_segment (uint8_t* add)
3521 return (uint8_t*)((size_t)(add + (gc_heap::min_segment_size - 1)) & ~(gc_heap::min_segment_size - 1));
3525 uint8_t* align_lower_segment (uint8_t* add)
3527 return (uint8_t*)((size_t)(add) & ~(gc_heap::min_segment_size - 1));
3530 size_t size_seg_mapping_table_of (uint8_t* from, uint8_t* end)
3532 from = align_lower_segment (from);
3533 end = align_on_segment (end);
3534 dprintf (1, ("from: %Ix, end: %Ix, size: %Ix", from, end, sizeof (seg_mapping)*(((size_t)(end - from) >> gc_heap::min_segment_size_shr))));
3535 return sizeof (seg_mapping)*((size_t)(end - from) >> gc_heap::min_segment_size_shr);
3538 // for seg_mapping_table we want it to start from a pointer sized address.
3540 size_t align_for_seg_mapping_table (size_t size)
3542 return ((size + (sizeof (uint8_t*) - 1)) &~ (sizeof (uint8_t*) - 1));
3546 size_t seg_mapping_word_of (uint8_t* add)
3548 return (size_t)add >> gc_heap::min_segment_size_shr;
3550 #else //GROWABLE_SEG_MAPPING_TABLE
3551 BOOL seg_mapping_table_init()
3554 uint64_t total_address_space = (uint64_t)8*1024*1024*1024*1024;
3556 uint64_t total_address_space = (uint64_t)4*1024*1024*1024;
3559 size_t num_entries = (size_t)(total_address_space >> gc_heap::min_segment_size_shr);
3560 seg_mapping_table = new seg_mapping[num_entries];
3562 if (seg_mapping_table)
3564 memset (seg_mapping_table, 0, num_entries * sizeof (seg_mapping));
3565 dprintf (1, ("created %d entries for heap mapping (%Id bytes)",
3566 num_entries, (num_entries * sizeof (seg_mapping))));
3571 dprintf (1, ("failed to create %d entries for heap mapping (%Id bytes)",
3572 num_entries, (num_entries * sizeof (seg_mapping))));
3576 #endif //GROWABLE_SEG_MAPPING_TABLE
3578 #ifdef FEATURE_BASICFREEZE
3580 size_t ro_seg_begin_index (heap_segment* seg)
3582 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3583 begin_index = max (begin_index, (size_t)g_gc_lowest_address >> gc_heap::min_segment_size_shr);
3588 size_t ro_seg_end_index (heap_segment* seg)
3590 size_t end_index = (size_t)(heap_segment_reserved (seg) - 1) >> gc_heap::min_segment_size_shr;
3591 end_index = min (end_index, (size_t)g_gc_highest_address >> gc_heap::min_segment_size_shr);
3595 void seg_mapping_table_add_ro_segment (heap_segment* seg)
3597 #ifdef GROWABLE_SEG_MAPPING_TABLE
3598 if ((heap_segment_reserved (seg) <= g_gc_lowest_address) || (heap_segment_mem (seg) >= g_gc_highest_address))
3600 #endif //GROWABLE_SEG_MAPPING_TABLE
3602 for (size_t entry_index = ro_seg_begin_index (seg); entry_index <= ro_seg_end_index (seg); entry_index++)
3603 seg_mapping_table[entry_index].seg1 = (heap_segment*)((size_t)seg_mapping_table[entry_index].seg1 | ro_in_entry);
3606 void seg_mapping_table_remove_ro_segment (heap_segment* seg)
3608 UNREFERENCED_PARAMETER(seg);
3610 // POSSIBLE PERF TODO: right now we are not doing anything because we can't simply remove the flag. If it proves
3611 // to be a perf problem, we can search in the current ro segs and see if any lands in this range and only
3612 // remove the flag if none lands in this range.
3616 heap_segment* ro_segment_lookup (uint8_t* o)
3618 uint8_t* ro_seg_start = o;
3619 heap_segment* seg = (heap_segment*)gc_heap::seg_table->lookup (ro_seg_start);
3621 if (ro_seg_start && in_range_for_segment (o, seg))
3627 #endif //FEATURE_BASICFREEZE
3629 void gc_heap::seg_mapping_table_add_segment (heap_segment* seg, gc_heap* hp)
3631 size_t seg_end = (size_t)(heap_segment_reserved (seg) - 1);
3632 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3633 seg_mapping* begin_entry = &seg_mapping_table[begin_index];
3634 size_t end_index = seg_end >> gc_heap::min_segment_size_shr;
3635 seg_mapping* end_entry = &seg_mapping_table[end_index];
3637 dprintf (1, ("adding seg %Ix(%d)-%Ix(%d)",
3638 seg, begin_index, heap_segment_reserved (seg), end_index));
3640 dprintf (1, ("before add: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3641 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3642 end_index, (seg_mapping_table[end_index].boundary + 1)));
3644 #ifdef MULTIPLE_HEAPS
3645 #ifdef SIMPLE_DPRINTF
3646 dprintf (1, ("begin %d: h0: %Ix(%d), h1: %Ix(%d); end %d: h0: %Ix(%d), h1: %Ix(%d)",
3647 begin_index, (uint8_t*)(begin_entry->h0), (begin_entry->h0 ? begin_entry->h0->heap_number : -1),
3648 (uint8_t*)(begin_entry->h1), (begin_entry->h1 ? begin_entry->h1->heap_number : -1),
3649 end_index, (uint8_t*)(end_entry->h0), (end_entry->h0 ? end_entry->h0->heap_number : -1),
3650 (uint8_t*)(end_entry->h1), (end_entry->h1 ? end_entry->h1->heap_number : -1)));
3651 #endif //SIMPLE_DPRINTF
3652 assert (end_entry->boundary == 0);
3653 assert (end_entry->h0 == 0);
3655 assert (begin_entry->h1 == 0);
3656 begin_entry->h1 = hp;
3658 UNREFERENCED_PARAMETER(hp);
3659 #endif //MULTIPLE_HEAPS
3661 end_entry->boundary = (uint8_t*)seg_end;
3663 dprintf (1, ("set entry %d seg1 and %d seg0 to %Ix", begin_index, end_index, seg));
3664 assert ((begin_entry->seg1 == 0) || ((size_t)(begin_entry->seg1) == ro_in_entry));
3665 begin_entry->seg1 = (heap_segment*)((size_t)(begin_entry->seg1) | (size_t)seg);
3666 end_entry->seg0 = seg;
3668 // for every entry inbetween we need to set its heap too.
3669 for (size_t entry_index = (begin_index + 1); entry_index <= (end_index - 1); entry_index++)
3671 assert (seg_mapping_table[entry_index].boundary == 0);
3672 #ifdef MULTIPLE_HEAPS
3673 assert (seg_mapping_table[entry_index].h0 == 0);
3674 seg_mapping_table[entry_index].h1 = hp;
3675 #endif //MULTIPLE_HEAPS
3676 seg_mapping_table[entry_index].seg1 = seg;
3679 dprintf (1, ("after add: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3680 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3681 end_index, (seg_mapping_table[end_index].boundary + 1)));
3682 #if defined(MULTIPLE_HEAPS) && defined(SIMPLE_DPRINTF)
3683 dprintf (1, ("begin %d: h0: %Ix(%d), h1: %Ix(%d); end: %d h0: %Ix(%d), h1: %Ix(%d)",
3684 begin_index, (uint8_t*)(begin_entry->h0), (begin_entry->h0 ? begin_entry->h0->heap_number : -1),
3685 (uint8_t*)(begin_entry->h1), (begin_entry->h1 ? begin_entry->h1->heap_number : -1),
3686 end_index, (uint8_t*)(end_entry->h0), (end_entry->h0 ? end_entry->h0->heap_number : -1),
3687 (uint8_t*)(end_entry->h1), (end_entry->h1 ? end_entry->h1->heap_number : -1)));
3688 #endif //MULTIPLE_HEAPS && SIMPLE_DPRINTF
3691 void gc_heap::seg_mapping_table_remove_segment (heap_segment* seg)
3693 size_t seg_end = (size_t)(heap_segment_reserved (seg) - 1);
3694 size_t begin_index = (size_t)seg >> gc_heap::min_segment_size_shr;
3695 seg_mapping* begin_entry = &seg_mapping_table[begin_index];
3696 size_t end_index = seg_end >> gc_heap::min_segment_size_shr;
3697 seg_mapping* end_entry = &seg_mapping_table[end_index];
3698 dprintf (1, ("removing seg %Ix(%d)-%Ix(%d)",
3699 seg, begin_index, heap_segment_reserved (seg), end_index));
3701 assert (end_entry->boundary == (uint8_t*)seg_end);
3702 end_entry->boundary = 0;
3704 #ifdef MULTIPLE_HEAPS
3705 gc_heap* hp = heap_segment_heap (seg);
3706 assert (end_entry->h0 == hp);
3708 assert (begin_entry->h1 == hp);
3709 begin_entry->h1 = 0;
3710 #endif //MULTIPLE_HEAPS
3712 assert (begin_entry->seg1 != 0);
3713 begin_entry->seg1 = (heap_segment*)((size_t)(begin_entry->seg1) & ro_in_entry);
3714 end_entry->seg0 = 0;
3716 // for every entry inbetween we need to reset its heap too.
3717 for (size_t entry_index = (begin_index + 1); entry_index <= (end_index - 1); entry_index++)
3719 assert (seg_mapping_table[entry_index].boundary == 0);
3720 #ifdef MULTIPLE_HEAPS
3721 assert (seg_mapping_table[entry_index].h0 == 0);
3722 assert (seg_mapping_table[entry_index].h1 == hp);
3723 seg_mapping_table[entry_index].h1 = 0;
3724 #endif //MULTIPLE_HEAPS
3725 seg_mapping_table[entry_index].seg1 = 0;
3728 dprintf (1, ("after remove: begin entry%d: boundary: %Ix; end entry: %d: boundary: %Ix",
3729 begin_index, (seg_mapping_table[begin_index].boundary + 1),
3730 end_index, (seg_mapping_table[end_index].boundary + 1)));
3731 #ifdef MULTIPLE_HEAPS
3732 dprintf (1, ("begin %d: h0: %Ix, h1: %Ix; end: %d h0: %Ix, h1: %Ix",
3733 begin_index, (uint8_t*)(begin_entry->h0), (uint8_t*)(begin_entry->h1),
3734 end_index, (uint8_t*)(end_entry->h0), (uint8_t*)(end_entry->h1)));
3735 #endif //MULTIPLE_HEAPS
3738 #ifdef MULTIPLE_HEAPS
3740 gc_heap* seg_mapping_table_heap_of_worker (uint8_t* o)
3742 size_t index = (size_t)o >> gc_heap::min_segment_size_shr;
3743 seg_mapping* entry = &seg_mapping_table[index];
3745 gc_heap* hp = ((o > entry->boundary) ? entry->h1 : entry->h0);
3747 dprintf (2, ("checking obj %Ix, index is %Id, entry: boundry: %Ix, h0: %Ix, seg0: %Ix, h1: %Ix, seg1: %Ix",
3748 o, index, (entry->boundary + 1),
3749 (uint8_t*)(entry->h0), (uint8_t*)(entry->seg0),
3750 (uint8_t*)(entry->h1), (uint8_t*)(entry->seg1)));
3753 heap_segment* seg = ((o > entry->boundary) ? entry->seg1 : entry->seg0);
3754 #ifdef FEATURE_BASICFREEZE
3755 if ((size_t)seg & ro_in_entry)
3756 seg = (heap_segment*)((size_t)seg & ~ro_in_entry);
3757 #endif //FEATURE_BASICFREEZE
3761 if (in_range_for_segment (o, seg))
3763 dprintf (2, ("obj %Ix belongs to segment %Ix(-%Ix)", o, seg, (uint8_t*)heap_segment_allocated (seg)));
3767 dprintf (2, ("found seg %Ix(-%Ix) for obj %Ix, but it's not on the seg",
3768 seg, (uint8_t*)heap_segment_allocated (seg), o));
3773 dprintf (2, ("could not find obj %Ix in any existing segments", o));
3780 gc_heap* seg_mapping_table_heap_of (uint8_t* o)
3782 #ifdef GROWABLE_SEG_MAPPING_TABLE
3783 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3785 #endif //GROWABLE_SEG_MAPPING_TABLE
3787 return seg_mapping_table_heap_of_worker (o);
3790 gc_heap* seg_mapping_table_heap_of_gc (uint8_t* o)
3792 #if defined(FEATURE_BASICFREEZE) && defined(GROWABLE_SEG_MAPPING_TABLE)
3793 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3795 #endif //FEATURE_BASICFREEZE || GROWABLE_SEG_MAPPING_TABLE
3797 return seg_mapping_table_heap_of_worker (o);
3799 #endif //MULTIPLE_HEAPS
3801 // Only returns a valid seg if we can actually find o on the seg.
3802 heap_segment* seg_mapping_table_segment_of (uint8_t* o)
3804 #if defined(FEATURE_BASICFREEZE) && defined(GROWABLE_SEG_MAPPING_TABLE)
3805 if ((o < g_gc_lowest_address) || (o >= g_gc_highest_address))
3806 #ifdef FEATURE_BASICFREEZE
3807 return ro_segment_lookup (o);
3810 #endif //FEATURE_BASICFREEZE
3811 #endif //FEATURE_BASICFREEZE || GROWABLE_SEG_MAPPING_TABLE
3813 size_t index = (size_t)o >> gc_heap::min_segment_size_shr;
3814 seg_mapping* entry = &seg_mapping_table[index];
3816 dprintf (2, ("checking obj %Ix, index is %Id, entry: boundry: %Ix, seg0: %Ix, seg1: %Ix",
3817 o, index, (entry->boundary + 1),
3818 (uint8_t*)(entry->seg0), (uint8_t*)(entry->seg1)));
3820 heap_segment* seg = ((o > entry->boundary) ? entry->seg1 : entry->seg0);
3821 #ifdef FEATURE_BASICFREEZE
3822 if ((size_t)seg & ro_in_entry)
3823 seg = (heap_segment*)((size_t)seg & ~ro_in_entry);
3824 #endif //FEATURE_BASICFREEZE
3828 if (in_range_for_segment (o, seg))
3830 dprintf (2, ("obj %Ix belongs to segment %Ix(-%Ix)", o, (uint8_t*)heap_segment_mem(seg), (uint8_t*)heap_segment_reserved(seg)));
3834 dprintf (2, ("found seg %Ix(-%Ix) for obj %Ix, but it's not on the seg, setting it to 0",
3835 (uint8_t*)heap_segment_mem(seg), (uint8_t*)heap_segment_reserved(seg), o));
3841 dprintf (2, ("could not find obj %Ix in any existing segments", o));
3844 #ifdef FEATURE_BASICFREEZE
3845 // TODO: This was originally written assuming that the seg_mapping_table would always contain entries for ro
3846 // segments whenever the ro segment falls into the [g_gc_lowest_address,g_gc_highest_address) range. I.e., it had an
3847 // extra "&& (size_t)(entry->seg1) & ro_in_entry" expression. However, at the moment, grow_brick_card_table does
3848 // not correctly go through the ro segments and add them back to the seg_mapping_table when the [lowest,highest)
3849 // range changes. We should probably go ahead and modify grow_brick_card_table and put back the
3850 // "&& (size_t)(entry->seg1) & ro_in_entry" here.
3853 seg = ro_segment_lookup (o);
3854 if (seg && !in_range_for_segment (o, seg))
3857 #endif //FEATURE_BASICFREEZE
3861 #endif //SEG_MAPPING_TABLE
3863 size_t gcard_of ( uint8_t*);
3865 #define memref(i) *(uint8_t**)(i)
3868 #define GC_MARKED (size_t)0x1
3869 #define slot(i, j) ((uint8_t**)(i))[j+1]
3871 #define free_object_base_size (plug_skew + sizeof(ArrayBase))
3873 class CObjectHeader : public Object
3877 #if defined(FEATURE_REDHAWK) || defined(BUILD_AS_STANDALONE)
3878 // The GC expects the following methods that are provided by the Object class in the CLR but not provided
3879 // by Redhawk's version of Object.
3880 uint32_t GetNumComponents()
3882 return ((ArrayBase *)this)->GetNumComponents();
3885 void Validate(BOOL bDeep=TRUE, BOOL bVerifyNextHeader = TRUE)
3887 UNREFERENCED_PARAMETER(bVerifyNextHeader);
3892 MethodTable * pMT = GetMethodTable();
3894 _ASSERTE(pMT->SanityCheck());
3896 bool noRangeChecks =
3897 (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_NO_RANGE_CHECKS) == GCConfig::HEAPVERIFY_NO_RANGE_CHECKS;
3899 BOOL fSmallObjectHeapPtr = FALSE, fLargeObjectHeapPtr = FALSE;
3902 fSmallObjectHeapPtr = g_theGCHeap->IsHeapPointer(this, TRUE);
3903 if (!fSmallObjectHeapPtr)
3904 fLargeObjectHeapPtr = g_theGCHeap->IsHeapPointer(this);
3906 _ASSERTE(fSmallObjectHeapPtr || fLargeObjectHeapPtr);
3909 #ifdef FEATURE_STRUCTALIGN
3910 _ASSERTE(IsStructAligned((uint8_t *)this, GetMethodTable()->GetBaseAlignment()));
3911 #endif // FEATURE_STRUCTALIGN
3913 #ifdef FEATURE_64BIT_ALIGNMENT
3914 if (pMT->RequiresAlign8())
3916 _ASSERTE((((size_t)this) & 0x7) == (pMT->IsValueType() ? 4U : 0U));
3918 #endif // FEATURE_64BIT_ALIGNMENT
3921 if (bDeep && (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC))
3922 g_theGCHeap->ValidateObjectMember(this);
3924 if (fSmallObjectHeapPtr)
3926 #ifdef FEATURE_BASICFREEZE
3927 _ASSERTE(!g_theGCHeap->IsLargeObject(pMT) || g_theGCHeap->IsInFrozenSegment(this));
3929 _ASSERTE(!g_theGCHeap->IsLargeObject(pMT));
3934 void ValidatePromote(ScanContext *sc, uint32_t flags)
3936 UNREFERENCED_PARAMETER(sc);
3937 UNREFERENCED_PARAMETER(flags);
3942 void ValidateHeap(Object *from, BOOL bDeep)
3944 UNREFERENCED_PARAMETER(from);
3946 Validate(bDeep, FALSE);
3949 ADIndex GetAppDomainIndex()
3951 return (ADIndex)RH_DEFAULT_DOMAIN_ID;
3953 #endif //FEATURE_REDHAWK
3957 // Header Status Information
3960 MethodTable *GetMethodTable() const
3962 return( (MethodTable *) (((size_t) RawGetMethodTable()) & (~(GC_MARKED))));
3967 RawSetMethodTable((MethodTable *) (((size_t) RawGetMethodTable()) | GC_MARKED));
3970 BOOL IsMarked() const
3972 return !!(((size_t)RawGetMethodTable()) & GC_MARKED);
3977 assert (!(gc_heap::settings.concurrent));
3978 GetHeader()->SetGCBit();
3981 BOOL IsPinned() const
3983 return !!((((CObjectHeader*)this)->GetHeader()->GetBits()) & BIT_SBLK_GC_RESERVE);
3988 RawSetMethodTable( GetMethodTable() );
3991 CGCDesc *GetSlotMap ()
3993 assert (GetMethodTable()->ContainsPointers());
3994 return CGCDesc::GetCGCDescFromMT(GetMethodTable());
3997 void SetFree(size_t size)
3999 assert (size >= free_object_base_size);
4001 assert (g_gc_pFreeObjectMethodTable->GetBaseSize() == free_object_base_size);
4002 assert (g_gc_pFreeObjectMethodTable->RawGetComponentSize() == 1);
4004 RawSetMethodTable( g_gc_pFreeObjectMethodTable );
4006 size_t* numComponentsPtr = (size_t*) &((uint8_t*) this)[ArrayBase::GetOffsetOfNumComponents()];
4007 *numComponentsPtr = size - free_object_base_size;
4009 //This introduces a bug in the free list management.
4010 //((void**) this)[-1] = 0; // clear the sync block,
4011 assert (*numComponentsPtr >= 0);
4012 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
4013 memset (((uint8_t*)this)+sizeof(ArrayBase), 0xcc, *numComponentsPtr);
4014 #endif //VERIFY_HEAP
4019 size_t size = free_object_base_size - plug_skew;
4021 // since we only need to clear 2 ptr size, we do it manually
4022 PTR_PTR m = (PTR_PTR) this;
4023 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
4027 BOOL IsFree () const
4029 return (GetMethodTable() == g_gc_pFreeObjectMethodTable);
4032 #ifdef FEATURE_STRUCTALIGN
4033 int GetRequiredAlignment () const
4035 return GetMethodTable()->GetRequiredAlignment();
4037 #endif // FEATURE_STRUCTALIGN
4039 BOOL ContainsPointers() const
4041 return GetMethodTable()->ContainsPointers();
4044 #ifdef COLLECTIBLE_CLASS
4045 BOOL Collectible() const
4047 return GetMethodTable()->Collectible();
4050 FORCEINLINE BOOL ContainsPointersOrCollectible() const
4052 MethodTable *pMethodTable = GetMethodTable();
4053 return (pMethodTable->ContainsPointers() || pMethodTable->Collectible());
4055 #endif //COLLECTIBLE_CLASS
4057 Object* GetObjectBase() const
4059 return (Object*) this;
4063 #define header(i) ((CObjectHeader*)(i))
4065 #define free_list_slot(x) ((uint8_t**)(x))[2]
4066 #define free_list_undo(x) ((uint8_t**)(x))[-1]
4067 #define UNDO_EMPTY ((uint8_t*)1)
4071 void set_plug_padded (uint8_t* node)
4073 header(node)->SetMarked();
4076 void clear_plug_padded (uint8_t* node)
4078 header(node)->ClearMarked();
4081 BOOL is_plug_padded (uint8_t* node)
4083 return header(node)->IsMarked();
4086 inline void set_plug_padded (uint8_t* node){}
4087 inline void clear_plug_padded (uint8_t* node){}
4089 BOOL is_plug_padded (uint8_t* node){return FALSE;}
4090 #endif //SHORT_PLUGS
4093 inline size_t unused_array_size(uint8_t * p)
4095 assert(((CObjectHeader*)p)->IsFree());
4097 size_t* numComponentsPtr = (size_t*)(p + ArrayBase::GetOffsetOfNumComponents());
4098 return free_object_base_size + *numComponentsPtr;
4101 heap_segment* heap_segment_rw (heap_segment* ns)
4103 if ((ns == 0) || !heap_segment_read_only_p (ns))
4111 ns = heap_segment_next (ns);
4112 } while ((ns != 0) && heap_segment_read_only_p (ns));
4117 //returns the next non ro segment.
4118 heap_segment* heap_segment_next_rw (heap_segment* seg)
4120 heap_segment* ns = heap_segment_next (seg);
4121 return heap_segment_rw (ns);
4124 // returns the segment before seg.
4125 heap_segment* heap_segment_prev_rw (heap_segment* begin, heap_segment* seg)
4127 assert (begin != 0);
4128 heap_segment* prev = begin;
4129 heap_segment* current = heap_segment_next_rw (begin);
4131 while (current && current != seg)
4134 current = heap_segment_next_rw (current);
4147 // returns the segment before seg.
4148 heap_segment* heap_segment_prev (heap_segment* begin, heap_segment* seg)
4150 assert (begin != 0);
4151 heap_segment* prev = begin;
4152 heap_segment* current = heap_segment_next (begin);
4154 while (current && current != seg)
4157 current = heap_segment_next (current);
4170 heap_segment* heap_segment_in_range (heap_segment* ns)
4172 if ((ns == 0) || heap_segment_in_range_p (ns))
4180 ns = heap_segment_next (ns);
4181 } while ((ns != 0) && !heap_segment_in_range_p (ns));
4186 heap_segment* heap_segment_next_in_range (heap_segment* seg)
4188 heap_segment* ns = heap_segment_next (seg);
4189 return heap_segment_in_range (ns);
4194 uint8_t* memory_base;
4199 imemory_data *initial_memory;
4200 imemory_data *initial_normal_heap; // points into initial_memory_array
4201 imemory_data *initial_large_heap; // points into initial_memory_array
4203 size_t block_size_normal;
4204 size_t block_size_large;
4206 size_t block_count; // # of blocks in each
4207 size_t current_block_normal;
4208 size_t current_block_large;
4217 size_t allocation_pattern;
4218 } initial_memory_details;
4220 initial_memory_details memory_details;
4222 BOOL reserve_initial_memory (size_t normal_size, size_t large_size, size_t num_heaps)
4224 BOOL reserve_success = FALSE;
4226 // should only be called once
4227 assert (memory_details.initial_memory == 0);
4229 memory_details.initial_memory = new (nothrow) imemory_data[num_heaps*2];
4230 if (memory_details.initial_memory == 0)
4232 dprintf (2, ("failed to reserve %Id bytes for imemory_data", num_heaps*2*sizeof(imemory_data)));
4236 memory_details.initial_normal_heap = memory_details.initial_memory;
4237 memory_details.initial_large_heap = memory_details.initial_memory + num_heaps;
4238 memory_details.block_size_normal = normal_size;
4239 memory_details.block_size_large = large_size;
4240 memory_details.block_count = num_heaps;
4242 memory_details.current_block_normal = 0;
4243 memory_details.current_block_large = 0;
4245 g_gc_lowest_address = MAX_PTR;
4246 g_gc_highest_address = 0;
4248 if (((size_t)MAX_PTR - large_size) < normal_size)
4250 // we are already overflowing with just one heap.
4251 dprintf (2, ("0x%Ix + 0x%Ix already overflow", normal_size, large_size));
4255 if (((size_t)MAX_PTR / memory_details.block_count) < (normal_size + large_size))
4257 dprintf (2, ("(0x%Ix + 0x%Ix)*0x%Ix overflow", normal_size, large_size, memory_details.block_count));
4261 size_t requestedMemory = memory_details.block_count * (normal_size + large_size);
4263 uint8_t* allatonce_block = (uint8_t*)virtual_alloc (requestedMemory);
4264 if (allatonce_block)
4266 g_gc_lowest_address = allatonce_block;
4267 g_gc_highest_address = allatonce_block + (memory_details.block_count * (large_size + normal_size));
4268 memory_details.allocation_pattern = initial_memory_details::ALLATONCE;
4270 for(size_t i = 0; i < memory_details.block_count; i++)
4272 memory_details.initial_normal_heap[i].memory_base = allatonce_block + (i*normal_size);
4273 memory_details.initial_large_heap[i].memory_base = allatonce_block +
4274 (memory_details.block_count*normal_size) + (i*large_size);
4275 reserve_success = TRUE;
4280 // try to allocate 2 blocks
4283 b1 = (uint8_t*)virtual_alloc (memory_details.block_count * normal_size);
4286 b2 = (uint8_t*)virtual_alloc (memory_details.block_count * large_size);
4289 memory_details.allocation_pattern = initial_memory_details::TWO_STAGE;
4290 g_gc_lowest_address = min(b1,b2);
4291 g_gc_highest_address = max(b1 + memory_details.block_count*normal_size,
4292 b2 + memory_details.block_count*large_size);
4293 for(size_t i = 0; i < memory_details.block_count; i++)
4295 memory_details.initial_normal_heap[i].memory_base = b1 + (i*normal_size);
4296 memory_details.initial_large_heap[i].memory_base = b2 + (i*large_size);
4297 reserve_success = TRUE;
4302 // b2 allocation failed, we'll go on to try allocating each block.
4303 // We could preserve the b1 alloc, but code complexity increases
4304 virtual_free (b1, memory_details.block_count * normal_size);
4308 if ((b2==NULL) && ( memory_details.block_count > 1))
4310 memory_details.allocation_pattern = initial_memory_details::EACH_BLOCK;
4312 imemory_data *current_block = memory_details.initial_memory;
4313 for(size_t i = 0; i < (memory_details.block_count*2); i++, current_block++)
4315 size_t block_size = ((i < memory_details.block_count) ?
4316 memory_details.block_size_normal :
4317 memory_details.block_size_large);
4318 current_block->memory_base =
4319 (uint8_t*)virtual_alloc (block_size);
4320 if (current_block->memory_base == 0)
4322 // Free the blocks that we've allocated so far
4323 current_block = memory_details.initial_memory;
4324 for(size_t j = 0; j < i; j++, current_block++){
4325 if (current_block->memory_base != 0){
4326 block_size = ((j < memory_details.block_count) ?
4327 memory_details.block_size_normal :
4328 memory_details.block_size_large);
4329 virtual_free (current_block->memory_base , block_size);
4332 reserve_success = FALSE;
4337 if (current_block->memory_base < g_gc_lowest_address)
4338 g_gc_lowest_address = current_block->memory_base;
4339 if (((uint8_t *) current_block->memory_base + block_size) > g_gc_highest_address)
4340 g_gc_highest_address = (current_block->memory_base + block_size);
4342 reserve_success = TRUE;
4347 return reserve_success;
4350 void destroy_initial_memory()
4352 if (memory_details.initial_memory != NULL)
4354 if (memory_details.allocation_pattern == initial_memory_details::ALLATONCE)
4356 virtual_free(memory_details.initial_memory[0].memory_base,
4357 memory_details.block_count*(memory_details.block_size_normal +
4358 memory_details.block_size_large));
4360 else if (memory_details.allocation_pattern == initial_memory_details::TWO_STAGE)
4362 virtual_free (memory_details.initial_normal_heap[0].memory_base,
4363 memory_details.block_count*memory_details.block_size_normal);
4365 virtual_free (memory_details.initial_large_heap[0].memory_base,
4366 memory_details.block_count*memory_details.block_size_large);
4370 assert (memory_details.allocation_pattern == initial_memory_details::EACH_BLOCK);
4371 imemory_data *current_block = memory_details.initial_memory;
4372 for(size_t i = 0; i < (memory_details.block_count*2); i++, current_block++)
4374 size_t block_size = (i < memory_details.block_count) ? memory_details.block_size_normal :
4375 memory_details.block_size_large;
4376 if (current_block->memory_base != NULL)
4378 virtual_free (current_block->memory_base, block_size);
4383 delete [] memory_details.initial_memory;
4384 memory_details.initial_memory = NULL;
4385 memory_details.initial_normal_heap = NULL;
4386 memory_details.initial_large_heap = NULL;
4390 void* next_initial_memory (size_t size)
4392 assert ((size == memory_details.block_size_normal) || (size == memory_details.block_size_large));
4395 if ((size != memory_details.block_size_normal) ||
4396 ((memory_details.current_block_normal == memory_details.block_count) &&
4397 (memory_details.block_size_normal == memory_details.block_size_large)))
4399 // If the block sizes are the same, flow block requests from normal to large
4400 assert (memory_details.current_block_large < memory_details.block_count);
4401 assert (memory_details.initial_large_heap != 0);
4403 res = memory_details.initial_large_heap[memory_details.current_block_large].memory_base;
4404 memory_details.current_block_large++;
4408 assert (memory_details.current_block_normal < memory_details.block_count);
4409 assert (memory_details.initial_normal_heap != NULL);
4411 res = memory_details.initial_normal_heap[memory_details.current_block_normal].memory_base;
4412 memory_details.current_block_normal++;
4418 heap_segment* get_initial_segment (size_t size, int h_number)
4420 void* mem = next_initial_memory (size);
4421 heap_segment* res = gc_heap::make_heap_segment ((uint8_t*)mem, size , h_number);
4426 void* virtual_alloc (size_t size)
4428 size_t requested_size = size;
4430 if ((gc_heap::reserved_memory_limit - gc_heap::reserved_memory) < requested_size)
4432 gc_heap::reserved_memory_limit =
4433 GCScan::AskForMoreReservedMemory (gc_heap::reserved_memory_limit, requested_size);
4434 if ((gc_heap::reserved_memory_limit - gc_heap::reserved_memory) < requested_size)
4440 uint32_t flags = VirtualReserveFlags::None;
4441 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
4442 if (virtual_alloc_hardware_write_watch)
4444 flags = VirtualReserveFlags::WriteWatch;
4446 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
4447 void* prgmem = GCToOSInterface::VirtualReserve (requested_size, card_size * card_word_width, flags);
4448 void *aligned_mem = prgmem;
4450 // We don't want (prgmem + size) to be right at the end of the address space
4451 // because we'd have to worry about that everytime we do (address + size).
4452 // We also want to make sure that we leave LARGE_OBJECT_SIZE at the end
4453 // so we allocate a small object we don't need to worry about overflow there
4454 // when we do alloc_ptr+size.
4457 uint8_t* end_mem = (uint8_t*)prgmem + requested_size;
4459 if ((end_mem == 0) || ((size_t)(MAX_PTR - end_mem) <= END_SPACE_AFTER_GC))
4461 GCToOSInterface::VirtualRelease (prgmem, requested_size);
4462 dprintf (2, ("Virtual Alloc size %Id returned memory right against 4GB [%Ix, %Ix[ - discarding",
4463 requested_size, (size_t)prgmem, (size_t)((uint8_t*)prgmem+requested_size)));
4471 gc_heap::reserved_memory += requested_size;
4474 dprintf (2, ("Virtual Alloc size %Id: [%Ix, %Ix[",
4475 requested_size, (size_t)prgmem, (size_t)((uint8_t*)prgmem+requested_size)));
4480 void virtual_free (void* add, size_t size)
4482 GCToOSInterface::VirtualRelease (add, size);
4483 gc_heap::reserved_memory -= size;
4484 dprintf (2, ("Virtual Free size %Id: [%Ix, %Ix[",
4485 size, (size_t)add, (size_t)((uint8_t*)add+size)));
4488 static size_t get_valid_segment_size (BOOL large_seg=FALSE)
4490 size_t seg_size, initial_seg_size;
4494 initial_seg_size = INITIAL_ALLOC;
4495 seg_size = static_cast<size_t>(GCConfig::GetSegmentSize());
4499 initial_seg_size = LHEAP_ALLOC;
4500 seg_size = static_cast<size_t>(GCConfig::GetSegmentSize()) / 2;
4503 #ifdef MULTIPLE_HEAPS
4508 if (g_num_processors > 4)
4509 initial_seg_size /= 2;
4510 if (g_num_processors > 8)
4511 initial_seg_size /= 2;
4513 #endif //MULTIPLE_HEAPS
4515 // if seg_size is small but not 0 (0 is default if config not set)
4516 // then set the segment to the minimum size
4517 if (!g_theGCHeap->IsValidSegmentSize(seg_size))
4519 // if requested size is between 1 byte and 4MB, use min
4520 if ((seg_size >> 1) && !(seg_size >> 22))
4521 seg_size = 1024*1024*4;
4523 seg_size = initial_seg_size;
4526 #ifdef SEG_MAPPING_TABLE
4528 seg_size = round_up_power2 (seg_size);
4530 seg_size = round_down_power2 (seg_size);
4532 #endif //SEG_MAPPING_TABLE
4538 gc_heap::compute_new_ephemeral_size()
4540 int eph_gen_max = max_generation - 1 - (settings.promotion ? 1 : 0);
4541 size_t padding_size = 0;
4543 for (int i = 0; i <= eph_gen_max; i++)
4545 dynamic_data* dd = dynamic_data_of (i);
4546 total_ephemeral_size += (dd_survived_size (dd) - dd_pinned_survived_size (dd));
4547 #ifdef RESPECT_LARGE_ALIGNMENT
4548 total_ephemeral_size += dd_num_npinned_plugs (dd) * switch_alignment_size (FALSE);
4549 #endif //RESPECT_LARGE_ALIGNMENT
4550 #ifdef FEATURE_STRUCTALIGN
4551 total_ephemeral_size += dd_num_npinned_plugs (dd) * MAX_STRUCTALIGN;
4552 #endif //FEATURE_STRUCTALIGN
4555 padding_size += dd_padding_size (dd);
4556 #endif //SHORT_PLUGS
4559 total_ephemeral_size += eph_gen_starts_size;
4561 #ifdef RESPECT_LARGE_ALIGNMENT
4562 size_t planned_ephemeral_size = heap_segment_plan_allocated (ephemeral_heap_segment) -
4563 generation_plan_allocation_start (generation_of (max_generation-1));
4564 total_ephemeral_size = min (total_ephemeral_size, planned_ephemeral_size);
4565 #endif //RESPECT_LARGE_ALIGNMENT
4568 total_ephemeral_size = Align ((size_t)((double)total_ephemeral_size * short_plugs_pad_ratio) + 1);
4569 total_ephemeral_size += Align (DESIRED_PLUG_LENGTH);
4570 #endif //SHORT_PLUGS
4572 dprintf (3, ("total ephemeral size is %Ix, padding %Ix(%Ix)",
4573 total_ephemeral_size,
4574 padding_size, (total_ephemeral_size - padding_size)));
4578 #pragma warning(disable:4706) // "assignment within conditional expression" is intentional in this function.
4582 gc_heap::soh_get_segment_to_expand()
4584 size_t size = soh_segment_size;
4586 ordered_plug_indices_init = FALSE;
4587 use_bestfit = FALSE;
4589 //compute the size of the new ephemeral heap segment.
4590 compute_new_ephemeral_size();
4592 if ((settings.pause_mode != pause_low_latency) &&
4593 (settings.pause_mode != pause_no_gc)
4594 #ifdef BACKGROUND_GC
4595 && (!recursive_gc_sync::background_running_p())
4596 #endif //BACKGROUND_GC
4599 allocator* gen_alloc = ((settings.condemned_generation == max_generation) ? 0 :
4600 generation_allocator (generation_of (max_generation)));
4601 dprintf (2, ("(gen%d)soh_get_segment_to_expand", settings.condemned_generation));
4603 // try to find one in the gen 2 segment list, search backwards because the first segments
4604 // tend to be more compact than the later ones.
4605 heap_segment* fseg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
4607 PREFIX_ASSUME(fseg != NULL);
4609 #ifdef SEG_REUSE_STATS
4611 #endif //SEG_REUSE_STATS
4613 heap_segment* seg = ephemeral_heap_segment;
4614 while ((seg = heap_segment_prev_rw (fseg, seg)) && (seg != fseg))
4616 #ifdef SEG_REUSE_STATS
4618 #endif //SEG_REUSE_STATS
4620 if (can_expand_into_p (seg, size/3, total_ephemeral_size, gen_alloc))
4622 get_gc_data_per_heap()->set_mechanism (gc_heap_expand,
4623 (use_bestfit ? expand_reuse_bestfit : expand_reuse_normal));
4624 if (settings.condemned_generation == max_generation)
4628 build_ordered_free_spaces (seg);
4629 dprintf (GTC_LOG, ("can use best fit"));
4632 #ifdef SEG_REUSE_STATS
4633 dprintf (SEG_REUSE_LOG_0, ("(gen%d)soh_get_segment_to_expand: found seg #%d to reuse",
4634 settings.condemned_generation, try_reuse));
4635 #endif //SEG_REUSE_STATS
4636 dprintf (GTC_LOG, ("max_gen: Found existing segment to expand into %Ix", (size_t)seg));
4641 #ifdef SEG_REUSE_STATS
4642 dprintf (SEG_REUSE_LOG_0, ("(gen%d)soh_get_segment_to_expand: found seg #%d to reuse - returning",
4643 settings.condemned_generation, try_reuse));
4644 #endif //SEG_REUSE_STATS
4645 dprintf (GTC_LOG, ("max_gen-1: Found existing segment to expand into %Ix", (size_t)seg));
4647 // If we return 0 here, the allocator will think since we are short on end
4648 // of seg we neeed to trigger a full compacting GC. So if sustained low latency
4649 // is set we should acquire a new seg instead, that way we wouldn't be short.
4650 // The real solution, of course, is to actually implement seg reuse in gen1.
4651 if (settings.pause_mode != pause_sustained_low_latency)
4653 dprintf (GTC_LOG, ("max_gen-1: SustainedLowLatency is set, acquire a new seg"));
4654 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, expand_next_full_gc);
4662 heap_segment* result = get_segment (size, FALSE);
4666 #ifdef BACKGROUND_GC
4667 if (current_c_gc_state == c_gc_state_planning)
4669 // When we expand heap during bgc sweep, we set the seg to be swept so
4670 // we'll always look at cards for objects on the new segment.
4671 result->flags |= heap_segment_flags_swept;
4673 #endif //BACKGROUND_GC
4675 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(result),
4676 (size_t)(heap_segment_reserved (result) - heap_segment_mem(result)),
4677 gc_etw_segment_small_object_heap);
4680 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, (result ? expand_new_seg : expand_no_memory));
4684 dprintf (2, ("h%d: failed to allocate a new segment!", heap_number));
4688 #ifdef MULTIPLE_HEAPS
4689 heap_segment_heap (result) = this;
4690 #endif //MULTIPLE_HEAPS
4693 dprintf (GTC_LOG, ("(gen%d)creating new segment %Ix", settings.condemned_generation, result));
4698 #pragma warning(default:4706)
4701 //returns 0 in case of allocation failure
4703 gc_heap::get_segment (size_t size, BOOL loh_p)
4705 heap_segment* result = 0;
4707 if (segment_standby_list != 0)
4709 result = segment_standby_list;
4710 heap_segment* last = 0;
4713 size_t hs = (size_t)(heap_segment_reserved (result) - (uint8_t*)result);
4714 if ((hs >= size) && ((hs / 2) < size))
4716 dprintf (2, ("Hoarded segment %Ix found", (size_t) result));
4719 heap_segment_next (last) = heap_segment_next (result);
4723 segment_standby_list = heap_segment_next (result);
4730 result = heap_segment_next (result);
4737 init_heap_segment (result);
4738 #ifdef BACKGROUND_GC
4739 if (should_commit_mark_array())
4741 dprintf (GC_TABLE_LOG, ("hoarded seg %Ix, mark_array is %Ix", result, mark_array));
4742 if (!commit_mark_array_new_seg (__this, result))
4744 dprintf (GC_TABLE_LOG, ("failed to commit mark array for hoarded seg"));
4745 // If we can't use it we need to thread it back.
4746 if (segment_standby_list != 0)
4748 heap_segment_next (result) = segment_standby_list;
4749 segment_standby_list = result;
4753 segment_standby_list = result;
4759 #endif //BACKGROUND_GC
4761 #ifdef SEG_MAPPING_TABLE
4763 seg_mapping_table_add_segment (result, __this);
4764 #endif //SEG_MAPPING_TABLE
4769 #ifndef SEG_MAPPING_TABLE
4770 if (!seg_table->ensure_space_for_insert ())
4772 #endif //SEG_MAPPING_TABLE
4773 void* mem = virtual_alloc (size);
4776 fgm_result.set_fgm (fgm_reserve_segment, size, loh_p);
4780 result = gc_heap::make_heap_segment ((uint8_t*)mem, size, heap_number);
4786 if (mem < g_gc_lowest_address)
4788 start = (uint8_t*)mem;
4792 start = (uint8_t*)g_gc_lowest_address;
4795 if (((uint8_t*)mem + size) > g_gc_highest_address)
4797 end = (uint8_t*)mem + size;
4801 end = (uint8_t*)g_gc_highest_address;
4804 if (gc_heap::grow_brick_card_tables (start, end, size, result, __this, loh_p) != 0)
4806 virtual_free (mem, size);
4812 fgm_result.set_fgm (fgm_commit_segment_beg, SEGMENT_INITIAL_COMMIT, loh_p);
4813 virtual_free (mem, size);
4818 #ifdef SEG_MAPPING_TABLE
4819 seg_mapping_table_add_segment (result, __this);
4820 #else //SEG_MAPPING_TABLE
4821 gc_heap::seg_table->insert ((uint8_t*)result, delta);
4822 #endif //SEG_MAPPING_TABLE
4826 #ifdef BACKGROUND_GC
4829 ::record_changed_seg ((uint8_t*)result, heap_segment_reserved (result),
4830 settings.gc_index, current_bgc_state,
4832 bgc_verify_mark_array_cleared (result);
4834 #endif //BACKGROUND_GC
4836 dprintf (GC_TABLE_LOG, ("h%d: new seg: %Ix-%Ix (%Id)", heap_number, result, ((uint8_t*)result + size), size));
4840 void release_segment (heap_segment* sg)
4842 ptrdiff_t delta = 0;
4843 FIRE_EVENT(GCFreeSegment_V1, heap_segment_mem(sg));
4844 virtual_free (sg, (uint8_t*)heap_segment_reserved (sg)-(uint8_t*)sg);
4847 heap_segment* gc_heap::get_segment_for_loh (size_t size
4848 #ifdef MULTIPLE_HEAPS
4850 #endif //MULTIPLE_HEAPS
4853 #ifndef MULTIPLE_HEAPS
4855 #endif //MULTIPLE_HEAPS
4856 heap_segment* res = hp->get_segment (size, TRUE);
4859 #ifdef MULTIPLE_HEAPS
4860 heap_segment_heap (res) = hp;
4861 #endif //MULTIPLE_HEAPS
4862 res->flags |= heap_segment_flags_loh;
4864 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(res), (size_t)(heap_segment_reserved (res) - heap_segment_mem(res)), gc_etw_segment_large_object_heap);
4866 GCToEEInterface::DiagUpdateGenerationBounds();
4868 #ifdef MULTIPLE_HEAPS
4869 hp->thread_loh_segment (res);
4871 thread_loh_segment (res);
4872 #endif //MULTIPLE_HEAPS
4878 void gc_heap::thread_loh_segment (heap_segment* new_seg)
4880 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
4882 while (heap_segment_next_rw (seg))
4883 seg = heap_segment_next_rw (seg);
4884 heap_segment_next (seg) = new_seg;
4888 gc_heap::get_large_segment (size_t size, BOOL* did_full_compact_gc)
4890 *did_full_compact_gc = FALSE;
4891 size_t last_full_compact_gc_count = get_full_compact_gc_count();
4893 //access to get_segment needs to be serialized
4894 add_saved_spinlock_info (me_release, mt_get_large_seg);
4896 dprintf (SPINLOCK_LOG, ("[%d]Seg: Lmsl", heap_number));
4897 leave_spin_lock (&more_space_lock);
4898 enter_spin_lock (&gc_heap::gc_lock);
4899 dprintf (SPINLOCK_LOG, ("[%d]Seg: Egc", heap_number));
4900 // if a GC happened between here and before we ask for a segment in
4901 // get_large_segment, we need to count that GC.
4902 size_t current_full_compact_gc_count = get_full_compact_gc_count();
4904 if (current_full_compact_gc_count > last_full_compact_gc_count)
4906 *did_full_compact_gc = TRUE;
4909 #ifdef BACKGROUND_GC
4910 while (current_c_gc_state == c_gc_state_planning)
4912 dprintf (3, ("lh state planning, waiting to get a large seg"));
4914 dprintf (SPINLOCK_LOG, ("[%d]Seg: P, Lgc", heap_number));
4915 leave_spin_lock (&gc_lock);
4916 background_gc_wait_lh (awr_get_loh_seg);
4917 enter_spin_lock (&gc_lock);
4918 dprintf (SPINLOCK_LOG, ("[%d]Seg: P, Egc", heap_number));
4920 assert ((current_c_gc_state == c_gc_state_free) ||
4921 (current_c_gc_state == c_gc_state_marking));
4922 #endif //BACKGROUND_GC
4924 heap_segment* res = get_segment_for_loh (size
4925 #ifdef MULTIPLE_HEAPS
4927 #endif //MULTIPLE_HEAPS
4930 dprintf (SPINLOCK_LOG, ("[%d]Seg: A Lgc", heap_number));
4931 leave_spin_lock (&gc_heap::gc_lock);
4932 enter_spin_lock (&more_space_lock);
4933 dprintf (SPINLOCK_LOG, ("[%d]Seg: A Emsl", heap_number));
4934 add_saved_spinlock_info (me_acquire, mt_get_large_seg);
4936 #ifdef BACKGROUND_GC
4937 wait_for_background_planning (awr_get_loh_seg);
4938 #endif //BACKGROUND_GC
4944 BOOL gc_heap::unprotect_segment (heap_segment* seg)
4946 uint8_t* start = align_lower_page (heap_segment_mem (seg));
4947 ptrdiff_t region_size = heap_segment_allocated (seg) - start;
4949 if (region_size != 0 )
4951 dprintf (3, ("unprotecting segment %Ix:", (size_t)seg));
4953 BOOL status = GCToOSInterface::VirtualUnprotect (start, region_size);
4961 #ifdef MULTIPLE_HEAPS
4964 #pragma warning(disable:4035)
4965 static ptrdiff_t get_cycle_count()
4969 #pragma warning(default:4035)
4970 #elif defined(__GNUC__)
4971 static ptrdiff_t get_cycle_count()
4975 __asm__ __volatile__
4976 ("rdtsc":"=a" (cycles), "=d" (cyclesHi));
4980 #error Unknown compiler
4982 #elif defined(_TARGET_AMD64_)
4984 extern "C" uint64_t __rdtsc();
4985 #pragma intrinsic(__rdtsc)
4986 static ptrdiff_t get_cycle_count()
4988 return (ptrdiff_t)__rdtsc();
4990 #elif defined(__clang__)
4991 static ptrdiff_t get_cycle_count()
4995 __asm__ __volatile__
4996 ("rdtsc":"=a" (cycles), "=d" (cyclesHi));
4997 return (cyclesHi << 32) | cycles;
5000 extern "C" ptrdiff_t get_cycle_count(void);
5002 #elif defined(_TARGET_ARM_)
5003 static ptrdiff_t get_cycle_count()
5005 // @ARMTODO: cycle counter is not exposed to user mode by CoreARM. For now (until we can show this
5006 // makes a difference on the ARM configurations on which we'll run) just return 0. This will result in
5007 // all buffer access times being reported as equal in access_time().
5010 #elif defined(_TARGET_ARM64_)
5011 static ptrdiff_t get_cycle_count()
5013 // @ARM64TODO: cycle counter is not exposed to user mode by CoreARM. For now (until we can show this
5014 // makes a difference on the ARM configurations on which we'll run) just return 0. This will result in
5015 // all buffer access times being reported as equal in access_time().
5019 #error NYI platform: get_cycle_count
5020 #endif //_TARGET_X86_
5025 static uint8_t* sniff_buffer;
5026 static unsigned n_sniff_buffers;
5027 static unsigned cur_sniff_index;
5029 static uint16_t proc_no_to_heap_no[MAX_SUPPORTED_CPUS];
5030 static uint16_t heap_no_to_proc_no[MAX_SUPPORTED_CPUS];
5031 static uint16_t heap_no_to_numa_node[MAX_SUPPORTED_CPUS];
5032 static uint16_t heap_no_to_cpu_group[MAX_SUPPORTED_CPUS];
5033 static uint16_t heap_no_to_group_proc[MAX_SUPPORTED_CPUS];
5034 static uint16_t numa_node_to_heap_map[MAX_SUPPORTED_CPUS+4];
5036 static int access_time(uint8_t *sniff_buffer, int heap_number, unsigned sniff_index, unsigned n_sniff_buffers)
5038 ptrdiff_t start_cycles = get_cycle_count();
5039 uint8_t sniff = sniff_buffer[(1 + heap_number*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE];
5040 assert (sniff == 0);
5041 ptrdiff_t elapsed_cycles = get_cycle_count() - start_cycles;
5042 // add sniff here just to defeat the optimizer
5043 elapsed_cycles += sniff;
5044 return (int) elapsed_cycles;
5048 static BOOL init(int n_heaps)
5050 assert (sniff_buffer == NULL && n_sniff_buffers == 0);
5051 if (!GCToOSInterface::CanGetCurrentProcessorNumber())
5053 n_sniff_buffers = n_heaps*2+1;
5054 size_t n_cache_lines = 1 + n_heaps * n_sniff_buffers + 1;
5055 size_t sniff_buf_size = n_cache_lines * HS_CACHE_LINE_SIZE;
5056 if (sniff_buf_size / HS_CACHE_LINE_SIZE != n_cache_lines) // check for overlow
5061 sniff_buffer = new (nothrow) uint8_t[sniff_buf_size];
5062 if (sniff_buffer == 0)
5064 memset(sniff_buffer, 0, sniff_buf_size*sizeof(uint8_t));
5067 //can not enable gc numa aware, force all heaps to be in
5068 //one numa node by filling the array with all 0s
5069 if (!GCToOSInterface::CanEnableGCNumaAware())
5070 memset(heap_no_to_numa_node, 0, sizeof (heap_no_to_numa_node));
5075 static void init_cpu_mapping(gc_heap * /*heap*/, int heap_number)
5077 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5079 uint32_t proc_no = GCToOSInterface::GetCurrentProcessorNumber() % gc_heap::n_heaps;
5080 // We can safely cast heap_number to a uint16_t 'cause GetCurrentProcessCpuCount
5081 // only returns up to MAX_SUPPORTED_CPUS procs right now. We only ever create at most
5082 // MAX_SUPPORTED_CPUS GC threads.
5083 proc_no_to_heap_no[proc_no] = (uint16_t)heap_number;
5087 static void mark_heap(int heap_number)
5089 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5092 for (unsigned sniff_index = 0; sniff_index < n_sniff_buffers; sniff_index++)
5093 sniff_buffer[(1 + heap_number*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE] &= 1;
5096 static int select_heap(alloc_context* acontext, int /*hint*/)
5098 UNREFERENCED_PARAMETER(acontext); // only referenced by dprintf
5100 if (GCToOSInterface::CanGetCurrentProcessorNumber())
5101 return proc_no_to_heap_no[GCToOSInterface::GetCurrentProcessorNumber() % gc_heap::n_heaps];
5103 unsigned sniff_index = Interlocked::Increment(&cur_sniff_index);
5104 sniff_index %= n_sniff_buffers;
5107 int best_access_time = 1000*1000*1000;
5108 int second_best_access_time = best_access_time;
5110 uint8_t *l_sniff_buffer = sniff_buffer;
5111 unsigned l_n_sniff_buffers = n_sniff_buffers;
5112 for (int heap_number = 0; heap_number < gc_heap::n_heaps; heap_number++)
5114 int this_access_time = access_time(l_sniff_buffer, heap_number, sniff_index, l_n_sniff_buffers);
5115 if (this_access_time < best_access_time)
5117 second_best_access_time = best_access_time;
5118 best_access_time = this_access_time;
5119 best_heap = heap_number;
5121 else if (this_access_time < second_best_access_time)
5123 second_best_access_time = this_access_time;
5127 if (best_access_time*2 < second_best_access_time)
5129 sniff_buffer[(1 + best_heap*n_sniff_buffers + sniff_index)*HS_CACHE_LINE_SIZE] &= 1;
5131 dprintf (3, ("select_heap yields crisp %d for context %p\n", best_heap, (void *)acontext));
5135 dprintf (3, ("select_heap yields vague %d for context %p\n", best_heap, (void *)acontext ));
5141 static bool can_find_heap_fast()
5143 return GCToOSInterface::CanGetCurrentProcessorNumber();
5146 static uint16_t find_proc_no_from_heap_no(int heap_number)
5148 return heap_no_to_proc_no[heap_number];
5151 static void set_proc_no_for_heap(int heap_number, uint16_t proc_no)
5153 heap_no_to_proc_no[heap_number] = proc_no;
5156 static uint16_t find_numa_node_from_heap_no(int heap_number)
5158 return heap_no_to_numa_node[heap_number];
5161 static void set_numa_node_for_heap(int heap_number, uint16_t numa_node)
5163 heap_no_to_numa_node[heap_number] = numa_node;
5166 static uint16_t find_cpu_group_from_heap_no(int heap_number)
5168 return heap_no_to_cpu_group[heap_number];
5171 static void set_cpu_group_for_heap(int heap_number, uint16_t group_number)
5173 heap_no_to_cpu_group[heap_number] = group_number;
5176 static uint16_t find_group_proc_from_heap_no(int heap_number)
5178 return heap_no_to_group_proc[heap_number];
5181 static void set_group_proc_for_heap(int heap_number, uint16_t group_proc)
5183 heap_no_to_group_proc[heap_number] = group_proc;
5186 static void init_numa_node_to_heap_map(int nheaps)
5187 { // called right after GCHeap::Init() for each heap is finished
5188 // when numa is not enabled, heap_no_to_numa_node[] are all filled
5189 // with 0s during initialization, and will be treated as one node
5190 numa_node_to_heap_map[0] = 0;
5193 for (int i=1; i < nheaps; i++)
5195 if (heap_no_to_numa_node[i] != heap_no_to_numa_node[i-1])
5196 numa_node_to_heap_map[node_index++] = (uint16_t)i;
5198 numa_node_to_heap_map[node_index] = (uint16_t)nheaps; //mark the end with nheaps
5201 static void get_heap_range_for_heap(int hn, int* start, int* end)
5202 { // 1-tier/no numa case: heap_no_to_numa_node[] all zeros,
5203 // and treated as in one node. thus: start=0, end=n_heaps
5204 uint16_t numa_node = heap_no_to_numa_node[hn];
5205 *start = (int)numa_node_to_heap_map[numa_node];
5206 *end = (int)(numa_node_to_heap_map[numa_node+1]);
5209 uint8_t* heap_select::sniff_buffer;
5210 unsigned heap_select::n_sniff_buffers;
5211 unsigned heap_select::cur_sniff_index;
5212 uint16_t heap_select::proc_no_to_heap_no[MAX_SUPPORTED_CPUS];
5213 uint16_t heap_select::heap_no_to_proc_no[MAX_SUPPORTED_CPUS];
5214 uint16_t heap_select::heap_no_to_numa_node[MAX_SUPPORTED_CPUS];
5215 uint16_t heap_select::heap_no_to_cpu_group[MAX_SUPPORTED_CPUS];
5216 uint16_t heap_select::heap_no_to_group_proc[MAX_SUPPORTED_CPUS];
5217 uint16_t heap_select::numa_node_to_heap_map[MAX_SUPPORTED_CPUS+4];
5219 BOOL gc_heap::create_thread_support (unsigned number_of_heaps)
5222 if (!gc_start_event.CreateOSManualEventNoThrow (FALSE))
5226 if (!ee_suspend_event.CreateOSAutoEventNoThrow (FALSE))
5230 if (!gc_t_join.init (number_of_heaps, join_flavor_server_gc))
5241 destroy_thread_support();
5247 void gc_heap::destroy_thread_support ()
5249 if (ee_suspend_event.IsValid())
5251 ee_suspend_event.CloseEvent();
5253 if (gc_start_event.IsValid())
5255 gc_start_event.CloseEvent();
5259 void set_thread_group_affinity_for_heap(int heap_number, GCThreadAffinity* affinity)
5261 affinity->Group = GCThreadAffinity::None;
5262 affinity->Processor = GCThreadAffinity::None;
5265 GCToOSInterface::GetGroupForProcessor((uint16_t)heap_number, &gn, &gpn);
5268 for (uintptr_t mask = 1; mask !=0; mask <<=1)
5270 if (bit_number == gpn)
5272 dprintf(3, ("using processor group %d, mask %Ix for heap %d\n", gn, mask, heap_number));
5273 affinity->Processor = gpn;
5274 affinity->Group = gn;
5275 heap_select::set_cpu_group_for_heap(heap_number, gn);
5276 heap_select::set_group_proc_for_heap(heap_number, gpn);
5277 if (GCToOSInterface::CanEnableGCNumaAware())
5279 PROCESSOR_NUMBER proc_no;
5281 proc_no.Number = (uint8_t)gpn;
5282 proc_no.Reserved = 0;
5284 uint16_t node_no = 0;
5285 if (GCToOSInterface::GetNumaProcessorNode(&proc_no, &node_no))
5286 heap_select::set_numa_node_for_heap(heap_number, node_no);
5289 { // no numa setting, each cpu group is treated as a node
5290 heap_select::set_numa_node_for_heap(heap_number, gn);
5298 void set_thread_affinity_mask_for_heap(int heap_number, GCThreadAffinity* affinity)
5300 affinity->Group = GCThreadAffinity::None;
5301 affinity->Processor = GCThreadAffinity::None;
5303 uintptr_t pmask, smask;
5304 if (GCToOSInterface::GetCurrentProcessAffinityMask(&pmask, &smask))
5308 uint8_t proc_number = 0;
5309 for (uintptr_t mask = 1; mask != 0; mask <<= 1)
5311 if ((mask & pmask) != 0)
5313 if (bit_number == heap_number)
5315 dprintf (3, ("Using processor %d for heap %d", proc_number, heap_number));
5316 affinity->Processor = proc_number;
5317 heap_select::set_proc_no_for_heap(heap_number, proc_number);
5318 if (GCToOSInterface::CanEnableGCNumaAware())
5320 uint16_t node_no = 0;
5321 PROCESSOR_NUMBER proc_no;
5323 proc_no.Number = (uint8_t)proc_number;
5324 proc_no.Reserved = 0;
5325 if (GCToOSInterface::GetNumaProcessorNode(&proc_no, &node_no))
5327 heap_select::set_numa_node_for_heap(heap_number, node_no);
5339 bool gc_heap::create_gc_thread ()
5341 dprintf (3, ("Creating gc thread\n"));
5342 return GCToEEInterface::CreateThread(gc_thread_stub, this, false, ".NET Server GC");
5346 #pragma warning(disable:4715) //IA64 xcompiler recognizes that without the 'break;' the while(1) will never end and therefore not return a value for that code path
5348 void gc_heap::gc_thread_function ()
5350 assert (gc_done_event.IsValid());
5351 assert (gc_start_event.IsValid());
5352 dprintf (3, ("gc thread started"));
5354 heap_select::init_cpu_mapping(this, heap_number);
5358 assert (!gc_t_join.joined());
5360 if (heap_number == 0)
5362 gc_heap::ee_suspend_event.Wait(INFINITE, FALSE);
5364 BEGIN_TIMING(suspend_ee_during_log);
5365 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC);
5366 END_TIMING(suspend_ee_during_log);
5368 proceed_with_gc_p = TRUE;
5370 if (!should_proceed_with_gc())
5372 update_collection_counts_for_no_gc();
5373 proceed_with_gc_p = FALSE;
5377 settings.init_mechanisms();
5378 gc_start_event.Set();
5380 dprintf (3, ("%d gc thread waiting...", heap_number));
5384 gc_start_event.Wait(INFINITE, FALSE);
5385 dprintf (3, ("%d gc thread waiting... Done", heap_number));
5388 assert ((heap_number == 0) || proceed_with_gc_p);
5390 if (proceed_with_gc_p)
5391 garbage_collect (GCHeap::GcCondemnedGeneration);
5393 if (heap_number == 0)
5395 if (proceed_with_gc_p && (!settings.concurrent))
5400 #ifdef BACKGROUND_GC
5401 recover_bgc_settings();
5402 #endif //BACKGROUND_GC
5404 #ifdef MULTIPLE_HEAPS
5405 for (int i = 0; i < gc_heap::n_heaps; i++)
5407 gc_heap* hp = gc_heap::g_heaps[i];
5408 hp->add_saved_spinlock_info (me_release, mt_block_gc);
5409 dprintf (SPINLOCK_LOG, ("[%d]GC Lmsl", i));
5410 leave_spin_lock(&hp->more_space_lock);
5412 #endif //MULTIPLE_HEAPS
5414 gc_heap::gc_started = FALSE;
5416 BEGIN_TIMING(restart_ee_during_log);
5417 GCToEEInterface::RestartEE(TRUE);
5418 END_TIMING(restart_ee_during_log);
5419 process_sync_log_stats();
5421 dprintf (SPINLOCK_LOG, ("GC Lgc"));
5422 leave_spin_lock (&gc_heap::gc_lock);
5424 gc_heap::internal_gc_done = true;
5426 if (proceed_with_gc_p)
5430 // If we didn't actually do a GC, it means we didn't wait up the other threads,
5431 // we still need to set the gc_done_event for those threads.
5432 for (int i = 0; i < gc_heap::n_heaps; i++)
5434 gc_heap* hp = gc_heap::g_heaps[i];
5441 int spin_count = 32 * (gc_heap::n_heaps - 1);
5443 // wait until RestartEE has progressed to a stage where we can restart user threads
5444 while (!gc_heap::internal_gc_done && !GCHeap::SafeToRestartManagedThreads())
5446 spin_and_switch (spin_count, (gc_heap::internal_gc_done || GCHeap::SafeToRestartManagedThreads()));
5453 #pragma warning(default:4715) //IA64 xcompiler recognizes that without the 'break;' the while(1) will never end and therefore not return a value for that code path
5456 #endif //MULTIPLE_HEAPS
5458 bool virtual_alloc_commit_for_heap(void* addr, size_t size, int h_number)
5460 #if defined(MULTIPLE_HEAPS) && !defined(FEATURE_REDHAWK)
5461 // Currently there is no way for us to specific the numa node to allocate on via hosting interfaces to
5462 // a host. This will need to be added later.
5463 #if !defined(FEATURE_CORECLR)
5464 if (!CLRMemoryHosted())
5467 if (GCToOSInterface::CanEnableGCNumaAware())
5469 uint32_t numa_node = heap_select::find_numa_node_from_heap_no(h_number);
5470 if (GCToOSInterface::VirtualCommit(addr, size, numa_node))
5475 UNREFERENCED_PARAMETER(h_number);
5478 //numa aware not enabled, or call failed --> fallback to VirtualCommit()
5479 return GCToOSInterface::VirtualCommit(addr, size);
5482 #ifndef SEG_MAPPING_TABLE
5484 heap_segment* gc_heap::segment_of (uint8_t* add, ptrdiff_t& delta, BOOL verify_p)
5486 uint8_t* sadd = add;
5487 heap_segment* hs = 0;
5488 heap_segment* hs1 = 0;
5489 if (!((add >= g_gc_lowest_address) && (add < g_gc_highest_address)))
5494 //repeat in case there is a concurrent insertion in the table.
5499 seg_table->lookup (sadd);
5500 hs1 = (heap_segment*)sadd;
5501 } while (hs1 && !in_range_for_segment (add, hs1) && (hs != hs1));
5506 (verify_p && (add > heap_segment_reserved ((heap_segment*)(sadd + delta)))))
5510 #endif //SEG_MAPPING_TABLE
5518 // If we want to save space we can have a pool of plug_and_gap's instead of
5519 // always having 2 allocated for each pinned plug.
5520 gap_reloc_pair saved_pre_plug;
5521 // If we decide to not compact, we need to restore the original values.
5522 gap_reloc_pair saved_pre_plug_reloc;
5524 gap_reloc_pair saved_post_plug;
5526 // Supposedly Pinned objects cannot have references but we are seeing some from pinvoke
5527 // frames. Also if it's an artificially pinned plug created by us, it can certainly
5529 // We know these cases will be rare so we can optimize this to be only allocated on decommand.
5530 gap_reloc_pair saved_post_plug_reloc;
5532 // We need to calculate this after we are done with plan phase and before compact
5533 // phase because compact phase will change the bricks so relocate_address will no
5535 uint8_t* saved_pre_plug_info_reloc_start;
5537 // We need to save this because we will have no way to calculate it, unlike the
5538 // pre plug info start which is right before this plug.
5539 uint8_t* saved_post_plug_info_start;
5542 uint8_t* allocation_context_start_region;
5543 #endif //SHORT_PLUGS
5545 // How the bits in these bytes are organized:
5547 // bit to indicate whether it's a short obj | 3 bits for refs in this short obj | 2 unused bits | bit to indicate if it's collectible | last bit
5548 // last bit indicates if there's pre or post info associated with this plug. If it's not set all other bits will be 0.
5553 // We are seeing this is getting corrupted for a PP with a NP after.
5554 // Save it when we first set it and make sure it doesn't change.
5555 gap_reloc_pair saved_post_plug_debug;
5558 size_t get_max_short_bits()
5560 return (sizeof (gap_reloc_pair) / sizeof (uint8_t*));
5564 size_t get_pre_short_start_bit ()
5566 return (sizeof (saved_pre_p) * 8 - 1 - (sizeof (gap_reloc_pair) / sizeof (uint8_t*)));
5571 return (saved_pre_p & (1 << (sizeof (saved_pre_p) * 8 - 1)));
5574 void set_pre_short()
5576 saved_pre_p |= (1 << (sizeof (saved_pre_p) * 8 - 1));
5579 void set_pre_short_bit (size_t bit)
5581 saved_pre_p |= 1 << (get_pre_short_start_bit() + bit);
5584 BOOL pre_short_bit_p (size_t bit)
5586 return (saved_pre_p & (1 << (get_pre_short_start_bit() + bit)));
5589 #ifdef COLLECTIBLE_CLASS
5590 void set_pre_short_collectible()
5595 BOOL pre_short_collectible_p()
5597 return (saved_pre_p & 2);
5599 #endif //COLLECTIBLE_CLASS
5602 size_t get_post_short_start_bit ()
5604 return (sizeof (saved_post_p) * 8 - 1 - (sizeof (gap_reloc_pair) / sizeof (uint8_t*)));
5609 return (saved_post_p & (1 << (sizeof (saved_post_p) * 8 - 1)));
5612 void set_post_short()
5614 saved_post_p |= (1 << (sizeof (saved_post_p) * 8 - 1));
5617 void set_post_short_bit (size_t bit)
5619 saved_post_p |= 1 << (get_post_short_start_bit() + bit);
5622 BOOL post_short_bit_p (size_t bit)
5624 return (saved_post_p & (1 << (get_post_short_start_bit() + bit)));
5627 #ifdef COLLECTIBLE_CLASS
5628 void set_post_short_collectible()
5633 BOOL post_short_collectible_p()
5635 return (saved_post_p & 2);
5637 #endif //COLLECTIBLE_CLASS
5639 uint8_t* get_plug_address() { return first; }
5641 BOOL has_pre_plug_info() { return saved_pre_p; }
5642 BOOL has_post_plug_info() { return saved_post_p; }
5644 gap_reloc_pair* get_pre_plug_reloc_info() { return &saved_pre_plug_reloc; }
5645 gap_reloc_pair* get_post_plug_reloc_info() { return &saved_post_plug_reloc; }
5646 void set_pre_plug_info_reloc_start (uint8_t* reloc) { saved_pre_plug_info_reloc_start = reloc; }
5647 uint8_t* get_post_plug_info_start() { return saved_post_plug_info_start; }
5649 // We need to temporarily recover the shortened plugs for compact phase so we can
5650 // copy over the whole plug and their related info (mark bits/cards). But we will
5651 // need to set the artificial gap back so compact phase can keep reading the plug info.
5652 // We also need to recover the saved info because we'll need to recover it later.
5654 // So we would call swap_p*_plug_and_saved once to recover the object info; then call
5655 // it again to recover the artificial gap.
5656 void swap_pre_plug_and_saved()
5658 gap_reloc_pair temp;
5659 memcpy (&temp, (first - sizeof (plug_and_gap)), sizeof (temp));
5660 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug_reloc, sizeof (saved_pre_plug_reloc));
5661 saved_pre_plug_reloc = temp;
5664 void swap_post_plug_and_saved()
5666 gap_reloc_pair temp;
5667 memcpy (&temp, saved_post_plug_info_start, sizeof (temp));
5668 memcpy (saved_post_plug_info_start, &saved_post_plug_reloc, sizeof (saved_post_plug_reloc));
5669 saved_post_plug_reloc = temp;
5672 void swap_pre_plug_and_saved_for_profiler()
5674 gap_reloc_pair temp;
5675 memcpy (&temp, (first - sizeof (plug_and_gap)), sizeof (temp));
5676 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug, sizeof (saved_pre_plug));
5677 saved_pre_plug = temp;
5680 void swap_post_plug_and_saved_for_profiler()
5682 gap_reloc_pair temp;
5683 memcpy (&temp, saved_post_plug_info_start, sizeof (temp));
5684 memcpy (saved_post_plug_info_start, &saved_post_plug, sizeof (saved_post_plug));
5685 saved_post_plug = temp;
5688 // We should think about whether it's really necessary to have to copy back the pre plug
5689 // info since it was already copied during compacting plugs. But if a plug doesn't move
5690 // by >= 3 ptr size (the size of gap_reloc_pair), it means we'd have to recover pre plug info.
5691 void recover_plug_info()
5695 if (gc_heap::settings.compaction)
5697 dprintf (3, ("%Ix: REC Pre: %Ix-%Ix",
5699 &saved_pre_plug_reloc,
5700 saved_pre_plug_info_reloc_start));
5701 memcpy (saved_pre_plug_info_reloc_start, &saved_pre_plug_reloc, sizeof (saved_pre_plug_reloc));
5705 dprintf (3, ("%Ix: REC Pre: %Ix-%Ix",
5708 (first - sizeof (plug_and_gap))));
5709 memcpy ((first - sizeof (plug_and_gap)), &saved_pre_plug, sizeof (saved_pre_plug));
5715 if (gc_heap::settings.compaction)
5717 dprintf (3, ("%Ix: REC Post: %Ix-%Ix",
5719 &saved_post_plug_reloc,
5720 saved_post_plug_info_start));
5721 memcpy (saved_post_plug_info_start, &saved_post_plug_reloc, sizeof (saved_post_plug_reloc));
5725 dprintf (3, ("%Ix: REC Post: %Ix-%Ix",
5728 saved_post_plug_info_start));
5729 memcpy (saved_post_plug_info_start, &saved_post_plug, sizeof (saved_post_plug));
5736 void gc_mechanisms::init_mechanisms()
5738 condemned_generation = 0;
5739 promotion = FALSE;//TRUE;
5741 #ifdef FEATURE_LOH_COMPACTION
5742 loh_compaction = gc_heap::should_compact_loh();
5744 loh_compaction = FALSE;
5745 #endif //FEATURE_LOH_COMPACTION
5746 heap_expansion = FALSE;
5749 elevation_reduced = FALSE;
5750 found_finalizers = FALSE;
5751 #ifdef BACKGROUND_GC
5752 background_p = recursive_gc_sync::background_running_p() != FALSE;
5753 allocations_allowed = TRUE;
5754 #endif //BACKGROUND_GC
5756 entry_memory_load = 0;
5757 exit_memory_load = 0;
5760 stress_induced = FALSE;
5761 #endif // STRESS_HEAP
5764 void gc_mechanisms::first_init()
5767 gen0_reduction_count = 0;
5768 should_lock_elevation = FALSE;
5769 elevation_locked_count = 0;
5770 reason = reason_empty;
5771 #ifdef BACKGROUND_GC
5772 pause_mode = gc_heap::gc_can_use_concurrent ? pause_interactive : pause_batch;
5774 int debug_pause_mode = static_cast<int>(GCConfig::GetLatencyMode());
5775 if (debug_pause_mode >= 0)
5777 assert (debug_pause_mode <= pause_sustained_low_latency);
5778 pause_mode = (gc_pause_mode)debug_pause_mode;
5781 #else //BACKGROUND_GC
5782 pause_mode = pause_batch;
5783 #endif //BACKGROUND_GC
5788 void gc_mechanisms::record (gc_history_global* history)
5790 #ifdef MULTIPLE_HEAPS
5791 history->num_heaps = gc_heap::n_heaps;
5793 history->num_heaps = 1;
5794 #endif //MULTIPLE_HEAPS
5796 history->condemned_generation = condemned_generation;
5797 history->gen0_reduction_count = gen0_reduction_count;
5798 history->reason = reason;
5799 history->pause_mode = (int)pause_mode;
5800 history->mem_pressure = entry_memory_load;
5801 history->global_mechanims_p = 0;
5803 // start setting the boolean values.
5805 history->set_mechanism_p (global_concurrent);
5808 history->set_mechanism_p (global_compaction);
5811 history->set_mechanism_p (global_promotion);
5814 history->set_mechanism_p (global_demotion);
5817 history->set_mechanism_p (global_card_bundles);
5819 if (elevation_reduced)
5820 history->set_mechanism_p (global_elevation);
5823 /**********************************
5824 called at the beginning of GC to fix the allocated size to
5825 what is really allocated, or to turn the free area into an unused object
5826 It needs to be called after all of the other allocation contexts have been
5827 fixed since it relies on alloc_allocated.
5828 ********************************/
5830 //for_gc_p indicates that the work is being done for GC,
5831 //as opposed to concurrent heap verification
5832 void gc_heap::fix_youngest_allocation_area (BOOL for_gc_p)
5834 UNREFERENCED_PARAMETER(for_gc_p);
5836 // The gen 0 alloc context is never used for allocation in the allocator path. It's
5837 // still used in the allocation path during GCs.
5838 assert (generation_allocation_pointer (youngest_generation) == nullptr);
5839 assert (generation_allocation_limit (youngest_generation) == nullptr);
5840 heap_segment_allocated (ephemeral_heap_segment) = alloc_allocated;
5843 void gc_heap::fix_large_allocation_area (BOOL for_gc_p)
5845 UNREFERENCED_PARAMETER(for_gc_p);
5848 alloc_context* acontext =
5850 generation_alloc_context (large_object_generation);
5851 assert (acontext->alloc_ptr == 0);
5852 assert (acontext->alloc_limit == 0);
5854 dprintf (3, ("Large object alloc context: ptr: %Ix, limit %Ix",
5855 (size_t)acontext->alloc_ptr, (size_t)acontext->alloc_limit));
5856 fix_allocation_context (acontext, FALSE, get_alignment_constant (FALSE));
5859 acontext->alloc_ptr = 0;
5860 acontext->alloc_limit = acontext->alloc_ptr;
5865 //for_gc_p indicates that the work is being done for GC,
5866 //as opposed to concurrent heap verification
5867 void gc_heap::fix_allocation_context (alloc_context* acontext, BOOL for_gc_p,
5870 dprintf (3, ("Fixing allocation context %Ix: ptr: %Ix, limit: %Ix",
5872 (size_t)acontext->alloc_ptr, (size_t)acontext->alloc_limit));
5874 if (((size_t)(alloc_allocated - acontext->alloc_limit) > Align (min_obj_size, align_const)) ||
5877 uint8_t* point = acontext->alloc_ptr;
5880 size_t size = (acontext->alloc_limit - acontext->alloc_ptr);
5881 // the allocation area was from the free list
5882 // it was shortened by Align (min_obj_size) to make room for
5883 // at least the shortest unused object
5884 size += Align (min_obj_size, align_const);
5885 assert ((size >= Align (min_obj_size)));
5887 dprintf(3,("Making unused area [%Ix, %Ix[", (size_t)point,
5888 (size_t)point + size ));
5889 make_unused_array (point, size);
5893 generation_free_obj_space (generation_of (0)) += size;
5894 alloc_contexts_used ++;
5900 alloc_allocated = acontext->alloc_ptr;
5901 assert (heap_segment_allocated (ephemeral_heap_segment) <=
5902 heap_segment_committed (ephemeral_heap_segment));
5903 alloc_contexts_used ++;
5908 // We need to update the alloc_bytes to reflect the portion that we have not used
5909 acontext->alloc_bytes -= (acontext->alloc_limit - acontext->alloc_ptr);
5910 acontext->alloc_ptr = 0;
5911 acontext->alloc_limit = acontext->alloc_ptr;
5915 //used by the heap verification for concurrent gc.
5916 //it nulls out the words set by fix_allocation_context for heap_verification
5917 void repair_allocation (gc_alloc_context* acontext, void*)
5919 uint8_t* point = acontext->alloc_ptr;
5923 dprintf (3, ("Clearing [%Ix, %Ix[", (size_t)acontext->alloc_ptr,
5924 (size_t)acontext->alloc_limit+Align(min_obj_size)));
5925 memclr (acontext->alloc_ptr - plug_skew,
5926 (acontext->alloc_limit - acontext->alloc_ptr)+Align (min_obj_size));
5930 void void_allocation (gc_alloc_context* acontext, void*)
5932 uint8_t* point = acontext->alloc_ptr;
5936 dprintf (3, ("Void [%Ix, %Ix[", (size_t)acontext->alloc_ptr,
5937 (size_t)acontext->alloc_limit+Align(min_obj_size)));
5938 acontext->alloc_ptr = 0;
5939 acontext->alloc_limit = acontext->alloc_ptr;
5943 void gc_heap::repair_allocation_contexts (BOOL repair_p)
5945 GCToEEInterface::GcEnumAllocContexts (repair_p ? repair_allocation : void_allocation, NULL);
5948 struct fix_alloc_context_args
5954 void fix_alloc_context(gc_alloc_context* acontext, void* param)
5956 fix_alloc_context_args* args = (fix_alloc_context_args*)param;
5957 g_theGCHeap->FixAllocContext(acontext, false, (void*)(size_t)(args->for_gc_p), args->heap);
5960 void gc_heap::fix_allocation_contexts(BOOL for_gc_p)
5962 fix_alloc_context_args args;
5963 args.for_gc_p = for_gc_p;
5966 GCToEEInterface::GcEnumAllocContexts(fix_alloc_context, &args);
5967 fix_youngest_allocation_area(for_gc_p);
5968 fix_large_allocation_area(for_gc_p);
5971 void gc_heap::fix_older_allocation_area (generation* older_gen)
5973 heap_segment* older_gen_seg = generation_allocation_segment (older_gen);
5974 if (generation_allocation_limit (older_gen) !=
5975 heap_segment_plan_allocated (older_gen_seg))
5977 uint8_t* point = generation_allocation_pointer (older_gen);
5979 size_t size = (generation_allocation_limit (older_gen) -
5980 generation_allocation_pointer (older_gen));
5983 assert ((size >= Align (min_obj_size)));
5984 dprintf(3,("Making unused area [%Ix, %Ix[", (size_t)point, (size_t)point+size));
5985 make_unused_array (point, size);
5990 assert (older_gen_seg != ephemeral_heap_segment);
5991 heap_segment_plan_allocated (older_gen_seg) =
5992 generation_allocation_pointer (older_gen);
5993 generation_allocation_limit (older_gen) =
5994 generation_allocation_pointer (older_gen);
5998 void gc_heap::set_allocation_heap_segment (generation* gen)
6000 uint8_t* p = generation_allocation_start (gen);
6002 heap_segment* seg = generation_allocation_segment (gen);
6003 if (in_range_for_segment (p, seg))
6006 // try ephemeral heap segment in case of heap expansion
6007 seg = ephemeral_heap_segment;
6008 if (!in_range_for_segment (p, seg))
6010 seg = heap_segment_rw (generation_start_segment (gen));
6012 PREFIX_ASSUME(seg != NULL);
6014 while (!in_range_for_segment (p, seg))
6016 seg = heap_segment_next_rw (seg);
6017 PREFIX_ASSUME(seg != NULL);
6021 generation_allocation_segment (gen) = seg;
6024 void gc_heap::reset_allocation_pointers (generation* gen, uint8_t* start)
6027 assert (Align ((size_t)start) == (size_t)start);
6028 generation_allocation_start (gen) = start;
6029 generation_allocation_pointer (gen) = 0;//start + Align (min_obj_size);
6030 generation_allocation_limit (gen) = 0;//generation_allocation_pointer (gen);
6031 set_allocation_heap_segment (gen);
6034 #ifdef BACKGROUND_GC
6035 //TODO BACKGROUND_GC this is for test only
6037 gc_heap::disallow_new_allocation (int gen_number)
6039 UNREFERENCED_PARAMETER(gen_number);
6040 settings.allocations_allowed = FALSE;
6043 gc_heap::allow_new_allocation (int gen_number)
6045 UNREFERENCED_PARAMETER(gen_number);
6046 settings.allocations_allowed = TRUE;
6049 #endif //BACKGROUND_GC
6051 bool gc_heap::new_allocation_allowed (int gen_number)
6053 #ifdef BACKGROUND_GC
6054 //TODO BACKGROUND_GC this is for test only
6055 if (!settings.allocations_allowed)
6057 dprintf (2, ("new allocation not allowed"));
6060 #endif //BACKGROUND_GC
6062 if (dd_new_allocation (dynamic_data_of (gen_number)) < 0)
6064 if (gen_number != 0)
6066 // For LOH we will give it more budget before we try a GC.
6067 if (settings.concurrent)
6069 dynamic_data* dd2 = dynamic_data_of (max_generation + 1 );
6071 if (dd_new_allocation (dd2) <= (ptrdiff_t)(-2 * dd_desired_allocation (dd2)))
6079 #ifndef MULTIPLE_HEAPS
6080 else if ((settings.pause_mode != pause_no_gc) && (gen_number == 0))
6082 dprintf (3, ("evaluating allocation rate"));
6083 dynamic_data* dd0 = dynamic_data_of (0);
6084 if ((allocation_running_amount - dd_new_allocation (dd0)) >
6087 uint32_t ctime = GCToOSInterface::GetLowPrecisionTimeStamp();
6088 if ((ctime - allocation_running_time) > 1000)
6090 dprintf (2, (">1s since last gen0 gc"));
6095 allocation_running_amount = dd_new_allocation (dd0);
6099 #endif //MULTIPLE_HEAPS
6104 ptrdiff_t gc_heap::get_desired_allocation (int gen_number)
6106 return dd_desired_allocation (dynamic_data_of (gen_number));
6110 ptrdiff_t gc_heap::get_new_allocation (int gen_number)
6112 return dd_new_allocation (dynamic_data_of (gen_number));
6115 //return the amount allocated so far in gen_number
6117 ptrdiff_t gc_heap::get_allocation (int gen_number)
6119 dynamic_data* dd = dynamic_data_of (gen_number);
6121 return dd_desired_allocation (dd) - dd_new_allocation (dd);
6125 BOOL grow_mark_stack (mark*& m, size_t& len, size_t init_len)
6127 size_t new_size = max (init_len, 2*len);
6128 mark* tmp = new (nothrow) mark [new_size];
6131 memcpy (tmp, m, len * sizeof (mark));
6139 dprintf (1, ("Failed to allocate %Id bytes for mark stack", (len * sizeof (mark))));
6145 uint8_t* pinned_plug (mark* m)
6151 size_t& pinned_len (mark* m)
6157 void set_new_pin_info (mark* m, uint8_t* pin_free_space_start)
6159 m->len = pinned_plug (m) - pin_free_space_start;
6161 m->allocation_context_start_region = pin_free_space_start;
6162 #endif //SHORT_PLUGS
6167 uint8_t*& pin_allocation_context_start_region (mark* m)
6169 return m->allocation_context_start_region;
6172 uint8_t* get_plug_start_in_saved (uint8_t* old_loc, mark* pinned_plug_entry)
6174 uint8_t* saved_pre_plug_info = (uint8_t*)(pinned_plug_entry->get_pre_plug_reloc_info());
6175 uint8_t* plug_start_in_saved = saved_pre_plug_info + (old_loc - (pinned_plug (pinned_plug_entry) - sizeof (plug_and_gap)));
6176 //dprintf (1, ("detected a very short plug: %Ix before PP %Ix, pad %Ix",
6177 // old_loc, pinned_plug (pinned_plug_entry), plug_start_in_saved));
6178 dprintf (1, ("EP: %Ix(%Ix), %Ix", old_loc, pinned_plug (pinned_plug_entry), plug_start_in_saved));
6179 return plug_start_in_saved;
6183 void set_padding_in_expand (uint8_t* old_loc,
6184 BOOL set_padding_on_saved_p,
6185 mark* pinned_plug_entry)
6187 if (set_padding_on_saved_p)
6189 set_plug_padded (get_plug_start_in_saved (old_loc, pinned_plug_entry));
6193 set_plug_padded (old_loc);
6198 void clear_padding_in_expand (uint8_t* old_loc,
6199 BOOL set_padding_on_saved_p,
6200 mark* pinned_plug_entry)
6202 if (set_padding_on_saved_p)
6204 clear_plug_padded (get_plug_start_in_saved (old_loc, pinned_plug_entry));
6208 clear_plug_padded (old_loc);
6211 #endif //SHORT_PLUGS
6213 void gc_heap::reset_pinned_queue()
6219 void gc_heap::reset_pinned_queue_bos()
6224 // last_pinned_plug is only for asserting purpose.
6225 void gc_heap::merge_with_last_pinned_plug (uint8_t* last_pinned_plug, size_t plug_size)
6227 if (last_pinned_plug)
6229 mark& last_m = mark_stack_array[mark_stack_tos - 1];
6230 assert (last_pinned_plug == last_m.first);
6231 if (last_m.saved_post_p)
6233 last_m.saved_post_p = FALSE;
6234 dprintf (3, ("setting last plug %Ix post to false", last_m.first));
6235 // We need to recover what the gap has overwritten.
6236 memcpy ((last_m.first + last_m.len - sizeof (plug_and_gap)), &(last_m.saved_post_plug), sizeof (gap_reloc_pair));
6238 last_m.len += plug_size;
6239 dprintf (3, ("recovered the last part of plug %Ix, setting its plug size to %Ix", last_m.first, last_m.len));
6243 void gc_heap::set_allocator_next_pin (uint8_t* alloc_pointer, uint8_t*& alloc_limit)
6245 dprintf (3, ("sanp: ptr: %Ix, limit: %Ix", alloc_pointer, alloc_limit));
6246 dprintf (3, ("oldest %Id: %Ix", mark_stack_bos, pinned_plug (oldest_pin())));
6247 if (!(pinned_plug_que_empty_p()))
6249 mark* oldest_entry = oldest_pin();
6250 uint8_t* plug = pinned_plug (oldest_entry);
6251 if ((plug >= alloc_pointer) && (plug < alloc_limit))
6253 alloc_limit = pinned_plug (oldest_entry);
6254 dprintf (3, ("now setting alloc context: %Ix->%Ix(%Id)",
6255 alloc_pointer, alloc_limit, (alloc_limit - alloc_pointer)));
6260 void gc_heap::set_allocator_next_pin (generation* gen)
6262 dprintf (3, ("SANP: gen%d, ptr; %Ix, limit: %Ix", gen->gen_num, generation_allocation_pointer (gen), generation_allocation_limit (gen)));
6263 if (!(pinned_plug_que_empty_p()))
6265 mark* oldest_entry = oldest_pin();
6266 uint8_t* plug = pinned_plug (oldest_entry);
6267 if ((plug >= generation_allocation_pointer (gen)) &&
6268 (plug < generation_allocation_limit (gen)))
6270 generation_allocation_limit (gen) = pinned_plug (oldest_entry);
6271 dprintf (3, ("SANP: get next pin free space in gen%d for alloc: %Ix->%Ix(%Id)",
6273 generation_allocation_pointer (gen), generation_allocation_limit (gen),
6274 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
6277 assert (!((plug < generation_allocation_pointer (gen)) &&
6278 (plug >= heap_segment_mem (generation_allocation_segment (gen)))));
6282 // After we set the info, we increase tos.
6283 void gc_heap::set_pinned_info (uint8_t* last_pinned_plug, size_t plug_len, uint8_t* alloc_pointer, uint8_t*& alloc_limit)
6285 UNREFERENCED_PARAMETER(last_pinned_plug);
6287 mark& m = mark_stack_array[mark_stack_tos];
6288 assert (m.first == last_pinned_plug);
6292 set_allocator_next_pin (alloc_pointer, alloc_limit);
6295 // After we set the info, we increase tos.
6296 void gc_heap::set_pinned_info (uint8_t* last_pinned_plug, size_t plug_len, generation* gen)
6298 UNREFERENCED_PARAMETER(last_pinned_plug);
6300 mark& m = mark_stack_array[mark_stack_tos];
6301 assert (m.first == last_pinned_plug);
6306 // Why are we checking here? gen is never 0.
6309 set_allocator_next_pin (gen);
6313 size_t gc_heap::deque_pinned_plug ()
6315 dprintf (3, ("dequed: %Id", mark_stack_bos));
6316 size_t m = mark_stack_bos;
6322 mark* gc_heap::pinned_plug_of (size_t bos)
6324 return &mark_stack_array [ bos ];
6328 mark* gc_heap::oldest_pin ()
6330 return pinned_plug_of (mark_stack_bos);
6334 BOOL gc_heap::pinned_plug_que_empty_p ()
6336 return (mark_stack_bos == mark_stack_tos);
6340 mark* gc_heap::before_oldest_pin()
6342 if (mark_stack_bos >= 1)
6343 return pinned_plug_of (mark_stack_bos-1);
6349 BOOL gc_heap::ephemeral_pointer_p (uint8_t* o)
6351 return ((o >= ephemeral_low) && (o < ephemeral_high));
6356 int& gc_heap::mark_stack_busy()
6358 return g_mark_stack_busy [(heap_number+2)*HS_CACHE_LINE_SIZE/sizeof(int)];
6362 void gc_heap::make_mark_stack (mark* arr)
6364 reset_pinned_queue();
6365 mark_stack_array = arr;
6366 mark_stack_array_length = MARK_STACK_INITIAL_LENGTH;
6368 mark_stack_busy() = 0;
6372 #ifdef BACKGROUND_GC
6374 size_t& gc_heap::bpromoted_bytes(int thread)
6376 #ifdef MULTIPLE_HEAPS
6377 return g_bpromoted [thread*16];
6378 #else //MULTIPLE_HEAPS
6379 UNREFERENCED_PARAMETER(thread);
6381 #endif //MULTIPLE_HEAPS
6384 void gc_heap::make_background_mark_stack (uint8_t** arr)
6386 background_mark_stack_array = arr;
6387 background_mark_stack_array_length = MARK_STACK_INITIAL_LENGTH;
6388 background_mark_stack_tos = arr;
6391 void gc_heap::make_c_mark_list (uint8_t** arr)
6394 c_mark_list_index = 0;
6395 c_mark_list_length = 1 + (OS_PAGE_SIZE / MIN_OBJECT_SIZE);
6397 #endif //BACKGROUND_GC
6402 // The card bundle keeps track of groups of card words.
6403 static const size_t card_bundle_word_width = 32;
6405 // How do we express the fact that 32 bits (card_word_width) is one uint32_t?
6406 static const size_t card_bundle_size = (size_t)(GC_PAGE_SIZE / (sizeof(uint32_t)*card_bundle_word_width));
6409 size_t card_bundle_word (size_t cardb)
6411 return cardb / card_bundle_word_width;
6415 uint32_t card_bundle_bit (size_t cardb)
6417 return (uint32_t)(cardb % card_bundle_word_width);
6420 size_t align_cardw_on_bundle (size_t cardw)
6422 return ((size_t)(cardw + card_bundle_size - 1) & ~(card_bundle_size - 1 ));
6425 // Get the card bundle representing a card word
6426 size_t cardw_card_bundle (size_t cardw)
6428 return cardw / card_bundle_size;
6431 // Get the first card word in a card bundle
6432 size_t card_bundle_cardw (size_t cardb)
6434 return cardb * card_bundle_size;
6437 // Clear the specified card bundle
6438 void gc_heap::card_bundle_clear (size_t cardb)
6440 card_bundle_table [card_bundle_word (cardb)] &= ~(1 << card_bundle_bit (cardb));
6441 dprintf (1,("Cleared card bundle %Ix [%Ix, %Ix[", cardb, (size_t)card_bundle_cardw (cardb),
6442 (size_t)card_bundle_cardw (cardb+1)));
6445 void gc_heap::card_bundle_set (size_t cardb)
6447 if (!card_bundle_set_p (cardb))
6449 card_bundle_table [card_bundle_word (cardb)] |= (1 << card_bundle_bit (cardb));
6453 // Set the card bundle bits between start_cardb and end_cardb
6454 void gc_heap::card_bundles_set (size_t start_cardb, size_t end_cardb)
6456 if (start_cardb == end_cardb)
6458 card_bundle_set(start_cardb);
6462 size_t start_word = card_bundle_word (start_cardb);
6463 size_t end_word = card_bundle_word (end_cardb);
6465 if (start_word < end_word)
6467 // Set the partial words
6468 card_bundle_table [start_word] |= highbits (~0u, card_bundle_bit (start_cardb));
6470 if (card_bundle_bit (end_cardb))
6471 card_bundle_table [end_word] |= lowbits (~0u, card_bundle_bit (end_cardb));
6473 // Set the full words
6474 for (size_t i = start_word + 1; i < end_word; i++)
6475 card_bundle_table [i] = ~0u;
6479 card_bundle_table [start_word] |= (highbits (~0u, card_bundle_bit (start_cardb)) &
6480 lowbits (~0u, card_bundle_bit (end_cardb)));
6484 // Indicates whether the specified bundle is set.
6485 BOOL gc_heap::card_bundle_set_p (size_t cardb)
6487 return (card_bundle_table[card_bundle_word(cardb)] & (1 << card_bundle_bit (cardb)));
6490 // Returns the size (in bytes) of a card bundle representing the region from 'from' to 'end'
6491 size_t size_card_bundle_of (uint8_t* from, uint8_t* end)
6493 // Number of heap bytes represented by a card bundle word
6494 size_t cbw_span = card_size * card_word_width * card_bundle_size * card_bundle_word_width;
6496 // Align the start of the region down
6497 from = (uint8_t*)((size_t)from & ~(cbw_span - 1));
6499 // Align the end of the region up
6500 end = (uint8_t*)((size_t)(end + (cbw_span - 1)) & ~(cbw_span - 1));
6502 // Make sure they're really aligned
6503 assert (((size_t)from & (cbw_span - 1)) == 0);
6504 assert (((size_t)end & (cbw_span - 1)) == 0);
6506 return ((end - from) / cbw_span) * sizeof (uint32_t);
6509 // Takes a pointer to a card bundle table and an address, and returns a pointer that represents
6510 // where a theoretical card bundle table that represents every address (starting from 0) would
6511 // start if the bundle word representing the address were to be located at the pointer passed in.
6512 // The returned 'translated' pointer makes it convenient/fast to calculate where the card bundle
6513 // for a given address is using a simple shift operation on the address.
6514 uint32_t* translate_card_bundle_table (uint32_t* cb, uint8_t* lowest_address)
6516 // The number of bytes of heap memory represented by a card bundle word
6517 const size_t heap_bytes_for_bundle_word = card_size * card_word_width * card_bundle_size * card_bundle_word_width;
6519 // Each card bundle word is 32 bits
6520 return (uint32_t*)((uint8_t*)cb - (((size_t)lowest_address / heap_bytes_for_bundle_word) * sizeof (uint32_t)));
6523 void gc_heap::enable_card_bundles ()
6525 if (can_use_write_watch_for_card_table() && (!card_bundles_enabled()))
6527 dprintf (1, ("Enabling card bundles"));
6529 // We initially set all of the card bundles
6530 card_bundles_set (cardw_card_bundle (card_word (card_of (lowest_address))),
6531 cardw_card_bundle (align_cardw_on_bundle (card_word (card_of (highest_address)))));
6532 settings.card_bundles = TRUE;
6536 BOOL gc_heap::card_bundles_enabled ()
6538 return settings.card_bundles;
6541 #endif // CARD_BUNDLE
6543 #if defined (_TARGET_AMD64_)
6544 #define brick_size ((size_t)4096)
6546 #define brick_size ((size_t)2048)
6547 #endif //_TARGET_AMD64_
6550 size_t gc_heap::brick_of (uint8_t* add)
6552 return (size_t)(add - lowest_address) / brick_size;
6556 uint8_t* gc_heap::brick_address (size_t brick)
6558 return lowest_address + (brick_size * brick);
6562 void gc_heap::clear_brick_table (uint8_t* from, uint8_t* end)
6564 for (size_t i = brick_of (from);i < brick_of (end); i++)
6568 //codes for the brick entries:
6569 //entry == 0 -> not assigned
6570 //entry >0 offset is entry-1
6571 //entry <0 jump back entry bricks
6575 void gc_heap::set_brick (size_t index, ptrdiff_t val)
6581 assert (val < 32767);
6583 brick_table [index] = (short)val+1;
6585 brick_table [index] = (short)val;
6589 int gc_heap::get_brick_entry (size_t index)
6591 #ifdef MULTIPLE_HEAPS
6592 return VolatileLoadWithoutBarrier(&brick_table [index]);
6594 return brick_table[index];
6600 uint8_t* align_on_brick (uint8_t* add)
6602 return (uint8_t*)((size_t)(add + brick_size - 1) & ~(brick_size - 1));
6606 uint8_t* align_lower_brick (uint8_t* add)
6608 return (uint8_t*)(((size_t)add) & ~(brick_size - 1));
6611 size_t size_brick_of (uint8_t* from, uint8_t* end)
6613 assert (((size_t)from & (brick_size-1)) == 0);
6614 assert (((size_t)end & (brick_size-1)) == 0);
6616 return ((end - from) / brick_size) * sizeof (short);
6620 uint8_t* gc_heap::card_address (size_t card)
6622 return (uint8_t*) (card_size * card);
6626 size_t gc_heap::card_of ( uint8_t* object)
6628 return (size_t)(object) / card_size;
6632 size_t gc_heap::card_to_brick (size_t card)
6634 return brick_of (card_address (card));
6638 uint8_t* align_on_card (uint8_t* add)
6640 return (uint8_t*)((size_t)(add + card_size - 1) & ~(card_size - 1 ));
6643 uint8_t* align_on_card_word (uint8_t* add)
6645 return (uint8_t*) ((size_t)(add + (card_size*card_word_width)-1) & ~(card_size*card_word_width - 1));
6649 uint8_t* align_lower_card (uint8_t* add)
6651 return (uint8_t*)((size_t)add & ~(card_size-1));
6655 void gc_heap::clear_card (size_t card)
6657 card_table [card_word (card)] =
6658 (card_table [card_word (card)] & ~(1 << card_bit (card)));
6659 dprintf (3,("Cleared card %Ix [%Ix, %Ix[", card, (size_t)card_address (card),
6660 (size_t)card_address (card+1)));
6664 void gc_heap::set_card (size_t card)
6666 size_t word = card_word (card);
6667 card_table[word] = (card_table [word] | (1 << card_bit (card)));
6669 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
6670 // Also set the card bundle that corresponds to the card
6671 size_t bundle_to_set = cardw_card_bundle(word);
6673 card_bundle_set(bundle_to_set);
6675 dprintf (3,("Set card %Ix [%Ix, %Ix[ and bundle %Ix", card, (size_t)card_address (card), (size_t)card_address (card+1), bundle_to_set));
6676 assert(card_bundle_set_p(bundle_to_set) != 0);
6681 BOOL gc_heap::card_set_p (size_t card)
6683 return ( card_table [ card_word (card) ] & (1 << card_bit (card)));
6686 // Returns the number of DWORDs in the card table that cover the
6687 // range of addresses [from, end[.
6688 size_t count_card_of (uint8_t* from, uint8_t* end)
6690 return card_word (gcard_of (end - 1)) - card_word (gcard_of (from)) + 1;
6693 // Returns the number of bytes to allocate for a card table
6694 // that covers the range of addresses [from, end[.
6695 size_t size_card_of (uint8_t* from, uint8_t* end)
6697 return count_card_of (from, end) * sizeof(uint32_t);
6700 // We don't store seg_mapping_table in card_table_info because there's only always one view.
6701 class card_table_info
6705 uint8_t* lowest_address;
6706 uint8_t* highest_address;
6710 uint32_t* card_bundle_table;
6711 #endif //CARD_BUNDLE
6713 // mark_array is always at the end of the data structure because we
6714 // want to be able to make one commit call for everything before it.
6716 uint32_t* mark_array;
6720 uint32_t* next_card_table;
6723 //These are accessors on untranslated cardtable
6725 unsigned& card_table_refcount (uint32_t* c_table)
6727 return *(unsigned*)((char*)c_table - sizeof (card_table_info));
6731 uint8_t*& card_table_lowest_address (uint32_t* c_table)
6733 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->lowest_address;
6736 uint32_t* translate_card_table (uint32_t* ct)
6738 return (uint32_t*)((uint8_t*)ct - card_word (gcard_of (card_table_lowest_address (ct))) * sizeof(uint32_t));
6742 uint8_t*& card_table_highest_address (uint32_t* c_table)
6744 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->highest_address;
6748 short*& card_table_brick_table (uint32_t* c_table)
6750 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->brick_table;
6754 // Get the card bundle table for the specified card table.
6756 uint32_t*& card_table_card_bundle_table (uint32_t* c_table)
6758 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->card_bundle_table;
6760 #endif //CARD_BUNDLE
6763 /* Support for mark_array */
6766 uint32_t*& card_table_mark_array (uint32_t* c_table)
6768 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->mark_array;
6772 #define mark_bit_pitch ((size_t)16)
6774 #define mark_bit_pitch ((size_t)8)
6776 #define mark_word_width ((size_t)32)
6777 #define mark_word_size (mark_word_width * mark_bit_pitch)
6780 uint8_t* align_on_mark_bit (uint8_t* add)
6782 return (uint8_t*)((size_t)(add + (mark_bit_pitch - 1)) & ~(mark_bit_pitch - 1));
6786 uint8_t* align_lower_mark_bit (uint8_t* add)
6788 return (uint8_t*)((size_t)(add) & ~(mark_bit_pitch - 1));
6792 BOOL is_aligned_on_mark_word (uint8_t* add)
6794 return ((size_t)add == ((size_t)(add) & ~(mark_word_size - 1)));
6798 uint8_t* align_on_mark_word (uint8_t* add)
6800 return (uint8_t*)((size_t)(add + mark_word_size - 1) & ~(mark_word_size - 1));
6804 uint8_t* align_lower_mark_word (uint8_t* add)
6806 return (uint8_t*)((size_t)(add) & ~(mark_word_size - 1));
6810 size_t mark_bit_of (uint8_t* add)
6812 return ((size_t)add / mark_bit_pitch);
6816 unsigned int mark_bit_bit (size_t mark_bit)
6818 return (unsigned int)(mark_bit % mark_word_width);
6822 size_t mark_bit_word (size_t mark_bit)
6824 return (mark_bit / mark_word_width);
6828 size_t mark_word_of (uint8_t* add)
6830 return ((size_t)add) / mark_word_size;
6833 uint8_t* mark_word_address (size_t wd)
6835 return (uint8_t*)(wd*mark_word_size);
6838 uint8_t* mark_bit_address (size_t mark_bit)
6840 return (uint8_t*)(mark_bit*mark_bit_pitch);
6844 size_t mark_bit_bit_of (uint8_t* add)
6846 return (((size_t)add / mark_bit_pitch) % mark_word_width);
6850 unsigned int gc_heap::mark_array_marked(uint8_t* add)
6852 return mark_array [mark_word_of (add)] & (1 << mark_bit_bit_of (add));
6856 BOOL gc_heap::is_mark_bit_set (uint8_t* add)
6858 return (mark_array [mark_word_of (add)] & (1 << mark_bit_bit_of (add)));
6862 void gc_heap::mark_array_set_marked (uint8_t* add)
6864 size_t index = mark_word_of (add);
6865 uint32_t val = (1 << mark_bit_bit_of (add));
6866 #ifdef MULTIPLE_HEAPS
6867 Interlocked::Or (&(mark_array [index]), val);
6869 mark_array [index] |= val;
6874 void gc_heap::mark_array_clear_marked (uint8_t* add)
6876 mark_array [mark_word_of (add)] &= ~(1 << mark_bit_bit_of (add));
6879 size_t size_mark_array_of (uint8_t* from, uint8_t* end)
6881 assert (((size_t)from & ((mark_word_size)-1)) == 0);
6882 assert (((size_t)end & ((mark_word_size)-1)) == 0);
6883 return sizeof (uint32_t)*(((end - from) / mark_word_size));
6886 //In order to eliminate the lowest_address in the mark array
6887 //computations (mark_word_of, etc) mark_array is offset
6888 // according to the lowest_address.
6889 uint32_t* translate_mark_array (uint32_t* ma)
6891 return (uint32_t*)((uint8_t*)ma - size_mark_array_of (0, g_gc_lowest_address));
6894 // from and end must be page aligned addresses.
6895 void gc_heap::clear_mark_array (uint8_t* from, uint8_t* end, BOOL check_only/*=TRUE*/
6896 #ifdef FEATURE_BASICFREEZE
6897 , BOOL read_only/*=FALSE*/
6898 #endif // FEATURE_BASICFREEZE
6901 if(!gc_can_use_concurrent)
6904 #ifdef FEATURE_BASICFREEZE
6906 #endif // FEATURE_BASICFREEZE
6908 assert (from == align_on_mark_word (from));
6910 assert (end == align_on_mark_word (end));
6912 #ifdef BACKGROUND_GC
6913 uint8_t* current_lowest_address = background_saved_lowest_address;
6914 uint8_t* current_highest_address = background_saved_highest_address;
6916 uint8_t* current_lowest_address = lowest_address;
6917 uint8_t* current_highest_address = highest_address;
6918 #endif //BACKGROUND_GC
6920 //there is a possibility of the addresses to be
6921 //outside of the covered range because of a newly allocated
6922 //large object segment
6923 if ((end <= current_highest_address) && (from >= current_lowest_address))
6925 size_t beg_word = mark_word_of (align_on_mark_word (from));
6926 MAYBE_UNUSED_VAR(beg_word);
6927 //align end word to make sure to cover the address
6928 size_t end_word = mark_word_of (align_on_mark_word (end));
6929 MAYBE_UNUSED_VAR(end_word);
6930 dprintf (3, ("Calling clearing mark array [%Ix, %Ix[ for addresses [%Ix, %Ix[(%s)",
6931 (size_t)mark_word_address (beg_word),
6932 (size_t)mark_word_address (end_word),
6933 (size_t)from, (size_t)end,
6934 (check_only ? "check_only" : "clear")));
6938 while (op < mark_word_address (beg_word))
6940 mark_array_clear_marked (op);
6941 op += mark_bit_pitch;
6944 memset (&mark_array[beg_word], 0, (end_word - beg_word)*sizeof (uint32_t));
6949 //Beware, it is assumed that the mark array word straddling
6950 //start has been cleared before
6951 //verify that the array is empty.
6952 size_t markw = mark_word_of (align_on_mark_word (from));
6953 size_t markw_end = mark_word_of (align_on_mark_word (end));
6954 while (markw < markw_end)
6956 assert (!(mark_array [markw]));
6959 uint8_t* p = mark_word_address (markw_end);
6962 assert (!(mark_array_marked (p)));
6971 //These work on untranslated card tables
6973 uint32_t*& card_table_next (uint32_t* c_table)
6975 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->next_card_table;
6979 size_t& card_table_size (uint32_t* c_table)
6981 return ((card_table_info*)((uint8_t*)c_table - sizeof (card_table_info)))->size;
6984 void own_card_table (uint32_t* c_table)
6986 card_table_refcount (c_table) += 1;
6989 void destroy_card_table (uint32_t* c_table);
6991 void delete_next_card_table (uint32_t* c_table)
6993 uint32_t* n_table = card_table_next (c_table);
6996 if (card_table_next (n_table))
6998 delete_next_card_table (n_table);
7000 if (card_table_refcount (n_table) == 0)
7002 destroy_card_table (n_table);
7003 card_table_next (c_table) = 0;
7008 void release_card_table (uint32_t* c_table)
7010 assert (card_table_refcount (c_table) >0);
7011 card_table_refcount (c_table) -= 1;
7012 if (card_table_refcount (c_table) == 0)
7014 delete_next_card_table (c_table);
7015 if (card_table_next (c_table) == 0)
7017 destroy_card_table (c_table);
7018 // sever the link from the parent
7019 if (&g_gc_card_table[card_word (gcard_of(g_gc_lowest_address))] == c_table)
7021 g_gc_card_table = 0;
7023 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7024 g_gc_card_bundle_table = 0;
7026 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7027 SoftwareWriteWatch::StaticClose();
7028 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7032 uint32_t* p_table = &g_gc_card_table[card_word (gcard_of(g_gc_lowest_address))];
7035 while (p_table && (card_table_next (p_table) != c_table))
7036 p_table = card_table_next (p_table);
7037 card_table_next (p_table) = 0;
7044 void destroy_card_table (uint32_t* c_table)
7046 // delete (uint32_t*)&card_table_refcount(c_table);
7048 GCToOSInterface::VirtualRelease (&card_table_refcount(c_table), card_table_size(c_table));
7049 dprintf (2, ("Table Virtual Free : %Ix", (size_t)&card_table_refcount(c_table)));
7052 uint32_t* gc_heap::make_card_table (uint8_t* start, uint8_t* end)
7054 assert (g_gc_lowest_address == start);
7055 assert (g_gc_highest_address == end);
7057 uint32_t virtual_reserve_flags = VirtualReserveFlags::None;
7059 size_t bs = size_brick_of (start, end);
7060 size_t cs = size_card_of (start, end);
7062 size_t ms = (gc_can_use_concurrent ?
7063 size_mark_array_of (start, end) :
7072 if (can_use_write_watch_for_card_table())
7074 cb = size_card_bundle_of (g_gc_lowest_address, g_gc_highest_address);
7075 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7076 // If we're not manually managing the card bundles, we will need to use OS write
7077 // watch APIs over this region to track changes.
7078 virtual_reserve_flags |= VirtualReserveFlags::WriteWatch;
7081 #endif //CARD_BUNDLE
7084 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7085 size_t sw_ww_table_offset = 0;
7086 if (gc_can_use_concurrent)
7088 size_t sw_ww_size_before_table = sizeof(card_table_info) + cs + bs + cb;
7089 sw_ww_table_offset = SoftwareWriteWatch::GetTableStartByteOffset(sw_ww_size_before_table);
7090 wws = sw_ww_table_offset - sw_ww_size_before_table + SoftwareWriteWatch::GetTableByteSize(start, end);
7092 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7094 #ifdef GROWABLE_SEG_MAPPING_TABLE
7095 size_t st = size_seg_mapping_table_of (g_gc_lowest_address, g_gc_highest_address);
7096 size_t st_table_offset = sizeof(card_table_info) + cs + bs + cb + wws;
7097 size_t st_table_offset_aligned = align_for_seg_mapping_table (st_table_offset);
7099 st += (st_table_offset_aligned - st_table_offset);
7100 #else //GROWABLE_SEG_MAPPING_TABLE
7102 #endif //GROWABLE_SEG_MAPPING_TABLE
7104 // it is impossible for alloc_size to overflow due bounds on each of
7106 size_t alloc_size = sizeof (uint8_t)*(sizeof(card_table_info) + cs + bs + cb + wws + st + ms);
7107 uint8_t* mem = (uint8_t*)GCToOSInterface::VirtualReserve (alloc_size, 0, virtual_reserve_flags);
7112 dprintf (2, ("Init - Card table alloc for %Id bytes: [%Ix, %Ix[",
7113 alloc_size, (size_t)mem, (size_t)(mem+alloc_size)));
7115 // mark array will be committed separately (per segment).
7116 size_t commit_size = alloc_size - ms;
7118 if (!GCToOSInterface::VirtualCommit (mem, commit_size))
7120 dprintf (2, ("Card table commit failed"));
7121 GCToOSInterface::VirtualRelease (mem, alloc_size);
7125 // initialize the ref count
7126 uint32_t* ct = (uint32_t*)(mem+sizeof (card_table_info));
7127 card_table_refcount (ct) = 0;
7128 card_table_lowest_address (ct) = start;
7129 card_table_highest_address (ct) = end;
7130 card_table_brick_table (ct) = (short*)((uint8_t*)ct + cs);
7131 card_table_size (ct) = alloc_size;
7132 card_table_next (ct) = 0;
7135 card_table_card_bundle_table (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs);
7137 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7138 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), g_gc_lowest_address);
7141 #endif //CARD_BUNDLE
7143 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7144 if (gc_can_use_concurrent)
7146 SoftwareWriteWatch::InitializeUntranslatedTable(mem + sw_ww_table_offset, start);
7148 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7150 #ifdef GROWABLE_SEG_MAPPING_TABLE
7151 seg_mapping_table = (seg_mapping*)(mem + st_table_offset_aligned);
7152 seg_mapping_table = (seg_mapping*)((uint8_t*)seg_mapping_table -
7153 size_seg_mapping_table_of (0, (align_lower_segment (g_gc_lowest_address))));
7154 #endif //GROWABLE_SEG_MAPPING_TABLE
7157 if (gc_can_use_concurrent)
7158 card_table_mark_array (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs + cb + wws + st);
7160 card_table_mark_array (ct) = NULL;
7163 return translate_card_table(ct);
7166 void gc_heap::set_fgm_result (failure_get_memory f, size_t s, BOOL loh_p)
7168 #ifdef MULTIPLE_HEAPS
7169 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
7171 gc_heap* hp = gc_heap::g_heaps [hn];
7172 hp->fgm_result.set_fgm (f, s, loh_p);
7174 #else //MULTIPLE_HEAPS
7175 fgm_result.set_fgm (f, s, loh_p);
7176 #endif //MULTIPLE_HEAPS
7179 //returns 0 for success, -1 otherwise
7180 // We are doing all the decommitting here because we want to make sure we have
7181 // enough memory to do so - if we do this during copy_brick_card_table and
7182 // and fail to decommit it would make the failure case very complicated to
7183 // handle. This way we can waste some decommit if we call this multiple
7184 // times before the next FGC but it's easier to handle the failure case.
7185 int gc_heap::grow_brick_card_tables (uint8_t* start,
7188 heap_segment* new_seg,
7192 uint8_t* la = g_gc_lowest_address;
7193 uint8_t* ha = g_gc_highest_address;
7194 uint8_t* saved_g_lowest_address = min (start, g_gc_lowest_address);
7195 uint8_t* saved_g_highest_address = max (end, g_gc_highest_address);
7196 seg_mapping* new_seg_mapping_table = nullptr;
7197 #ifdef BACKGROUND_GC
7198 // This value is only for logging purpose - it's not necessarily exactly what we
7199 // would commit for mark array but close enough for diagnostics purpose.
7200 size_t logging_ma_commit_size = size_mark_array_of (0, (uint8_t*)size);
7201 #endif //BACKGROUND_GC
7203 // See if the address is already covered
7204 if ((la != saved_g_lowest_address ) || (ha != saved_g_highest_address))
7207 //modify the higest address so the span covered
7208 //is twice the previous one.
7209 uint8_t* top = (uint8_t*)0 + Align (GCToOSInterface::GetVirtualMemoryLimit());
7210 // On non-Windows systems, we get only an approximate value that can possibly be
7211 // slightly lower than the saved_g_highest_address.
7212 // In such case, we set the top to the saved_g_highest_address so that the
7213 // card and brick tables always cover the whole new range.
7214 if (top < saved_g_highest_address)
7216 top = saved_g_highest_address;
7220 if (ps > (uint64_t)200*1024*1024*1024)
7221 ps += (uint64_t)100*1024*1024*1024;
7226 if (saved_g_lowest_address < g_gc_lowest_address)
7228 if (ps > (size_t)g_gc_lowest_address)
7229 saved_g_lowest_address = (uint8_t*)(size_t)OS_PAGE_SIZE;
7232 assert (((size_t)g_gc_lowest_address - ps) >= OS_PAGE_SIZE);
7233 saved_g_lowest_address = min (saved_g_lowest_address, (g_gc_lowest_address - ps));
7237 if (saved_g_highest_address > g_gc_highest_address)
7239 saved_g_highest_address = max ((saved_g_lowest_address + ps), saved_g_highest_address);
7240 if (saved_g_highest_address > top)
7241 saved_g_highest_address = top;
7244 dprintf (GC_TABLE_LOG, ("Growing card table [%Ix, %Ix[",
7245 (size_t)saved_g_lowest_address,
7246 (size_t)saved_g_highest_address));
7248 bool write_barrier_updated = false;
7249 uint32_t virtual_reserve_flags = VirtualReserveFlags::None;
7250 uint32_t* saved_g_card_table = g_gc_card_table;
7252 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7253 uint32_t* saved_g_card_bundle_table = g_gc_card_bundle_table;
7257 uint32_t* translated_ct = 0;
7260 size_t cs = size_card_of (saved_g_lowest_address, saved_g_highest_address);
7261 size_t bs = size_brick_of (saved_g_lowest_address, saved_g_highest_address);
7264 size_t ms = (gc_heap::gc_can_use_concurrent ?
7265 size_mark_array_of (saved_g_lowest_address, saved_g_highest_address) :
7274 if (can_use_write_watch_for_card_table())
7276 cb = size_card_bundle_of (saved_g_lowest_address, saved_g_highest_address);
7278 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7279 // If we're not manually managing the card bundles, we will need to use OS write
7280 // watch APIs over this region to track changes.
7281 virtual_reserve_flags |= VirtualReserveFlags::WriteWatch;
7284 #endif //CARD_BUNDLE
7287 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7288 size_t sw_ww_table_offset = 0;
7289 if (gc_can_use_concurrent)
7291 size_t sw_ww_size_before_table = sizeof(card_table_info) + cs + bs + cb;
7292 sw_ww_table_offset = SoftwareWriteWatch::GetTableStartByteOffset(sw_ww_size_before_table);
7294 sw_ww_table_offset -
7295 sw_ww_size_before_table +
7296 SoftwareWriteWatch::GetTableByteSize(saved_g_lowest_address, saved_g_highest_address);
7298 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7300 #ifdef GROWABLE_SEG_MAPPING_TABLE
7301 size_t st = size_seg_mapping_table_of (saved_g_lowest_address, saved_g_highest_address);
7302 size_t st_table_offset = sizeof(card_table_info) + cs + bs + cb + wws;
7303 size_t st_table_offset_aligned = align_for_seg_mapping_table (st_table_offset);
7304 st += (st_table_offset_aligned - st_table_offset);
7305 #else //GROWABLE_SEG_MAPPING_TABLE
7307 #endif //GROWABLE_SEG_MAPPING_TABLE
7309 // it is impossible for alloc_size to overflow due bounds on each of
7311 size_t alloc_size = sizeof (uint8_t)*(sizeof(card_table_info) + cs + bs + cb + wws + st + ms);
7312 dprintf (GC_TABLE_LOG, ("card table: %Id; brick table: %Id; card bundle: %Id; sw ww table: %Id; seg table: %Id; mark array: %Id",
7313 cs, bs, cb, wws, st, ms));
7315 uint8_t* mem = (uint8_t*)GCToOSInterface::VirtualReserve (alloc_size, 0, virtual_reserve_flags);
7319 set_fgm_result (fgm_grow_table, alloc_size, loh_p);
7323 dprintf (GC_TABLE_LOG, ("Table alloc for %Id bytes: [%Ix, %Ix[",
7324 alloc_size, (size_t)mem, (size_t)((uint8_t*)mem+alloc_size)));
7327 // mark array will be committed separately (per segment).
7328 size_t commit_size = alloc_size - ms;
7330 if (!GCToOSInterface::VirtualCommit (mem, commit_size))
7332 dprintf (GC_TABLE_LOG, ("Table commit failed"));
7333 set_fgm_result (fgm_commit_table, commit_size, loh_p);
7338 ct = (uint32_t*)(mem + sizeof (card_table_info));
7339 card_table_refcount (ct) = 0;
7340 card_table_lowest_address (ct) = saved_g_lowest_address;
7341 card_table_highest_address (ct) = saved_g_highest_address;
7342 card_table_next (ct) = &g_gc_card_table[card_word (gcard_of (la))];
7344 //clear the card table
7346 memclr ((uint8_t*)ct,
7347 (((saved_g_highest_address - saved_g_lowest_address)*sizeof (uint32_t) /
7348 (card_size * card_word_width))
7349 + sizeof (uint32_t)));
7352 bt = (short*)((uint8_t*)ct + cs);
7354 // No initialization needed, will be done in copy_brick_card
7356 card_table_brick_table (ct) = bt;
7359 card_table_card_bundle_table (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs);
7360 //set all bundle to look at all of the cards
7361 memset(card_table_card_bundle_table (ct), 0xFF, cb);
7362 #endif //CARD_BUNDLE
7364 #ifdef GROWABLE_SEG_MAPPING_TABLE
7366 new_seg_mapping_table = (seg_mapping*)(mem + st_table_offset_aligned);
7367 new_seg_mapping_table = (seg_mapping*)((uint8_t*)new_seg_mapping_table -
7368 size_seg_mapping_table_of (0, (align_lower_segment (saved_g_lowest_address))));
7369 memcpy(&new_seg_mapping_table[seg_mapping_word_of(g_gc_lowest_address)],
7370 &seg_mapping_table[seg_mapping_word_of(g_gc_lowest_address)],
7371 size_seg_mapping_table_of(g_gc_lowest_address, g_gc_highest_address));
7373 // new_seg_mapping_table gets assigned to seg_mapping_table at the bottom of this function,
7374 // not here. The reason for this is that, if we fail at mark array committing (OOM) and we've
7375 // already switched seg_mapping_table to point to the new mapping table, we'll decommit it and
7376 // run into trouble. By not assigning here, we're making sure that we will not change seg_mapping_table
7377 // if an OOM occurs.
7379 #endif //GROWABLE_SEG_MAPPING_TABLE
7382 if(gc_can_use_concurrent)
7383 card_table_mark_array (ct) = (uint32_t*)((uint8_t*)card_table_brick_table (ct) + bs + cb + wws + st);
7385 card_table_mark_array (ct) = NULL;
7388 translated_ct = translate_card_table (ct);
7390 dprintf (GC_TABLE_LOG, ("card table: %Ix(translated: %Ix), seg map: %Ix, mark array: %Ix",
7391 (size_t)ct, (size_t)translated_ct, (size_t)new_seg_mapping_table, (size_t)card_table_mark_array (ct)));
7393 #ifdef BACKGROUND_GC
7394 if (hp->should_commit_mark_array())
7396 dprintf (GC_TABLE_LOG, ("new low: %Ix, new high: %Ix, latest mark array is %Ix(translate: %Ix)",
7397 saved_g_lowest_address, saved_g_highest_address,
7398 card_table_mark_array (ct),
7399 translate_mark_array (card_table_mark_array (ct))));
7400 uint32_t* new_mark_array = (uint32_t*)((uint8_t*)card_table_mark_array (ct) - size_mark_array_of (0, saved_g_lowest_address));
7401 if (!commit_new_mark_array_global (new_mark_array))
7403 dprintf (GC_TABLE_LOG, ("failed to commit portions in the mark array for existing segments"));
7404 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7408 if (!commit_mark_array_new_seg (hp, new_seg, translated_ct, saved_g_lowest_address))
7410 dprintf (GC_TABLE_LOG, ("failed to commit mark array for the new seg"));
7411 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7417 clear_commit_flag_global();
7419 #endif //BACKGROUND_GC
7421 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7422 if (gc_can_use_concurrent)
7424 // The current design of software write watch requires that the runtime is suspended during resize. Suspending
7425 // on resize is preferred because it is a far less frequent operation than GetWriteWatch() / ResetWriteWatch().
7426 // Suspending here allows copying dirty state from the old table into the new table, and not have to merge old
7427 // table info lazily as done for card tables.
7429 // Either this thread was the thread that did the suspension which means we are suspended; or this is called
7430 // from a GC thread which means we are in a blocking GC and also suspended.
7431 bool is_runtime_suspended = GCToEEInterface::IsGCThread();
7432 if (!is_runtime_suspended)
7434 // Note on points where the runtime is suspended anywhere in this function. Upon an attempt to suspend the
7435 // runtime, a different thread may suspend first, causing this thread to block at the point of the suspend call.
7436 // So, at any suspend point, externally visible state needs to be consistent, as code that depends on that state
7437 // may run while this thread is blocked. This includes updates to g_gc_card_table, g_gc_lowest_address, and
7438 // g_gc_highest_address.
7442 g_gc_card_table = translated_ct;
7444 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7445 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), saved_g_lowest_address);
7448 SoftwareWriteWatch::SetResizedUntranslatedTable(
7449 mem + sw_ww_table_offset,
7450 saved_g_lowest_address,
7451 saved_g_highest_address);
7453 // Since the runtime is already suspended, update the write barrier here as well.
7454 // This passes a bool telling whether we need to switch to the post
7455 // grow version of the write barrier. This test tells us if the new
7456 // segment was allocated at a lower address than the old, requiring
7457 // that we start doing an upper bounds check in the write barrier.
7458 g_gc_lowest_address = saved_g_lowest_address;
7459 g_gc_highest_address = saved_g_highest_address;
7460 stomp_write_barrier_resize(true, la != saved_g_lowest_address);
7461 write_barrier_updated = true;
7463 if (!is_runtime_suspended)
7469 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
7471 g_gc_card_table = translated_ct;
7473 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7474 g_gc_card_bundle_table = translate_card_bundle_table(card_table_card_bundle_table(ct), saved_g_lowest_address);
7478 seg_mapping_table = new_seg_mapping_table;
7480 GCToOSInterface::FlushProcessWriteBuffers();
7481 g_gc_lowest_address = saved_g_lowest_address;
7482 g_gc_highest_address = saved_g_highest_address;
7484 if (!write_barrier_updated)
7486 // This passes a bool telling whether we need to switch to the post
7487 // grow version of the write barrier. This test tells us if the new
7488 // segment was allocated at a lower address than the old, requiring
7489 // that we start doing an upper bounds check in the write barrier.
7490 // This will also suspend the runtime if the write barrier type needs
7491 // to be changed, so we are doing this after all global state has
7492 // been updated. See the comment above suspend_EE() above for more
7494 stomp_write_barrier_resize(GCToEEInterface::IsGCThread(), la != saved_g_lowest_address);
7501 //cleanup mess and return -1;
7505 assert(g_gc_card_table == saved_g_card_table);
7507 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7508 assert(g_gc_card_bundle_table == saved_g_card_bundle_table);
7511 //delete (uint32_t*)((uint8_t*)ct - sizeof(card_table_info));
7512 if (!GCToOSInterface::VirtualRelease (mem, alloc_size))
7514 dprintf (GC_TABLE_LOG, ("GCToOSInterface::VirtualRelease failed"));
7515 assert (!"release failed");
7523 #ifdef BACKGROUND_GC
7524 if (hp->should_commit_mark_array())
7526 dprintf (GC_TABLE_LOG, ("in range new seg %Ix, mark_array is %Ix", new_seg, hp->mark_array));
7527 if (!commit_mark_array_new_seg (hp, new_seg))
7529 dprintf (GC_TABLE_LOG, ("failed to commit mark array for the new seg in range"));
7530 set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p);
7534 #endif //BACKGROUND_GC
7540 //copy all of the arrays managed by the card table for a page aligned range
7541 void gc_heap::copy_brick_card_range (uint8_t* la, uint32_t* old_card_table,
7542 short* old_brick_table,
7544 uint8_t* start, uint8_t* end)
7546 ptrdiff_t brick_offset = brick_of (start) - brick_of (la);
7549 dprintf (2, ("copying tables for range [%Ix %Ix[", (size_t)start, (size_t)end));
7552 short* brick_start = &brick_table [brick_of (start)];
7553 if (old_brick_table)
7555 // segments are always on page boundaries
7556 memcpy (brick_start, &old_brick_table[brick_offset],
7557 size_brick_of (start, end));
7562 // This is a large heap, just clear the brick table
7565 uint32_t* old_ct = &old_card_table[card_word (card_of (la))];
7567 #ifdef BACKGROUND_GC
7568 UNREFERENCED_PARAMETER(seg);
7569 if (recursive_gc_sync::background_running_p())
7571 uint32_t* old_mark_array = card_table_mark_array (old_ct);
7573 // We don't need to go through all the card tables here because
7574 // we only need to copy from the GC version of the mark array - when we
7575 // mark (even in allocate_large_object) we always use that mark array.
7576 if ((card_table_highest_address (old_ct) >= start) &&
7577 (card_table_lowest_address (old_ct) <= end))
7579 if ((background_saved_highest_address >= start) &&
7580 (background_saved_lowest_address <= end))
7582 //copy the mark bits
7583 // segments are always on page boundaries
7584 uint8_t* m_start = max (background_saved_lowest_address, start);
7585 uint8_t* m_end = min (background_saved_highest_address, end);
7586 memcpy (&mark_array[mark_word_of (m_start)],
7587 &old_mark_array[mark_word_of (m_start) - mark_word_of (la)],
7588 size_mark_array_of (m_start, m_end));
7593 //only large segments can be out of range
7594 assert (old_brick_table == 0);
7597 #else //BACKGROUND_GC
7599 clear_mark_array (start, heap_segment_committed(seg));
7600 #endif //BACKGROUND_GC
7603 // n way merge with all of the card table ever used in between
7604 uint32_t* ct = card_table_next (&card_table[card_word (card_of(lowest_address))]);
7607 while (card_table_next (old_ct) != ct)
7609 //copy if old card table contained [start, end[
7610 if ((card_table_highest_address (ct) >= end) &&
7611 (card_table_lowest_address (ct) <= start))
7613 // or the card_tables
7615 size_t start_word = card_word (card_of (start));
7617 uint32_t* dest = &card_table[start_word];
7618 uint32_t* src = &((translate_card_table (ct))[start_word]);
7619 ptrdiff_t count = count_card_of (start, end);
7620 for (int x = 0; x < count; x++)
7624 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
7627 card_bundle_set(cardw_card_bundle(start_word+x));
7635 ct = card_table_next (ct);
7639 //initialize all of the arrays managed by the card table for a page aligned range when an existing ro segment becomes in range
7640 void gc_heap::init_brick_card_range (heap_segment* seg)
7642 dprintf (2, ("initialising tables for range [%Ix %Ix[",
7643 (size_t)heap_segment_mem (seg),
7644 (size_t)heap_segment_allocated (seg)));
7646 // initialize the brick table
7647 for (size_t b = brick_of (heap_segment_mem (seg));
7648 b < brick_of (align_on_brick (heap_segment_allocated (seg)));
7655 if (recursive_gc_sync::background_running_p() && (seg->flags & heap_segment_flags_ma_committed))
7658 clear_mark_array (heap_segment_mem (seg), heap_segment_committed(seg));
7662 clear_card_for_addresses (heap_segment_mem (seg),
7663 heap_segment_allocated (seg));
7666 void gc_heap::copy_brick_card_table()
7668 uint8_t* la = lowest_address;
7669 uint8_t* ha = highest_address;
7670 MAYBE_UNUSED_VAR(ha);
7671 uint32_t* old_card_table = card_table;
7672 short* old_brick_table = brick_table;
7674 assert (la == card_table_lowest_address (&old_card_table[card_word (card_of (la))]));
7675 assert (ha == card_table_highest_address (&old_card_table[card_word (card_of (la))]));
7677 /* todo: Need a global lock for this */
7678 uint32_t* ct = &g_gc_card_table[card_word (gcard_of (g_gc_lowest_address))];
7679 own_card_table (ct);
7680 card_table = translate_card_table (ct);
7681 /* End of global lock */
7682 highest_address = card_table_highest_address (ct);
7683 lowest_address = card_table_lowest_address (ct);
7685 brick_table = card_table_brick_table (ct);
7688 if (gc_can_use_concurrent)
7690 mark_array = translate_mark_array (card_table_mark_array (ct));
7691 assert (mark_word_of (g_gc_highest_address) ==
7692 mark_word_of (align_on_mark_word (g_gc_highest_address)));
7699 #if defined(MARK_ARRAY) && defined(_DEBUG)
7700 #ifdef GROWABLE_SEG_MAPPING_TABLE
7701 size_t st = size_seg_mapping_table_of (g_gc_lowest_address, g_gc_highest_address);
7702 #else //GROWABLE_SEG_MAPPING_TABLE
7704 #endif //GROWABLE_SEG_MAPPING_TABLE
7705 #endif //MARK_ARRAY && _DEBUG
7706 card_bundle_table = translate_card_bundle_table (card_table_card_bundle_table (ct), g_gc_lowest_address);
7708 // Ensure that the word that represents g_gc_lowest_address in the translated table is located at the
7709 // start of the untranslated table.
7710 assert (&card_bundle_table [card_bundle_word (cardw_card_bundle (card_word (card_of (g_gc_lowest_address))))] ==
7711 card_table_card_bundle_table (ct));
7713 //set the card table if we are in a heap growth scenario
7714 if (card_bundles_enabled())
7716 card_bundles_set (cardw_card_bundle (card_word (card_of (lowest_address))),
7717 cardw_card_bundle (align_cardw_on_bundle (card_word (card_of (highest_address)))));
7719 //check if we need to turn on card_bundles.
7720 #ifdef MULTIPLE_HEAPS
7721 // use INT64 arithmetic here because of possible overflow on 32p
7722 uint64_t th = (uint64_t)MH_TH_CARD_BUNDLE*gc_heap::n_heaps;
7724 // use INT64 arithmetic here because of possible overflow on 32p
7725 uint64_t th = (uint64_t)SH_TH_CARD_BUNDLE;
7726 #endif //MULTIPLE_HEAPS
7727 if (reserved_memory >= th)
7729 enable_card_bundles();
7732 #endif //CARD_BUNDLE
7734 // for each of the segments and heaps, copy the brick table and
7735 // or the card table
7736 heap_segment* seg = generation_start_segment (generation_of (max_generation));
7739 if (heap_segment_read_only_p (seg) && !heap_segment_in_range_p (seg))
7741 //check if it became in range
7742 if ((heap_segment_reserved (seg) > lowest_address) &&
7743 (heap_segment_mem (seg) < highest_address))
7745 set_ro_segment_in_range (seg);
7751 uint8_t* end = align_on_page (heap_segment_allocated (seg));
7752 copy_brick_card_range (la, old_card_table,
7755 align_lower_page (heap_segment_mem (seg)),
7758 seg = heap_segment_next (seg);
7761 seg = generation_start_segment (large_object_generation);
7764 if (heap_segment_read_only_p (seg) && !heap_segment_in_range_p (seg))
7766 //check if it became in range
7767 if ((heap_segment_reserved (seg) > lowest_address) &&
7768 (heap_segment_mem (seg) < highest_address))
7770 set_ro_segment_in_range (seg);
7775 uint8_t* end = align_on_page (heap_segment_allocated (seg));
7776 copy_brick_card_range (la, old_card_table,
7779 align_lower_page (heap_segment_mem (seg)),
7782 seg = heap_segment_next (seg);
7785 release_card_table (&old_card_table[card_word (card_of(la))]);
7788 #ifdef FEATURE_BASICFREEZE
7789 BOOL gc_heap::insert_ro_segment (heap_segment* seg)
7791 enter_spin_lock (&gc_heap::gc_lock);
7793 if (!gc_heap::seg_table->ensure_space_for_insert ()
7794 || (should_commit_mark_array() && !commit_mark_array_new_seg(__this, seg)))
7796 leave_spin_lock(&gc_heap::gc_lock);
7800 //insert at the head of the segment list
7801 generation* gen2 = generation_of (max_generation);
7802 heap_segment* oldhead = generation_start_segment (gen2);
7803 heap_segment_next (seg) = oldhead;
7804 generation_start_segment (gen2) = seg;
7806 seg_table->insert (heap_segment_mem(seg), (size_t)seg);
7808 #ifdef SEG_MAPPING_TABLE
7809 seg_mapping_table_add_ro_segment (seg);
7810 #endif //SEG_MAPPING_TABLE
7813 if ((heap_segment_reserved (seg) > lowest_address) &&
7814 (heap_segment_mem (seg) < highest_address))
7816 set_ro_segment_in_range (seg);
7819 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(seg), (size_t)(heap_segment_reserved (seg) - heap_segment_mem(seg)), gc_etw_segment_read_only_heap);
7821 leave_spin_lock (&gc_heap::gc_lock);
7825 // No one is calling this function right now. If this is getting called we need
7826 // to take care of decommitting the mark array for it - we will need to remember
7827 // which portion of the mark array was committed and only decommit that.
7828 void gc_heap::remove_ro_segment (heap_segment* seg)
7830 //clear the mark bits so a new segment allocated in its place will have a clear mark bits
7832 if (gc_can_use_concurrent)
7834 clear_mark_array (align_lower_mark_word (max (heap_segment_mem (seg), lowest_address)),
7835 align_on_card_word (min (heap_segment_allocated (seg), highest_address)),
7836 false); // read_only segments need the mark clear
7840 enter_spin_lock (&gc_heap::gc_lock);
7842 seg_table->remove ((uint8_t*)seg);
7844 #ifdef SEG_MAPPING_TABLE
7845 seg_mapping_table_remove_ro_segment (seg);
7846 #endif //SEG_MAPPING_TABLE
7848 // Locate segment (and previous segment) in the list.
7849 generation* gen2 = generation_of (max_generation);
7850 heap_segment* curr_seg = generation_start_segment (gen2);
7851 heap_segment* prev_seg = NULL;
7853 while (curr_seg && curr_seg != seg)
7855 prev_seg = curr_seg;
7856 curr_seg = heap_segment_next (curr_seg);
7858 assert (curr_seg == seg);
7860 // Patch previous segment (or list head if there is none) to skip the removed segment.
7862 heap_segment_next (prev_seg) = heap_segment_next (curr_seg);
7864 generation_start_segment (gen2) = heap_segment_next (curr_seg);
7866 leave_spin_lock (&gc_heap::gc_lock);
7868 #endif //FEATURE_BASICFREEZE
7870 BOOL gc_heap::set_ro_segment_in_range (heap_segment* seg)
7873 seg->flags |= heap_segment_flags_inrange;
7874 // init_brick_card_range (seg);
7875 ro_segments_in_range = TRUE;
7876 //right now, segments aren't protected
7877 //unprotect_segment (seg);
7883 uint8_t** make_mark_list (size_t size)
7885 uint8_t** mark_list = new (nothrow) uint8_t* [size];
7889 #define swap(a,b){uint8_t* t; t = a; a = b; b = t;}
7891 void verify_qsort_array (uint8_t* *low, uint8_t* *high)
7895 for (i = low+1; i <= high; i++)
7904 #ifndef USE_INTROSORT
7905 void qsort1( uint8_t* *low, uint8_t* *high, unsigned int depth)
7907 if (((low + 16) >= high) || (depth > 100))
7911 for (i = low+1; i <= high; i++)
7914 for (j=i;j >low && val<*(j-1);j--)
7923 uint8_t *pivot, **left, **right;
7925 //sort low middle and high
7926 if (*(low+((high-low)/2)) < *low)
7927 swap (*(low+((high-low)/2)), *low);
7930 if (*high < *(low+((high-low)/2)))
7931 swap (*(low+((high-low)/2)), *high);
7933 swap (*(low+((high-low)/2)), *(high-1));
7935 left = low; right = high-1;
7937 while (*(--right) > pivot);
7938 while (*(++left) < pivot);
7941 swap(*left, *right);
7946 swap (*left, *(high-1));
7947 qsort1(low, left-1, depth+1);
7948 qsort1(left+1, high, depth+1);
7951 #endif //USE_INTROSORT
7952 void rqsort1( uint8_t* *low, uint8_t* *high)
7954 if ((low + 16) >= high)
7958 for (i = low+1; i <= high; i++)
7961 for (j=i;j >low && val>*(j-1);j--)
7970 uint8_t *pivot, **left, **right;
7972 //sort low middle and high
7973 if (*(low+((high-low)/2)) > *low)
7974 swap (*(low+((high-low)/2)), *low);
7977 if (*high > *(low+((high-low)/2)))
7978 swap (*(low+((high-low)/2)), *high);
7980 swap (*(low+((high-low)/2)), *(high-1));
7982 left = low; right = high-1;
7984 while (*(--right) < pivot);
7985 while (*(++left) > pivot);
7988 swap(*left, *right);
7993 swap (*left, *(high-1));
7994 rqsort1(low, left-1);
7995 rqsort1(left+1, high);
7999 #ifdef USE_INTROSORT
8004 static const int size_threshold = 64;
8005 static const int max_depth = 100;
8008 inline static void swap_elements(uint8_t** i,uint8_t** j)
8016 static void sort (uint8_t** begin, uint8_t** end, int ignored)
8019 introsort_loop (begin, end, max_depth);
8020 insertionsort (begin, end);
8025 static void introsort_loop (uint8_t** lo, uint8_t** hi, int depth_limit)
8027 while (hi-lo >= size_threshold)
8029 if (depth_limit == 0)
8034 uint8_t** p=median_partition (lo, hi);
8035 depth_limit=depth_limit-1;
8036 introsort_loop (p, hi, depth_limit);
8041 static uint8_t** median_partition (uint8_t** low, uint8_t** high)
8043 uint8_t *pivot, **left, **right;
8045 //sort low middle and high
8046 if (*(low+((high-low)/2)) < *low)
8047 swap_elements ((low+((high-low)/2)), low);
8049 swap_elements (low, high);
8050 if (*high < *(low+((high-low)/2)))
8051 swap_elements ((low+((high-low)/2)), high);
8053 swap_elements ((low+((high-low)/2)), (high-1));
8055 left = low; right = high-1;
8057 while (*(--right) > pivot);
8058 while (*(++left) < pivot);
8061 swap_elements(left, right);
8066 swap_elements (left, (high-1));
8071 static void insertionsort (uint8_t** lo, uint8_t** hi)
8073 for (uint8_t** i=lo+1; i <= hi; i++)
8077 while((j > lo) && (t <*(j-1)))
8086 static void heapsort (uint8_t** lo, uint8_t** hi)
8088 size_t n = hi - lo + 1;
8089 for (size_t i=n / 2; i >= 1; i--)
8093 for (size_t i = n; i > 1; i--)
8095 swap_elements (lo, lo + i - 1);
8096 downheap(1, i - 1, lo);
8100 static void downheap (size_t i, size_t n, uint8_t** lo)
8102 uint8_t* d = *(lo + i - 1);
8107 if (child < n && *(lo + child - 1)<(*(lo + child)))
8111 if (!(d<*(lo + child - 1)))
8115 *(lo + i - 1) = *(lo + child - 1);
8123 #endif //USE_INTROSORT
8125 #ifdef MULTIPLE_HEAPS
8126 #ifdef PARALLEL_MARK_LIST_SORT
8127 void gc_heap::sort_mark_list()
8129 // if this heap had a mark list overflow, we don't do anything
8130 if (mark_list_index > mark_list_end)
8132 // printf("sort_mark_list: overflow on heap %d\n", heap_number);
8136 // if any other heap had a mark list overflow, we fake one too,
8137 // so we don't use an incomplete mark list by mistake
8138 for (int i = 0; i < n_heaps; i++)
8140 if (g_heaps[i]->mark_list_index > g_heaps[i]->mark_list_end)
8142 mark_list_index = mark_list_end + 1;
8143 // printf("sort_mark_list: overflow on heap %d\n", i);
8148 // unsigned long start = GetCycleCount32();
8150 dprintf (3, ("Sorting mark lists"));
8151 if (mark_list_index > mark_list)
8152 _sort (mark_list, mark_list_index - 1, 0);
8154 // printf("first phase of sort_mark_list for heap %d took %u cycles to sort %u entries\n", this->heap_number, GetCycleCount32() - start, mark_list_index - mark_list);
8155 // start = GetCycleCount32();
8157 // first set the pieces for all heaps to empty
8159 for (heap_num = 0; heap_num < n_heaps; heap_num++)
8161 mark_list_piece_start[heap_num] = NULL;
8162 mark_list_piece_end[heap_num] = NULL;
8165 uint8_t** x = mark_list;
8167 // predicate means: x is still within the mark list, and within the bounds of this heap
8168 #define predicate(x) (((x) < mark_list_index) && (*(x) < heap->ephemeral_high))
8171 while (x < mark_list_index)
8174 // find the heap x points into - searching cyclically from the last heap,
8175 // because in many cases the right heap is the next one or comes soon after
8176 int last_heap_num = heap_num;
8177 MAYBE_UNUSED_VAR(last_heap_num);
8181 if (heap_num >= n_heaps)
8183 assert(heap_num != last_heap_num); // we should always find the heap - infinite loop if not!
8184 heap = g_heaps[heap_num];
8186 while (!(*x >= heap->ephemeral_low && *x < heap->ephemeral_high));
8188 // x is the start of the mark list piece for this heap
8189 mark_list_piece_start[heap_num] = x;
8191 // to find the end of the mark list piece for this heap, find the first x
8192 // that has !predicate(x), i.e. that is either not in this heap, or beyond the end of the list
8195 // let's see if we get lucky and the whole rest belongs to this piece
8196 if (predicate(mark_list_index-1))
8198 x = mark_list_index;
8199 mark_list_piece_end[heap_num] = x;
8203 // we play a variant of binary search to find the point sooner.
8204 // the first loop advances by increasing steps until the predicate turns false.
8205 // then we retreat the last step, and the second loop advances by decreasing steps, keeping the predicate true.
8210 uint8_t** temp_x = x;
8217 while (predicate(x));
8218 // we know that only the last step was wrong, so we undo it
8222 // loop invariant - predicate holds at x, but not x + inc
8223 assert (predicate(x) && !(((x + inc) > x) && predicate(x + inc)));
8225 if (((x + inc) > x) && predicate(x + inc))
8231 // the termination condition and the loop invariant together imply this:
8232 assert(predicate(x) && !predicate(x + inc) && (inc == 1));
8233 // so the spot we're looking for is one further
8236 mark_list_piece_end[heap_num] = x;
8241 // printf("second phase of sort_mark_list for heap %d took %u cycles\n", this->heap_number, GetCycleCount32() - start);
8244 void gc_heap::append_to_mark_list(uint8_t **start, uint8_t **end)
8246 size_t slots_needed = end - start;
8247 size_t slots_available = mark_list_end + 1 - mark_list_index;
8248 size_t slots_to_copy = min(slots_needed, slots_available);
8249 memcpy(mark_list_index, start, slots_to_copy*sizeof(*start));
8250 mark_list_index += slots_to_copy;
8251 // printf("heap %d: appended %Id slots to mark_list\n", heap_number, slots_to_copy);
8254 void gc_heap::merge_mark_lists()
8256 uint8_t** source[MAX_SUPPORTED_CPUS];
8257 uint8_t** source_end[MAX_SUPPORTED_CPUS];
8258 int source_heap[MAX_SUPPORTED_CPUS];
8259 int source_count = 0;
8261 // in case of mark list overflow, don't bother
8262 if (mark_list_index > mark_list_end)
8264 // printf("merge_mark_lists: overflow\n");
8268 dprintf(3, ("merge_mark_lists: heap_number = %d starts out with %Id entries", heap_number, mark_list_index - mark_list));
8269 // unsigned long start = GetCycleCount32();
8270 for (int i = 0; i < n_heaps; i++)
8272 gc_heap* heap = g_heaps[i];
8273 if (heap->mark_list_piece_start[heap_number] < heap->mark_list_piece_end[heap_number])
8275 source[source_count] = heap->mark_list_piece_start[heap_number];
8276 source_end[source_count] = heap->mark_list_piece_end[heap_number];
8277 source_heap[source_count] = i;
8278 if (source_count < MAX_SUPPORTED_CPUS)
8282 // printf("first phase of merge_mark_lists for heap %d took %u cycles\n", heap_number, GetCycleCount32() - start);
8284 dprintf(3, ("heap_number = %d has %d sources\n", heap_number, source_count));
8285 #if defined(_DEBUG) || defined(TRACE_GC)
8286 for (int j = 0; j < source_count; j++)
8288 dprintf(3, ("heap_number = %d ", heap_number));
8289 dprintf(3, (" source from heap %d = %Ix .. %Ix (%Id entries)",
8290 (size_t)(source_heap[j]), (size_t)(source[j][0]), (size_t)(source_end[j][-1]), (size_t)(source_end[j] - source[j])));
8291 // the sources should all be sorted
8292 for (uint8_t **x = source[j]; x < source_end[j] - 1; x++)
8296 dprintf(3, ("oops, mark_list from source %d for heap %d isn't sorted\n", j, heap_number));
8301 #endif //_DEBUG || TRACE_GC
8303 // start = GetCycleCount32();
8305 mark_list = &g_mark_list_copy [heap_number*mark_list_size];
8306 mark_list_index = mark_list;
8307 mark_list_end = &mark_list [mark_list_size-1];
8308 int piece_count = 0;
8309 if (source_count == 0)
8313 else if (source_count == 1)
8315 mark_list = source[0];
8316 mark_list_index = source_end[0];
8317 mark_list_end = mark_list_index;
8322 while (source_count > 1)
8324 // find the lowest and second lowest value in the sources we're merging from
8325 int lowest_source = 0;
8326 uint8_t *lowest = *source[0];
8327 uint8_t *second_lowest = *source[1];
8328 for (int i = 1; i < source_count; i++)
8330 if (lowest > *source[i])
8332 second_lowest = lowest;
8333 lowest = *source[i];
8336 else if (second_lowest > *source[i])
8338 second_lowest = *source[i];
8342 // find the point in the lowest source where it either runs out or is not <= second_lowest anymore
8344 // let's first try to get lucky and see if the whole source is <= second_lowest -- this is actually quite common
8346 if (source_end[lowest_source][-1] <= second_lowest)
8347 x = source_end[lowest_source];
8350 // use linear search to find the end -- could also use binary search as in sort_mark_list,
8351 // but saw no improvement doing that
8352 for (x = source[lowest_source]; x < source_end[lowest_source] && *x <= second_lowest; x++)
8356 // blast this piece to the mark list
8357 append_to_mark_list(source[lowest_source], x);
8360 source[lowest_source] = x;
8362 // check whether this source is now exhausted
8363 if (x >= source_end[lowest_source])
8365 // if it's not the source with the highest index, copy the source with the highest index
8366 // over it so the non-empty sources are always at the beginning
8367 if (lowest_source < source_count-1)
8369 source[lowest_source] = source[source_count-1];
8370 source_end[lowest_source] = source_end[source_count-1];
8375 // we're left with just one source that we copy
8376 append_to_mark_list(source[0], source_end[0]);
8380 // printf("second phase of merge_mark_lists for heap %d took %u cycles to merge %d pieces\n", heap_number, GetCycleCount32() - start, piece_count);
8382 #if defined(_DEBUG) || defined(TRACE_GC)
8383 // the final mark list must be sorted
8384 for (uint8_t **x = mark_list; x < mark_list_index - 1; x++)
8388 dprintf(3, ("oops, mark_list for heap %d isn't sorted at the end of merge_mark_lists", heap_number));
8392 #endif //defined(_DEBUG) || defined(TRACE_GC)
8394 #else //PARALLEL_MARK_LIST_SORT
8395 void gc_heap::combine_mark_lists()
8397 dprintf (3, ("Combining mark lists"));
8398 //verify if a heap has overflowed its mark list
8399 BOOL use_mark_list = TRUE;
8400 for (int i = 0; i < n_heaps; i++)
8402 if (g_heaps [i]->mark_list_index > g_heaps [i]->mark_list_end)
8404 use_mark_list = FALSE;
8411 dprintf (3, ("Using mark list"));
8412 //compact the gaps out of the mark list
8414 uint8_t** current_gap = g_heaps [gn]->mark_list_index;
8415 uint8_t** current_gap_end = g_heaps[gn]->mark_list_end + 1;
8416 uint8_t** dst_last = current_gap-1;
8418 int srcn = n_heaps-1;
8419 gc_heap* srch = g_heaps [srcn];
8420 uint8_t** src = srch->mark_list_index - 1;
8421 uint8_t** src_beg = srch->mark_list;
8423 while (current_gap <= src)
8425 while ((gn < n_heaps-1) && (current_gap >= current_gap_end))
8427 //go to the next gap
8429 dprintf (3, ("Going to the next gap %d", gn));
8430 assert (gn < n_heaps);
8431 current_gap = g_heaps [gn]->mark_list_index;
8432 current_gap_end = g_heaps[gn]->mark_list_end + 1;
8433 assert ((gn == (n_heaps-1)) || (current_gap_end == g_heaps[gn+1]->mark_list));
8435 while ((srcn > 0) && (src < src_beg))
8437 //go to the previous source
8439 dprintf (3, ("going to the previous source %d", srcn));
8441 gc_heap* srch = g_heaps [srcn];
8442 src = srch->mark_list_index - 1;
8443 src_beg = srch->mark_list;
8445 if (current_gap < src)
8447 dst_last = current_gap;
8448 *current_gap++ = *src--;
8451 dprintf (3, ("src: %Ix dst_last: %Ix", (size_t)src, (size_t)dst_last));
8453 uint8_t** end_of_list = max (src, dst_last);
8455 //sort the resulting compacted list
8456 assert (end_of_list < &g_mark_list [n_heaps*mark_list_size]);
8457 if (end_of_list > &g_mark_list[0])
8458 _sort (&g_mark_list[0], end_of_list, 0);
8459 //adjust the mark_list to the begining of the resulting mark list.
8460 for (int i = 0; i < n_heaps; i++)
8462 g_heaps [i]->mark_list = g_mark_list;
8463 g_heaps [i]->mark_list_index = end_of_list + 1;
8464 g_heaps [i]->mark_list_end = end_of_list + 1;
8469 uint8_t** end_of_list = g_mark_list;
8470 //adjust the mark_list to the begining of the resulting mark list.
8471 //put the index beyond the end to turn off mark list processing
8472 for (int i = 0; i < n_heaps; i++)
8474 g_heaps [i]->mark_list = g_mark_list;
8475 g_heaps [i]->mark_list_index = end_of_list + 1;
8476 g_heaps [i]->mark_list_end = end_of_list;
8480 #endif // PARALLEL_MARK_LIST_SORT
8481 #endif //MULTIPLE_HEAPS
8484 class seg_free_spaces
8486 struct seg_free_space
8492 struct free_space_bucket
8494 seg_free_space* free_space;
8495 ptrdiff_t count_add; // Assigned when we first contruct the array.
8496 ptrdiff_t count_fit; // How many items left when we are fitting plugs.
8499 void move_bucket (int old_power2, int new_power2)
8501 // PREFAST warning 22015: old_power2 could be negative
8502 assert (old_power2 >= 0);
8503 assert (old_power2 >= new_power2);
8505 if (old_power2 == new_power2)
8510 seg_free_space* src_index = free_space_buckets[old_power2].free_space;
8511 for (int i = old_power2; i > new_power2; i--)
8513 seg_free_space** dest = &(free_space_buckets[i].free_space);
8516 seg_free_space* dest_index = free_space_buckets[i - 1].free_space;
8517 if (i > (new_power2 + 1))
8519 seg_free_space temp = *src_index;
8520 *src_index = *dest_index;
8523 src_index = dest_index;
8526 free_space_buckets[old_power2].count_fit--;
8527 free_space_buckets[new_power2].count_fit++;
8532 void dump_free_space (seg_free_space* item)
8539 mark* m = (mark*)(item->start);
8540 len = pinned_len (m);
8541 addr = pinned_plug (m) - len;
8545 heap_segment* seg = (heap_segment*)(item->start);
8546 addr = heap_segment_plan_allocated (seg);
8547 len = heap_segment_committed (seg) - addr;
8550 dprintf (SEG_REUSE_LOG_1, ("[%d]0x%Ix %Id", heap_num, addr, len));
8555 seg_free_space* item = NULL;
8558 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------\nnow the free spaces look like:", heap_num));
8559 for (i = 0; i < (free_space_bucket_count - 1); i++)
8561 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces for 2^%d bucket:", heap_num, (base_power2 + i)));
8562 dprintf (SEG_REUSE_LOG_1, ("[%d]%s %s", heap_num, "start", "len"));
8563 item = free_space_buckets[i].free_space;
8564 while (item < free_space_buckets[i + 1].free_space)
8566 dump_free_space (item);
8569 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------", heap_num));
8572 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces for 2^%d bucket:", heap_num, (base_power2 + i)));
8573 dprintf (SEG_REUSE_LOG_1, ("[%d]%s %s", heap_num, "start", "len"));
8574 item = free_space_buckets[i].free_space;
8576 while (item <= &seg_free_space_array[free_space_item_count - 1])
8578 dump_free_space (item);
8581 dprintf (SEG_REUSE_LOG_1, ("[%d]----------------------------------", heap_num));
8586 free_space_bucket* free_space_buckets;
8587 seg_free_space* seg_free_space_array;
8588 ptrdiff_t free_space_bucket_count;
8589 ptrdiff_t free_space_item_count;
8593 BOOL has_end_of_seg;
8598 seg_free_spaces (int h_number)
8600 heap_num = h_number;
8605 size_t total_prealloc_size =
8606 MAX_NUM_BUCKETS * sizeof (free_space_bucket) +
8607 MAX_NUM_FREE_SPACES * sizeof (seg_free_space);
8609 free_space_buckets = (free_space_bucket*) new (nothrow) uint8_t[total_prealloc_size];
8611 return (!!free_space_buckets);
8614 // We take the ordered free space array we got from the 1st pass,
8615 // and feed the portion that we decided to use to this method, ie,
8616 // the largest item_count free spaces.
8617 void add_buckets (int base, size_t* ordered_free_spaces, int bucket_count, size_t item_count)
8619 assert (free_space_buckets);
8620 assert (item_count <= (size_t)MAX_PTR);
8622 free_space_bucket_count = bucket_count;
8623 free_space_item_count = item_count;
8626 has_end_of_seg = FALSE;
8629 ptrdiff_t total_item_count = 0;
8632 seg_free_space_array = (seg_free_space*)(free_space_buckets + free_space_bucket_count);
8634 for (i = 0; i < (ptrdiff_t)item_count; i++)
8636 seg_free_space_array[i].start = 0;
8637 seg_free_space_array[i].is_plug = FALSE;
8640 for (i = 0; i < bucket_count; i++)
8642 free_space_buckets[i].count_add = ordered_free_spaces[i];
8643 free_space_buckets[i].count_fit = ordered_free_spaces[i];
8644 free_space_buckets[i].free_space = &seg_free_space_array[total_item_count];
8645 total_item_count += free_space_buckets[i].count_add;
8648 assert (total_item_count == (ptrdiff_t)item_count);
8651 // If we are adding a free space before a plug we pass the
8652 // mark stack position so we can update the length; we could
8653 // also be adding the free space after the last plug in which
8654 // case start is the segment which we'll need to update the
8655 // heap_segment_plan_allocated.
8656 void add (void* start, BOOL plug_p, BOOL first_p)
8658 size_t size = (plug_p ?
8659 pinned_len ((mark*)start) :
8660 (heap_segment_committed ((heap_segment*)start) -
8661 heap_segment_plan_allocated ((heap_segment*)start)));
8665 dprintf (SEG_REUSE_LOG_1, ("[%d]Adding a free space before plug: %Id", heap_num, size));
8669 dprintf (SEG_REUSE_LOG_1, ("[%d]Adding a free space at end of seg: %Id", heap_num, size));
8671 has_end_of_seg = TRUE;
8677 size_t eph_gen_starts = gc_heap::eph_gen_starts_size;
8678 size -= eph_gen_starts;
8681 mark* m = (mark*)(start);
8682 pinned_len (m) -= eph_gen_starts;
8686 heap_segment* seg = (heap_segment*)start;
8687 heap_segment_plan_allocated (seg) += eph_gen_starts;
8691 int bucket_power2 = index_of_set_bit (round_down_power2 (size));
8692 if (bucket_power2 < base_power2)
8697 free_space_bucket* bucket = &free_space_buckets[bucket_power2 - base_power2];
8699 seg_free_space* bucket_free_space = bucket->free_space;
8700 assert (plug_p || (!plug_p && bucket->count_add));
8702 if (bucket->count_add == 0)
8704 dprintf (SEG_REUSE_LOG_1, ("[%d]Already have enough of 2^%d", heap_num, bucket_power2));
8708 ptrdiff_t index = bucket->count_add - 1;
8710 dprintf (SEG_REUSE_LOG_1, ("[%d]Building free spaces: adding %Ix; len: %Id (2^%d)",
8713 (pinned_plug ((mark*)start) - pinned_len ((mark*)start)) :
8714 heap_segment_plan_allocated ((heap_segment*)start)),
8720 bucket_free_space[index].is_plug = TRUE;
8723 bucket_free_space[index].start = start;
8724 bucket->count_add--;
8729 // Do a consistency check after all free spaces are added.
8733 int end_of_seg_count = 0;
8735 for (i = 0; i < free_space_item_count; i++)
8737 assert (seg_free_space_array[i].start);
8738 if (!(seg_free_space_array[i].is_plug))
8746 assert (end_of_seg_count == 1);
8750 assert (end_of_seg_count == 0);
8753 for (i = 0; i < free_space_bucket_count; i++)
8755 assert (free_space_buckets[i].count_add == 0);
8761 uint8_t* fit (uint8_t* old_loc,
8763 BOOL set_padding_on_saved_p,
8764 mark* pinned_plug_entry,
8765 #endif //SHORT_PLUGS
8767 REQD_ALIGN_AND_OFFSET_DCL)
8772 assert (!is_plug_padded (old_loc));
8773 #endif //SHORT_PLUGS
8774 assert (!node_realigned (old_loc));
8777 size_t saved_plug_size = plug_size;
8779 #ifdef FEATURE_STRUCTALIGN
8780 // BARTOKTODO (4841): this code path is disabled (see can_fit_all_blocks_p) until we take alignment requirements into account
8781 _ASSERTE(requiredAlignment == DATA_ALIGNMENT && false);
8782 #endif // FEATURE_STRUCTALIGN
8783 // TODO: this is also not large alignment ready. We would need to consider alignment when chosing the
8786 size_t plug_size_to_fit = plug_size;
8788 int pad_in_front = (old_loc != 0) ? USE_PADDING_FRONT : 0;
8791 plug_size_to_fit += (pad_in_front ? Align(min_obj_size) : 0);
8792 #endif //SHORT_PLUGS
8794 int plug_power2 = index_of_set_bit (round_up_power2 (plug_size_to_fit + Align(min_obj_size)));
8796 uint8_t* new_address = 0;
8798 if (plug_power2 < base_power2)
8800 plug_power2 = base_power2;
8803 int chosen_power2 = plug_power2 - base_power2;
8805 for (i = chosen_power2; i < free_space_bucket_count; i++)
8807 if (free_space_buckets[i].count_fit != 0)
8814 dprintf (SEG_REUSE_LOG_1, ("[%d]Fitting plug len %Id (2^%d) using 2^%d free space",
8818 (chosen_power2 + base_power2)));
8820 assert (i < free_space_bucket_count);
8822 seg_free_space* bucket_free_space = free_space_buckets[chosen_power2].free_space;
8823 ptrdiff_t free_space_count = free_space_buckets[chosen_power2].count_fit;
8824 size_t new_free_space_size = 0;
8825 BOOL can_fit = FALSE;
8828 for (i = 0; i < free_space_count; i++)
8830 size_t free_space_size = 0;
8833 BOOL short_plugs_padding_p = FALSE;
8834 #endif //SHORT_PLUGS
8835 BOOL realign_padding_p = FALSE;
8837 if (bucket_free_space[i].is_plug)
8839 mark* m = (mark*)(bucket_free_space[i].start);
8840 uint8_t* plug_free_space_start = pinned_plug (m) - pinned_len (m);
8843 if ((pad_in_front & USE_PADDING_FRONT) &&
8844 (((plug_free_space_start - pin_allocation_context_start_region (m))==0) ||
8845 ((plug_free_space_start - pin_allocation_context_start_region (m))>=DESIRED_PLUG_LENGTH)))
8847 pad = Align (min_obj_size);
8848 short_plugs_padding_p = TRUE;
8850 #endif //SHORT_PLUGS
8852 if (!((old_loc == 0) || same_large_alignment_p (old_loc, plug_free_space_start+pad)))
8854 pad += switch_alignment_size (pad != 0);
8855 realign_padding_p = TRUE;
8858 plug_size = saved_plug_size + pad;
8860 free_space_size = pinned_len (m);
8861 new_address = pinned_plug (m) - pinned_len (m);
8863 if (free_space_size >= (plug_size + Align (min_obj_size)) ||
8864 free_space_size == plug_size)
8866 new_free_space_size = free_space_size - plug_size;
8867 pinned_len (m) = new_free_space_size;
8868 #ifdef SIMPLE_DPRINTF
8869 dprintf (SEG_REUSE_LOG_0, ("[%d]FP: 0x%Ix->0x%Ix(%Ix)(%Ix), [0x%Ix (2^%d) -> [0x%Ix (2^%d)",
8876 index_of_set_bit (round_down_power2 (free_space_size)),
8877 (pinned_plug (m) - pinned_len (m)),
8878 index_of_set_bit (round_down_power2 (new_free_space_size))));
8879 #endif //SIMPLE_DPRINTF
8882 if (short_plugs_padding_p)
8884 pin_allocation_context_start_region (m) = plug_free_space_start;
8885 set_padding_in_expand (old_loc, set_padding_on_saved_p, pinned_plug_entry);
8887 #endif //SHORT_PLUGS
8889 if (realign_padding_p)
8891 set_node_realigned (old_loc);
8899 heap_segment* seg = (heap_segment*)(bucket_free_space[i].start);
8900 free_space_size = heap_segment_committed (seg) - heap_segment_plan_allocated (seg);
8902 if (!((old_loc == 0) || same_large_alignment_p (old_loc, heap_segment_plan_allocated (seg))))
8904 pad = switch_alignment_size (FALSE);
8905 realign_padding_p = TRUE;
8908 plug_size = saved_plug_size + pad;
8910 if (free_space_size >= (plug_size + Align (min_obj_size)) ||
8911 free_space_size == plug_size)
8913 new_address = heap_segment_plan_allocated (seg);
8914 new_free_space_size = free_space_size - plug_size;
8915 heap_segment_plan_allocated (seg) = new_address + plug_size;
8916 #ifdef SIMPLE_DPRINTF
8917 dprintf (SEG_REUSE_LOG_0, ("[%d]FS: 0x%Ix-> 0x%Ix(%Ix) (2^%d) -> 0x%Ix (2^%d)",
8922 index_of_set_bit (round_down_power2 (free_space_size)),
8923 heap_segment_plan_allocated (seg),
8924 index_of_set_bit (round_down_power2 (new_free_space_size))));
8925 #endif //SIMPLE_DPRINTF
8927 if (realign_padding_p)
8928 set_node_realigned (old_loc);
8942 assert (chosen_power2 == 0);
8952 assert ((chosen_power2 && (i == 0)) ||
8953 (!chosen_power2) && (i < free_space_count));
8956 int new_bucket_power2 = index_of_set_bit (round_down_power2 (new_free_space_size));
8958 if (new_bucket_power2 < base_power2)
8960 new_bucket_power2 = base_power2;
8963 move_bucket (chosen_power2, new_bucket_power2 - base_power2);
8972 if (free_space_buckets)
8974 delete [] free_space_buckets;
8976 if (seg_free_space_array)
8978 delete [] seg_free_space_array;
8984 #define marked(i) header(i)->IsMarked()
8985 #define set_marked(i) header(i)->SetMarked()
8986 #define clear_marked(i) header(i)->ClearMarked()
8987 #define pinned(i) header(i)->IsPinned()
8988 #define set_pinned(i) header(i)->SetPinned()
8989 #define clear_pinned(i) header(i)->GetHeader()->ClrGCBit();
8991 inline size_t my_get_size (Object* ob)
8993 MethodTable* mT = header(ob)->GetMethodTable();
8994 return (mT->GetBaseSize() +
8995 (mT->HasComponentSize() ?
8996 ((size_t)((CObjectHeader*)ob)->GetNumComponents() * mT->RawGetComponentSize()) : 0));
8999 //#define size(i) header(i)->GetSize()
9000 #define size(i) my_get_size (header(i))
9002 #define contain_pointers(i) header(i)->ContainsPointers()
9003 #ifdef COLLECTIBLE_CLASS
9004 #define contain_pointers_or_collectible(i) header(i)->ContainsPointersOrCollectible()
9006 #define get_class_object(i) GCToEEInterface::GetLoaderAllocatorObjectForGC((Object *)i)
9007 #define is_collectible(i) method_table(i)->Collectible()
9008 #else //COLLECTIBLE_CLASS
9009 #define contain_pointers_or_collectible(i) header(i)->ContainsPointers()
9010 #endif //COLLECTIBLE_CLASS
9012 #if defined (MARK_ARRAY) && defined (BACKGROUND_GC)
9014 void gc_heap::seg_clear_mark_array_bits_soh (heap_segment* seg)
9016 uint8_t* range_beg = 0;
9017 uint8_t* range_end = 0;
9018 if (bgc_mark_array_range (seg, FALSE, &range_beg, &range_end))
9020 clear_mark_array (range_beg, align_on_mark_word (range_end), FALSE
9021 #ifdef FEATURE_BASICFREEZE
9023 #endif // FEATURE_BASICFREEZE
9028 void gc_heap::clear_batch_mark_array_bits (uint8_t* start, uint8_t* end)
9030 if ((start < background_saved_highest_address) &&
9031 (end > background_saved_lowest_address))
9033 start = max (start, background_saved_lowest_address);
9034 end = min (end, background_saved_highest_address);
9036 size_t start_mark_bit = mark_bit_of (start);
9037 size_t end_mark_bit = mark_bit_of (end);
9038 unsigned int startbit = mark_bit_bit (start_mark_bit);
9039 unsigned int endbit = mark_bit_bit (end_mark_bit);
9040 size_t startwrd = mark_bit_word (start_mark_bit);
9041 size_t endwrd = mark_bit_word (end_mark_bit);
9043 dprintf (3, ("Clearing all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
9044 (size_t)start, (size_t)start_mark_bit,
9045 (size_t)end, (size_t)end_mark_bit));
9047 unsigned int firstwrd = lowbits (~0, startbit);
9048 unsigned int lastwrd = highbits (~0, endbit);
9050 if (startwrd == endwrd)
9052 unsigned int wrd = firstwrd | lastwrd;
9053 mark_array[startwrd] &= wrd;
9057 // clear the first mark word.
9060 mark_array[startwrd] &= firstwrd;
9064 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
9066 mark_array[wrdtmp] = 0;
9069 // clear the last mark word.
9072 mark_array[endwrd] &= lastwrd;
9077 void gc_heap::bgc_clear_batch_mark_array_bits (uint8_t* start, uint8_t* end)
9079 if ((start < background_saved_highest_address) &&
9080 (end > background_saved_lowest_address))
9082 start = max (start, background_saved_lowest_address);
9083 end = min (end, background_saved_highest_address);
9085 clear_batch_mark_array_bits (start, end);
9089 void gc_heap::clear_mark_array_by_objects (uint8_t* from, uint8_t* end, BOOL loh_p)
9091 dprintf (3, ("clearing mark array bits by objects for addr [%Ix,[%Ix",
9093 int align_const = get_alignment_constant (!loh_p);
9099 uint8_t* next_o = o + Align (size (o), align_const);
9101 if (background_object_marked (o, TRUE))
9103 dprintf (3, ("%Ix was marked by bgc, is now cleared", o));
9109 #endif //MARK_ARRAY && BACKGROUND_GC
9112 BOOL gc_heap::is_mark_set (uint8_t* o)
9117 #if defined (_MSC_VER) && defined (_TARGET_X86_)
9118 #pragma optimize("y", on) // Small critical routines, don't put in EBP frame
9119 #endif //_MSC_VER && _TARGET_X86_
9121 // return the generation number of an object.
9122 // It is assumed that the object is valid.
9123 //Note that this will return max_generation for a LOH object
9124 int gc_heap::object_gennum (uint8_t* o)
9126 if (in_range_for_segment (o, ephemeral_heap_segment) &&
9127 (o >= generation_allocation_start (generation_of (max_generation-1))))
9129 // in an ephemeral generation.
9130 for ( int i = 0; i < max_generation-1; i++)
9132 if ((o >= generation_allocation_start (generation_of (i))))
9135 return max_generation-1;
9139 return max_generation;
9143 int gc_heap::object_gennum_plan (uint8_t* o)
9145 if (in_range_for_segment (o, ephemeral_heap_segment))
9147 for (int i = 0; i <= max_generation-1; i++)
9149 uint8_t* plan_start = generation_plan_allocation_start (generation_of (i));
9150 if (plan_start && (o >= plan_start))
9156 return max_generation;
9159 #if defined(_MSC_VER) && defined(_TARGET_X86_)
9160 #pragma optimize("", on) // Go back to command line default optimizations
9161 #endif //_MSC_VER && _TARGET_X86_
9163 heap_segment* gc_heap::make_heap_segment (uint8_t* new_pages, size_t size, int h_number)
9165 size_t initial_commit = SEGMENT_INITIAL_COMMIT;
9167 //Commit the first page
9168 if (!virtual_alloc_commit_for_heap (new_pages, initial_commit, h_number))
9173 //overlay the heap_segment
9174 heap_segment* new_segment = (heap_segment*)new_pages;
9176 uint8_t* start = new_pages + segment_info_size;
9177 heap_segment_mem (new_segment) = start;
9178 heap_segment_used (new_segment) = start;
9179 heap_segment_reserved (new_segment) = new_pages + size;
9180 heap_segment_committed (new_segment) = new_pages + initial_commit;
9181 init_heap_segment (new_segment);
9182 dprintf (2, ("Creating heap segment %Ix", (size_t)new_segment));
9186 void gc_heap::init_heap_segment (heap_segment* seg)
9189 heap_segment_next (seg) = 0;
9190 heap_segment_plan_allocated (seg) = heap_segment_mem (seg);
9191 heap_segment_allocated (seg) = heap_segment_mem (seg);
9192 #ifdef BACKGROUND_GC
9193 heap_segment_background_allocated (seg) = 0;
9194 heap_segment_saved_bg_allocated (seg) = 0;
9195 #endif //BACKGROUND_GC
9198 //Releases the segment to the OS.
9199 // this is always called on one thread only so calling seg_table->remove is fine.
9200 void gc_heap::delete_heap_segment (heap_segment* seg, BOOL consider_hoarding)
9202 if (!heap_segment_loh_p (seg))
9204 //cleanup the brick table back to the empty value
9205 clear_brick_table (heap_segment_mem (seg), heap_segment_reserved (seg));
9208 if (consider_hoarding)
9210 assert ((heap_segment_mem (seg) - (uint8_t*)seg) <= ptrdiff_t(2*OS_PAGE_SIZE));
9211 size_t ss = (size_t) (heap_segment_reserved (seg) - (uint8_t*)seg);
9212 //Don't keep the big ones.
9213 if (ss <= INITIAL_ALLOC)
9215 dprintf (2, ("Hoarding segment %Ix", (size_t)seg));
9216 #ifdef BACKGROUND_GC
9217 // We don't need to clear the decommitted flag because when this segment is used
9218 // for a new segment the flags will be cleared.
9219 if (!heap_segment_decommitted_p (seg))
9220 #endif //BACKGROUND_GC
9222 decommit_heap_segment (seg);
9225 #ifdef SEG_MAPPING_TABLE
9226 seg_mapping_table_remove_segment (seg);
9227 #endif //SEG_MAPPING_TABLE
9229 heap_segment_next (seg) = segment_standby_list;
9230 segment_standby_list = seg;
9237 dprintf (2, ("h%d: del seg: [%Ix, %Ix[",
9238 heap_number, (size_t)seg,
9239 (size_t)(heap_segment_reserved (seg))));
9241 #ifdef BACKGROUND_GC
9242 ::record_changed_seg ((uint8_t*)seg, heap_segment_reserved (seg),
9243 settings.gc_index, current_bgc_state,
9245 decommit_mark_array_by_seg (seg);
9246 #endif //BACKGROUND_GC
9248 #ifdef SEG_MAPPING_TABLE
9249 seg_mapping_table_remove_segment (seg);
9250 #else //SEG_MAPPING_TABLE
9251 seg_table->remove ((uint8_t*)seg);
9252 #endif //SEG_MAPPING_TABLE
9254 release_segment (seg);
9258 //resets the pages beyond allocates size so they won't be swapped out and back in
9260 void gc_heap::reset_heap_segment_pages (heap_segment* seg)
9262 size_t page_start = align_on_page ((size_t)heap_segment_allocated (seg));
9263 size_t size = (size_t)heap_segment_committed (seg) - page_start;
9265 GCToOSInterface::VirtualReset((void*)page_start, size, false /* unlock */);
9268 void gc_heap::decommit_heap_segment_pages (heap_segment* seg,
9271 uint8_t* page_start = align_on_page (heap_segment_allocated(seg));
9272 size_t size = heap_segment_committed (seg) - page_start;
9273 extra_space = align_on_page (extra_space);
9274 if (size >= max ((extra_space + 2*OS_PAGE_SIZE), 100*OS_PAGE_SIZE))
9276 page_start += max(extra_space, 32*OS_PAGE_SIZE);
9277 size -= max (extra_space, 32*OS_PAGE_SIZE);
9279 GCToOSInterface::VirtualDecommit (page_start, size);
9280 dprintf (3, ("Decommitting heap segment [%Ix, %Ix[(%d)",
9282 (size_t)(page_start + size),
9284 heap_segment_committed (seg) = page_start;
9285 if (heap_segment_used (seg) > heap_segment_committed (seg))
9287 heap_segment_used (seg) = heap_segment_committed (seg);
9292 //decommit all pages except one or 2
9293 void gc_heap::decommit_heap_segment (heap_segment* seg)
9295 uint8_t* page_start = align_on_page (heap_segment_mem (seg));
9297 dprintf (3, ("Decommitting heap segment %Ix", (size_t)seg));
9299 #ifdef BACKGROUND_GC
9300 page_start += OS_PAGE_SIZE;
9301 #endif //BACKGROUND_GC
9303 size_t size = heap_segment_committed (seg) - page_start;
9304 GCToOSInterface::VirtualDecommit (page_start, size);
9306 //re-init the segment object
9307 heap_segment_committed (seg) = page_start;
9308 if (heap_segment_used (seg) > heap_segment_committed (seg))
9310 heap_segment_used (seg) = heap_segment_committed (seg);
9314 void gc_heap::clear_gen0_bricks()
9316 if (!gen0_bricks_cleared)
9318 gen0_bricks_cleared = TRUE;
9319 //initialize brick table for gen 0
9320 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
9321 b < brick_of (align_on_brick
9322 (heap_segment_allocated (ephemeral_heap_segment)));
9330 #ifdef BACKGROUND_GC
9331 void gc_heap::rearrange_small_heap_segments()
9333 heap_segment* seg = freeable_small_heap_segment;
9336 heap_segment* next_seg = heap_segment_next (seg);
9337 // TODO: we need to consider hoarding here.
9338 delete_heap_segment (seg, FALSE);
9341 freeable_small_heap_segment = 0;
9343 #endif //BACKGROUND_GC
9345 void gc_heap::rearrange_large_heap_segments()
9347 dprintf (2, ("deleting empty large segments"));
9348 heap_segment* seg = freeable_large_heap_segment;
9351 heap_segment* next_seg = heap_segment_next (seg);
9352 delete_heap_segment (seg, GCConfig::GetRetainVM());
9355 freeable_large_heap_segment = 0;
9358 void gc_heap::rearrange_heap_segments(BOOL compacting)
9361 generation_start_segment (generation_of (max_generation));
9363 heap_segment* prev_seg = 0;
9364 heap_segment* next_seg = 0;
9367 next_seg = heap_segment_next (seg);
9369 //link ephemeral segment when expanding
9370 if ((next_seg == 0) && (seg != ephemeral_heap_segment))
9372 seg->next = ephemeral_heap_segment;
9373 next_seg = heap_segment_next (seg);
9376 //re-used expanded heap segment
9377 if ((seg == ephemeral_heap_segment) && next_seg)
9379 heap_segment_next (prev_seg) = next_seg;
9380 heap_segment_next (seg) = 0;
9384 uint8_t* end_segment = (compacting ?
9385 heap_segment_plan_allocated (seg) :
9386 heap_segment_allocated (seg));
9387 // check if the segment was reached by allocation
9388 if ((end_segment == heap_segment_mem (seg))&&
9389 !heap_segment_read_only_p (seg))
9391 //if not, unthread and delete
9393 assert (seg != ephemeral_heap_segment);
9394 heap_segment_next (prev_seg) = next_seg;
9395 delete_heap_segment (seg, GCConfig::GetRetainVM());
9397 dprintf (2, ("Deleting heap segment %Ix", (size_t)seg));
9401 if (!heap_segment_read_only_p (seg))
9405 heap_segment_allocated (seg) =
9406 heap_segment_plan_allocated (seg);
9409 // reset the pages between allocated and committed.
9410 if (seg != ephemeral_heap_segment)
9412 decommit_heap_segment_pages (seg, 0);
9426 uint8_t* g_addresses [array_size+2]; // to get around the bug in GetWriteWatch
9428 #ifdef TIME_WRITE_WATCH
9429 static unsigned int tot_cycles = 0;
9430 #endif //TIME_WRITE_WATCH
9434 inline void gc_heap::verify_card_bundle_bits_set(size_t first_card_word, size_t last_card_word)
9437 for (size_t x = cardw_card_bundle (first_card_word); x < cardw_card_bundle (last_card_word); x++)
9439 if (!card_bundle_set_p (x))
9441 assert (!"Card bundle not set");
9442 dprintf (3, ("Card bundle %Ix not set", x));
9448 // Verifies that any bundles that are not set represent only cards that are not set.
9449 inline void gc_heap::verify_card_bundles()
9452 size_t lowest_card = card_word (card_of (lowest_address));
9453 size_t highest_card = card_word (card_of (highest_address));
9454 size_t cardb = cardw_card_bundle (lowest_card);
9455 size_t end_cardb = cardw_card_bundle (align_cardw_on_bundle (highest_card));
9457 while (cardb < end_cardb)
9459 uint32_t* card_word = &card_table[max(card_bundle_cardw (cardb), lowest_card)];
9460 uint32_t* card_word_end = &card_table[min(card_bundle_cardw (cardb+1), highest_card)];
9462 if (card_bundle_set_p (cardb) == 0)
9464 // Verify that no card is set
9465 while (card_word < card_word_end)
9467 if (*card_word != 0)
9469 dprintf (3, ("gc: %d, Card word %Ix for address %Ix set, card_bundle %Ix clear",
9470 dd_collection_count (dynamic_data_of (0)),
9471 (size_t)(card_word-&card_table[0]),
9472 (size_t)(card_address ((size_t)(card_word-&card_table[0]) * card_word_width)), cardb));
9475 assert((*card_word)==0);
9485 // If card bundles are enabled, use write watch to find pages in the card table that have
9486 // been dirtied, and set the corresponding card bundle bits.
9487 void gc_heap::update_card_table_bundle()
9489 if (card_bundles_enabled())
9491 // The address of the card word containing the card representing the lowest heap address
9492 uint8_t* base_address = (uint8_t*)(&card_table[card_word (card_of (lowest_address))]);
9494 // The address of the card word containing the card representing the highest heap address
9495 uint8_t* high_address = (uint8_t*)(&card_table[card_word (card_of (highest_address))]);
9497 uint8_t* saved_base_address = base_address;
9498 uintptr_t bcount = array_size;
9499 size_t saved_region_size = align_on_page (high_address) - saved_base_address;
9503 size_t region_size = align_on_page (high_address) - base_address;
9505 dprintf (3,("Probing card table pages [%Ix, %Ix[", (size_t)base_address, (size_t)base_address+region_size));
9506 bool success = GCToOSInterface::GetWriteWatch(false /* resetState */,
9509 (void**)g_addresses,
9511 assert (success && "GetWriteWatch failed!");
9513 dprintf (3,("Found %d pages written", bcount));
9514 for (unsigned i = 0; i < bcount; i++)
9516 // Offset of the dirty page from the start of the card table (clamped to base_address)
9517 size_t bcardw = (uint32_t*)(max(g_addresses[i],base_address)) - &card_table[0];
9519 // Offset of the end of the page from the start of the card table (clamped to high addr)
9520 size_t ecardw = (uint32_t*)(min(g_addresses[i]+OS_PAGE_SIZE, high_address)) - &card_table[0];
9521 assert (bcardw >= card_word (card_of (g_gc_lowest_address)));
9523 // Set the card bundle bits representing the dirty card table page
9524 card_bundles_set (cardw_card_bundle (bcardw), cardw_card_bundle (align_cardw_on_bundle (ecardw)));
9525 dprintf (3,("Set Card bundle [%Ix, %Ix[", cardw_card_bundle (bcardw), cardw_card_bundle (align_cardw_on_bundle (ecardw))));
9527 verify_card_bundle_bits_set(bcardw, ecardw);
9530 if (bcount >= array_size)
9532 base_address = g_addresses [array_size-1] + OS_PAGE_SIZE;
9533 bcount = array_size;
9536 } while ((bcount >= array_size) && (base_address < high_address));
9538 // Now that we've updated the card bundle bits, reset the write-tracking state.
9539 GCToOSInterface::ResetWriteWatch (saved_base_address, saved_region_size);
9542 #endif //CARD_BUNDLE
9545 void gc_heap::reset_write_watch_for_gc_heap(void* base_address, size_t region_size)
9547 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9548 SoftwareWriteWatch::ClearDirty(base_address, region_size);
9549 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9550 GCToOSInterface::ResetWriteWatch(base_address, region_size);
9551 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9555 void gc_heap::get_write_watch_for_gc_heap(bool reset, void *base_address, size_t region_size, void** dirty_pages, uintptr_t* dirty_page_count_ref, bool is_runtime_suspended)
9557 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9558 SoftwareWriteWatch::GetDirty(base_address, region_size, dirty_pages, dirty_page_count_ref, reset, is_runtime_suspended);
9559 #else // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9560 UNREFERENCED_PARAMETER(is_runtime_suspended);
9561 bool success = GCToOSInterface::GetWriteWatch(reset, base_address, region_size, dirty_pages, dirty_page_count_ref);
9563 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9566 const size_t ww_reset_quantum = 128*1024*1024;
9569 void gc_heap::switch_one_quantum()
9571 enable_preemptive ();
9572 GCToOSInterface::Sleep (1);
9573 disable_preemptive (true);
9576 void gc_heap::reset_ww_by_chunk (uint8_t* start_address, size_t total_reset_size)
9578 size_t reset_size = 0;
9579 size_t remaining_reset_size = 0;
9580 size_t next_reset_size = 0;
9582 while (reset_size != total_reset_size)
9584 remaining_reset_size = total_reset_size - reset_size;
9585 next_reset_size = ((remaining_reset_size >= ww_reset_quantum) ? ww_reset_quantum : remaining_reset_size);
9586 if (next_reset_size)
9588 reset_write_watch_for_gc_heap(start_address, next_reset_size);
9589 reset_size += next_reset_size;
9591 switch_one_quantum();
9595 assert (reset_size == total_reset_size);
9598 // This does a Sleep(1) for every reset ww_reset_quantum bytes of reset
9599 // we do concurrently.
9600 void gc_heap::switch_on_reset (BOOL concurrent_p, size_t* current_total_reset_size, size_t last_reset_size)
9604 *current_total_reset_size += last_reset_size;
9606 dprintf (2, ("reset %Id bytes so far", *current_total_reset_size));
9608 if (*current_total_reset_size > ww_reset_quantum)
9610 switch_one_quantum();
9612 *current_total_reset_size = 0;
9617 void gc_heap::reset_write_watch (BOOL concurrent_p)
9619 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9620 // Software write watch currently requires the runtime to be suspended during reset. See SoftwareWriteWatch::ClearDirty().
9621 assert(!concurrent_p);
9622 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9624 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
9626 PREFIX_ASSUME(seg != NULL);
9628 size_t reset_size = 0;
9629 size_t region_size = 0;
9631 dprintf (2, ("bgc lowest: %Ix, bgc highest: %Ix", background_saved_lowest_address, background_saved_highest_address));
9635 uint8_t* base_address = align_lower_page (heap_segment_mem (seg));
9636 base_address = max (base_address, background_saved_lowest_address);
9638 uint8_t* high_address = 0;
9639 high_address = ((seg == ephemeral_heap_segment) ? alloc_allocated : heap_segment_allocated (seg));
9640 high_address = min (high_address, background_saved_highest_address);
9642 if (base_address < high_address)
9644 region_size = high_address - base_address;
9646 #ifdef TIME_WRITE_WATCH
9647 unsigned int time_start = GetCycleCount32();
9648 #endif //TIME_WRITE_WATCH
9649 dprintf (3, ("h%d: soh ww: [%Ix(%Id)", heap_number, (size_t)base_address, region_size));
9650 //reset_ww_by_chunk (base_address, region_size);
9651 reset_write_watch_for_gc_heap(base_address, region_size);
9653 #ifdef TIME_WRITE_WATCH
9654 unsigned int time_stop = GetCycleCount32();
9655 tot_cycles += time_stop - time_start;
9656 printf ("ResetWriteWatch Duration: %d, total: %d\n",
9657 time_stop - time_start, tot_cycles);
9658 #endif //TIME_WRITE_WATCH
9660 switch_on_reset (concurrent_p, &reset_size, region_size);
9663 seg = heap_segment_next_rw (seg);
9665 concurrent_print_time_delta ("CRWW soh");
9668 //concurrent_print_time_delta ("CRW soh");
9670 seg = heap_segment_rw (generation_start_segment (large_object_generation));
9672 PREFIX_ASSUME(seg != NULL);
9676 uint8_t* base_address = align_lower_page (heap_segment_mem (seg));
9677 uint8_t* high_address = heap_segment_allocated (seg);
9679 base_address = max (base_address, background_saved_lowest_address);
9680 high_address = min (high_address, background_saved_highest_address);
9682 if (base_address < high_address)
9684 region_size = high_address - base_address;
9686 #ifdef TIME_WRITE_WATCH
9687 unsigned int time_start = GetCycleCount32();
9688 #endif //TIME_WRITE_WATCH
9689 dprintf (3, ("h%d: loh ww: [%Ix(%Id)", heap_number, (size_t)base_address, region_size));
9690 //reset_ww_by_chunk (base_address, region_size);
9691 reset_write_watch_for_gc_heap(base_address, region_size);
9693 #ifdef TIME_WRITE_WATCH
9694 unsigned int time_stop = GetCycleCount32();
9695 tot_cycles += time_stop - time_start;
9696 printf ("ResetWriteWatch Duration: %d, total: %d\n",
9697 time_stop - time_start, tot_cycles);
9698 #endif //TIME_WRITE_WATCH
9700 switch_on_reset (concurrent_p, &reset_size, region_size);
9703 seg = heap_segment_next_rw (seg);
9705 concurrent_print_time_delta ("CRWW loh");
9708 #ifdef DEBUG_WRITE_WATCH
9709 debug_write_watch = (uint8_t**)~0;
9710 #endif //DEBUG_WRITE_WATCH
9713 #endif //WRITE_WATCH
9715 #ifdef BACKGROUND_GC
9716 void gc_heap::restart_vm()
9718 //assert (generation_allocation_pointer (youngest_generation) == 0);
9719 dprintf (3, ("Restarting EE"));
9720 STRESS_LOG0(LF_GC, LL_INFO10000, "Concurrent GC: Retarting EE\n");
9721 ee_proceed_event.Set();
9725 void fire_alloc_wait_event (alloc_wait_reason awr, BOOL begin_p)
9727 if (awr != awr_ignored)
9731 FIRE_EVENT(BGCAllocWaitBegin, awr);
9735 FIRE_EVENT(BGCAllocWaitEnd, awr);
9741 void gc_heap::fire_alloc_wait_event_begin (alloc_wait_reason awr)
9743 fire_alloc_wait_event (awr, TRUE);
9747 void gc_heap::fire_alloc_wait_event_end (alloc_wait_reason awr)
9749 fire_alloc_wait_event (awr, FALSE);
9751 #endif //BACKGROUND_GC
9752 void gc_heap::make_generation (generation& gen, heap_segment* seg, uint8_t* start, uint8_t* pointer)
9754 gen.allocation_start = start;
9755 gen.allocation_context.alloc_ptr = pointer;
9756 gen.allocation_context.alloc_limit = pointer;
9757 gen.allocation_context.alloc_bytes = 0;
9758 gen.allocation_context.alloc_bytes_loh = 0;
9759 gen.allocation_context_start_region = pointer;
9760 gen.start_segment = seg;
9761 gen.allocation_segment = seg;
9762 gen.plan_allocation_start = 0;
9763 gen.free_list_space = 0;
9764 gen.pinned_allocated = 0;
9765 gen.free_list_allocated = 0;
9766 gen.end_seg_allocated = 0;
9767 gen.condemned_allocated = 0;
9768 gen.free_obj_space = 0;
9769 gen.allocation_size = 0;
9770 gen.pinned_allocation_sweep_size = 0;
9771 gen.pinned_allocation_compact_size = 0;
9772 gen.allocate_end_seg_p = FALSE;
9773 gen.free_list_allocator.clear();
9775 #ifdef FREE_USAGE_STATS
9776 memset (gen.gen_free_spaces, 0, sizeof (gen.gen_free_spaces));
9777 memset (gen.gen_current_pinned_free_spaces, 0, sizeof (gen.gen_current_pinned_free_spaces));
9778 memset (gen.gen_plugs, 0, sizeof (gen.gen_plugs));
9779 #endif //FREE_USAGE_STATS
9782 void gc_heap::adjust_ephemeral_limits ()
9784 ephemeral_low = generation_allocation_start (generation_of (max_generation - 1));
9785 ephemeral_high = heap_segment_reserved (ephemeral_heap_segment);
9787 dprintf (3, ("new ephemeral low: %Ix new ephemeral high: %Ix",
9788 (size_t)ephemeral_low, (size_t)ephemeral_high))
9790 #ifndef MULTIPLE_HEAPS
9791 // This updates the write barrier helpers with the new info.
9792 stomp_write_barrier_ephemeral(ephemeral_low, ephemeral_high);
9793 #endif // MULTIPLE_HEAPS
9796 #if defined(TRACE_GC) || defined(GC_CONFIG_DRIVEN)
9797 FILE* CreateLogFile(const GCConfigStringHolder& temp_logfile_name, bool is_config)
9801 if (!temp_logfile_name.Get())
9806 char logfile_name[MAX_LONGPATH+1];
9807 uint32_t pid = GCToOSInterface::GetCurrentProcessId();
9808 const char* suffix = is_config ? ".config.log" : ".log";
9809 _snprintf_s(logfile_name, MAX_LONGPATH+1, _TRUNCATE, "%s.%d%s", temp_logfile_name.Get(), pid, suffix);
9810 logFile = fopen(logfile_name, "wb");
9813 #endif //TRACE_GC || GC_CONFIG_DRIVEN
9815 HRESULT gc_heap::initialize_gc (size_t segment_size,
9817 #ifdef MULTIPLE_HEAPS
9818 ,unsigned number_of_heaps
9819 #endif //MULTIPLE_HEAPS
9823 if (GCConfig::GetLogEnabled())
9825 gc_log = CreateLogFile(GCConfig::GetLogFile(), false);
9830 // GCLogFileSize in MBs.
9831 gc_log_file_size = static_cast<size_t>(GCConfig::GetLogFileSize());
9833 if (gc_log_file_size <= 0 || gc_log_file_size > 500)
9839 gc_log_lock.Initialize();
9840 gc_log_buffer = new (nothrow) uint8_t [gc_log_buffer_size];
9847 memset (gc_log_buffer, '*', gc_log_buffer_size);
9849 max_gc_buffers = gc_log_file_size * 1024 * 1024 / gc_log_buffer_size;
9853 #ifdef GC_CONFIG_DRIVEN
9854 if (GCConfig::GetConfigLogEnabled())
9856 gc_config_log = CreateLogFile(GCConfig::GetConfigLogFile(), true);
9858 if (gc_config_log == NULL)
9861 gc_config_log_buffer = new (nothrow) uint8_t [gc_config_log_buffer_size];
9862 if (!gc_config_log_buffer)
9864 fclose(gc_config_log);
9868 compact_ratio = static_cast<int>(GCConfig::GetCompactRatio());
9870 // h# | GC | gen | C | EX | NF | BF | ML | DM || PreS | PostS | Merge | Conv | Pre | Post | PrPo | PreP | PostP |
9871 cprintf (("%2s | %6s | %1s | %1s | %2s | %2s | %2s | %2s | %2s || %5s | %5s | %5s | %5s | %5s | %5s | %5s | %5s | %5s |",
9875 "C", // compaction (empty means sweeping), 'M' means it was mandatory, 'W' means it was not
9876 "EX", // heap expansion
9878 "BF", // best fit (if it indicates neither NF nor BF it means it had to acquire a new seg.
9881 "PreS", // short object before pinned plug
9882 "PostS", // short object after pinned plug
9883 "Merge", // merged pinned plugs
9884 "Conv", // converted to pinned plug
9885 "Pre", // plug before pinned plug but not after
9886 "Post", // plug after pinned plug but not before
9887 "PrPo", // plug both before and after pinned plug
9888 "PreP", // pre short object padded
9889 "PostP" // post short object padded
9892 #endif //GC_CONFIG_DRIVEN
9895 GCConfigStringHolder logFileName = GCConfig::GetMixLogFile();
9896 if (logFileName.Get() != nullptr)
9898 GCStatistics::logFileName = _strdup(logFileName.Get());
9899 GCStatistics::logFile = fopen(GCStatistics::logFileName, "a");
9900 if (!GCStatistics::logFile)
9907 HRESULT hres = S_OK;
9910 hardware_write_watch_api_supported();
9911 #ifdef BACKGROUND_GC
9912 if (can_use_write_watch_for_gc_heap() && GCConfig::GetConcurrentGC())
9914 gc_can_use_concurrent = true;
9915 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9916 virtual_alloc_hardware_write_watch = true;
9917 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
9921 gc_can_use_concurrent = false;
9923 #endif //BACKGROUND_GC
9924 #endif //WRITE_WATCH
9926 #ifdef BACKGROUND_GC
9927 // leave the first page to contain only segment info
9928 // because otherwise we could need to revisit the first page frequently in
9930 segment_info_size = OS_PAGE_SIZE;
9932 segment_info_size = Align (sizeof (heap_segment), get_alignment_constant (FALSE));
9933 #endif //BACKGROUND_GC
9935 reserved_memory = 0;
9936 unsigned block_count;
9937 #ifdef MULTIPLE_HEAPS
9938 reserved_memory_limit = (segment_size + heap_size) * number_of_heaps;
9939 block_count = number_of_heaps;
9940 #else //MULTIPLE_HEAPS
9941 reserved_memory_limit = segment_size + heap_size;
9943 #endif //MULTIPLE_HEAPS
9945 if (!reserve_initial_memory(segment_size,heap_size,block_count))
9946 return E_OUTOFMEMORY;
9949 //check if we need to turn on card_bundles.
9950 #ifdef MULTIPLE_HEAPS
9951 // use INT64 arithmetic here because of possible overflow on 32p
9952 uint64_t th = (uint64_t)MH_TH_CARD_BUNDLE*number_of_heaps;
9954 // use INT64 arithmetic here because of possible overflow on 32p
9955 uint64_t th = (uint64_t)SH_TH_CARD_BUNDLE;
9956 #endif //MULTIPLE_HEAPS
9958 if (can_use_write_watch_for_card_table() && reserved_memory >= th)
9960 settings.card_bundles = TRUE;
9964 settings.card_bundles = FALSE;
9966 #endif //CARD_BUNDLE
9968 settings.first_init();
9970 int latency_level_from_config = static_cast<int>(GCConfig::GetLatencyLevel());
9971 if (latency_level_from_config >= latency_level_first && latency_level_from_config <= latency_level_last)
9973 gc_heap::latency_level = static_cast<gc_latency_level>(latency_level_from_config);
9978 g_gc_card_table = make_card_table (g_gc_lowest_address, g_gc_highest_address);
9980 if (!g_gc_card_table)
9981 return E_OUTOFMEMORY;
9985 #ifdef MULTIPLE_HEAPS
9986 n_heaps = number_of_heaps;
9988 g_heaps = new (nothrow) gc_heap* [number_of_heaps];
9990 return E_OUTOFMEMORY;
9993 #pragma warning(push)
9994 #pragma warning(disable:22011) // Suppress PREFast warning about integer underflow/overflow
9996 g_promoted = new (nothrow) size_t [number_of_heaps*16];
9997 g_bpromoted = new (nothrow) size_t [number_of_heaps*16];
9999 g_mark_stack_busy = new (nothrow) int[(number_of_heaps+2)*HS_CACHE_LINE_SIZE/sizeof(int)];
10000 #endif //MH_SC_MARK
10002 #pragma warning(pop)
10003 #endif // _PREFAST_
10004 if (!g_promoted || !g_bpromoted)
10005 return E_OUTOFMEMORY;
10008 if (!g_mark_stack_busy)
10009 return E_OUTOFMEMORY;
10010 #endif //MH_SC_MARK
10012 if (!create_thread_support (number_of_heaps))
10013 return E_OUTOFMEMORY;
10015 if (!heap_select::init (number_of_heaps))
10016 return E_OUTOFMEMORY;
10018 #endif //MULTIPLE_HEAPS
10020 if (!init_semi_shared())
10028 //Initializes PER_HEAP_ISOLATED data members.
10030 gc_heap::init_semi_shared()
10034 // This is used for heap expansion - it's to fix exactly the start for gen 0
10035 // through (max_generation-1). When we expand the heap we allocate all these
10036 // gen starts at the beginning of the new ephemeral seg.
10037 eph_gen_starts_size = (Align (min_obj_size)) * max_generation;
10040 #ifdef MULTIPLE_HEAPS
10041 mark_list_size = min (150*1024, max (8192, soh_segment_size/(2*10*32)));
10042 g_mark_list = make_mark_list (mark_list_size*n_heaps);
10044 min_balance_threshold = alloc_quantum_balance_units * CLR_SIZE * 2;
10045 #ifdef PARALLEL_MARK_LIST_SORT
10046 g_mark_list_copy = make_mark_list (mark_list_size*n_heaps);
10047 if (!g_mark_list_copy)
10051 #endif //PARALLEL_MARK_LIST_SORT
10053 #else //MULTIPLE_HEAPS
10055 mark_list_size = max (8192, soh_segment_size/(64*32));
10056 g_mark_list = make_mark_list (mark_list_size);
10058 #endif //MULTIPLE_HEAPS
10060 dprintf (3, ("mark_list_size: %d", mark_list_size));
10068 #if defined(SEG_MAPPING_TABLE) && !defined(GROWABLE_SEG_MAPPING_TABLE)
10069 if (!seg_mapping_table_init())
10071 #endif //SEG_MAPPING_TABLE && !GROWABLE_SEG_MAPPING_TABLE
10073 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
10074 seg_table = sorted_table::make_sorted_table();
10078 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
10080 segment_standby_list = 0;
10082 if (!full_gc_approach_event.CreateManualEventNoThrow(FALSE))
10086 if (!full_gc_end_event.CreateManualEventNoThrow(FALSE))
10091 fgn_maxgen_percent = 0;
10092 fgn_loh_percent = 0;
10093 full_gc_approach_event_set = false;
10095 memset (full_gc_counts, 0, sizeof (full_gc_counts));
10098 should_expand_in_full_gc = FALSE;
10100 #ifdef FEATURE_LOH_COMPACTION
10101 loh_compaction_always_p = GCConfig::GetLOHCompactionMode() != 0;
10102 loh_compaction_mode = loh_compaction_default;
10103 #endif //FEATURE_LOH_COMPACTION
10105 #ifdef BACKGROUND_GC
10106 memset (ephemeral_fgc_counts, 0, sizeof (ephemeral_fgc_counts));
10107 bgc_alloc_spin_count = static_cast<uint32_t>(GCConfig::GetBGCSpinCount());
10108 bgc_alloc_spin = static_cast<uint32_t>(GCConfig::GetBGCSpin());
10111 int number_bgc_threads = 1;
10112 #ifdef MULTIPLE_HEAPS
10113 number_bgc_threads = n_heaps;
10114 #endif //MULTIPLE_HEAPS
10115 if (!create_bgc_threads_support (number_bgc_threads))
10120 #endif //BACKGROUND_GC
10122 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
10124 #ifdef GC_CONFIG_DRIVEN
10125 compact_or_sweep_gcs[0] = 0;
10126 compact_or_sweep_gcs[1] = 0;
10127 #endif //GC_CONFIG_DRIVEN
10130 short_plugs_pad_ratio = (double)DESIRED_PLUG_LENGTH / (double)(DESIRED_PLUG_LENGTH - Align (min_obj_size));
10131 #endif //SHORT_PLUGS
10139 if (full_gc_approach_event.IsValid())
10141 full_gc_approach_event.CloseEvent();
10143 if (full_gc_end_event.IsValid())
10145 full_gc_end_event.CloseEvent();
10152 gc_heap* gc_heap::make_gc_heap (
10153 #ifdef MULTIPLE_HEAPS
10156 #endif //MULTIPLE_HEAPS
10161 #ifdef MULTIPLE_HEAPS
10162 res = new (nothrow) gc_heap;
10166 res->vm_heap = vm_hp;
10167 res->alloc_context_count = 0;
10170 #ifdef PARALLEL_MARK_LIST_SORT
10171 res->mark_list_piece_start = new (nothrow) uint8_t**[n_heaps];
10172 if (!res->mark_list_piece_start)
10176 #pragma warning(push)
10177 #pragma warning(disable:22011) // Suppress PREFast warning about integer underflow/overflow
10178 #endif // _PREFAST_
10179 res->mark_list_piece_end = new (nothrow) uint8_t**[n_heaps + 32]; // +32 is padding to reduce false sharing
10181 #pragma warning(pop)
10182 #endif // _PREFAST_
10184 if (!res->mark_list_piece_end)
10186 #endif //PARALLEL_MARK_LIST_SORT
10190 #endif //MULTIPLE_HEAPS
10192 if (res->init_gc_heap (
10193 #ifdef MULTIPLE_HEAPS
10195 #else //MULTIPLE_HEAPS
10197 #endif //MULTIPLE_HEAPS
10203 #ifdef MULTIPLE_HEAPS
10206 return (gc_heap*)1;
10207 #endif //MULTIPLE_HEAPS
10211 gc_heap::wait_for_gc_done(int32_t timeOut)
10213 bool cooperative_mode = enable_preemptive ();
10215 uint32_t dwWaitResult = NOERROR;
10217 gc_heap* wait_heap = NULL;
10218 while (gc_heap::gc_started)
10220 #ifdef MULTIPLE_HEAPS
10221 wait_heap = GCHeap::GetHeap(heap_select::select_heap(NULL, 0))->pGenGCHeap;
10222 dprintf(2, ("waiting for the gc_done_event on heap %d", wait_heap->heap_number));
10223 #endif // MULTIPLE_HEAPS
10226 PREFIX_ASSUME(wait_heap != NULL);
10227 #endif // _PREFAST_
10229 dwWaitResult = wait_heap->gc_done_event.Wait(timeOut, FALSE);
10231 disable_preemptive (cooperative_mode);
10233 return dwWaitResult;
10237 gc_heap::set_gc_done()
10239 enter_gc_done_event_lock();
10240 if (!gc_done_event_set)
10242 gc_done_event_set = true;
10243 dprintf (2, ("heap %d: setting gc_done_event", heap_number));
10244 gc_done_event.Set();
10246 exit_gc_done_event_lock();
10250 gc_heap::reset_gc_done()
10252 enter_gc_done_event_lock();
10253 if (gc_done_event_set)
10255 gc_done_event_set = false;
10256 dprintf (2, ("heap %d: resetting gc_done_event", heap_number));
10257 gc_done_event.Reset();
10259 exit_gc_done_event_lock();
10263 gc_heap::enter_gc_done_event_lock()
10265 uint32_t dwSwitchCount = 0;
10268 if (Interlocked::CompareExchange(&gc_done_event_lock, 0, -1) >= 0)
10270 while (gc_done_event_lock >= 0)
10272 if (g_num_processors > 1)
10274 int spin_count = 32 * g_num_processors;
10275 for (int j = 0; j < spin_count; j++)
10277 if (gc_done_event_lock < 0)
10279 YieldProcessor(); // indicate to the processor that we are spining
10281 if (gc_done_event_lock >= 0)
10282 GCToOSInterface::YieldThread(++dwSwitchCount);
10285 GCToOSInterface::YieldThread(++dwSwitchCount);
10292 gc_heap::exit_gc_done_event_lock()
10294 gc_done_event_lock = -1;
10297 #ifndef MULTIPLE_HEAPS
10299 #ifdef RECORD_LOH_STATE
10300 int gc_heap::loh_state_index = 0;
10301 gc_heap::loh_state_info gc_heap::last_loh_states[max_saved_loh_states];
10302 #endif //RECORD_LOH_STATE
10304 VOLATILE(int32_t) gc_heap::gc_done_event_lock;
10305 VOLATILE(bool) gc_heap::gc_done_event_set;
10306 GCEvent gc_heap::gc_done_event;
10307 #endif //!MULTIPLE_HEAPS
10308 VOLATILE(bool) gc_heap::internal_gc_done;
10310 void gc_heap::add_saved_spinlock_info (
10311 msl_enter_state enter_state,
10312 msl_take_state take_state)
10315 #ifdef SPINLOCK_HISTORY
10316 spinlock_info* current = &last_spinlock_info[spinlock_info_index];
10318 current->enter_state = enter_state;
10319 current->take_state = take_state;
10320 current->thread_id.SetToCurrentThread();
10322 spinlock_info_index++;
10324 assert (spinlock_info_index <= max_saved_spinlock_info);
10326 if (spinlock_info_index >= max_saved_spinlock_info)
10328 spinlock_info_index = 0;
10331 MAYBE_UNUSED_VAR(enter_state);
10332 MAYBE_UNUSED_VAR(take_state);
10333 #endif //SPINLOCK_HISTORY
10337 gc_heap::init_gc_heap (int h_number)
10339 #ifdef MULTIPLE_HEAPS
10343 #ifdef SPINLOCK_HISTORY
10344 spinlock_info_index = 0;
10345 memset (last_spinlock_info, 0, sizeof(last_spinlock_info));
10346 #endif //SPINLOCK_HISTORY
10348 // initialize per heap members.
10349 ephemeral_low = (uint8_t*)1;
10351 ephemeral_high = MAX_PTR;
10353 ephemeral_heap_segment = 0;
10355 freeable_large_heap_segment = 0;
10357 condemned_generation_num = 0;
10359 blocking_collection = FALSE;
10361 generation_skip_ratio = 100;
10363 mark_stack_tos = 0;
10365 mark_stack_bos = 0;
10367 mark_stack_array_length = 0;
10369 mark_stack_array = 0;
10371 verify_pinned_queue_p = FALSE;
10373 loh_pinned_queue_tos = 0;
10375 loh_pinned_queue_bos = 0;
10377 loh_pinned_queue_length = 0;
10379 loh_pinned_queue_decay = LOH_PIN_DECAY;
10381 loh_pinned_queue = 0;
10383 min_overflow_address = MAX_PTR;
10385 max_overflow_address = 0;
10387 gen0_bricks_cleared = FALSE;
10389 gen0_must_clear_bricks = 0;
10391 allocation_quantum = CLR_SIZE;
10393 more_space_lock = gc_lock;
10395 ro_segments_in_range = FALSE;
10397 loh_alloc_since_cg = 0;
10399 new_heap_segment = NULL;
10401 #ifdef RECORD_LOH_STATE
10402 loh_state_index = 0;
10403 #endif //RECORD_LOH_STATE
10404 #endif //MULTIPLE_HEAPS
10406 #ifdef MULTIPLE_HEAPS
10407 if (h_number > n_heaps)
10409 assert (!"Number of heaps exceeded");
10413 heap_number = h_number;
10414 #endif //MULTIPLE_HEAPS
10416 memset (&oom_info, 0, sizeof (oom_info));
10417 memset (&fgm_result, 0, sizeof (fgm_result));
10418 if (!gc_done_event.CreateManualEventNoThrow(FALSE))
10422 gc_done_event_lock = -1;
10423 gc_done_event_set = false;
10425 #ifndef SEG_MAPPING_TABLE
10426 if (!gc_heap::seg_table->ensure_space_for_insert ())
10430 #endif //!SEG_MAPPING_TABLE
10432 heap_segment* seg = get_initial_segment (soh_segment_size, h_number);
10436 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(seg),
10437 (size_t)(heap_segment_reserved (seg) - heap_segment_mem(seg)),
10438 gc_etw_segment_small_object_heap);
10440 #ifdef SEG_MAPPING_TABLE
10441 seg_mapping_table_add_segment (seg, __this);
10442 #else //SEG_MAPPING_TABLE
10443 seg_table->insert ((uint8_t*)seg, sdelta);
10444 #endif //SEG_MAPPING_TABLE
10446 #ifdef MULTIPLE_HEAPS
10447 heap_segment_heap (seg) = this;
10448 #endif //MULTIPLE_HEAPS
10450 /* todo: Need a global lock for this */
10451 uint32_t* ct = &g_gc_card_table [card_word (card_of (g_gc_lowest_address))];
10452 own_card_table (ct);
10453 card_table = translate_card_table (ct);
10454 /* End of global lock */
10456 brick_table = card_table_brick_table (ct);
10457 highest_address = card_table_highest_address (ct);
10458 lowest_address = card_table_lowest_address (ct);
10461 card_bundle_table = translate_card_bundle_table (card_table_card_bundle_table (ct), g_gc_lowest_address);
10462 assert (&card_bundle_table [card_bundle_word (cardw_card_bundle (card_word (card_of (g_gc_lowest_address))))] ==
10463 card_table_card_bundle_table (ct));
10464 #endif //CARD_BUNDLE
10467 if (gc_can_use_concurrent)
10468 mark_array = translate_mark_array (card_table_mark_array (&g_gc_card_table[card_word (card_of (g_gc_lowest_address))]));
10471 #endif //MARK_ARRAY
10473 uint8_t* start = heap_segment_mem (seg);
10475 for (int i = 0; i < 1 + max_generation; i++)
10477 make_generation (generation_table [ (max_generation - i) ],
10479 generation_table [(max_generation - i)].gen_num = max_generation - i;
10480 start += Align (min_obj_size);
10483 heap_segment_allocated (seg) = start;
10484 alloc_allocated = start;
10485 heap_segment_used (seg) = start - plug_skew;
10487 ephemeral_heap_segment = seg;
10489 #ifndef SEG_MAPPING_TABLE
10490 if (!gc_heap::seg_table->ensure_space_for_insert ())
10494 #endif //!SEG_MAPPING_TABLE
10495 //Create the large segment generation
10496 heap_segment* lseg = get_initial_segment(min_loh_segment_size, h_number);
10499 lseg->flags |= heap_segment_flags_loh;
10501 FIRE_EVENT(GCCreateSegment_V1, heap_segment_mem(lseg),
10502 (size_t)(heap_segment_reserved (lseg) - heap_segment_mem(lseg)),
10503 gc_etw_segment_large_object_heap);
10505 #ifdef SEG_MAPPING_TABLE
10506 seg_mapping_table_add_segment (lseg, __this);
10507 #else //SEG_MAPPING_TABLE
10508 seg_table->insert ((uint8_t*)lseg, sdelta);
10509 #endif //SEG_MAPPING_TABLE
10511 generation_table [max_generation].free_list_allocator = allocator(NUM_GEN2_ALIST, BASE_GEN2_ALIST, gen2_alloc_list);
10512 //assign the alloc_list for the large generation
10513 generation_table [max_generation+1].free_list_allocator = allocator(NUM_LOH_ALIST, BASE_LOH_ALIST, loh_alloc_list);
10514 generation_table [max_generation+1].gen_num = max_generation+1;
10515 make_generation (generation_table [max_generation+1],lseg, heap_segment_mem (lseg), 0);
10516 heap_segment_allocated (lseg) = heap_segment_mem (lseg) + Align (min_obj_size, get_alignment_constant (FALSE));
10517 heap_segment_used (lseg) = heap_segment_allocated (lseg) - plug_skew;
10519 for (int gen_num = 0; gen_num <= 1 + max_generation; gen_num++)
10521 generation* gen = generation_of (gen_num);
10522 make_unused_array (generation_allocation_start (gen), Align (min_obj_size));
10525 #ifdef MULTIPLE_HEAPS
10526 heap_segment_heap (lseg) = this;
10528 //initialize the alloc context heap
10529 generation_alloc_context (generation_of (0))->set_alloc_heap(vm_heap);
10531 //initialize the alloc context heap
10532 generation_alloc_context (generation_of (max_generation+1))->set_alloc_heap(vm_heap);
10534 #endif //MULTIPLE_HEAPS
10536 //Do this only once
10537 #ifdef MULTIPLE_HEAPS
10539 #endif //MULTIPLE_HEAPS
10541 #ifndef INTERIOR_POINTERS
10542 //set the brick_table for large objects
10543 //but default value is clearded
10544 //clear_brick_table ((uint8_t*)heap_segment_mem (lseg),
10545 // (uint8_t*)heap_segment_reserved (lseg));
10547 #else //INTERIOR_POINTERS
10549 //Because of the interior pointer business, we have to clear
10550 //the whole brick table
10551 //but the default value is cleared
10552 // clear_brick_table (lowest_address, highest_address);
10553 #endif //INTERIOR_POINTERS
10556 if (!init_dynamic_data())
10561 etw_allocation_running_amount[0] = 0;
10562 etw_allocation_running_amount[1] = 0;
10564 //needs to be done after the dynamic data has been initialized
10565 #ifndef MULTIPLE_HEAPS
10566 allocation_running_amount = dd_min_size (dynamic_data_of (0));
10567 #endif //!MULTIPLE_HEAPS
10569 fgn_last_alloc = dd_min_size (dynamic_data_of (0));
10571 mark* arr = new (nothrow) (mark [MARK_STACK_INITIAL_LENGTH]);
10575 make_mark_stack(arr);
10577 #ifdef BACKGROUND_GC
10578 freeable_small_heap_segment = 0;
10579 gchist_index_per_heap = 0;
10580 uint8_t** b_arr = new (nothrow) (uint8_t* [MARK_STACK_INITIAL_LENGTH]);
10584 make_background_mark_stack (b_arr);
10585 #endif //BACKGROUND_GC
10587 ephemeral_low = generation_allocation_start(generation_of(max_generation - 1));
10588 ephemeral_high = heap_segment_reserved(ephemeral_heap_segment);
10589 if (heap_number == 0)
10591 stomp_write_barrier_initialize(
10592 #ifdef MULTIPLE_HEAPS
10593 reinterpret_cast<uint8_t*>(1), reinterpret_cast<uint8_t*>(~0)
10595 ephemeral_low, ephemeral_high
10596 #endif //!MULTIPLE_HEAPS
10601 // why would we clear the mark array for this page? it should be cleared..
10602 // clear the first committed page
10603 //if(gc_can_use_concurrent)
10605 // clear_mark_array (align_lower_page (heap_segment_mem (seg)), heap_segment_committed (seg));
10607 #endif //MARK_ARRAY
10609 #ifdef MULTIPLE_HEAPS
10610 //register the heap in the heaps array
10612 if (!create_gc_thread ())
10615 g_heaps [heap_number] = this;
10617 #endif //MULTIPLE_HEAPS
10619 #ifdef FEATURE_PREMORTEM_FINALIZATION
10620 HRESULT hr = AllocateCFinalize(&finalize_queue);
10623 #endif // FEATURE_PREMORTEM_FINALIZATION
10625 max_free_space_items = MAX_NUM_FREE_SPACES;
10627 bestfit_seg = new (nothrow) seg_free_spaces (heap_number);
10634 if (!bestfit_seg->alloc())
10639 last_gc_before_oom = FALSE;
10641 #ifdef MULTIPLE_HEAPS
10643 #ifdef HEAP_ANALYZE
10645 heap_analyze_success = TRUE;
10647 internal_root_array = 0;
10649 internal_root_array_index = 0;
10651 internal_root_array_length = initial_internal_roots;
10655 current_obj_size = 0;
10657 #endif //HEAP_ANALYZE
10659 #endif // MULTIPLE_HEAPS
10661 #ifdef BACKGROUND_GC
10662 bgc_thread_id.Clear();
10664 if (!create_bgc_thread_support())
10669 bgc_alloc_lock = new (nothrow) exclusive_sync;
10670 if (!bgc_alloc_lock)
10675 bgc_alloc_lock->init();
10679 if (!recursive_gc_sync::init())
10683 bgc_thread_running = 0;
10685 bgc_threads_timeout_cs.Initialize();
10686 expanded_in_fgc = 0;
10687 current_bgc_state = bgc_not_in_process;
10688 background_soh_alloc_count = 0;
10689 background_loh_alloc_count = 0;
10690 bgc_overflow_count = 0;
10691 end_loh_size = dd_min_size (dynamic_data_of (max_generation + 1));
10692 #endif //BACKGROUND_GC
10694 #ifdef GC_CONFIG_DRIVEN
10695 memset (interesting_data_per_heap, 0, sizeof (interesting_data_per_heap));
10696 memset(compact_reasons_per_heap, 0, sizeof (compact_reasons_per_heap));
10697 memset(expand_mechanisms_per_heap, 0, sizeof (expand_mechanisms_per_heap));
10698 memset(interesting_mechanism_bits_per_heap, 0, sizeof (interesting_mechanism_bits_per_heap));
10699 #endif //GC_CONFIG_DRIVEN
10705 gc_heap::destroy_semi_shared()
10707 //TODO: will need to move this to per heap
10708 //#ifdef BACKGROUND_GC
10709 // if (c_mark_list)
10710 // delete c_mark_list;
10711 //#endif //BACKGROUND_GC
10715 delete g_mark_list;
10718 #if defined(SEG_MAPPING_TABLE) && !defined(GROWABLE_SEG_MAPPING_TABLE)
10719 if (seg_mapping_table)
10720 delete seg_mapping_table;
10721 #endif //SEG_MAPPING_TABLE && !GROWABLE_SEG_MAPPING_TABLE
10723 #if !defined(SEG_MAPPING_TABLE) || defined(FEATURE_BASICFREEZE)
10724 //destroy the segment map
10725 seg_table->delete_sorted_table();
10726 #endif //!SEG_MAPPING_TABLE || FEATURE_BASICFREEZE
10730 gc_heap::self_destroy()
10732 #ifdef BACKGROUND_GC
10734 #endif //BACKGROUND_GC
10736 if (gc_done_event.IsValid())
10738 gc_done_event.CloseEvent();
10741 // destroy every segment.
10742 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
10744 PREFIX_ASSUME(seg != NULL);
10746 heap_segment* next_seg;
10749 next_seg = heap_segment_next_rw (seg);
10750 delete_heap_segment (seg);
10754 seg = heap_segment_rw (generation_start_segment (generation_of (max_generation+1)));
10756 PREFIX_ASSUME(seg != NULL);
10760 next_seg = heap_segment_next_rw (seg);
10761 delete_heap_segment (seg);
10765 // get rid of the card table
10766 release_card_table (card_table);
10768 // destroy the mark stack
10769 delete mark_stack_array;
10771 #ifdef FEATURE_PREMORTEM_FINALIZATION
10772 if (finalize_queue)
10773 delete finalize_queue;
10774 #endif // FEATURE_PREMORTEM_FINALIZATION
10778 gc_heap::destroy_gc_heap(gc_heap* heap)
10780 heap->self_destroy();
10784 // Destroys resources owned by gc. It is assumed that a last GC has been performed and that
10785 // the finalizer queue has been drained.
10786 void gc_heap::shutdown_gc()
10788 destroy_semi_shared();
10790 #ifdef MULTIPLE_HEAPS
10791 //delete the heaps array
10793 destroy_thread_support();
10795 #endif //MULTIPLE_HEAPS
10796 //destroy seg_manager
10798 destroy_initial_memory();
10800 GCToOSInterface::Shutdown();
10804 BOOL gc_heap::size_fit_p (size_t size REQD_ALIGN_AND_OFFSET_DCL, uint8_t* alloc_pointer, uint8_t* alloc_limit,
10805 uint8_t* old_loc, int use_padding)
10807 BOOL already_padded = FALSE;
10809 if ((old_loc != 0) && (use_padding & USE_PADDING_FRONT))
10811 alloc_pointer = alloc_pointer + Align (min_obj_size);
10812 already_padded = TRUE;
10814 #endif //SHORT_PLUGS
10816 if (!((old_loc == 0) || same_large_alignment_p (old_loc, alloc_pointer)))
10817 size = size + switch_alignment_size (already_padded);
10819 #ifdef FEATURE_STRUCTALIGN
10820 alloc_pointer = StructAlign(alloc_pointer, requiredAlignment, alignmentOffset);
10821 #endif // FEATURE_STRUCTALIGN
10823 // in allocate_in_condemned_generation we can have this when we
10824 // set the alloc_limit to plan_allocated which could be less than
10826 if (alloc_limit < alloc_pointer)
10833 return (((size_t)(alloc_limit - alloc_pointer) >= (size + ((use_padding & USE_PADDING_TAIL)? Align(min_obj_size) : 0)))
10835 ||((!(use_padding & USE_PADDING_FRONT)) && ((alloc_pointer + size) == alloc_limit))
10836 #else //SHORT_PLUGS
10837 ||((alloc_pointer + size) == alloc_limit)
10838 #endif //SHORT_PLUGS
10843 assert (size == Align (min_obj_size));
10844 return ((size_t)(alloc_limit - alloc_pointer) >= size);
10849 BOOL gc_heap::a_size_fit_p (size_t size, uint8_t* alloc_pointer, uint8_t* alloc_limit,
10852 // We could have run into cases where this is true when alloc_allocated is the
10853 // the same as the seg committed.
10854 if (alloc_limit < alloc_pointer)
10859 return ((size_t)(alloc_limit - alloc_pointer) >= (size + Align(min_obj_size, align_const)));
10862 // Grow by committing more pages
10863 BOOL gc_heap::grow_heap_segment (heap_segment* seg, uint8_t* high_address)
10865 assert (high_address <= heap_segment_reserved (seg));
10867 //return 0 if we are at the end of the segment.
10868 if (align_on_page (high_address) > heap_segment_reserved (seg))
10871 if (high_address <= heap_segment_committed (seg))
10874 size_t c_size = align_on_page ((size_t)(high_address - heap_segment_committed (seg)));
10875 c_size = max (c_size, 16*OS_PAGE_SIZE);
10876 c_size = min (c_size, (size_t)(heap_segment_reserved (seg) - heap_segment_committed (seg)));
10881 STRESS_LOG2(LF_GC, LL_INFO10000,
10882 "Growing heap_segment: %Ix high address: %Ix\n",
10883 (size_t)seg, (size_t)high_address);
10885 dprintf(3, ("Growing segment allocation %Ix %Ix", (size_t)heap_segment_committed(seg),c_size));
10887 if (!virtual_alloc_commit_for_heap(heap_segment_committed (seg), c_size, heap_number))
10889 dprintf(3, ("Cannot grow heap segment"));
10893 #ifndef BACKGROUND_GC
10894 clear_mark_array (heap_segment_committed (seg),
10895 heap_segment_committed (seg)+c_size, TRUE);
10896 #endif //BACKGROUND_GC
10897 #endif //MARK_ARRAY
10898 heap_segment_committed (seg) += c_size;
10899 STRESS_LOG1(LF_GC, LL_INFO10000, "New commit: %Ix",
10900 (size_t)heap_segment_committed (seg));
10902 assert (heap_segment_committed (seg) <= heap_segment_reserved (seg));
10904 assert (high_address <= heap_segment_committed (seg));
10910 int gc_heap::grow_heap_segment (heap_segment* seg, uint8_t* allocated, uint8_t* old_loc, size_t size, BOOL pad_front_p REQD_ALIGN_AND_OFFSET_DCL)
10913 if ((old_loc != 0) && pad_front_p)
10915 allocated = allocated + Align (min_obj_size);
10917 #endif //SHORT_PLUGS
10919 if (!((old_loc == 0) || same_large_alignment_p (old_loc, allocated)))
10920 size = size + switch_alignment_size (FALSE);
10921 #ifdef FEATURE_STRUCTALIGN
10922 size_t pad = ComputeStructAlignPad(allocated, requiredAlignment, alignmentOffset);
10923 return grow_heap_segment (seg, allocated + pad + size);
10924 #else // FEATURE_STRUCTALIGN
10925 return grow_heap_segment (seg, allocated + size);
10926 #endif // FEATURE_STRUCTALIGN
10929 //used only in older generation allocation (i.e during gc).
10930 void gc_heap::adjust_limit (uint8_t* start, size_t limit_size, generation* gen,
10933 UNREFERENCED_PARAMETER(gennum);
10934 dprintf (3, ("gc Expanding segment allocation"));
10935 heap_segment* seg = generation_allocation_segment (gen);
10936 if ((generation_allocation_limit (gen) != start) || (start != heap_segment_plan_allocated (seg)))
10938 if (generation_allocation_limit (gen) == heap_segment_plan_allocated (seg))
10940 assert (generation_allocation_pointer (gen) >= heap_segment_mem (seg));
10941 assert (generation_allocation_pointer (gen) <= heap_segment_committed (seg));
10942 heap_segment_plan_allocated (generation_allocation_segment (gen)) = generation_allocation_pointer (gen);
10946 uint8_t* hole = generation_allocation_pointer (gen);
10947 size_t size = (generation_allocation_limit (gen) - generation_allocation_pointer (gen));
10951 dprintf (3, ("filling up hole: %Ix, size %Ix", hole, size));
10952 size_t allocated_size = generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen);
10953 if (size >= Align (min_free_list))
10955 if (allocated_size < min_free_list)
10957 if (size >= (Align (min_free_list) + Align (min_obj_size)))
10959 //split hole into min obj + threadable free item
10960 make_unused_array (hole, min_obj_size);
10961 generation_free_obj_space (gen) += Align (min_obj_size);
10962 make_unused_array (hole + Align (min_obj_size), size - Align (min_obj_size));
10963 generation_free_list_space (gen) += size - Align (min_obj_size);
10964 generation_allocator(gen)->thread_item_front (hole + Align (min_obj_size),
10965 size - Align (min_obj_size));
10966 add_gen_free (gen->gen_num, (size - Align (min_obj_size)));
10970 dprintf (3, ("allocated size too small, can't put back rest on free list %Ix", allocated_size));
10971 make_unused_array (hole, size);
10972 generation_free_obj_space (gen) += size;
10977 dprintf (3, ("threading hole in front of free list"));
10978 make_unused_array (hole, size);
10979 generation_free_list_space (gen) += size;
10980 generation_allocator(gen)->thread_item_front (hole, size);
10981 add_gen_free (gen->gen_num, size);
10986 make_unused_array (hole, size);
10987 generation_free_obj_space (gen) += size;
10991 generation_allocation_pointer (gen) = start;
10992 generation_allocation_context_start_region (gen) = start;
10994 generation_allocation_limit (gen) = (start + limit_size);
10997 void verify_mem_cleared (uint8_t* start, size_t size)
10999 if (!Aligned (size))
11004 PTR_PTR curr_ptr = (PTR_PTR) start;
11005 for (size_t i = 0; i < size / sizeof(PTR_PTR); i++)
11007 if (*(curr_ptr++) != 0)
11014 #if defined (VERIFY_HEAP) && defined (BACKGROUND_GC)
11015 void gc_heap::set_batch_mark_array_bits (uint8_t* start, uint8_t* end)
11017 size_t start_mark_bit = mark_bit_of (start);
11018 size_t end_mark_bit = mark_bit_of (end);
11019 unsigned int startbit = mark_bit_bit (start_mark_bit);
11020 unsigned int endbit = mark_bit_bit (end_mark_bit);
11021 size_t startwrd = mark_bit_word (start_mark_bit);
11022 size_t endwrd = mark_bit_word (end_mark_bit);
11024 dprintf (3, ("Setting all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
11025 (size_t)start, (size_t)start_mark_bit,
11026 (size_t)end, (size_t)end_mark_bit));
11028 unsigned int firstwrd = ~(lowbits (~0, startbit));
11029 unsigned int lastwrd = ~(highbits (~0, endbit));
11031 if (startwrd == endwrd)
11033 unsigned int wrd = firstwrd & lastwrd;
11034 mark_array[startwrd] |= wrd;
11038 // set the first mark word.
11041 mark_array[startwrd] |= firstwrd;
11045 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
11047 mark_array[wrdtmp] = ~(unsigned int)0;
11050 // set the last mark word.
11053 mark_array[endwrd] |= lastwrd;
11057 // makes sure that the mark array bits between start and end are 0.
11058 void gc_heap::check_batch_mark_array_bits (uint8_t* start, uint8_t* end)
11060 size_t start_mark_bit = mark_bit_of (start);
11061 size_t end_mark_bit = mark_bit_of (end);
11062 unsigned int startbit = mark_bit_bit (start_mark_bit);
11063 unsigned int endbit = mark_bit_bit (end_mark_bit);
11064 size_t startwrd = mark_bit_word (start_mark_bit);
11065 size_t endwrd = mark_bit_word (end_mark_bit);
11067 //dprintf (3, ("Setting all mark array bits between [%Ix:%Ix-[%Ix:%Ix",
11068 // (size_t)start, (size_t)start_mark_bit,
11069 // (size_t)end, (size_t)end_mark_bit));
11071 unsigned int firstwrd = ~(lowbits (~0, startbit));
11072 unsigned int lastwrd = ~(highbits (~0, endbit));
11074 if (startwrd == endwrd)
11076 unsigned int wrd = firstwrd & lastwrd;
11077 if (mark_array[startwrd] & wrd)
11079 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11081 mark_array [startwrd], mark_word_address (startwrd)));
11087 // set the first mark word.
11090 if (mark_array[startwrd] & firstwrd)
11092 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11093 firstwrd, startwrd,
11094 mark_array [startwrd], mark_word_address (startwrd)));
11101 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
11103 if (mark_array[wrdtmp])
11105 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11107 mark_array [wrdtmp], mark_word_address (wrdtmp)));
11112 // set the last mark word.
11115 if (mark_array[endwrd] & lastwrd)
11117 dprintf (3, ("The %Ix portion of mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
11119 mark_array [lastwrd], mark_word_address (lastwrd)));
11124 #endif //VERIFY_HEAP && BACKGROUND_GC
11126 allocator::allocator (unsigned int num_b, size_t fbs, alloc_list* b)
11128 assert (num_b < MAX_BUCKET_COUNT);
11129 num_buckets = num_b;
11130 frst_bucket_size = fbs;
11134 alloc_list& allocator::alloc_list_of (unsigned int bn)
11136 assert (bn < num_buckets);
11138 return first_bucket;
11140 return buckets [bn-1];
11143 size_t& allocator::alloc_list_damage_count_of (unsigned int bn)
11145 assert (bn < num_buckets);
11147 return first_bucket.alloc_list_damage_count();
11149 return buckets [bn-1].alloc_list_damage_count();
11152 void allocator::unlink_item (unsigned int bn, uint8_t* item, uint8_t* prev_item, BOOL use_undo_p)
11154 //unlink the free_item
11155 alloc_list* al = &alloc_list_of (bn);
11158 if (use_undo_p && (free_list_undo (prev_item) == UNDO_EMPTY))
11160 assert (item == free_list_slot (prev_item));
11161 free_list_undo (prev_item) = item;
11162 alloc_list_damage_count_of (bn)++;
11164 free_list_slot (prev_item) = free_list_slot(item);
11168 al->alloc_list_head() = (uint8_t*)free_list_slot(item);
11170 if (al->alloc_list_tail() == item)
11172 al->alloc_list_tail() = prev_item;
11176 void allocator::clear()
11178 for (unsigned int i = 0; i < num_buckets; i++)
11180 alloc_list_head_of (i) = 0;
11181 alloc_list_tail_of (i) = 0;
11185 //always thread to the end.
11186 void allocator::thread_free_item (uint8_t* item, uint8_t*& head, uint8_t*& tail)
11188 free_list_slot (item) = 0;
11189 free_list_undo (item) = UNDO_EMPTY;
11190 assert (item != head);
11196 //TODO: This shouldn't happen anymore - verify that's the case.
11197 //the following is necessary because the last free element
11198 //may have been truncated, and tail isn't updated.
11199 else if (free_list_slot (head) == 0)
11201 free_list_slot (head) = item;
11205 assert (item != tail);
11206 assert (free_list_slot(tail) == 0);
11207 free_list_slot (tail) = item;
11212 void allocator::thread_item (uint8_t* item, size_t size)
11214 size_t sz = frst_bucket_size;
11215 unsigned int a_l_number = 0;
11217 for (; a_l_number < (num_buckets-1); a_l_number++)
11225 alloc_list* al = &alloc_list_of (a_l_number);
11226 thread_free_item (item,
11227 al->alloc_list_head(),
11228 al->alloc_list_tail());
11231 void allocator::thread_item_front (uint8_t* item, size_t size)
11233 //find right free list
11234 size_t sz = frst_bucket_size;
11235 unsigned int a_l_number = 0;
11236 for (; a_l_number < (num_buckets-1); a_l_number++)
11244 alloc_list* al = &alloc_list_of (a_l_number);
11245 free_list_slot (item) = al->alloc_list_head();
11246 free_list_undo (item) = UNDO_EMPTY;
11248 if (al->alloc_list_tail() == 0)
11250 al->alloc_list_tail() = al->alloc_list_head();
11252 al->alloc_list_head() = item;
11253 if (al->alloc_list_tail() == 0)
11255 al->alloc_list_tail() = item;
11259 void allocator::copy_to_alloc_list (alloc_list* toalist)
11261 for (unsigned int i = 0; i < num_buckets; i++)
11263 toalist [i] = alloc_list_of (i);
11264 #ifdef FL_VERIFICATION
11265 uint8_t* free_item = alloc_list_head_of (i);
11270 free_item = free_list_slot (free_item);
11273 toalist[i].item_count = count;
11274 #endif //FL_VERIFICATION
11278 void allocator::copy_from_alloc_list (alloc_list* fromalist)
11280 BOOL repair_list = !discard_if_no_fit_p ();
11281 for (unsigned int i = 0; i < num_buckets; i++)
11283 size_t count = alloc_list_damage_count_of (i);
11284 alloc_list_of (i) = fromalist [i];
11285 assert (alloc_list_damage_count_of (i) == 0);
11289 //repair the the list
11290 //new items may have been added during the plan phase
11291 //items may have been unlinked.
11292 uint8_t* free_item = alloc_list_head_of (i);
11293 while (free_item && count)
11295 assert (((CObjectHeader*)free_item)->IsFree());
11296 if ((free_list_undo (free_item) != UNDO_EMPTY))
11299 free_list_slot (free_item) = free_list_undo (free_item);
11300 free_list_undo (free_item) = UNDO_EMPTY;
11303 free_item = free_list_slot (free_item);
11306 #ifdef FL_VERIFICATION
11307 free_item = alloc_list_head_of (i);
11308 size_t item_count = 0;
11312 free_item = free_list_slot (free_item);
11315 assert (item_count == alloc_list_of (i).item_count);
11316 #endif //FL_VERIFICATION
11319 uint8_t* tail_item = alloc_list_tail_of (i);
11320 assert ((tail_item == 0) || (free_list_slot (tail_item) == 0));
11325 void allocator::commit_alloc_list_changes()
11327 BOOL repair_list = !discard_if_no_fit_p ();
11330 for (unsigned int i = 0; i < num_buckets; i++)
11332 //remove the undo info from list.
11333 uint8_t* free_item = alloc_list_head_of (i);
11334 size_t count = alloc_list_damage_count_of (i);
11335 while (free_item && count)
11337 assert (((CObjectHeader*)free_item)->IsFree());
11339 if (free_list_undo (free_item) != UNDO_EMPTY)
11341 free_list_undo (free_item) = UNDO_EMPTY;
11345 free_item = free_list_slot (free_item);
11348 alloc_list_damage_count_of (i) = 0;
11353 void gc_heap::adjust_limit_clr (uint8_t* start, size_t limit_size,
11354 alloc_context* acontext, heap_segment* seg,
11355 int align_const, int gen_number)
11357 size_t aligned_min_obj_size = Align(min_obj_size, align_const);
11359 //probably should pass seg==0 for free lists.
11362 assert (heap_segment_used (seg) <= heap_segment_committed (seg));
11365 dprintf (3, ("Expanding segment allocation [%Ix, %Ix[", (size_t)start,
11366 (size_t)start + limit_size - aligned_min_obj_size));
11368 if ((acontext->alloc_limit != start) &&
11369 (acontext->alloc_limit + aligned_min_obj_size)!= start)
11371 uint8_t* hole = acontext->alloc_ptr;
11374 size_t size = (acontext->alloc_limit - acontext->alloc_ptr);
11375 dprintf (3, ("filling up hole [%Ix, %Ix[", (size_t)hole, (size_t)hole + size + Align (min_obj_size, align_const)));
11376 // when we are finishing an allocation from a free list
11377 // we know that the free area was Align(min_obj_size) larger
11378 acontext->alloc_bytes -= size;
11379 size_t free_obj_size = size + aligned_min_obj_size;
11380 make_unused_array (hole, free_obj_size);
11381 generation_free_obj_space (generation_of (gen_number)) += free_obj_size;
11383 acontext->alloc_ptr = start;
11387 // If the next alloc context is right up against the current one it means we are absorbing the min
11388 // object, so need to account for that.
11389 acontext->alloc_bytes += (start - acontext->alloc_limit);
11392 acontext->alloc_limit = (start + limit_size - aligned_min_obj_size);
11393 acontext->alloc_bytes += limit_size - ((gen_number < max_generation + 1) ? aligned_min_obj_size : 0);
11395 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
11398 AppDomain* alloc_appdomain = GetAppDomain();
11399 alloc_appdomain->RecordAllocBytes (limit_size, heap_number);
11401 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
11403 uint8_t* saved_used = 0;
11407 saved_used = heap_segment_used (seg);
11410 if (seg == ephemeral_heap_segment)
11412 //Sometimes the allocated size is advanced without clearing the
11413 //memory. Let's catch up here
11414 if (heap_segment_used (seg) < (alloc_allocated - plug_skew))
11417 #ifndef BACKGROUND_GC
11418 clear_mark_array (heap_segment_used (seg) + plug_skew, alloc_allocated);
11419 #endif //BACKGROUND_GC
11420 #endif //MARK_ARRAY
11421 heap_segment_used (seg) = alloc_allocated - plug_skew;
11424 #ifdef BACKGROUND_GC
11427 uint8_t* old_allocated = heap_segment_allocated (seg) - plug_skew - limit_size;
11428 #ifdef FEATURE_LOH_COMPACTION
11429 old_allocated -= Align (loh_padding_obj_size, align_const);
11430 #endif //FEATURE_LOH_COMPACTION
11432 assert (heap_segment_used (seg) >= old_allocated);
11434 #endif //BACKGROUND_GC
11436 (start - plug_skew + limit_size) <= heap_segment_used (seg))
11438 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to clear memory(1)", heap_number));
11439 add_saved_spinlock_info (me_release, mt_clr_mem);
11440 leave_spin_lock (&more_space_lock);
11441 dprintf (3, ("clearing memory at %Ix for %d bytes", (start - plug_skew), limit_size));
11442 memclr (start - plug_skew, limit_size);
11446 uint8_t* used = heap_segment_used (seg);
11447 heap_segment_used (seg) = start + limit_size - plug_skew;
11449 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to clear memory", heap_number));
11450 add_saved_spinlock_info (me_release, mt_clr_mem);
11451 leave_spin_lock (&more_space_lock);
11452 if ((start - plug_skew) < used)
11454 if (used != saved_used)
11459 dprintf (2, ("clearing memory before used at %Ix for %Id bytes",
11460 (start - plug_skew), (plug_skew + used - start)));
11461 memclr (start - plug_skew, used - (start - plug_skew));
11465 //this portion can be done after we release the lock
11466 if (seg == ephemeral_heap_segment)
11468 #ifdef FFIND_OBJECT
11469 if (gen0_must_clear_bricks > 0)
11471 //set the brick table to speed up find_object
11472 size_t b = brick_of (acontext->alloc_ptr);
11473 set_brick (b, acontext->alloc_ptr - brick_address (b));
11475 dprintf (3, ("Allocation Clearing bricks [%Ix, %Ix[",
11476 b, brick_of (align_on_brick (start + limit_size))));
11477 volatile short* x = &brick_table [b];
11478 short* end_x = &brick_table [brick_of (align_on_brick (start + limit_size))];
11480 for (;x < end_x;x++)
11484 #endif //FFIND_OBJECT
11486 gen0_bricks_cleared = FALSE;
11490 // verifying the memory is completely cleared.
11491 //verify_mem_cleared (start - plug_skew, limit_size);
11494 /* in order to make the allocator faster, allocate returns a
11495 * 0 filled object. Care must be taken to set the allocation limit to the
11496 * allocation pointer after gc
11499 size_t gc_heap::limit_from_size (size_t size, size_t room, int gen_number,
11502 size_t new_limit = new_allocation_limit ((size + Align (min_obj_size, align_const)),
11503 min (room,max (size + Align (min_obj_size, align_const),
11504 ((gen_number < max_generation+1) ?
11505 allocation_quantum :
11508 assert (new_limit >= (size + Align (min_obj_size, align_const)));
11509 dprintf (100, ("requested to allocate %Id bytes, actual size is %Id", size, new_limit));
11513 void gc_heap::handle_oom (int heap_num, oom_reason reason, size_t alloc_size,
11514 uint8_t* allocated, uint8_t* reserved)
11516 dprintf (1, ("total committed on the heap is %Id", get_total_committed_size()));
11518 UNREFERENCED_PARAMETER(heap_num);
11520 if (reason == oom_budget)
11522 alloc_size = dd_min_size (dynamic_data_of (0)) / 2;
11525 if ((reason == oom_budget) && ((!fgm_result.loh_p) && (fgm_result.fgm != fgm_no_failure)))
11527 // This means during the last GC we needed to reserve and/or commit more memory
11528 // but we couldn't. We proceeded with the GC and ended up not having enough
11529 // memory at the end. This is a legitimate OOM situtation. Otherwise we
11530 // probably made a mistake and didn't expand the heap when we should have.
11531 reason = oom_low_mem;
11534 oom_info.reason = reason;
11535 oom_info.allocated = allocated;
11536 oom_info.reserved = reserved;
11537 oom_info.alloc_size = alloc_size;
11538 oom_info.gc_index = settings.gc_index;
11539 oom_info.fgm = fgm_result.fgm;
11540 oom_info.size = fgm_result.size;
11541 oom_info.available_pagefile_mb = fgm_result.available_pagefile_mb;
11542 oom_info.loh_p = fgm_result.loh_p;
11544 fgm_result.fgm = fgm_no_failure;
11546 // Break early - before the more_space_lock is release so no other threads
11547 // could have allocated on the same heap when OOM happened.
11548 if (GCConfig::GetBreakOnOOM())
11550 GCToOSInterface::DebugBreak();
11554 #ifdef BACKGROUND_GC
11555 BOOL gc_heap::background_allowed_p()
11557 return ( gc_can_use_concurrent && ((settings.pause_mode == pause_interactive) || (settings.pause_mode == pause_sustained_low_latency)) );
11559 #endif //BACKGROUND_GC
11561 void gc_heap::check_for_full_gc (int gen_num, size_t size)
11563 BOOL should_notify = FALSE;
11564 // if we detect full gc because of the allocation budget specified this is TRUE;
11565 // it's FALSE if it's due to other factors.
11566 BOOL alloc_factor = TRUE;
11569 int n_initial = gen_num;
11570 BOOL local_blocking_collection = FALSE;
11571 BOOL local_elevation_requested = FALSE;
11572 int new_alloc_remain_percent = 0;
11574 if (full_gc_approach_event_set)
11579 if (gen_num != (max_generation + 1))
11581 gen_num = max_generation;
11584 dynamic_data* dd_full = dynamic_data_of (gen_num);
11585 ptrdiff_t new_alloc_remain = 0;
11586 uint32_t pct = ((gen_num == (max_generation + 1)) ? fgn_loh_percent : fgn_maxgen_percent);
11588 for (int gen_index = 0; gen_index <= (max_generation + 1); gen_index++)
11590 dprintf (2, ("FGN: h#%d: gen%d: %Id(%Id)",
11591 heap_number, gen_index,
11592 dd_new_allocation (dynamic_data_of (gen_index)),
11593 dd_desired_allocation (dynamic_data_of (gen_index))));
11596 // For small object allocations we only check every fgn_check_quantum bytes.
11597 if (n_initial == 0)
11599 dprintf (2, ("FGN: gen0 last recorded alloc: %Id", fgn_last_alloc));
11600 dynamic_data* dd_0 = dynamic_data_of (n_initial);
11601 if (((fgn_last_alloc - dd_new_allocation (dd_0)) < fgn_check_quantum) &&
11602 (dd_new_allocation (dd_0) >= 0))
11608 fgn_last_alloc = dd_new_allocation (dd_0);
11609 dprintf (2, ("FGN: gen0 last recorded alloc is now: %Id", fgn_last_alloc));
11612 // We don't consider the size that came from soh 'cause it doesn't contribute to the
11617 for (i = n+1; i <= max_generation; i++)
11619 if (get_new_allocation (i) <= 0)
11621 n = min (i, max_generation);
11627 dprintf (2, ("FGN: h#%d: gen%d budget exceeded", heap_number, n));
11628 if (gen_num == max_generation)
11630 // If it's small object heap we should first see if we will even be looking at gen2 budget
11631 // in the next GC or not. If not we should go directly to checking other factors.
11632 if (n < (max_generation - 1))
11634 goto check_other_factors;
11638 new_alloc_remain = dd_new_allocation (dd_full) - size;
11640 new_alloc_remain_percent = (int)(((float)(new_alloc_remain) / (float)dd_desired_allocation (dd_full)) * 100);
11642 dprintf (2, ("FGN: alloc threshold for gen%d is %d%%, current threshold is %d%%",
11643 gen_num, pct, new_alloc_remain_percent));
11645 if (new_alloc_remain_percent <= (int)pct)
11647 #ifdef BACKGROUND_GC
11648 // If background GC is enabled, we still want to check whether this will
11649 // be a blocking GC or not because we only want to notify when it's a
11650 // blocking full GC.
11651 if (background_allowed_p())
11653 goto check_other_factors;
11655 #endif //BACKGROUND_GC
11657 should_notify = TRUE;
11661 check_other_factors:
11663 dprintf (2, ("FGC: checking other factors"));
11664 n = generation_to_condemn (n,
11665 &local_blocking_collection,
11666 &local_elevation_requested,
11669 if (local_elevation_requested && (n == max_generation))
11671 if (settings.should_lock_elevation)
11673 int local_elevation_locked_count = settings.elevation_locked_count + 1;
11674 if (local_elevation_locked_count != 6)
11676 dprintf (2, ("FGN: lock count is %d - Condemning max_generation-1",
11677 local_elevation_locked_count));
11678 n = max_generation - 1;
11683 dprintf (2, ("FGN: we estimate gen%d will be collected", n));
11685 #ifdef BACKGROUND_GC
11686 // When background GC is enabled it decreases the accuracy of our predictability -
11687 // by the time the GC happens, we may not be under BGC anymore. If we try to
11688 // predict often enough it should be ok.
11689 if ((n == max_generation) &&
11690 (recursive_gc_sync::background_running_p()))
11692 n = max_generation - 1;
11693 dprintf (2, ("FGN: bgc - 1 instead of 2"));
11696 if ((n == max_generation) && !local_blocking_collection)
11698 if (!background_allowed_p())
11700 local_blocking_collection = TRUE;
11703 #endif //BACKGROUND_GC
11705 dprintf (2, ("FGN: we estimate gen%d will be collected: %s",
11707 (local_blocking_collection ? "blocking" : "background")));
11709 if ((n == max_generation) && local_blocking_collection)
11711 alloc_factor = FALSE;
11712 should_notify = TRUE;
11720 dprintf (2, ("FGN: gen%d detecting full GC approaching(%s) (GC#%d) (%Id%% left in gen%d)",
11722 (alloc_factor ? "alloc" : "other"),
11723 dd_collection_count (dynamic_data_of (0)),
11724 new_alloc_remain_percent,
11727 send_full_gc_notification (n_initial, alloc_factor);
11731 void gc_heap::send_full_gc_notification (int gen_num, BOOL due_to_alloc_p)
11733 if (!full_gc_approach_event_set)
11735 assert (full_gc_approach_event.IsValid());
11736 FIRE_EVENT(GCFullNotify_V1, gen_num, due_to_alloc_p);
11738 full_gc_end_event.Reset();
11739 full_gc_approach_event.Set();
11740 full_gc_approach_event_set = true;
11744 wait_full_gc_status gc_heap::full_gc_wait (GCEvent *event, int time_out_ms)
11746 if (fgn_maxgen_percent == 0)
11748 return wait_full_gc_na;
11751 uint32_t wait_result = user_thread_wait(event, FALSE, time_out_ms);
11753 if ((wait_result == WAIT_OBJECT_0) || (wait_result == WAIT_TIMEOUT))
11755 if (fgn_maxgen_percent == 0)
11757 return wait_full_gc_cancelled;
11760 if (wait_result == WAIT_OBJECT_0)
11762 #ifdef BACKGROUND_GC
11763 if (fgn_last_gc_was_concurrent)
11765 fgn_last_gc_was_concurrent = FALSE;
11766 return wait_full_gc_na;
11769 #endif //BACKGROUND_GC
11771 return wait_full_gc_success;
11776 return wait_full_gc_timeout;
11781 return wait_full_gc_failed;
11785 size_t gc_heap::get_full_compact_gc_count()
11787 return full_gc_counts[gc_type_compacting];
11790 // DTREVIEW - we should check this in dt_low_ephemeral_space_p
11793 BOOL gc_heap::short_on_end_of_seg (int gen_number,
11797 UNREFERENCED_PARAMETER(gen_number);
11798 uint8_t* allocated = heap_segment_allocated(seg);
11800 return (!a_size_fit_p (end_space_after_gc(),
11802 heap_segment_reserved (seg),
11807 #pragma warning(disable:4706) // "assignment within conditional expression" is intentional in this function.
11811 BOOL gc_heap::a_fit_free_list_p (int gen_number,
11813 alloc_context* acontext,
11816 BOOL can_fit = FALSE;
11817 generation* gen = generation_of (gen_number);
11818 allocator* gen_allocator = generation_allocator (gen);
11819 size_t sz_list = gen_allocator->first_bucket_size();
11820 for (unsigned int a_l_idx = 0; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
11822 if ((size < sz_list) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
11824 uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
11825 uint8_t* prev_free_item = 0;
11827 while (free_list != 0)
11829 dprintf (3, ("considering free list %Ix", (size_t)free_list));
11830 size_t free_list_size = unused_array_size (free_list);
11831 if ((size + Align (min_obj_size, align_const)) <= free_list_size)
11833 dprintf (3, ("Found adequate unused area: [%Ix, size: %Id",
11834 (size_t)free_list, free_list_size));
11836 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
11837 // We ask for more Align (min_obj_size)
11838 // to make sure that we can insert a free object
11839 // in adjust_limit will set the limit lower
11840 size_t limit = limit_from_size (size, free_list_size, gen_number, align_const);
11842 uint8_t* remain = (free_list + limit);
11843 size_t remain_size = (free_list_size - limit);
11844 if (remain_size >= Align(min_free_list, align_const))
11846 make_unused_array (remain, remain_size);
11847 gen_allocator->thread_item_front (remain, remain_size);
11848 assert (remain_size >= Align (min_obj_size, align_const));
11852 //absorb the entire free list
11853 limit += remain_size;
11855 generation_free_list_space (gen) -= limit;
11857 adjust_limit_clr (free_list, limit, acontext, 0, align_const, gen_number);
11862 else if (gen_allocator->discard_if_no_fit_p())
11864 assert (prev_free_item == 0);
11865 dprintf (3, ("couldn't use this free area, discarding"));
11866 generation_free_obj_space (gen) += free_list_size;
11868 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
11869 generation_free_list_space (gen) -= free_list_size;
11873 prev_free_item = free_list;
11875 free_list = free_list_slot (free_list);
11878 sz_list = sz_list * 2;
11885 #ifdef BACKGROUND_GC
11886 void gc_heap::bgc_loh_alloc_clr (uint8_t* alloc_start,
11888 alloc_context* acontext,
11894 make_unused_array (alloc_start, size);
11896 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
11899 AppDomain* alloc_appdomain = GetAppDomain();
11900 alloc_appdomain->RecordAllocBytes (size, heap_number);
11902 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
11904 size_t size_of_array_base = sizeof(ArrayBase);
11906 bgc_alloc_lock->loh_alloc_done_with_index (lock_index);
11908 // clear memory while not holding the lock.
11909 size_t size_to_skip = size_of_array_base;
11910 size_t size_to_clear = size - size_to_skip - plug_skew;
11911 size_t saved_size_to_clear = size_to_clear;
11914 uint8_t* end = alloc_start + size - plug_skew;
11915 uint8_t* used = heap_segment_used (seg);
11918 if ((alloc_start + size_to_skip) < used)
11920 size_to_clear = used - (alloc_start + size_to_skip);
11926 dprintf (2, ("bgc loh: setting used to %Ix", end));
11927 heap_segment_used (seg) = end;
11930 dprintf (2, ("bgc loh: used: %Ix, alloc: %Ix, end of alloc: %Ix, clear %Id bytes",
11931 used, alloc_start, end, size_to_clear));
11935 dprintf (2, ("bgc loh: [%Ix-[%Ix(%Id)", alloc_start, alloc_start+size, size));
11939 // since we filled in 0xcc for free object when we verify heap,
11940 // we need to make sure we clear those bytes.
11941 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
11943 if (size_to_clear < saved_size_to_clear)
11945 size_to_clear = saved_size_to_clear;
11948 #endif //VERIFY_HEAP
11950 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to clear large obj", heap_number));
11951 add_saved_spinlock_info (me_release, mt_clr_large_mem);
11952 leave_spin_lock (&more_space_lock);
11953 memclr (alloc_start + size_to_skip, size_to_clear);
11955 bgc_alloc_lock->loh_alloc_set (alloc_start);
11957 acontext->alloc_ptr = alloc_start;
11958 acontext->alloc_limit = (alloc_start + size - Align (min_obj_size, align_const));
11960 // need to clear the rest of the object before we hand it out.
11961 clear_unused_array(alloc_start, size);
11963 #endif //BACKGROUND_GC
11965 BOOL gc_heap::a_fit_free_list_large_p (size_t size,
11966 alloc_context* acontext,
11969 #ifdef BACKGROUND_GC
11970 wait_for_background_planning (awr_loh_alloc_during_plan);
11971 #endif //BACKGROUND_GC
11973 BOOL can_fit = FALSE;
11974 int gen_number = max_generation + 1;
11975 generation* gen = generation_of (gen_number);
11976 allocator* loh_allocator = generation_allocator (gen);
11978 #ifdef FEATURE_LOH_COMPACTION
11979 size_t loh_pad = Align (loh_padding_obj_size, align_const);
11980 #endif //FEATURE_LOH_COMPACTION
11982 #ifdef BACKGROUND_GC
11984 #endif //BACKGROUND_GC
11985 size_t sz_list = loh_allocator->first_bucket_size();
11986 for (unsigned int a_l_idx = 0; a_l_idx < loh_allocator->number_of_buckets(); a_l_idx++)
11988 if ((size < sz_list) || (a_l_idx == (loh_allocator->number_of_buckets()-1)))
11990 uint8_t* free_list = loh_allocator->alloc_list_head_of (a_l_idx);
11991 uint8_t* prev_free_item = 0;
11992 while (free_list != 0)
11994 dprintf (3, ("considering free list %Ix", (size_t)free_list));
11996 size_t free_list_size = unused_array_size(free_list);
11998 #ifdef FEATURE_LOH_COMPACTION
11999 if ((size + loh_pad) <= free_list_size)
12001 if (((size + Align (min_obj_size, align_const)) <= free_list_size)||
12002 (size == free_list_size))
12003 #endif //FEATURE_LOH_COMPACTION
12005 #ifdef BACKGROUND_GC
12006 cookie = bgc_alloc_lock->loh_alloc_set (free_list);
12007 #endif //BACKGROUND_GC
12009 //unlink the free_item
12010 loh_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
12012 // Substract min obj size because limit_from_size adds it. Not needed for LOH
12013 size_t limit = limit_from_size (size - Align(min_obj_size, align_const), free_list_size,
12014 gen_number, align_const);
12016 #ifdef FEATURE_LOH_COMPACTION
12017 make_unused_array (free_list, loh_pad);
12019 free_list += loh_pad;
12020 free_list_size -= loh_pad;
12021 #endif //FEATURE_LOH_COMPACTION
12023 uint8_t* remain = (free_list + limit);
12024 size_t remain_size = (free_list_size - limit);
12025 if (remain_size != 0)
12027 assert (remain_size >= Align (min_obj_size, align_const));
12028 make_unused_array (remain, remain_size);
12030 if (remain_size >= Align(min_free_list, align_const))
12032 loh_thread_gap_front (remain, remain_size, gen);
12033 assert (remain_size >= Align (min_obj_size, align_const));
12037 generation_free_obj_space (gen) += remain_size;
12039 generation_free_list_space (gen) -= free_list_size;
12040 dprintf (3, ("found fit on loh at %Ix", free_list));
12041 #ifdef BACKGROUND_GC
12044 bgc_loh_alloc_clr (free_list, limit, acontext, align_const, cookie, FALSE, 0);
12047 #endif //BACKGROUND_GC
12049 adjust_limit_clr (free_list, limit, acontext, 0, align_const, gen_number);
12052 //fix the limit to compensate for adjust_limit_clr making it too short
12053 acontext->alloc_limit += Align (min_obj_size, align_const);
12057 prev_free_item = free_list;
12058 free_list = free_list_slot (free_list);
12061 sz_list = sz_list * 2;
12068 #pragma warning(default:4706)
12071 BOOL gc_heap::a_fit_segment_end_p (int gen_number,
12074 alloc_context* acontext,
12076 BOOL* commit_failed_p)
12078 *commit_failed_p = FALSE;
12080 #ifdef BACKGROUND_GC
12082 #endif //BACKGROUND_GC
12084 uint8_t*& allocated = ((gen_number == 0) ?
12086 heap_segment_allocated(seg));
12088 size_t pad = Align (min_obj_size, align_const);
12090 #ifdef FEATURE_LOH_COMPACTION
12091 if (gen_number == (max_generation + 1))
12093 pad += Align (loh_padding_obj_size, align_const);
12095 #endif //FEATURE_LOH_COMPACTION
12097 uint8_t* end = heap_segment_committed (seg) - pad;
12099 if (a_size_fit_p (size, allocated, end, align_const))
12101 limit = limit_from_size (size,
12103 gen_number, align_const);
12107 end = heap_segment_reserved (seg) - pad;
12109 if (a_size_fit_p (size, allocated, end, align_const))
12111 limit = limit_from_size (size,
12113 gen_number, align_const);
12114 if (grow_heap_segment (seg, allocated + limit))
12120 dprintf (2, ("can't grow segment, doing a full gc"));
12121 *commit_failed_p = TRUE;
12128 #ifdef BACKGROUND_GC
12129 if (gen_number != 0)
12131 cookie = bgc_alloc_lock->loh_alloc_set (allocated);
12133 #endif //BACKGROUND_GC
12135 uint8_t* old_alloc;
12136 old_alloc = allocated;
12137 #ifdef FEATURE_LOH_COMPACTION
12138 if (gen_number == (max_generation + 1))
12140 size_t loh_pad = Align (loh_padding_obj_size, align_const);
12141 make_unused_array (old_alloc, loh_pad);
12142 old_alloc += loh_pad;
12143 allocated += loh_pad;
12146 #endif //FEATURE_LOH_COMPACTION
12148 #if defined (VERIFY_HEAP) && defined (_DEBUG)
12149 ((void**) allocated)[-1] = 0; //clear the sync block
12150 #endif //VERIFY_HEAP && _DEBUG
12151 allocated += limit;
12153 dprintf (3, ("found fit at end of seg: %Ix", old_alloc));
12155 #ifdef BACKGROUND_GC
12158 bgc_loh_alloc_clr (old_alloc, limit, acontext, align_const, cookie, TRUE, seg);
12161 #endif //BACKGROUND_GC
12163 adjust_limit_clr (old_alloc, limit, acontext, seg, align_const, gen_number);
12173 BOOL gc_heap::loh_a_fit_segment_end_p (int gen_number,
12175 alloc_context* acontext,
12177 BOOL* commit_failed_p,
12180 *commit_failed_p = FALSE;
12181 heap_segment* seg = generation_allocation_segment (generation_of (gen_number));
12182 BOOL can_allocate_p = FALSE;
12186 if (a_fit_segment_end_p (gen_number, seg, (size - Align (min_obj_size, align_const)),
12187 acontext, align_const, commit_failed_p))
12189 acontext->alloc_limit += Align (min_obj_size, align_const);
12190 can_allocate_p = TRUE;
12195 if (*commit_failed_p)
12197 *oom_r = oom_cant_commit;
12202 seg = heap_segment_next_rw (seg);
12207 return can_allocate_p;
12210 #ifdef BACKGROUND_GC
12212 void gc_heap::wait_for_background (alloc_wait_reason awr)
12214 dprintf (2, ("BGC is already in progress, waiting for it to finish"));
12215 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to wait for bgc done", heap_number));
12216 add_saved_spinlock_info (me_release, mt_wait_bgc);
12217 leave_spin_lock (&more_space_lock);
12218 background_gc_wait (awr);
12219 enter_spin_lock (&more_space_lock);
12220 add_saved_spinlock_info (me_acquire, mt_wait_bgc);
12221 dprintf (SPINLOCK_LOG, ("[%d]Emsl after waiting for bgc done", heap_number));
12224 void gc_heap::wait_for_bgc_high_memory (alloc_wait_reason awr)
12226 if (recursive_gc_sync::background_running_p())
12228 uint32_t memory_load;
12229 get_memory_info (&memory_load);
12230 if (memory_load >= 95)
12232 dprintf (GTC_LOG, ("high mem - wait for BGC to finish, wait reason: %d", awr));
12233 wait_for_background (awr);
12238 #endif //BACKGROUND_GC
12240 // We request to trigger an ephemeral GC but we may get a full compacting GC.
12241 // return TRUE if that's the case.
12242 BOOL gc_heap::trigger_ephemeral_gc (gc_reason gr)
12244 #ifdef BACKGROUND_GC
12245 wait_for_bgc_high_memory (awr_loh_oos_bgc);
12246 #endif //BACKGROUND_GC
12248 BOOL did_full_compact_gc = FALSE;
12250 dprintf (2, ("triggering a gen1 GC"));
12251 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12252 vm_heap->GarbageCollectGeneration(max_generation - 1, gr);
12254 #ifdef MULTIPLE_HEAPS
12255 enter_spin_lock (&more_space_lock);
12256 add_saved_spinlock_info (me_acquire, mt_t_eph_gc);
12257 dprintf (SPINLOCK_LOG, ("[%d]Emsl after a GC", heap_number));
12258 #endif //MULTIPLE_HEAPS
12260 size_t current_full_compact_gc_count = get_full_compact_gc_count();
12262 if (current_full_compact_gc_count > last_full_compact_gc_count)
12264 dprintf (2, ("attempted to trigger an ephemeral GC and got a full compacting GC"));
12265 did_full_compact_gc = TRUE;
12268 return did_full_compact_gc;
12271 BOOL gc_heap::soh_try_fit (int gen_number,
12273 alloc_context* acontext,
12275 BOOL* commit_failed_p,
12276 BOOL* short_seg_end_p)
12278 BOOL can_allocate = TRUE;
12279 if (short_seg_end_p)
12281 *short_seg_end_p = FALSE;
12284 can_allocate = a_fit_free_list_p (gen_number, size, acontext, align_const);
12287 if (short_seg_end_p)
12289 *short_seg_end_p = short_on_end_of_seg (gen_number, ephemeral_heap_segment, align_const);
12291 // If the caller doesn't care, we always try to fit at the end of seg;
12292 // otherwise we would only try if we are actually not short at end of seg.
12293 if (!short_seg_end_p || !(*short_seg_end_p))
12295 can_allocate = a_fit_segment_end_p (gen_number, ephemeral_heap_segment, size,
12296 acontext, align_const, commit_failed_p);
12300 return can_allocate;
12303 BOOL gc_heap::allocate_small (int gen_number,
12305 alloc_context* acontext,
12308 #if defined (BACKGROUND_GC) && !defined (MULTIPLE_HEAPS)
12309 if (recursive_gc_sync::background_running_p())
12311 background_soh_alloc_count++;
12312 if ((background_soh_alloc_count % bgc_alloc_spin_count) == 0)
12314 add_saved_spinlock_info (me_release, mt_alloc_small);
12315 dprintf (SPINLOCK_LOG, ("[%d]spin Lmsl", heap_number));
12316 leave_spin_lock (&more_space_lock);
12317 bool cooperative_mode = enable_preemptive ();
12318 GCToOSInterface::Sleep (bgc_alloc_spin);
12319 disable_preemptive (cooperative_mode);
12320 enter_spin_lock (&more_space_lock);
12321 add_saved_spinlock_info (me_acquire, mt_alloc_small);
12322 dprintf (SPINLOCK_LOG, ("[%d]spin Emsl", heap_number));
12326 //GCToOSInterface::YieldThread (0);
12329 #endif //BACKGROUND_GC && !MULTIPLE_HEAPS
12331 gc_reason gr = reason_oos_soh;
12332 oom_reason oom_r = oom_no_failure;
12334 // No variable values should be "carried over" from one state to the other.
12335 // That's why there are local variable for each state
12337 allocation_state soh_alloc_state = a_state_start;
12339 // If we can get a new seg it means allocation will succeed.
12342 dprintf (3, ("[h%d]soh state is %s", heap_number, allocation_state_str[soh_alloc_state]));
12343 switch (soh_alloc_state)
12345 case a_state_can_allocate:
12346 case a_state_cant_allocate:
12350 case a_state_start:
12352 soh_alloc_state = a_state_try_fit;
12355 case a_state_try_fit:
12357 BOOL commit_failed_p = FALSE;
12358 BOOL can_use_existing_p = FALSE;
12360 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12361 align_const, &commit_failed_p,
12363 soh_alloc_state = (can_use_existing_p ?
12364 a_state_can_allocate :
12366 a_state_trigger_full_compact_gc :
12367 a_state_trigger_ephemeral_gc));
12370 case a_state_try_fit_after_bgc:
12372 BOOL commit_failed_p = FALSE;
12373 BOOL can_use_existing_p = FALSE;
12374 BOOL short_seg_end_p = FALSE;
12376 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12377 align_const, &commit_failed_p,
12379 soh_alloc_state = (can_use_existing_p ?
12380 a_state_can_allocate :
12382 a_state_trigger_2nd_ephemeral_gc :
12383 a_state_trigger_full_compact_gc));
12386 case a_state_try_fit_after_cg:
12388 BOOL commit_failed_p = FALSE;
12389 BOOL can_use_existing_p = FALSE;
12390 BOOL short_seg_end_p = FALSE;
12392 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12393 align_const, &commit_failed_p,
12395 if (short_seg_end_p)
12397 soh_alloc_state = a_state_cant_allocate;
12398 oom_r = oom_budget;
12402 if (can_use_existing_p)
12404 soh_alloc_state = a_state_can_allocate;
12408 #ifdef MULTIPLE_HEAPS
12409 if (!commit_failed_p)
12411 // some other threads already grabbed the more space lock and allocated
12412 // so we should attempt an ephemeral GC again.
12413 assert (heap_segment_allocated (ephemeral_heap_segment) < alloc_allocated);
12414 soh_alloc_state = a_state_trigger_ephemeral_gc;
12417 #endif //MULTIPLE_HEAPS
12419 assert (commit_failed_p);
12420 soh_alloc_state = a_state_cant_allocate;
12421 oom_r = oom_cant_commit;
12427 case a_state_check_and_wait_for_bgc:
12429 BOOL bgc_in_progress_p = FALSE;
12430 BOOL did_full_compacting_gc = FALSE;
12432 bgc_in_progress_p = check_and_wait_for_bgc (awr_gen0_oos_bgc, &did_full_compacting_gc);
12433 soh_alloc_state = (did_full_compacting_gc ?
12434 a_state_try_fit_after_cg :
12435 a_state_try_fit_after_bgc);
12438 case a_state_trigger_ephemeral_gc:
12440 BOOL commit_failed_p = FALSE;
12441 BOOL can_use_existing_p = FALSE;
12442 BOOL short_seg_end_p = FALSE;
12443 BOOL bgc_in_progress_p = FALSE;
12444 BOOL did_full_compacting_gc = FALSE;
12446 did_full_compacting_gc = trigger_ephemeral_gc (gr);
12447 if (did_full_compacting_gc)
12449 soh_alloc_state = a_state_try_fit_after_cg;
12453 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12454 align_const, &commit_failed_p,
12456 #ifdef BACKGROUND_GC
12457 bgc_in_progress_p = recursive_gc_sync::background_running_p();
12458 #endif //BACKGROUND_GC
12460 if (short_seg_end_p)
12462 soh_alloc_state = (bgc_in_progress_p ?
12463 a_state_check_and_wait_for_bgc :
12464 a_state_trigger_full_compact_gc);
12466 if (fgn_maxgen_percent)
12468 dprintf (2, ("FGN: doing last GC before we throw OOM"));
12469 send_full_gc_notification (max_generation, FALSE);
12474 if (can_use_existing_p)
12476 soh_alloc_state = a_state_can_allocate;
12480 #ifdef MULTIPLE_HEAPS
12481 if (!commit_failed_p)
12483 // some other threads already grabbed the more space lock and allocated
12484 // so we should attempt an ephemeral GC again.
12485 assert (heap_segment_allocated (ephemeral_heap_segment) < alloc_allocated);
12486 soh_alloc_state = a_state_trigger_ephemeral_gc;
12489 #endif //MULTIPLE_HEAPS
12491 soh_alloc_state = a_state_trigger_full_compact_gc;
12492 if (fgn_maxgen_percent)
12494 dprintf (2, ("FGN: failed to commit, doing full compacting GC"));
12495 send_full_gc_notification (max_generation, FALSE);
12503 case a_state_trigger_2nd_ephemeral_gc:
12505 BOOL commit_failed_p = FALSE;
12506 BOOL can_use_existing_p = FALSE;
12507 BOOL short_seg_end_p = FALSE;
12508 BOOL did_full_compacting_gc = FALSE;
12511 did_full_compacting_gc = trigger_ephemeral_gc (gr);
12513 if (did_full_compacting_gc)
12515 soh_alloc_state = a_state_try_fit_after_cg;
12519 can_use_existing_p = soh_try_fit (gen_number, size, acontext,
12520 align_const, &commit_failed_p,
12522 if (short_seg_end_p || commit_failed_p)
12524 soh_alloc_state = a_state_trigger_full_compact_gc;
12528 assert (can_use_existing_p);
12529 soh_alloc_state = a_state_can_allocate;
12534 case a_state_trigger_full_compact_gc:
12536 BOOL got_full_compacting_gc = FALSE;
12538 got_full_compacting_gc = trigger_full_compact_gc (gr, &oom_r);
12539 soh_alloc_state = (got_full_compacting_gc ? a_state_try_fit_after_cg : a_state_cant_allocate);
12544 assert (!"Invalid state!");
12551 if (soh_alloc_state == a_state_cant_allocate)
12553 assert (oom_r != oom_no_failure);
12554 handle_oom (heap_number,
12557 heap_segment_allocated (ephemeral_heap_segment),
12558 heap_segment_reserved (ephemeral_heap_segment));
12560 dprintf (SPINLOCK_LOG, ("[%d]Lmsl for oom", heap_number));
12561 add_saved_spinlock_info (me_release, mt_alloc_small_cant);
12562 leave_spin_lock (&more_space_lock);
12565 return (soh_alloc_state == a_state_can_allocate);
12568 #ifdef BACKGROUND_GC
12570 void gc_heap::wait_for_background_planning (alloc_wait_reason awr)
12572 while (current_c_gc_state == c_gc_state_planning)
12574 dprintf (3, ("lh state planning, cannot allocate"));
12576 dprintf (SPINLOCK_LOG, ("[%d]Lmsl to wait for bgc plan", heap_number));
12577 add_saved_spinlock_info (me_release, mt_wait_bgc_plan);
12578 leave_spin_lock (&more_space_lock);
12579 background_gc_wait_lh (awr);
12580 enter_spin_lock (&more_space_lock);
12581 add_saved_spinlock_info (me_acquire, mt_wait_bgc_plan);
12582 dprintf (SPINLOCK_LOG, ("[%d]Emsl after waiting for bgc plan", heap_number));
12584 assert ((current_c_gc_state == c_gc_state_free) ||
12585 (current_c_gc_state == c_gc_state_marking));
12588 BOOL gc_heap::bgc_loh_should_allocate()
12590 size_t min_gc_size = dd_min_size(dynamic_data_of (max_generation + 1));
12592 if ((bgc_begin_loh_size + bgc_loh_size_increased) < (min_gc_size * 10))
12597 if (((bgc_begin_loh_size / end_loh_size) >= 2) || (bgc_loh_size_increased >= bgc_begin_loh_size))
12599 if ((bgc_begin_loh_size / end_loh_size) > 2)
12601 dprintf (3, ("alloc-ed too much before bgc started"));
12605 dprintf (3, ("alloc-ed too much after bgc started"));
12611 bgc_alloc_spin_loh = (uint32_t)(((float)bgc_loh_size_increased / (float)bgc_begin_loh_size) * 10);
12615 #endif //BACKGROUND_GC
12617 size_t gc_heap::get_large_seg_size (size_t size)
12619 size_t default_seg_size = min_loh_segment_size;
12620 #ifdef SEG_MAPPING_TABLE
12621 size_t align_size = default_seg_size;
12622 #else //SEG_MAPPING_TABLE
12623 size_t align_size = default_seg_size / 2;
12624 #endif //SEG_MAPPING_TABLE
12625 int align_const = get_alignment_constant (FALSE);
12626 size_t large_seg_size = align_on_page (
12627 max (default_seg_size,
12628 ((size + 2 * Align(min_obj_size, align_const) + OS_PAGE_SIZE +
12629 align_size) / align_size * align_size)));
12630 return large_seg_size;
12633 BOOL gc_heap::loh_get_new_seg (generation* gen,
12636 BOOL* did_full_compact_gc,
12639 UNREFERENCED_PARAMETER(gen);
12640 UNREFERENCED_PARAMETER(align_const);
12642 *did_full_compact_gc = FALSE;
12644 size_t seg_size = get_large_seg_size (size);
12646 heap_segment* new_seg = get_large_segment (seg_size, did_full_compact_gc);
12650 loh_alloc_since_cg += seg_size;
12657 return (new_seg != 0);
12660 BOOL gc_heap::retry_full_compact_gc (size_t size)
12662 size_t seg_size = get_large_seg_size (size);
12664 if (loh_alloc_since_cg >= (2 * (uint64_t)seg_size))
12669 #ifdef MULTIPLE_HEAPS
12670 uint64_t total_alloc_size = 0;
12671 for (int i = 0; i < n_heaps; i++)
12673 total_alloc_size += g_heaps[i]->loh_alloc_since_cg;
12676 if (total_alloc_size >= (2 * (uint64_t)seg_size))
12680 #endif //MULTIPLE_HEAPS
12685 BOOL gc_heap::check_and_wait_for_bgc (alloc_wait_reason awr,
12686 BOOL* did_full_compact_gc)
12688 BOOL bgc_in_progress = FALSE;
12689 *did_full_compact_gc = FALSE;
12690 #ifdef BACKGROUND_GC
12691 if (recursive_gc_sync::background_running_p())
12693 bgc_in_progress = TRUE;
12694 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12695 wait_for_background (awr);
12696 size_t current_full_compact_gc_count = get_full_compact_gc_count();
12697 if (current_full_compact_gc_count > last_full_compact_gc_count)
12699 *did_full_compact_gc = TRUE;
12702 #endif //BACKGROUND_GC
12704 return bgc_in_progress;
12707 BOOL gc_heap::loh_try_fit (int gen_number,
12709 alloc_context* acontext,
12711 BOOL* commit_failed_p,
12714 BOOL can_allocate = TRUE;
12716 if (!a_fit_free_list_large_p (size, acontext, align_const))
12718 can_allocate = loh_a_fit_segment_end_p (gen_number, size,
12719 acontext, align_const,
12720 commit_failed_p, oom_r);
12722 #ifdef BACKGROUND_GC
12723 if (can_allocate && recursive_gc_sync::background_running_p())
12725 bgc_loh_size_increased += size;
12727 #endif //BACKGROUND_GC
12729 #ifdef BACKGROUND_GC
12732 if (recursive_gc_sync::background_running_p())
12734 bgc_loh_allocated_in_free += size;
12737 #endif //BACKGROUND_GC
12739 return can_allocate;
12742 BOOL gc_heap::trigger_full_compact_gc (gc_reason gr,
12745 BOOL did_full_compact_gc = FALSE;
12747 size_t last_full_compact_gc_count = get_full_compact_gc_count();
12749 // Set this so the next GC will be a full compacting GC.
12750 if (!last_gc_before_oom)
12752 last_gc_before_oom = TRUE;
12755 #ifdef BACKGROUND_GC
12756 if (recursive_gc_sync::background_running_p())
12758 wait_for_background ((gr == reason_oos_soh) ? awr_gen0_oos_bgc : awr_loh_oos_bgc);
12759 dprintf (2, ("waited for BGC - done"));
12761 #endif //BACKGROUND_GC
12763 size_t current_full_compact_gc_count = get_full_compact_gc_count();
12764 if (current_full_compact_gc_count > last_full_compact_gc_count)
12766 dprintf (3, ("a full compacting GC triggered while waiting for BGC (%d->%d)", last_full_compact_gc_count, current_full_compact_gc_count));
12767 assert (current_full_compact_gc_count > last_full_compact_gc_count);
12768 did_full_compact_gc = TRUE;
12772 dprintf (3, ("h%d full GC", heap_number));
12773 vm_heap->GarbageCollectGeneration(max_generation, gr);
12775 #ifdef MULTIPLE_HEAPS
12776 enter_spin_lock (&more_space_lock);
12777 dprintf (SPINLOCK_LOG, ("[%d]Emsl after full gc", heap_number));
12778 add_saved_spinlock_info (me_acquire, mt_t_full_gc);
12779 #endif //MULTIPLE_HEAPS
12781 current_full_compact_gc_count = get_full_compact_gc_count();
12783 if (current_full_compact_gc_count == last_full_compact_gc_count)
12785 dprintf (2, ("attempted to trigger a full compacting GC but didn't get it"));
12786 // We requested a full GC but didn't get because of the elevation logic
12787 // which means we should fail.
12788 *oom_r = oom_unproductive_full_gc;
12792 dprintf (3, ("h%d: T full compacting GC (%d->%d)",
12794 last_full_compact_gc_count,
12795 current_full_compact_gc_count));
12797 assert (current_full_compact_gc_count > last_full_compact_gc_count);
12798 did_full_compact_gc = TRUE;
12802 return did_full_compact_gc;
12805 #ifdef RECORD_LOH_STATE
12806 void gc_heap::add_saved_loh_state (allocation_state loh_state_to_save, EEThreadId thread_id)
12808 // When the state is can_allocate we already have released the more
12809 // space lock. So we are not logging states here since this code
12810 // is not thread safe.
12811 if (loh_state_to_save != a_state_can_allocate)
12813 last_loh_states[loh_state_index].alloc_state = loh_state_to_save;
12814 last_loh_states[loh_state_index].thread_id = thread_id;
12817 if (loh_state_index == max_saved_loh_states)
12819 loh_state_index = 0;
12822 assert (loh_state_index < max_saved_loh_states);
12825 #endif //RECORD_LOH_STATE
12827 BOOL gc_heap::allocate_large (int gen_number,
12829 alloc_context* acontext,
12832 #ifdef BACKGROUND_GC
12833 if (recursive_gc_sync::background_running_p() && (current_c_gc_state != c_gc_state_planning))
12835 background_loh_alloc_count++;
12836 //if ((background_loh_alloc_count % bgc_alloc_spin_count_loh) == 0)
12838 if (bgc_loh_should_allocate())
12840 if (!bgc_alloc_spin_loh)
12842 add_saved_spinlock_info (me_release, mt_alloc_large);
12843 dprintf (SPINLOCK_LOG, ("[%d]spin Lmsl loh", heap_number));
12844 leave_spin_lock (&more_space_lock);
12845 bool cooperative_mode = enable_preemptive ();
12846 GCToOSInterface::YieldThread (bgc_alloc_spin_loh);
12847 disable_preemptive (cooperative_mode);
12848 enter_spin_lock (&more_space_lock);
12849 add_saved_spinlock_info (me_acquire, mt_alloc_large);
12850 dprintf (SPINLOCK_LOG, ("[%d]spin Emsl loh", heap_number));
12855 wait_for_background (awr_loh_alloc_during_bgc);
12859 #endif //BACKGROUND_GC
12861 gc_reason gr = reason_oos_loh;
12862 generation* gen = generation_of (gen_number);
12863 oom_reason oom_r = oom_no_failure;
12864 size_t current_full_compact_gc_count = 0;
12866 // No variable values should be "carried over" from one state to the other.
12867 // That's why there are local variable for each state
12868 allocation_state loh_alloc_state = a_state_start;
12869 #ifdef RECORD_LOH_STATE
12870 EEThreadId current_thread_id;
12871 current_thread_id.SetToCurrentThread();
12872 #endif //RECORD_LOH_STATE
12874 // If we can get a new seg it means allocation will succeed.
12877 dprintf (3, ("[h%d]loh state is %s", heap_number, allocation_state_str[loh_alloc_state]));
12879 #ifdef RECORD_LOH_STATE
12880 add_saved_loh_state (loh_alloc_state, current_thread_id);
12881 #endif //RECORD_LOH_STATE
12882 switch (loh_alloc_state)
12884 case a_state_can_allocate:
12885 case a_state_cant_allocate:
12889 case a_state_start:
12891 loh_alloc_state = a_state_try_fit;
12894 case a_state_try_fit:
12896 BOOL commit_failed_p = FALSE;
12897 BOOL can_use_existing_p = FALSE;
12899 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12900 align_const, &commit_failed_p, &oom_r);
12901 loh_alloc_state = (can_use_existing_p ?
12902 a_state_can_allocate :
12904 a_state_trigger_full_compact_gc :
12905 a_state_acquire_seg));
12906 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12909 case a_state_try_fit_new_seg:
12911 BOOL commit_failed_p = FALSE;
12912 BOOL can_use_existing_p = FALSE;
12914 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12915 align_const, &commit_failed_p, &oom_r);
12916 // Even after we got a new seg it doesn't necessarily mean we can allocate,
12917 // another LOH allocating thread could have beat us to acquire the msl so
12918 // we need to try again.
12919 loh_alloc_state = (can_use_existing_p ? a_state_can_allocate : a_state_try_fit);
12920 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12923 case a_state_try_fit_new_seg_after_cg:
12925 BOOL commit_failed_p = FALSE;
12926 BOOL can_use_existing_p = FALSE;
12928 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12929 align_const, &commit_failed_p, &oom_r);
12930 // Even after we got a new seg it doesn't necessarily mean we can allocate,
12931 // another LOH allocating thread could have beat us to acquire the msl so
12932 // we need to try again. However, if we failed to commit, which means we
12933 // did have space on the seg, we bail right away 'cause we already did a
12934 // full compacting GC.
12935 loh_alloc_state = (can_use_existing_p ?
12936 a_state_can_allocate :
12938 a_state_cant_allocate :
12939 a_state_acquire_seg_after_cg));
12940 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12943 case a_state_try_fit_no_seg:
12945 BOOL commit_failed_p = FALSE;
12946 BOOL can_use_existing_p = FALSE;
12948 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12949 align_const, &commit_failed_p, &oom_r);
12950 loh_alloc_state = (can_use_existing_p ? a_state_can_allocate : a_state_cant_allocate);
12951 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12952 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
12955 case a_state_try_fit_after_cg:
12957 BOOL commit_failed_p = FALSE;
12958 BOOL can_use_existing_p = FALSE;
12960 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12961 align_const, &commit_failed_p, &oom_r);
12962 loh_alloc_state = (can_use_existing_p ?
12963 a_state_can_allocate :
12965 a_state_cant_allocate :
12966 a_state_acquire_seg_after_cg));
12967 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12970 case a_state_try_fit_after_bgc:
12972 BOOL commit_failed_p = FALSE;
12973 BOOL can_use_existing_p = FALSE;
12975 can_use_existing_p = loh_try_fit (gen_number, size, acontext,
12976 align_const, &commit_failed_p, &oom_r);
12977 loh_alloc_state = (can_use_existing_p ?
12978 a_state_can_allocate :
12980 a_state_trigger_full_compact_gc :
12981 a_state_acquire_seg_after_bgc));
12982 assert ((loh_alloc_state == a_state_can_allocate) == (acontext->alloc_ptr != 0));
12985 case a_state_acquire_seg:
12987 BOOL can_get_new_seg_p = FALSE;
12988 BOOL did_full_compacting_gc = FALSE;
12990 current_full_compact_gc_count = get_full_compact_gc_count();
12992 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
12993 loh_alloc_state = (can_get_new_seg_p ?
12994 a_state_try_fit_new_seg :
12995 (did_full_compacting_gc ?
12996 a_state_check_retry_seg :
12997 a_state_check_and_wait_for_bgc));
13000 case a_state_acquire_seg_after_cg:
13002 BOOL can_get_new_seg_p = FALSE;
13003 BOOL did_full_compacting_gc = FALSE;
13005 current_full_compact_gc_count = get_full_compact_gc_count();
13007 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
13008 // Since we release the msl before we try to allocate a seg, other
13009 // threads could have allocated a bunch of segments before us so
13010 // we might need to retry.
13011 loh_alloc_state = (can_get_new_seg_p ?
13012 a_state_try_fit_new_seg_after_cg :
13013 a_state_check_retry_seg);
13016 case a_state_acquire_seg_after_bgc:
13018 BOOL can_get_new_seg_p = FALSE;
13019 BOOL did_full_compacting_gc = FALSE;
13021 current_full_compact_gc_count = get_full_compact_gc_count();
13023 can_get_new_seg_p = loh_get_new_seg (gen, size, align_const, &did_full_compacting_gc, &oom_r);
13024 loh_alloc_state = (can_get_new_seg_p ?
13025 a_state_try_fit_new_seg :
13026 (did_full_compacting_gc ?
13027 a_state_check_retry_seg :
13028 a_state_trigger_full_compact_gc));
13029 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13032 case a_state_check_and_wait_for_bgc:
13034 BOOL bgc_in_progress_p = FALSE;
13035 BOOL did_full_compacting_gc = FALSE;
13037 if (fgn_maxgen_percent)
13039 dprintf (2, ("FGN: failed to acquire seg, may need to do a full blocking GC"));
13040 send_full_gc_notification (max_generation, FALSE);
13043 bgc_in_progress_p = check_and_wait_for_bgc (awr_loh_oos_bgc, &did_full_compacting_gc);
13044 loh_alloc_state = (!bgc_in_progress_p ?
13045 a_state_trigger_full_compact_gc :
13046 (did_full_compacting_gc ?
13047 a_state_try_fit_after_cg :
13048 a_state_try_fit_after_bgc));
13051 case a_state_trigger_full_compact_gc:
13053 BOOL got_full_compacting_gc = FALSE;
13055 got_full_compacting_gc = trigger_full_compact_gc (gr, &oom_r);
13056 loh_alloc_state = (got_full_compacting_gc ? a_state_try_fit_after_cg : a_state_cant_allocate);
13057 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13060 case a_state_check_retry_seg:
13062 BOOL should_retry_gc = retry_full_compact_gc (size);
13063 BOOL should_retry_get_seg = FALSE;
13064 if (!should_retry_gc)
13066 size_t last_full_compact_gc_count = current_full_compact_gc_count;
13067 current_full_compact_gc_count = get_full_compact_gc_count();
13069 if (current_full_compact_gc_count > (last_full_compact_gc_count + 1))
13071 should_retry_get_seg = TRUE;
13075 loh_alloc_state = (should_retry_gc ?
13076 a_state_trigger_full_compact_gc :
13077 (should_retry_get_seg ?
13078 a_state_acquire_seg_after_cg :
13079 a_state_cant_allocate));
13080 assert ((loh_alloc_state != a_state_cant_allocate) || (oom_r != oom_no_failure));
13085 assert (!"Invalid state!");
13092 if (loh_alloc_state == a_state_cant_allocate)
13094 assert (oom_r != oom_no_failure);
13095 handle_oom (heap_number,
13101 add_saved_spinlock_info (me_release, mt_alloc_large_cant);
13102 dprintf (SPINLOCK_LOG, ("[%d]Lmsl for loh oom", heap_number));
13103 leave_spin_lock (&more_space_lock);
13106 return (loh_alloc_state == a_state_can_allocate);
13109 int gc_heap::try_allocate_more_space (alloc_context* acontext, size_t size,
13112 if (gc_heap::gc_started)
13114 wait_for_gc_done();
13118 #ifdef SYNCHRONIZATION_STATS
13119 int64_t msl_acquire_start = GCToOSInterface::QueryPerformanceCounter();
13120 #endif //SYNCHRONIZATION_STATS
13121 enter_spin_lock (&more_space_lock);
13122 add_saved_spinlock_info (me_acquire, mt_try_alloc);
13123 dprintf (SPINLOCK_LOG, ("[%d]Emsl for alloc", heap_number));
13124 #ifdef SYNCHRONIZATION_STATS
13125 int64_t msl_acquire = GCToOSInterface::QueryPerformanceCounter() - msl_acquire_start;
13126 total_msl_acquire += msl_acquire;
13127 num_msl_acquired++;
13128 if (msl_acquire > 200)
13130 num_high_msl_acquire++;
13134 num_low_msl_acquire++;
13136 #endif //SYNCHRONIZATION_STATS
13139 // We are commenting this out 'cause we don't see the point - we already
13140 // have checked gc_started when we were acquiring the msl - no need to check
13141 // again. This complicates the logic in bgc_suspend_EE 'cause that one would
13142 // need to release msl which causes all sorts of trouble.
13143 if (gc_heap::gc_started)
13145 #ifdef SYNCHRONIZATION_STATS
13147 #endif //SYNCHRONIZATION_STATS
13148 BOOL fStress = (g_pConfig->GetGCStressLevel() & GCConfig::GCSTRESS_TRANSITION) != 0;
13151 //Rendez vous early (MP scaling issue)
13152 //dprintf (1, ("[%d]waiting for gc", heap_number));
13153 wait_for_gc_done();
13154 #ifdef MULTIPLE_HEAPS
13156 #endif //MULTIPLE_HEAPS
13161 dprintf (3, ("requested to allocate %d bytes on gen%d", size, gen_number));
13163 int align_const = get_alignment_constant (gen_number != (max_generation+1));
13165 if (fgn_maxgen_percent)
13167 check_for_full_gc (gen_number, size);
13170 if (!(new_allocation_allowed (gen_number)))
13172 if (fgn_maxgen_percent && (gen_number == 0))
13174 // We only check gen0 every so often, so take this opportunity to check again.
13175 check_for_full_gc (gen_number, size);
13178 #ifdef BACKGROUND_GC
13179 wait_for_bgc_high_memory (awr_gen0_alloc);
13180 #endif //BACKGROUND_GC
13182 #ifdef SYNCHRONIZATION_STATS
13184 #endif //SYNCHRONIZATION_STATS
13185 dprintf (/*100*/ 2, ("running out of budget on gen%d, gc", gen_number));
13187 if (!settings.concurrent || (gen_number == 0))
13189 vm_heap->GarbageCollectGeneration (0, ((gen_number == 0) ? reason_alloc_soh : reason_alloc_loh));
13190 #ifdef MULTIPLE_HEAPS
13191 enter_spin_lock (&more_space_lock);
13192 add_saved_spinlock_info (me_acquire, mt_try_budget);
13193 dprintf (SPINLOCK_LOG, ("[%d]Emsl out budget", heap_number));
13194 #endif //MULTIPLE_HEAPS
13198 BOOL can_allocate = ((gen_number == 0) ?
13199 allocate_small (gen_number, size, acontext, align_const) :
13200 allocate_large (gen_number, size, acontext, align_const));
13204 size_t alloc_context_bytes = acontext->alloc_limit + Align (min_obj_size, align_const) - acontext->alloc_ptr;
13205 int etw_allocation_index = ((gen_number == 0) ? 0 : 1);
13207 etw_allocation_running_amount[etw_allocation_index] += alloc_context_bytes;
13210 if (etw_allocation_running_amount[etw_allocation_index] > etw_allocation_tick)
13212 #ifdef FEATURE_REDHAWK
13213 FIRE_EVENT(GCAllocationTick_V1, (uint32_t)etw_allocation_running_amount[etw_allocation_index],
13214 (gen_number == 0) ? gc_etw_alloc_soh : gc_etw_alloc_loh);
13216 // Unfortunately some of the ETW macros do not check whether the ETW feature is enabled.
13217 // The ones that do are much less efficient.
13218 #if defined(FEATURE_EVENT_TRACE)
13219 if (EVENT_ENABLED(GCAllocationTick_V3))
13221 fire_etw_allocation_event (etw_allocation_running_amount[etw_allocation_index], gen_number, acontext->alloc_ptr);
13223 #endif //FEATURE_EVENT_TRACE
13224 #endif //FEATURE_REDHAWK
13225 etw_allocation_running_amount[etw_allocation_index] = 0;
13229 return (int)can_allocate;
13232 #ifdef MULTIPLE_HEAPS
13233 void gc_heap::balance_heaps (alloc_context* acontext)
13236 if (acontext->alloc_count < 4)
13238 if (acontext->alloc_count == 0)
13240 acontext->set_home_heap(GCHeap::GetHeap( heap_select::select_heap(acontext, 0) ));
13241 gc_heap* hp = acontext->get_home_heap()->pGenGCHeap;
13242 dprintf (3, ("First allocation for context %Ix on heap %d\n", (size_t)acontext, (size_t)hp->heap_number));
13243 acontext->set_alloc_heap(acontext->get_home_heap());
13244 hp->alloc_context_count++;
13249 BOOL set_home_heap = FALSE;
13252 if (heap_select::can_find_heap_fast())
13254 if (acontext->get_home_heap() != NULL)
13255 hint = acontext->get_home_heap()->pGenGCHeap->heap_number;
13256 if (acontext->get_home_heap() != GCHeap::GetHeap(hint = heap_select::select_heap(acontext, hint)) || ((acontext->alloc_count & 15) == 0))
13258 set_home_heap = TRUE;
13264 if ((acontext->alloc_count & 3) == 0)
13265 set_home_heap = TRUE;
13271 // Since we are balancing up to MAX_SUPPORTED_CPUS, no need for this.
13272 if (n_heaps > MAX_SUPPORTED_CPUS)
13274 // on machines with many processors cache affinity is really king, so don't even try
13275 // to balance on these.
13276 acontext->home_heap = GCHeap::GetHeap( heap_select::select_heap(acontext, hint) );
13277 acontext->alloc_heap = acontext->home_heap;
13282 gc_heap* org_hp = acontext->get_alloc_heap()->pGenGCHeap;
13284 dynamic_data* dd = org_hp->dynamic_data_of (0);
13285 ptrdiff_t org_size = dd_new_allocation (dd);
13286 int org_alloc_context_count;
13287 int max_alloc_context_count;
13289 ptrdiff_t max_size;
13290 size_t delta = dd_min_size (dd)/4;
13292 int start, end, finish;
13293 heap_select::get_heap_range_for_heap(org_hp->heap_number, &start, &end);
13294 finish = start + n_heaps;
13300 max_size = org_size + delta;
13301 acontext->set_home_heap(GCHeap::GetHeap( heap_select::select_heap(acontext, hint) ));
13303 if (org_hp == acontext->get_home_heap()->pGenGCHeap)
13304 max_size = max_size + delta;
13306 org_alloc_context_count = org_hp->alloc_context_count;
13307 max_alloc_context_count = org_alloc_context_count;
13308 if (max_alloc_context_count > 1)
13309 max_size /= max_alloc_context_count;
13311 for (int i = start; i < end; i++)
13313 gc_heap* hp = GCHeap::GetHeap(i%n_heaps)->pGenGCHeap;
13314 dd = hp->dynamic_data_of (0);
13315 ptrdiff_t size = dd_new_allocation (dd);
13316 if (hp == acontext->get_home_heap()->pGenGCHeap)
13317 size = size + delta;
13318 int hp_alloc_context_count = hp->alloc_context_count;
13319 if (hp_alloc_context_count > 0)
13320 size /= (hp_alloc_context_count + 1);
13321 if (size > max_size)
13325 max_alloc_context_count = hp_alloc_context_count;
13329 while (org_alloc_context_count != org_hp->alloc_context_count ||
13330 max_alloc_context_count != max_hp->alloc_context_count);
13332 if ((max_hp == org_hp) && (end < finish))
13334 start = end; end = finish;
13335 delta = dd_min_size(dd)/2; // Make it twice as hard to balance to remote nodes on NUMA.
13339 if (max_hp != org_hp)
13341 org_hp->alloc_context_count--;
13342 max_hp->alloc_context_count++;
13343 acontext->set_alloc_heap(GCHeap::GetHeap(max_hp->heap_number));
13344 if (GCToOSInterface::CanEnableGCCPUGroups())
13345 { //only set ideal processor when max_hp and org_hp are in the same cpu
13346 //group. DO NOT MOVE THREADS ACROSS CPU GROUPS
13347 uint16_t org_gn = heap_select::find_cpu_group_from_heap_no(org_hp->heap_number);
13348 uint16_t max_gn = heap_select::find_cpu_group_from_heap_no(max_hp->heap_number);
13349 if (org_gn == max_gn) //only set within CPU group, so SetThreadIdealProcessor is enough
13351 uint16_t group_proc_no = heap_select::find_group_proc_from_heap_no(max_hp->heap_number);
13353 GCThreadAffinity affinity;
13354 affinity.Processor = group_proc_no;
13355 affinity.Group = org_gn;
13356 if (!GCToOSInterface::SetCurrentThreadIdealAffinity(&affinity))
13358 dprintf (3, ("Failed to set the ideal processor and group for heap %d.",
13359 org_hp->heap_number));
13365 uint16_t proc_no = heap_select::find_proc_no_from_heap_no(max_hp->heap_number);
13367 GCThreadAffinity affinity;
13368 affinity.Processor = proc_no;
13369 affinity.Group = GCThreadAffinity::None;
13371 if (!GCToOSInterface::SetCurrentThreadIdealAffinity(&affinity))
13373 dprintf (3, ("Failed to set the ideal processor for heap %d.",
13374 org_hp->heap_number));
13377 dprintf (3, ("Switching context %p (home heap %d) ",
13379 acontext->get_home_heap()->pGenGCHeap->heap_number));
13380 dprintf (3, (" from heap %d (%Id free bytes, %d contexts) ",
13381 org_hp->heap_number,
13383 org_alloc_context_count));
13384 dprintf (3, (" to heap %d (%Id free bytes, %d contexts)\n",
13385 max_hp->heap_number,
13386 dd_new_allocation(max_hp->dynamic_data_of(0)),
13387 max_alloc_context_count));
13392 acontext->alloc_count++;
13395 gc_heap* gc_heap::balance_heaps_loh (alloc_context* acontext, size_t /*size*/)
13397 gc_heap* org_hp = acontext->get_alloc_heap()->pGenGCHeap;
13398 //dprintf (1, ("LA: %Id", size));
13400 //if (size > 128*1024)
13403 dynamic_data* dd = org_hp->dynamic_data_of (max_generation + 1);
13405 ptrdiff_t org_size = dd_new_allocation (dd);
13407 ptrdiff_t max_size;
13408 size_t delta = dd_min_size (dd) * 4;
13410 int start, end, finish;
13411 heap_select::get_heap_range_for_heap(org_hp->heap_number, &start, &end);
13412 finish = start + n_heaps;
13417 max_size = org_size + delta;
13418 dprintf (3, ("orig hp: %d, max size: %d",
13419 org_hp->heap_number,
13422 for (int i = start; i < end; i++)
13424 gc_heap* hp = GCHeap::GetHeap(i%n_heaps)->pGenGCHeap;
13425 dd = hp->dynamic_data_of (max_generation + 1);
13426 ptrdiff_t size = dd_new_allocation (dd);
13427 dprintf (3, ("hp: %d, size: %d",
13430 if (size > max_size)
13434 dprintf (3, ("max hp: %d, max size: %d",
13435 max_hp->heap_number,
13441 if ((max_hp == org_hp) && (end < finish))
13443 start = end; end = finish;
13444 delta = dd_min_size(dd) * 4; // Need to tuning delta
13448 if (max_hp != org_hp)
13450 dprintf (3, ("loh: %d(%Id)->%d(%Id)",
13451 org_hp->heap_number, dd_new_allocation (org_hp->dynamic_data_of (max_generation + 1)),
13452 max_hp->heap_number, dd_new_allocation (max_hp->dynamic_data_of (max_generation + 1))));
13462 #endif //MULTIPLE_HEAPS
13464 BOOL gc_heap::allocate_more_space(alloc_context* acontext, size_t size,
13465 int alloc_generation_number)
13470 #ifdef MULTIPLE_HEAPS
13471 if (alloc_generation_number == 0)
13473 balance_heaps (acontext);
13474 status = acontext->get_alloc_heap()->pGenGCHeap->try_allocate_more_space (acontext, size, alloc_generation_number);
13478 gc_heap* alloc_heap = balance_heaps_loh (acontext, size);
13479 status = alloc_heap->try_allocate_more_space (acontext, size, alloc_generation_number);
13482 status = try_allocate_more_space (acontext, size, alloc_generation_number);
13483 #endif //MULTIPLE_HEAPS
13485 while (status == -1);
13487 return (status != 0);
13491 CObjectHeader* gc_heap::allocate (size_t jsize, alloc_context* acontext)
13493 size_t size = Align (jsize);
13494 assert (size >= Align (min_obj_size));
13497 uint8_t* result = acontext->alloc_ptr;
13498 acontext->alloc_ptr+=size;
13499 if (acontext->alloc_ptr <= acontext->alloc_limit)
13501 CObjectHeader* obj = (CObjectHeader*)result;
13507 acontext->alloc_ptr -= size;
13510 #pragma inline_depth(0)
13513 if (! allocate_more_space (acontext, size, 0))
13517 #pragma inline_depth(20)
13526 CObjectHeader* gc_heap::try_fast_alloc (size_t jsize)
13528 size_t size = Align (jsize);
13529 assert (size >= Align (min_obj_size));
13530 generation* gen = generation_of (0);
13531 uint8_t* result = generation_allocation_pointer (gen);
13532 generation_allocation_pointer (gen) += size;
13533 if (generation_allocation_pointer (gen) <=
13534 generation_allocation_limit (gen))
13536 return (CObjectHeader*)result;
13540 generation_allocation_pointer (gen) -= size;
13544 void gc_heap::leave_allocation_segment (generation* gen)
13546 adjust_limit (0, 0, gen, max_generation);
13549 void gc_heap::init_free_and_plug()
13551 #ifdef FREE_USAGE_STATS
13552 for (int i = 0; i <= settings.condemned_generation; i++)
13554 generation* gen = generation_of (i);
13555 memset (gen->gen_free_spaces, 0, sizeof (gen->gen_free_spaces));
13556 memset (gen->gen_plugs, 0, sizeof (gen->gen_plugs));
13557 memset (gen->gen_current_pinned_free_spaces, 0, sizeof (gen->gen_current_pinned_free_spaces));
13560 if (settings.condemned_generation != max_generation)
13562 for (int i = (settings.condemned_generation + 1); i <= max_generation; i++)
13564 generation* gen = generation_of (i);
13565 memset (gen->gen_plugs, 0, sizeof (gen->gen_plugs));
13568 #endif //FREE_USAGE_STATS
13571 void gc_heap::print_free_and_plug (const char* msg)
13573 #if defined(FREE_USAGE_STATS) && defined(SIMPLE_DPRINTF)
13574 int older_gen = ((settings.condemned_generation == max_generation) ? max_generation : (settings.condemned_generation + 1));
13575 for (int i = 0; i <= older_gen; i++)
13577 generation* gen = generation_of (i);
13578 for (int j = 0; j < NUM_GEN_POWER2; j++)
13580 if ((gen->gen_free_spaces[j] != 0) || (gen->gen_plugs[j] != 0))
13582 dprintf (2, ("[%s][h%d][%s#%d]gen%d: 2^%d: F: %Id, P: %Id",
13585 (settings.concurrent ? "BGC" : "GC"),
13588 (j + 9), gen->gen_free_spaces[j], gen->gen_plugs[j]));
13593 UNREFERENCED_PARAMETER(msg);
13594 #endif //FREE_USAGE_STATS && SIMPLE_DPRINTF
13597 void gc_heap::add_gen_plug (int gen_number, size_t plug_size)
13599 #ifdef FREE_USAGE_STATS
13600 dprintf (3, ("adding plug size %Id to gen%d", plug_size, gen_number));
13601 generation* gen = generation_of (gen_number);
13602 size_t sz = BASE_GEN_SIZE;
13605 for (; i < NUM_GEN_POWER2; i++)
13607 if (plug_size < sz)
13614 (gen->gen_plugs[i])++;
13616 UNREFERENCED_PARAMETER(gen_number);
13617 UNREFERENCED_PARAMETER(plug_size);
13618 #endif //FREE_USAGE_STATS
13621 void gc_heap::add_item_to_current_pinned_free (int gen_number, size_t free_size)
13623 #ifdef FREE_USAGE_STATS
13624 generation* gen = generation_of (gen_number);
13625 size_t sz = BASE_GEN_SIZE;
13628 for (; i < NUM_GEN_POWER2; i++)
13630 if (free_size < sz)
13637 (gen->gen_current_pinned_free_spaces[i])++;
13638 generation_pinned_free_obj_space (gen) += free_size;
13639 dprintf (3, ("left pin free %Id(2^%d) to gen%d, total %Id bytes (%Id)",
13640 free_size, (i + 10), gen_number,
13641 generation_pinned_free_obj_space (gen),
13642 gen->gen_current_pinned_free_spaces[i]));
13644 UNREFERENCED_PARAMETER(gen_number);
13645 UNREFERENCED_PARAMETER(free_size);
13646 #endif //FREE_USAGE_STATS
13649 void gc_heap::add_gen_free (int gen_number, size_t free_size)
13651 #ifdef FREE_USAGE_STATS
13652 dprintf (3, ("adding free size %Id to gen%d", free_size, gen_number));
13653 generation* gen = generation_of (gen_number);
13654 size_t sz = BASE_GEN_SIZE;
13657 for (; i < NUM_GEN_POWER2; i++)
13659 if (free_size < sz)
13666 (gen->gen_free_spaces[i])++;
13668 UNREFERENCED_PARAMETER(gen_number);
13669 UNREFERENCED_PARAMETER(free_size);
13670 #endif //FREE_USAGE_STATS
13673 void gc_heap::remove_gen_free (int gen_number, size_t free_size)
13675 #ifdef FREE_USAGE_STATS
13676 dprintf (3, ("removing free %Id from gen%d", free_size, gen_number));
13677 generation* gen = generation_of (gen_number);
13678 size_t sz = BASE_GEN_SIZE;
13681 for (; i < NUM_GEN_POWER2; i++)
13683 if (free_size < sz)
13690 (gen->gen_free_spaces[i])--;
13692 UNREFERENCED_PARAMETER(gen_number);
13693 UNREFERENCED_PARAMETER(free_size);
13694 #endif //FREE_USAGE_STATS
13697 uint8_t* gc_heap::allocate_in_older_generation (generation* gen, size_t size,
13698 int from_gen_number,
13699 uint8_t* old_loc REQD_ALIGN_AND_OFFSET_DCL)
13701 size = Align (size);
13702 assert (size >= Align (min_obj_size));
13703 assert (from_gen_number < max_generation);
13704 assert (from_gen_number >= 0);
13705 assert (generation_of (from_gen_number + 1) == gen);
13707 allocator* gen_allocator = generation_allocator (gen);
13708 BOOL discard_p = gen_allocator->discard_if_no_fit_p ();
13709 int pad_in_front = (old_loc != 0)? USE_PADDING_FRONT : 0;
13711 size_t real_size = size + Align (min_obj_size);
13713 real_size += Align (min_obj_size);
13715 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
13716 generation_allocation_limit (gen), old_loc, USE_PADDING_TAIL | pad_in_front)))
13718 size_t sz_list = gen_allocator->first_bucket_size();
13719 for (unsigned int a_l_idx = 0; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
13721 if ((real_size < (sz_list / 2)) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
13723 uint8_t* free_list = gen_allocator->alloc_list_head_of (a_l_idx);
13724 uint8_t* prev_free_item = 0;
13725 while (free_list != 0)
13727 dprintf (3, ("considering free list %Ix", (size_t)free_list));
13729 size_t free_list_size = unused_array_size (free_list);
13731 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, free_list, (free_list + free_list_size),
13732 old_loc, USE_PADDING_TAIL | pad_in_front))
13734 dprintf (4, ("F:%Ix-%Id",
13735 (size_t)free_list, free_list_size));
13737 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, !discard_p);
13738 generation_free_list_space (gen) -= free_list_size;
13739 remove_gen_free (gen->gen_num, free_list_size);
13741 adjust_limit (free_list, free_list_size, gen, from_gen_number+1);
13744 // We do first fit on bucket 0 because we are not guaranteed to find a fit there.
13745 else if (discard_p || (a_l_idx == 0))
13747 dprintf (3, ("couldn't use this free area, discarding"));
13748 generation_free_obj_space (gen) += free_list_size;
13750 gen_allocator->unlink_item (a_l_idx, free_list, prev_free_item, FALSE);
13751 generation_free_list_space (gen) -= free_list_size;
13752 remove_gen_free (gen->gen_num, free_list_size);
13756 prev_free_item = free_list;
13758 free_list = free_list_slot (free_list);
13761 sz_list = sz_list * 2;
13763 //go back to the beginning of the segment list
13764 generation_allocate_end_seg_p (gen) = TRUE;
13765 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
13766 if (seg != generation_allocation_segment (gen))
13768 leave_allocation_segment (gen);
13769 generation_allocation_segment (gen) = seg;
13771 while (seg != ephemeral_heap_segment)
13773 if (size_fit_p(size REQD_ALIGN_AND_OFFSET_ARG, heap_segment_plan_allocated (seg),
13774 heap_segment_committed (seg), old_loc, USE_PADDING_TAIL | pad_in_front))
13776 dprintf (3, ("using what's left in committed"));
13777 adjust_limit (heap_segment_plan_allocated (seg),
13778 heap_segment_committed (seg) -
13779 heap_segment_plan_allocated (seg),
13780 gen, from_gen_number+1);
13781 // dformat (t, 3, "Expanding segment allocation");
13782 heap_segment_plan_allocated (seg) =
13783 heap_segment_committed (seg);
13788 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, heap_segment_plan_allocated (seg),
13789 heap_segment_reserved (seg), old_loc, USE_PADDING_TAIL | pad_in_front) &&
13790 grow_heap_segment (seg, heap_segment_plan_allocated (seg), old_loc, size, pad_in_front REQD_ALIGN_AND_OFFSET_ARG))
13792 dprintf (3, ("using what's left in reserved"));
13793 adjust_limit (heap_segment_plan_allocated (seg),
13794 heap_segment_committed (seg) -
13795 heap_segment_plan_allocated (seg),
13796 gen, from_gen_number+1);
13797 heap_segment_plan_allocated (seg) =
13798 heap_segment_committed (seg);
13804 leave_allocation_segment (gen);
13805 heap_segment* next_seg = heap_segment_next_rw (seg);
13808 dprintf (3, ("getting next segment"));
13809 generation_allocation_segment (gen) = next_seg;
13810 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
13811 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
13820 seg = generation_allocation_segment (gen);
13822 //No need to fix the last region. Will be done later
13833 uint8_t* result = generation_allocation_pointer (gen);
13837 if ((pad_in_front & USE_PADDING_FRONT) &&
13838 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
13839 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
13841 pad = Align (min_obj_size);
13842 set_plug_padded (old_loc);
13844 #endif //SHORT_PLUGS
13846 #ifdef FEATURE_STRUCTALIGN
13847 _ASSERTE(!old_loc || alignmentOffset != 0);
13848 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
13851 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
13852 set_node_aligninfo (old_loc, requiredAlignment, pad1);
13855 #else // FEATURE_STRUCTALIGN
13856 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
13858 pad += switch_alignment_size (is_plug_padded (old_loc));
13859 set_node_realigned (old_loc);
13860 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
13861 (size_t)old_loc, (size_t)(result+pad)));
13862 assert (same_large_alignment_p (result + pad, old_loc));
13864 #endif // FEATURE_STRUCTALIGN
13865 dprintf (3, ("Allocate %Id bytes", size));
13867 if ((old_loc == 0) || (pad != 0))
13869 //allocating a non plug or a gap, so reset the start region
13870 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
13873 generation_allocation_pointer (gen) += size + pad;
13874 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
13875 if (generation_allocate_end_seg_p (gen))
13877 generation_end_seg_allocated (gen) += size;
13881 generation_free_list_allocated (gen) += size;
13883 generation_allocation_size (gen) += size;
13885 dprintf (3, ("aio: ptr: %Ix, limit: %Ix, sr: %Ix",
13886 generation_allocation_pointer (gen), generation_allocation_limit (gen),
13887 generation_allocation_context_start_region (gen)));
13889 return result + pad;;
13893 void gc_heap::repair_allocation_in_expanded_heap (generation* consing_gen)
13895 //make sure that every generation has a planned allocation start
13896 int gen_number = max_generation - 1;
13897 while (gen_number>= 0)
13899 generation* gen = generation_of (gen_number);
13900 if (0 == generation_plan_allocation_start (gen))
13902 realloc_plan_generation_start (gen, consing_gen);
13904 assert (generation_plan_allocation_start (gen));
13909 // now we know the planned allocation size
13910 size_t size = (generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
13911 heap_segment* seg = generation_allocation_segment (consing_gen);
13912 if (generation_allocation_limit (consing_gen) == heap_segment_plan_allocated (seg))
13916 heap_segment_plan_allocated (seg) = generation_allocation_pointer (consing_gen);
13921 assert (settings.condemned_generation == max_generation);
13922 uint8_t* first_address = generation_allocation_limit (consing_gen);
13923 //look through the pinned plugs for relevant ones.
13924 //Look for the right pinned plug to start from.
13927 while (mi != mark_stack_tos)
13929 m = pinned_plug_of (mi);
13930 if ((pinned_plug (m) == first_address))
13935 assert (mi != mark_stack_tos);
13936 pinned_len (m) = size;
13940 //tododefrag optimize for new segment (plan_allocated == mem)
13941 uint8_t* gc_heap::allocate_in_expanded_heap (generation* gen,
13946 BOOL set_padding_on_saved_p,
13947 mark* pinned_plug_entry,
13948 #endif //SHORT_PLUGS
13949 BOOL consider_bestfit,
13950 int active_new_gen_number
13951 REQD_ALIGN_AND_OFFSET_DCL)
13953 UNREFERENCED_PARAMETER(active_new_gen_number);
13954 dprintf (3, ("aie: P: %Ix, size: %Ix", old_loc, size));
13956 size = Align (size);
13957 assert (size >= Align (min_obj_size));
13958 int pad_in_front = (old_loc != 0) ? USE_PADDING_FRONT : 0;
13960 if (consider_bestfit && use_bestfit)
13962 assert (bestfit_seg);
13963 dprintf (SEG_REUSE_LOG_1, ("reallocating 0x%Ix in expanded heap, size: %Id",
13965 return bestfit_seg->fit (old_loc,
13967 set_padding_on_saved_p,
13969 #endif //SHORT_PLUGS
13970 size REQD_ALIGN_AND_OFFSET_ARG);
13973 heap_segment* seg = generation_allocation_segment (gen);
13975 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
13976 generation_allocation_limit (gen), old_loc,
13977 ((generation_allocation_limit (gen) !=
13978 heap_segment_plan_allocated (seg))? USE_PADDING_TAIL : 0) | pad_in_front)))
13980 dprintf (3, ("aie: can't fit: ptr: %Ix, limit: %Ix", generation_allocation_pointer (gen),
13981 generation_allocation_limit (gen)));
13984 uint8_t* first_address = (generation_allocation_limit (gen) ?
13985 generation_allocation_limit (gen) :
13986 heap_segment_mem (seg));
13987 assert (in_range_for_segment (first_address, seg));
13989 uint8_t* end_address = heap_segment_reserved (seg);
13991 dprintf (3, ("aie: first_addr: %Ix, gen alloc limit: %Ix, end_address: %Ix",
13992 first_address, generation_allocation_limit (gen), end_address));
13997 if (heap_segment_allocated (seg) != heap_segment_mem (seg))
13999 assert (settings.condemned_generation == max_generation);
14000 //look through the pinned plugs for relevant ones.
14001 //Look for the right pinned plug to start from.
14002 while (mi != mark_stack_tos)
14004 m = pinned_plug_of (mi);
14005 if ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address))
14007 dprintf (3, ("aie: found pin: %Ix", pinned_plug (m)));
14013 if (mi != mark_stack_tos)
14015 //fix old free list.
14016 size_t hsize = (generation_allocation_limit (gen) - generation_allocation_pointer (gen));
14018 dprintf(3,("gc filling up hole"));
14019 ptrdiff_t mi1 = (ptrdiff_t)mi;
14020 while ((mi1 >= 0) &&
14021 (pinned_plug (pinned_plug_of(mi1)) != generation_allocation_limit (gen)))
14023 dprintf (3, ("aie: checking pin %Ix", pinned_plug (pinned_plug_of(mi1))));
14028 size_t saved_pinned_len = pinned_len (pinned_plug_of(mi1));
14029 pinned_len (pinned_plug_of(mi1)) = hsize;
14030 dprintf (3, ("changing %Ix len %Ix->%Ix",
14031 pinned_plug (pinned_plug_of(mi1)),
14032 saved_pinned_len, pinned_len (pinned_plug_of(mi1))));
14039 assert (generation_allocation_limit (gen) ==
14040 generation_allocation_pointer (gen));
14041 mi = mark_stack_tos;
14044 while ((mi != mark_stack_tos) && in_range_for_segment (pinned_plug (m), seg))
14046 size_t len = pinned_len (m);
14047 uint8_t* free_list = (pinned_plug (m) - len);
14048 dprintf (3, ("aie: testing free item: %Ix->%Ix(%Ix)",
14049 free_list, (free_list + len), len));
14050 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, free_list, (free_list + len), old_loc, USE_PADDING_TAIL | pad_in_front))
14052 dprintf (3, ("aie: Found adequate unused area: %Ix, size: %Id",
14053 (size_t)free_list, len));
14055 generation_allocation_pointer (gen) = free_list;
14056 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14057 generation_allocation_limit (gen) = (free_list + len);
14059 goto allocate_in_free;
14062 m = pinned_plug_of (mi);
14065 //switch to the end of the segment.
14066 generation_allocation_pointer (gen) = heap_segment_plan_allocated (seg);
14067 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14068 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14069 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14070 dprintf (3, ("aie: switching to end of seg: %Ix->%Ix(%Ix)",
14071 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14072 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
14074 if (!size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14075 generation_allocation_limit (gen), old_loc, USE_PADDING_TAIL | pad_in_front))
14077 dprintf (3, ("aie: ptr: %Ix, limit: %Ix, can't alloc", generation_allocation_pointer (gen),
14078 generation_allocation_limit (gen)));
14079 assert (!"Can't allocate if no free space");
14090 uint8_t* result = generation_allocation_pointer (gen);
14094 if ((pad_in_front & USE_PADDING_FRONT) &&
14095 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
14096 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
14099 pad = Align (min_obj_size);
14100 set_padding_in_expand (old_loc, set_padding_on_saved_p, pinned_plug_entry);
14102 #endif //SHORT_PLUGS
14104 #ifdef FEATURE_STRUCTALIGN
14105 _ASSERTE(!old_loc || alignmentOffset != 0);
14106 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
14109 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
14110 set_node_aligninfo (old_loc, requiredAlignment, pad1);
14114 #else // FEATURE_STRUCTALIGN
14115 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
14117 pad += switch_alignment_size (is_plug_padded (old_loc));
14118 set_node_realigned (old_loc);
14119 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
14120 (size_t)old_loc, (size_t)(result+pad)));
14121 assert (same_large_alignment_p (result + pad, old_loc));
14124 #endif // FEATURE_STRUCTALIGN
14126 if ((old_loc == 0) || (pad != 0))
14128 //allocating a non plug or a gap, so reset the start region
14129 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14132 generation_allocation_pointer (gen) += size + pad;
14133 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
14134 dprintf (3, ("Allocated in expanded heap %Ix:%Id", (size_t)(result+pad), size));
14136 dprintf (3, ("aie: ptr: %Ix, limit: %Ix, sr: %Ix",
14137 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14138 generation_allocation_context_start_region (gen)));
14140 return result + pad;
14144 generation* gc_heap::ensure_ephemeral_heap_segment (generation* consing_gen)
14146 heap_segment* seg = generation_allocation_segment (consing_gen);
14147 if (seg != ephemeral_heap_segment)
14149 assert (generation_allocation_pointer (consing_gen)>= heap_segment_mem (seg));
14150 assert (generation_allocation_pointer (consing_gen)<= heap_segment_committed (seg));
14152 //fix the allocated size of the segment.
14153 heap_segment_plan_allocated (seg) = generation_allocation_pointer (consing_gen);
14155 generation* new_consing_gen = generation_of (max_generation - 1);
14156 generation_allocation_pointer (new_consing_gen) =
14157 heap_segment_mem (ephemeral_heap_segment);
14158 generation_allocation_limit (new_consing_gen) =
14159 generation_allocation_pointer (new_consing_gen);
14160 generation_allocation_context_start_region (new_consing_gen) =
14161 generation_allocation_pointer (new_consing_gen);
14162 generation_allocation_segment (new_consing_gen) = ephemeral_heap_segment;
14164 return new_consing_gen;
14167 return consing_gen;
14170 uint8_t* gc_heap::allocate_in_condemned_generations (generation* gen,
14172 int from_gen_number,
14174 BOOL* convert_to_pinned_p,
14175 uint8_t* next_pinned_plug,
14176 heap_segment* current_seg,
14177 #endif //SHORT_PLUGS
14179 REQD_ALIGN_AND_OFFSET_DCL)
14181 // Make sure that the youngest generation gap hasn't been allocated
14182 if (settings.promotion)
14184 assert (generation_plan_allocation_start (youngest_generation) == 0);
14187 size = Align (size);
14188 assert (size >= Align (min_obj_size));
14189 int to_gen_number = from_gen_number;
14190 if (from_gen_number != (int)max_generation)
14192 to_gen_number = from_gen_number + (settings.promotion ? 1 : 0);
14195 dprintf (3, ("aic gen%d: s: %Id, %d->%d, %Ix->%Ix", gen->gen_num, size, from_gen_number,
14196 to_gen_number, generation_allocation_pointer(gen), generation_allocation_limit(gen)));
14198 int pad_in_front = (old_loc != 0) ? USE_PADDING_FRONT : 0;
14200 if ((from_gen_number != -1) && (from_gen_number != (int)max_generation) && settings.promotion)
14202 generation_condemned_allocated (generation_of (from_gen_number + (settings.promotion ? 1 : 0))) += size;
14203 generation_allocation_size (generation_of (from_gen_number + (settings.promotion ? 1 : 0))) += size;
14207 heap_segment* seg = generation_allocation_segment (gen);
14208 if (! (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14209 generation_allocation_limit (gen), old_loc,
14210 ((generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))?USE_PADDING_TAIL:0)|pad_in_front)))
14212 if ((! (pinned_plug_que_empty_p()) &&
14213 (generation_allocation_limit (gen) ==
14214 pinned_plug (oldest_pin()))))
14216 size_t entry = deque_pinned_plug();
14217 mark* pinned_plug_entry = pinned_plug_of (entry);
14218 size_t len = pinned_len (pinned_plug_entry);
14219 uint8_t* plug = pinned_plug (pinned_plug_entry);
14220 set_new_pin_info (pinned_plug_entry, generation_allocation_pointer (gen));
14222 #ifdef FREE_USAGE_STATS
14223 generation_allocated_in_pinned_free (gen) += generation_allocated_since_last_pin (gen);
14224 dprintf (3, ("allocated %Id so far within pin %Ix, total->%Id",
14225 generation_allocated_since_last_pin (gen),
14227 generation_allocated_in_pinned_free (gen)));
14228 generation_allocated_since_last_pin (gen) = 0;
14230 add_item_to_current_pinned_free (gen->gen_num, pinned_len (pinned_plug_of (entry)));
14231 #endif //FREE_USAGE_STATS
14233 dprintf (3, ("mark stack bos: %Id, tos: %Id, aic: p %Ix len: %Ix->%Ix",
14234 mark_stack_bos, mark_stack_tos, plug, len, pinned_len (pinned_plug_of (entry))));
14236 assert(mark_stack_array[entry].len == 0 ||
14237 mark_stack_array[entry].len >= Align(min_obj_size));
14238 generation_allocation_pointer (gen) = plug + len;
14239 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14240 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14241 set_allocator_next_pin (gen);
14243 //Add the size of the pinned plug to the right pinned allocations
14244 //find out which gen this pinned plug came from
14245 int frgn = object_gennum (plug);
14246 if ((frgn != (int)max_generation) && settings.promotion)
14248 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
14249 int togn = object_gennum_plan (plug);
14252 generation_pinned_allocation_compact_size (generation_of (togn)) += len;
14258 if (generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))
14260 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14261 dprintf (3, ("changed limit to plan alloc: %Ix", generation_allocation_limit (gen)));
14265 if (heap_segment_plan_allocated (seg) != heap_segment_committed (seg))
14267 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14268 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14269 dprintf (3, ("changed limit to commit: %Ix", generation_allocation_limit (gen)));
14273 #ifndef RESPECT_LARGE_ALIGNMENT
14274 assert (gen != youngest_generation);
14275 #endif //RESPECT_LARGE_ALIGNMENT
14277 if (size_fit_p (size REQD_ALIGN_AND_OFFSET_ARG, generation_allocation_pointer (gen),
14278 heap_segment_reserved (seg), old_loc, USE_PADDING_TAIL | pad_in_front) &&
14279 (grow_heap_segment (seg, generation_allocation_pointer (gen), old_loc,
14280 size, pad_in_front REQD_ALIGN_AND_OFFSET_ARG)))
14282 dprintf (3, ("Expanded segment allocation by committing more memory"));
14283 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
14284 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
14288 heap_segment* next_seg = heap_segment_next (seg);
14289 assert (generation_allocation_pointer (gen)>=
14290 heap_segment_mem (seg));
14291 // Verify that all pinned plugs for this segment are consumed
14292 if (!pinned_plug_que_empty_p() &&
14293 ((pinned_plug (oldest_pin()) <
14294 heap_segment_allocated (seg)) &&
14295 (pinned_plug (oldest_pin()) >=
14296 generation_allocation_pointer (gen))))
14298 LOG((LF_GC, LL_INFO10, "remaining pinned plug %Ix while leaving segment on allocation",
14299 pinned_plug (oldest_pin())));
14302 assert (generation_allocation_pointer (gen)>=
14303 heap_segment_mem (seg));
14304 assert (generation_allocation_pointer (gen)<=
14305 heap_segment_committed (seg));
14306 heap_segment_plan_allocated (seg) = generation_allocation_pointer (gen);
14310 generation_allocation_segment (gen) = next_seg;
14311 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
14312 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
14313 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14317 return 0; //should only happen during allocation of generation 0 gap
14318 // in that case we are going to grow the heap anyway
14323 set_allocator_next_pin (gen);
14330 assert (generation_allocation_pointer (gen)>=
14331 heap_segment_mem (generation_allocation_segment (gen)));
14332 uint8_t* result = generation_allocation_pointer (gen);
14335 if ((pad_in_front & USE_PADDING_FRONT) &&
14336 (((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))==0) ||
14337 ((generation_allocation_pointer (gen) - generation_allocation_context_start_region (gen))>=DESIRED_PLUG_LENGTH)))
14339 ptrdiff_t dist = old_loc - result;
14342 dprintf (3, ("old alloc: %Ix, same as new alloc, not padding", old_loc));
14347 if ((dist > 0) && (dist < (ptrdiff_t)Align (min_obj_size)))
14349 dprintf (3, ("old alloc: %Ix, only %d bytes > new alloc! Shouldn't happen", old_loc, dist));
14353 pad = Align (min_obj_size);
14354 set_plug_padded (old_loc);
14357 #endif //SHORT_PLUGS
14358 #ifdef FEATURE_STRUCTALIGN
14359 _ASSERTE(!old_loc || alignmentOffset != 0);
14360 _ASSERTE(old_loc || requiredAlignment == DATA_ALIGNMENT);
14361 if ((old_loc != 0))
14363 size_t pad1 = ComputeStructAlignPad(result+pad, requiredAlignment, alignmentOffset);
14364 set_node_aligninfo (old_loc, requiredAlignment, pad1);
14367 #else // FEATURE_STRUCTALIGN
14368 if (!((old_loc == 0) || same_large_alignment_p (old_loc, result+pad)))
14370 pad += switch_alignment_size (is_plug_padded (old_loc));
14371 set_node_realigned(old_loc);
14372 dprintf (3, ("Allocation realignment old_loc: %Ix, new_loc:%Ix",
14373 (size_t)old_loc, (size_t)(result+pad)));
14374 assert (same_large_alignment_p (result + pad, old_loc));
14376 #endif // FEATURE_STRUCTALIGN
14379 if ((next_pinned_plug != 0) && (pad != 0) && (generation_allocation_segment (gen) == current_seg))
14381 assert (old_loc != 0);
14382 ptrdiff_t dist_to_next_pin = (ptrdiff_t)(next_pinned_plug - (generation_allocation_pointer (gen) + size + pad));
14383 assert (dist_to_next_pin >= 0);
14385 if ((dist_to_next_pin >= 0) && (dist_to_next_pin < (ptrdiff_t)Align (min_obj_size)))
14387 dprintf (3, ("%Ix->(%Ix,%Ix),%Ix(%Ix)(%Ix),NP->PP",
14389 generation_allocation_pointer (gen),
14390 generation_allocation_limit (gen),
14393 dist_to_next_pin));
14394 clear_plug_padded (old_loc);
14396 *convert_to_pinned_p = TRUE;
14397 record_interesting_data_point (idp_converted_pin);
14402 #endif //SHORT_PLUGS
14404 if ((old_loc == 0) || (pad != 0))
14406 //allocating a non plug or a gap, so reset the start region
14407 generation_allocation_context_start_region (gen) = generation_allocation_pointer (gen);
14410 generation_allocation_pointer (gen) += size + pad;
14411 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
14413 #ifdef FREE_USAGE_STATS
14414 generation_allocated_since_last_pin (gen) += size;
14415 #endif //FREE_USAGE_STATS
14417 dprintf (3, ("aic: ptr: %Ix, limit: %Ix, sr: %Ix",
14418 generation_allocation_pointer (gen), generation_allocation_limit (gen),
14419 generation_allocation_context_start_region (gen)));
14421 assert (result + pad);
14422 return result + pad;
14426 inline int power (int x, int y)
14429 for (int i = 0; i < y; i++)
14436 int gc_heap::joined_generation_to_condemn (BOOL should_evaluate_elevation,
14438 BOOL* blocking_collection_p
14439 STRESS_HEAP_ARG(int n_original))
14442 #ifdef MULTIPLE_HEAPS
14443 BOOL blocking_p = *blocking_collection_p;
14446 for (int i = 0; i < n_heaps; i++)
14448 if (g_heaps[i]->last_gc_before_oom)
14450 dprintf (GTC_LOG, ("h%d is setting blocking to TRUE", i));
14451 *blocking_collection_p = TRUE;
14456 #endif //MULTIPLE_HEAPS
14458 if (should_evaluate_elevation && (n == max_generation))
14460 dprintf (GTC_LOG, ("lock: %d(%d)",
14461 (settings.should_lock_elevation ? 1 : 0),
14462 settings.elevation_locked_count));
14464 if (settings.should_lock_elevation)
14466 settings.elevation_locked_count++;
14467 if (settings.elevation_locked_count == 6)
14469 settings.elevation_locked_count = 0;
14473 n = max_generation - 1;
14474 settings.elevation_reduced = TRUE;
14479 settings.elevation_locked_count = 0;
14484 settings.should_lock_elevation = FALSE;
14485 settings.elevation_locked_count = 0;
14489 #ifdef BACKGROUND_GC
14490 // We can only do Concurrent GC Stress if the caller did not explicitly ask for all
14491 // generations to be collected,
14493 // [LOCALGC TODO] STRESS_HEAP is not defined for a standalone GC so there are multiple
14494 // things that need to be fixed in this code block.
14495 if (n_original != max_generation &&
14496 g_pConfig->GetGCStressLevel() && gc_can_use_concurrent)
14498 #ifndef FEATURE_REDHAWK
14499 // for the GC stress mix mode throttle down gen2 collections
14500 if (g_pConfig->IsGCStressMix())
14502 size_t current_gc_count = 0;
14504 #ifdef MULTIPLE_HEAPS
14505 current_gc_count = (size_t)dd_collection_count (g_heaps[0]->dynamic_data_of (0));
14507 current_gc_count = (size_t)dd_collection_count (dynamic_data_of (0));
14508 #endif //MULTIPLE_HEAPS
14509 // in gc stress, only escalate every 10th non-gen2 collection to a gen2...
14510 if ((current_gc_count % 10) == 0)
14512 n = max_generation;
14515 // for traditional GC stress
14517 #endif // !FEATURE_REDHAWK
14518 if (*blocking_collection_p)
14520 // We call StressHeap() a lot for Concurrent GC Stress. However,
14521 // if we can not do a concurrent collection, no need to stress anymore.
14522 // @TODO: Enable stress when the memory pressure goes down again
14523 GCStressPolicy::GlobalDisable();
14527 n = max_generation;
14530 #endif //BACKGROUND_GC
14531 #endif //STRESS_HEAP
14537 size_t get_survived_size (gc_history_per_heap* hist)
14539 size_t surv_size = 0;
14540 gc_generation_data* gen_data;
14542 for (int gen_number = 0; gen_number <= (max_generation + 1); gen_number++)
14544 gen_data = &(hist->gen_data[gen_number]);
14545 surv_size += (gen_data->size_after -
14546 gen_data->free_list_space_after -
14547 gen_data->free_obj_space_after);
14553 size_t gc_heap::get_total_survived_size()
14555 size_t total_surv_size = 0;
14556 #ifdef MULTIPLE_HEAPS
14557 for (int i = 0; i < gc_heap::n_heaps; i++)
14559 gc_heap* hp = gc_heap::g_heaps[i];
14560 gc_history_per_heap* current_gc_data_per_heap = hp->get_gc_data_per_heap();
14561 total_surv_size += get_survived_size (current_gc_data_per_heap);
14564 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
14565 total_surv_size = get_survived_size (current_gc_data_per_heap);
14566 #endif //MULTIPLE_HEAPS
14567 return total_surv_size;
14570 // Gets what's allocated on both SOH and LOH that hasn't been collected.
14571 size_t gc_heap::get_current_allocated()
14573 dynamic_data* dd = dynamic_data_of (0);
14574 size_t current_alloc = dd_desired_allocation (dd) - dd_new_allocation (dd);
14575 dd = dynamic_data_of (max_generation + 1);
14576 current_alloc += dd_desired_allocation (dd) - dd_new_allocation (dd);
14578 return current_alloc;
14581 size_t gc_heap::get_total_allocated()
14583 size_t total_current_allocated = 0;
14584 #ifdef MULTIPLE_HEAPS
14585 for (int i = 0; i < gc_heap::n_heaps; i++)
14587 gc_heap* hp = gc_heap::g_heaps[i];
14588 total_current_allocated += hp->get_current_allocated();
14591 total_current_allocated = get_current_allocated();
14592 #endif //MULTIPLE_HEAPS
14593 return total_current_allocated;
14596 size_t gc_heap::current_generation_size (int gen_number)
14598 dynamic_data* dd = dynamic_data_of (gen_number);
14599 size_t gen_size = (dd_current_size (dd) + dd_desired_allocation (dd)
14600 - dd_new_allocation (dd));
14606 #pragma warning(push)
14607 #pragma warning(disable:6326) // "Potential comparison of a constant with another constant" is intentional in this function.
14611 This is called by when we are actually doing a GC, or when we are just checking whether
14612 we would do a full blocking GC, in which case check_only_p is TRUE.
14614 The difference between calling this with check_only_p TRUE and FALSE is that when it's
14616 settings.reason is ignored
14617 budgets are not checked (since they are checked before this is called)
14618 it doesn't change anything non local like generation_skip_ratio
14620 int gc_heap::generation_to_condemn (int n_initial,
14621 BOOL* blocking_collection_p,
14622 BOOL* elevation_requested_p,
14625 gc_mechanisms temp_settings = settings;
14626 gen_to_condemn_tuning temp_condemn_reasons;
14627 gc_mechanisms* local_settings = (check_only_p ? &temp_settings : &settings);
14628 gen_to_condemn_tuning* local_condemn_reasons = (check_only_p ? &temp_condemn_reasons : &gen_to_condemn_reasons);
14631 if ((local_settings->reason == reason_oos_soh) || (local_settings->reason == reason_oos_loh))
14633 assert (n_initial >= 1);
14636 assert (settings.reason != reason_empty);
14639 local_condemn_reasons->init();
14643 if (heap_number == 0)
14645 dprintf (GTC_LOG, ("init: %d(%d)", n_initial, settings.reason));
14649 BOOL low_memory_detected = g_low_memory_status;
14650 uint32_t memory_load = 0;
14651 uint64_t available_physical = 0;
14652 uint64_t available_page_file = 0;
14653 BOOL check_memory = FALSE;
14654 BOOL high_fragmentation = FALSE;
14655 BOOL v_high_memory_load = FALSE;
14656 BOOL high_memory_load = FALSE;
14657 BOOL low_ephemeral_space = FALSE;
14658 BOOL evaluate_elevation = TRUE;
14659 *elevation_requested_p = FALSE;
14660 *blocking_collection_p = FALSE;
14662 BOOL check_max_gen_alloc = TRUE;
14666 #endif //STRESS_HEAP
14670 dd_fragmentation (dynamic_data_of (0)) =
14671 generation_free_list_space (youngest_generation) +
14672 generation_free_obj_space (youngest_generation);
14674 dd_fragmentation (dynamic_data_of (max_generation + 1)) =
14675 generation_free_list_space (large_object_generation) +
14676 generation_free_obj_space (large_object_generation);
14678 //save new_allocation
14679 for (i = 0; i <= max_generation+1; i++)
14681 dynamic_data* dd = dynamic_data_of (i);
14682 dprintf (GTC_LOG, ("h%d: g%d: l: %Id (%Id)",
14684 dd_new_allocation (dd),
14685 dd_desired_allocation (dd)));
14686 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
14689 local_condemn_reasons->set_gen (gen_initial, n);
14692 #ifdef BACKGROUND_GC
14693 if (recursive_gc_sync::background_running_p())
14695 dprintf (GTC_LOG, ("bgc in prog, 1"));
14696 check_max_gen_alloc = FALSE;
14698 #endif //BACKGROUND_GC
14700 if (check_max_gen_alloc)
14702 //figure out if large objects need to be collected.
14703 if (get_new_allocation (max_generation+1) <= 0)
14705 n = max_generation;
14706 local_condemn_reasons->set_gen (gen_alloc_budget, n);
14710 //figure out which generation ran out of allocation
14711 for (i = n+1; i <= (check_max_gen_alloc ? max_generation : (max_generation - 1)); i++)
14713 if (get_new_allocation (i) <= 0)
14724 local_condemn_reasons->set_gen (gen_alloc_budget, n);
14727 dprintf (GTC_LOG, ("h%d: g%d budget", heap_number, ((get_new_allocation (max_generation+1) <= 0) ? 3 : n)));
14731 #if defined(BACKGROUND_GC) && !defined(MULTIPLE_HEAPS)
14732 //time based tuning
14733 // if enough time has elapsed since the last gc
14734 // and the number of gc is too low (1/10 of lower gen) then collect
14735 // This should also be enabled if we have memory concerns
14736 int n_time_max = max_generation;
14740 if (recursive_gc_sync::background_running_p())
14742 n_time_max = max_generation - 1;
14746 if ((local_settings->pause_mode == pause_interactive) ||
14747 (local_settings->pause_mode == pause_sustained_low_latency))
14749 dynamic_data* dd0 = dynamic_data_of (0);
14750 size_t now = GetHighPrecisionTimeStamp();
14752 for (i = (temp_gen+1); i <= n_time_max; i++)
14754 dynamic_data* dd = dynamic_data_of (i);
14755 if ((now > dd_time_clock(dd) + dd_time_clock_interval(dd)) &&
14756 (dd_gc_clock (dd0) > (dd_gc_clock (dd) + dd_gc_clock_interval(dd))) &&
14757 ((n < max_generation) || ((dd_current_size (dd) < dd_max_size (dd0)))))
14759 n = min (i, n_time_max);
14760 dprintf (GTC_LOG, ("time %d", n));
14765 local_condemn_reasons->set_gen (gen_time_tuning, n);
14771 dprintf (GTC_LOG, ("Condemning %d based on time tuning and fragmentation", n));
14773 #endif //BACKGROUND_GC && !MULTIPLE_HEAPS
14775 if (n < (max_generation - 1))
14777 if (dt_low_card_table_efficiency_p (tuning_deciding_condemned_gen))
14779 n = max (n, max_generation - 1);
14780 local_settings->promotion = TRUE;
14781 dprintf (GTC_LOG, ("h%d: skip %d, c %d",
14782 heap_number, generation_skip_ratio, n));
14783 local_condemn_reasons->set_condition (gen_low_card_p);
14789 generation_skip_ratio = 100;
14792 if (dt_low_ephemeral_space_p (check_only_p ?
14793 tuning_deciding_full_gc :
14794 tuning_deciding_condemned_gen))
14796 low_ephemeral_space = TRUE;
14798 n = max (n, max_generation - 1);
14799 local_condemn_reasons->set_condition (gen_low_ephemeral_p);
14800 dprintf (GTC_LOG, ("h%d: low eph", heap_number));
14802 #ifdef BACKGROUND_GC
14803 if (!gc_can_use_concurrent || (generation_free_list_space (generation_of (max_generation)) == 0))
14804 #endif //BACKGROUND_GC
14806 //It is better to defragment first if we are running out of space for
14807 //the ephemeral generation but we have enough fragmentation to make up for it
14808 //in the non ephemeral generation. Essentially we are trading a gen2 for
14809 // having to expand heap in ephemeral collections.
14810 if (dt_high_frag_p (tuning_deciding_condemned_gen,
14811 max_generation - 1,
14814 high_fragmentation = TRUE;
14815 local_condemn_reasons->set_condition (gen_max_high_frag_e_p);
14816 dprintf (GTC_LOG, ("heap%d: gen1 frag", heap_number));
14821 //figure out which ephemeral generation is too fragramented
14823 for (i = n+1; i < max_generation; i++)
14825 if (dt_high_frag_p (tuning_deciding_condemned_gen, i))
14827 dprintf (GTC_LOG, ("h%d g%d too frag", heap_number, i));
14834 if (low_ephemeral_space)
14837 local_settings->promotion = TRUE;
14842 local_condemn_reasons->set_condition (gen_eph_high_frag_p);
14847 if (settings.pause_mode == pause_low_latency)
14849 if (!is_induced (settings.reason))
14851 n = min (n, max_generation - 1);
14852 dprintf (GTC_LOG, ("low latency mode is enabled, condemning %d", n));
14853 evaluate_elevation = FALSE;
14859 // It's hard to catch when we get to the point that the memory load is so high
14860 // we get an induced GC from the finalizer thread so we are checking the memory load
14861 // for every gen0 GC.
14862 check_memory = (check_only_p ?
14864 ((n >= 1) || low_memory_detected));
14868 //find out if we are short on memory
14869 get_memory_info (&memory_load, &available_physical, &available_page_file);
14870 if (heap_number == 0)
14872 dprintf (GTC_LOG, ("ml: %d", memory_load));
14875 // Need to get it early enough for all heaps to use.
14876 entry_available_physical_mem = available_physical;
14877 local_settings->entry_memory_load = memory_load;
14879 // @TODO: Force compaction more often under GCSTRESS
14880 if (memory_load >= high_memory_load_th || low_memory_detected)
14882 #ifdef SIMPLE_DPRINTF
14883 // stress log can't handle any parameter that's bigger than a void*.
14884 if (heap_number == 0)
14886 dprintf (GTC_LOG, ("tp: %I64d, ap: %I64d", total_physical_mem, available_physical));
14888 #endif //SIMPLE_DPRINTF
14890 high_memory_load = TRUE;
14892 if (memory_load >= v_high_memory_load_th || low_memory_detected)
14894 // TODO: Perhaps in 64-bit we should be estimating gen1's fragmentation as well since
14895 // gen1/gen0 may take a lot more memory than gen2.
14896 if (!high_fragmentation)
14898 high_fragmentation = dt_estimate_reclaim_space_p (tuning_deciding_condemned_gen, max_generation);
14900 v_high_memory_load = TRUE;
14904 if (!high_fragmentation)
14906 high_fragmentation = dt_estimate_high_frag_p (tuning_deciding_condemned_gen, max_generation, available_physical);
14910 if (high_fragmentation)
14912 if (high_memory_load)
14914 local_condemn_reasons->set_condition (gen_max_high_frag_m_p);
14916 else if (v_high_memory_load)
14918 local_condemn_reasons->set_condition (gen_max_high_frag_vm_p);
14924 dprintf (GTC_LOG, ("h%d: le: %d, hm: %d, vm: %d, f: %d",
14925 heap_number, low_ephemeral_space, high_memory_load, v_high_memory_load,
14926 high_fragmentation));
14928 if (should_expand_in_full_gc)
14930 dprintf (GTC_LOG, ("h%d: expand_in_full - BLOCK", heap_number));
14931 *blocking_collection_p = TRUE;
14934 should_expand_in_full_gc = FALSE;
14936 evaluate_elevation = FALSE;
14937 n = max_generation;
14938 local_condemn_reasons->set_condition (gen_expand_fullgc_p);
14941 if (last_gc_before_oom)
14943 dprintf (GTC_LOG, ("h%d: alloc full - BLOCK", heap_number));
14944 n = max_generation;
14945 *blocking_collection_p = TRUE;
14946 if ((local_settings->reason == reason_oos_loh) ||
14947 (local_settings->reason == reason_alloc_loh))
14948 evaluate_elevation = FALSE;
14950 local_condemn_reasons->set_condition (gen_before_oom);
14955 if (is_induced_blocking (settings.reason) &&
14956 n_initial == max_generation
14957 IN_STRESS_HEAP( && !settings.stress_induced ))
14959 if (heap_number == 0)
14961 dprintf (GTC_LOG, ("induced - BLOCK"));
14964 *blocking_collection_p = TRUE;
14965 local_condemn_reasons->set_condition (gen_induced_fullgc_p);
14966 evaluate_elevation = FALSE;
14969 if (settings.reason == reason_induced_noforce)
14971 local_condemn_reasons->set_condition (gen_induced_noforce_p);
14972 evaluate_elevation = FALSE;
14976 if (evaluate_elevation && (low_ephemeral_space || high_memory_load || v_high_memory_load))
14978 *elevation_requested_p = TRUE;
14980 // if we are in high memory load and have consumed 10% of the gen2 budget, do a gen2 now.
14981 if (high_memory_load || v_high_memory_load)
14983 dynamic_data* dd_max = dynamic_data_of (max_generation);
14984 if (((float)dd_new_allocation (dd_max) / (float)dd_desired_allocation (dd_max)) < 0.9)
14986 dprintf (GTC_LOG, ("%Id left in gen2 alloc (%Id)",
14987 dd_new_allocation (dd_max), dd_desired_allocation (dd_max)));
14988 n = max_generation;
14989 local_condemn_reasons->set_condition (gen_almost_max_alloc);
14993 if (n <= max_generation)
14996 if (high_fragmentation)
14998 //elevate to max_generation
14999 n = max_generation;
15000 dprintf (GTC_LOG, ("h%d: f full", heap_number));
15002 #ifdef BACKGROUND_GC
15003 if (high_memory_load || v_high_memory_load)
15005 // For background GC we want to do blocking collections more eagerly because we don't
15006 // want to get into the situation where the memory load becomes high while we are in
15007 // a background GC and we'd have to wait for the background GC to finish to start
15008 // a blocking collection (right now the implemenation doesn't handle converting
15009 // a background GC to a blocking collection midway.
15010 dprintf (GTC_LOG, ("h%d: bgc - BLOCK", heap_number));
15011 *blocking_collection_p = TRUE;
15014 if (v_high_memory_load)
15016 dprintf (GTC_LOG, ("h%d: - BLOCK", heap_number));
15017 *blocking_collection_p = TRUE;
15019 #endif //BACKGROUND_GC
15023 n = max (n, max_generation - 1);
15024 dprintf (GTC_LOG, ("h%d: nf c %d", heap_number, n));
15031 if ((n == (max_generation - 1)) && (n_alloc < (max_generation -1)))
15033 dprintf (GTC_LOG, ("h%d: budget %d, check 2",
15034 heap_number, n_alloc));
15035 if (get_new_allocation (max_generation) <= 0)
15037 dprintf (GTC_LOG, ("h%d: budget alloc", heap_number));
15038 n = max_generation;
15039 local_condemn_reasons->set_condition (gen_max_gen1);
15043 //figure out if max_generation is too fragmented -> blocking collection
15044 if (n == max_generation)
15046 if (dt_high_frag_p (tuning_deciding_condemned_gen, n))
15048 dprintf (GTC_LOG, ("h%d: g%d too frag", heap_number, n));
15049 local_condemn_reasons->set_condition (gen_max_high_frag_p);
15050 if (local_settings->pause_mode != pause_sustained_low_latency)
15052 *blocking_collection_p = TRUE;
15057 #ifdef BACKGROUND_GC
15058 if (n == max_generation)
15060 if (heap_number == 0)
15062 BOOL bgc_heap_too_small = TRUE;
15063 size_t gen2size = 0;
15064 size_t gen3size = 0;
15065 #ifdef MULTIPLE_HEAPS
15066 for (int i = 0; i < n_heaps; i++)
15068 if (((g_heaps[i]->current_generation_size (max_generation)) > bgc_min_per_heap) ||
15069 ((g_heaps[i]->current_generation_size (max_generation + 1)) > bgc_min_per_heap))
15071 bgc_heap_too_small = FALSE;
15075 #else //MULTIPLE_HEAPS
15076 if ((current_generation_size (max_generation) > bgc_min_per_heap) ||
15077 (current_generation_size (max_generation + 1) > bgc_min_per_heap))
15079 bgc_heap_too_small = FALSE;
15081 #endif //MULTIPLE_HEAPS
15083 if (bgc_heap_too_small)
15085 dprintf (GTC_LOG, ("gen2 and gen3 too small"));
15088 // do not turn stress-induced collections into blocking GCs
15089 if (!settings.stress_induced)
15090 #endif //STRESS_HEAP
15092 *blocking_collection_p = TRUE;
15095 local_condemn_reasons->set_condition (gen_gen2_too_small);
15099 #endif //BACKGROUND_GC
15105 #ifdef BACKGROUND_GC
15106 // We can only do Concurrent GC Stress if the caller did not explicitly ask for all
15107 // generations to be collected,
15109 if (orig_gen != max_generation &&
15110 g_pConfig->GetGCStressLevel() && gc_can_use_concurrent)
15112 *elevation_requested_p = FALSE;
15114 #endif //BACKGROUND_GC
15115 #endif //STRESS_HEAP
15119 fgm_result.available_pagefile_mb = (size_t)(available_page_file / (1024 * 1024));
15122 local_condemn_reasons->set_gen (gen_final_per_heap, n);
15123 get_gc_data_per_heap()->gen_to_condemn_reasons.init (local_condemn_reasons);
15126 local_condemn_reasons->print (heap_number);
15129 if ((local_settings->reason == reason_oos_soh) ||
15130 (local_settings->reason == reason_oos_loh))
15136 if (n == max_generation && GCToEEInterface::ForceFullGCToBeBlocking())
15138 #ifdef BACKGROUND_GC
15139 // do not turn stress-induced collections into blocking GCs, unless there
15140 // have already been more full BGCs than full NGCs
15142 // This exposes DevDiv 94129, so we'll leave this out for now
15143 if (!settings.stress_induced ||
15144 full_gc_counts[gc_type_blocking] <= full_gc_counts[gc_type_background])
15146 #endif // BACKGROUND_GC
15148 *blocking_collection_p = TRUE;
15156 #pragma warning(pop)
15160 size_t gc_heap::min_reclaim_fragmentation_threshold (uint32_t num_heaps)
15162 // if the memory load is higher, the threshold we'd want to collect gets lower.
15163 size_t min_mem_based_on_available =
15164 (500 - (settings.entry_memory_load - high_memory_load_th) * 40) * 1024 * 1024 / num_heaps;
15165 size_t ten_percent_size = (size_t)((float)generation_size (max_generation) * 0.10);
15166 uint64_t three_percent_mem = mem_one_percent * 3 / num_heaps;
15168 #ifdef SIMPLE_DPRINTF
15169 dprintf (GTC_LOG, ("min av: %Id, 10%% gen2: %Id, 3%% mem: %I64d",
15170 min_mem_based_on_available, ten_percent_size, three_percent_mem));
15171 #endif //SIMPLE_DPRINTF
15172 return (size_t)(min (min_mem_based_on_available, min (ten_percent_size, three_percent_mem)));
15176 uint64_t gc_heap::min_high_fragmentation_threshold(uint64_t available_mem, uint32_t num_heaps)
15178 return min (available_mem, (256*1024*1024)) / num_heaps;
15182 CORINFO_EXCEPTION_GC = 0xE0004743 // 'GC'
15186 #ifdef BACKGROUND_GC
15187 void gc_heap::init_background_gc ()
15189 //reset the allocation so foreground gc can allocate into older (max_generation) generation
15190 generation* gen = generation_of (max_generation);
15191 generation_allocation_pointer (gen)= 0;
15192 generation_allocation_limit (gen) = 0;
15193 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
15195 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
15197 //reset the plan allocation for each segment
15198 for (heap_segment* seg = generation_allocation_segment (gen); seg != ephemeral_heap_segment;
15199 seg = heap_segment_next_rw (seg))
15201 heap_segment_plan_allocated (seg) = heap_segment_allocated (seg);
15204 if (heap_number == 0)
15206 dprintf (2, ("heap%d: bgc lowest: %Ix, highest: %Ix",
15208 background_saved_lowest_address,
15209 background_saved_highest_address));
15212 gc_lh_block_event.Reset();
15215 #endif //BACKGROUND_GC
15218 void fire_drain_mark_list_event (size_t mark_list_objects)
15220 FIRE_EVENT(BGCDrainMark, mark_list_objects);
15224 void fire_revisit_event (size_t dirtied_pages,
15225 size_t marked_objects,
15226 BOOL large_objects_p)
15228 FIRE_EVENT(BGCRevisit, dirtied_pages, marked_objects, large_objects_p);
15232 void fire_overflow_event (uint8_t* overflow_min,
15233 uint8_t* overflow_max,
15234 size_t marked_objects,
15235 int large_objects_p)
15237 FIRE_EVENT(BGCOverflow, (uint64_t)overflow_min, (uint64_t)overflow_max, marked_objects, large_objects_p);
15240 void gc_heap::concurrent_print_time_delta (const char* msg)
15243 size_t current_time = GetHighPrecisionTimeStamp();
15244 size_t elapsed_time = current_time - time_bgc_last;
15245 time_bgc_last = current_time;
15247 dprintf (2, ("h%d: %s T %Id ms", heap_number, msg, elapsed_time));
15249 UNREFERENCED_PARAMETER(msg);
15253 void gc_heap::free_list_info (int gen_num, const char* msg)
15255 UNREFERENCED_PARAMETER(gen_num);
15256 #if defined (BACKGROUND_GC) && defined (TRACE_GC)
15257 dprintf (3, ("h%d: %s", heap_number, msg));
15258 for (int i = 0; i <= (max_generation + 1); i++)
15260 generation* gen = generation_of (i);
15261 if ((generation_allocation_size (gen) == 0) &&
15262 (generation_free_list_space (gen) == 0) &&
15263 (generation_free_obj_space (gen) == 0))
15265 // don't print if everything is 0.
15269 dprintf (3, ("h%d: g%d: a-%Id, fl-%Id, fo-%Id",
15271 generation_allocation_size (gen),
15272 generation_free_list_space (gen),
15273 generation_free_obj_space (gen)));
15277 UNREFERENCED_PARAMETER(msg);
15278 #endif // BACKGROUND_GC && TRACE_GC
15281 void gc_heap::update_collection_counts_for_no_gc()
15283 assert (settings.pause_mode == pause_no_gc);
15285 settings.condemned_generation = max_generation;
15286 #ifdef MULTIPLE_HEAPS
15287 for (int i = 0; i < n_heaps; i++)
15288 g_heaps[i]->update_collection_counts();
15289 #else //MULTIPLE_HEAPS
15290 update_collection_counts();
15291 #endif //MULTIPLE_HEAPS
15293 full_gc_counts[gc_type_blocking]++;
15296 BOOL gc_heap::should_proceed_with_gc()
15298 if (gc_heap::settings.pause_mode == pause_no_gc)
15300 if (current_no_gc_region_info.started)
15302 // The no_gc mode was already in progress yet we triggered another GC,
15303 // this effectively exits the no_gc mode.
15304 restore_data_for_no_gc();
15307 return should_proceed_for_no_gc();
15313 //internal part of gc used by the serial and concurrent version
15314 void gc_heap::gc1()
15316 #ifdef BACKGROUND_GC
15317 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15318 #endif //BACKGROUND_GC
15321 mark_time = plan_time = reloc_time = compact_time = sweep_time = 0;
15324 verify_soh_segment_list();
15326 int n = settings.condemned_generation;
15328 update_collection_counts ();
15330 #ifdef BACKGROUND_GC
15331 bgc_alloc_lock->check();
15332 #endif //BACKGROUND_GC
15334 free_list_info (max_generation, "beginning");
15336 vm_heap->GcCondemnedGeneration = settings.condemned_generation;
15338 assert (g_gc_card_table == card_table);
15340 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
15341 assert (g_gc_card_bundle_table == card_bundle_table);
15345 if (n == max_generation)
15347 gc_low = lowest_address;
15348 gc_high = highest_address;
15352 gc_low = generation_allocation_start (generation_of (n));
15353 gc_high = heap_segment_reserved (ephemeral_heap_segment);
15355 #ifdef BACKGROUND_GC
15356 if (settings.concurrent)
15359 time_bgc_last = GetHighPrecisionTimeStamp();
15362 FIRE_EVENT(BGCBegin);
15364 concurrent_print_time_delta ("BGC");
15366 //#ifdef WRITE_WATCH
15367 //reset_write_watch (FALSE);
15368 //#endif //WRITE_WATCH
15370 concurrent_print_time_delta ("RW");
15371 background_mark_phase();
15372 free_list_info (max_generation, "after mark phase");
15374 background_sweep();
15375 free_list_info (max_generation, "after sweep phase");
15378 #endif //BACKGROUND_GC
15380 mark_phase (n, FALSE);
15382 GCScan::GcRuntimeStructuresValid (FALSE);
15384 GCScan::GcRuntimeStructuresValid (TRUE);
15388 size_t end_gc_time = GetHighPrecisionTimeStamp();
15389 // printf ("generation: %d, elapsed time: %Id\n", n, end_gc_time - dd_time_clock (dynamic_data_of (0)));
15391 //adjust the allocation size from the pinned quantities.
15392 for (int gen_number = 0; gen_number <= min (max_generation,n+1); gen_number++)
15394 generation* gn = generation_of (gen_number);
15395 if (settings.compaction)
15397 generation_pinned_allocated (gn) += generation_pinned_allocation_compact_size (gn);
15398 generation_allocation_size (generation_of (gen_number)) += generation_pinned_allocation_compact_size (gn);
15402 generation_pinned_allocated (gn) += generation_pinned_allocation_sweep_size (gn);
15403 generation_allocation_size (generation_of (gen_number)) += generation_pinned_allocation_sweep_size (gn);
15405 generation_pinned_allocation_sweep_size (gn) = 0;
15406 generation_pinned_allocation_compact_size (gn) = 0;
15409 #ifdef BACKGROUND_GC
15410 if (settings.concurrent)
15412 dynamic_data* dd = dynamic_data_of (n);
15413 dd_gc_elapsed_time (dd) = end_gc_time - dd_time_clock (dd);
15415 free_list_info (max_generation, "after computing new dynamic data");
15417 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
15419 for (int gen_number = 0; gen_number < max_generation; gen_number++)
15421 dprintf (2, ("end of BGC: gen%d new_alloc: %Id",
15422 gen_number, dd_desired_allocation (dynamic_data_of (gen_number))));
15423 current_gc_data_per_heap->gen_data[gen_number].size_after = generation_size (gen_number);
15424 current_gc_data_per_heap->gen_data[gen_number].free_list_space_after = generation_free_list_space (generation_of (gen_number));
15425 current_gc_data_per_heap->gen_data[gen_number].free_obj_space_after = generation_free_obj_space (generation_of (gen_number));
15429 #endif //BACKGROUND_GC
15431 free_list_info (max_generation, "end");
15432 for (int gen_number = 0; gen_number <= n; gen_number++)
15434 dynamic_data* dd = dynamic_data_of (gen_number);
15435 dd_gc_elapsed_time (dd) = end_gc_time - dd_time_clock (dd);
15436 compute_new_dynamic_data (gen_number);
15439 if (n != max_generation)
15441 int gen_num_for_data = ((n < (max_generation - 1)) ? (n + 1) : (max_generation + 1));
15442 for (int gen_number = (n + 1); gen_number <= gen_num_for_data; gen_number++)
15444 get_gc_data_per_heap()->gen_data[gen_number].size_after = generation_size (gen_number);
15445 get_gc_data_per_heap()->gen_data[gen_number].free_list_space_after = generation_free_list_space (generation_of (gen_number));
15446 get_gc_data_per_heap()->gen_data[gen_number].free_obj_space_after = generation_free_obj_space (generation_of (gen_number));
15450 get_gc_data_per_heap()->maxgen_size_info.running_free_list_efficiency = (uint32_t)(generation_allocator_efficiency (generation_of (max_generation)) * 100);
15452 free_list_info (max_generation, "after computing new dynamic data");
15454 if (heap_number == 0)
15456 dprintf (GTC_LOG, ("GC#%d(gen%d) took %Idms",
15457 dd_collection_count (dynamic_data_of (0)),
15458 settings.condemned_generation,
15459 dd_gc_elapsed_time (dynamic_data_of (0))));
15462 for (int gen_number = 0; gen_number <= (max_generation + 1); gen_number++)
15464 dprintf (2, ("end of FGC/NGC: gen%d new_alloc: %Id",
15465 gen_number, dd_desired_allocation (dynamic_data_of (gen_number))));
15469 if (n < max_generation)
15471 compute_promoted_allocation (1 + n);
15473 dynamic_data* dd = dynamic_data_of (1 + n);
15474 size_t new_fragmentation = generation_free_list_space (generation_of (1 + n)) +
15475 generation_free_obj_space (generation_of (1 + n));
15477 #ifdef BACKGROUND_GC
15478 if (current_c_gc_state != c_gc_state_planning)
15479 #endif //BACKGROUND_GC
15481 if (settings.promotion)
15483 dd_fragmentation (dd) = new_fragmentation;
15487 //assert (dd_fragmentation (dd) == new_fragmentation);
15492 #ifdef BACKGROUND_GC
15493 if (!settings.concurrent)
15494 #endif //BACKGROUND_GC
15496 #ifndef FEATURE_REDHAWK
15497 // GCToEEInterface::IsGCThread() always returns false on CoreRT, but this assert is useful in CoreCLR.
15498 assert(GCToEEInterface::IsGCThread());
15499 #endif // FEATURE_REDHAWK
15500 adjust_ephemeral_limits();
15503 #ifdef BACKGROUND_GC
15504 assert (ephemeral_low == generation_allocation_start (generation_of ( max_generation -1)));
15505 assert (ephemeral_high == heap_segment_reserved (ephemeral_heap_segment));
15506 #endif //BACKGROUND_GC
15508 if (fgn_maxgen_percent)
15510 if (settings.condemned_generation == (max_generation - 1))
15512 check_for_full_gc (max_generation - 1, 0);
15514 else if (settings.condemned_generation == max_generation)
15516 if (full_gc_approach_event_set
15517 #ifdef MULTIPLE_HEAPS
15518 && (heap_number == 0)
15519 #endif //MULTIPLE_HEAPS
15522 dprintf (2, ("FGN-GC: setting gen2 end event"));
15524 full_gc_approach_event.Reset();
15525 #ifdef BACKGROUND_GC
15526 // By definition WaitForFullGCComplete only succeeds if it's full, *blocking* GC, otherwise need to return N/A
15527 fgn_last_gc_was_concurrent = settings.concurrent ? TRUE : FALSE;
15528 #endif //BACKGROUND_GC
15529 full_gc_end_event.Set();
15530 full_gc_approach_event_set = false;
15535 #ifdef BACKGROUND_GC
15536 if (!settings.concurrent)
15537 #endif //BACKGROUND_GC
15539 //decide on the next allocation quantum
15540 if (alloc_contexts_used >= 1)
15542 allocation_quantum = Align (min ((size_t)CLR_SIZE,
15543 (size_t)max (1024, get_new_allocation (0) / (2 * alloc_contexts_used))),
15544 get_alignment_constant(FALSE));
15545 dprintf (3, ("New allocation quantum: %d(0x%Ix)", allocation_quantum, allocation_quantum));
15549 descr_generations (FALSE);
15551 verify_soh_segment_list();
15553 #ifdef BACKGROUND_GC
15554 add_to_history_per_heap();
15555 if (heap_number == 0)
15559 #endif // BACKGROUND_GC
15562 if (GCStatistics::Enabled() && heap_number == 0)
15563 g_GCStatistics.AddGCStats(settings,
15564 dd_gc_elapsed_time(dynamic_data_of(settings.condemned_generation)));
15568 fprintf (stdout, "%d,%d,%d,%d,%d,%d\n",
15569 n, mark_time, plan_time, reloc_time, compact_time, sweep_time);
15572 #ifdef BACKGROUND_GC
15573 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15574 #endif //BACKGROUND_GC
15576 #if defined(VERIFY_HEAP) || (defined (FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC))
15579 // Note that right now g_pConfig->GetHeapVerifyLevel always returns the same
15580 // value. If we ever allow randomly adjusting this as the process runs,
15581 // we cannot call it this way as joins need to match - we must have the same
15582 // value for all heaps like we do with bgc_heap_walk_for_etw_p.
15583 || (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
15585 #if defined(FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC)
15586 || (bgc_heap_walk_for_etw_p && settings.concurrent)
15590 #ifdef BACKGROUND_GC
15591 bool cooperative_mode = true;
15593 if (settings.concurrent)
15595 cooperative_mode = enable_preemptive ();
15597 #ifdef MULTIPLE_HEAPS
15598 bgc_t_join.join(this, gc_join_suspend_ee_verify);
15599 if (bgc_t_join.joined())
15601 bgc_threads_sync_event.Reset();
15603 dprintf(2, ("Joining BGC threads to suspend EE for verify heap"));
15604 bgc_t_join.restart();
15606 if (heap_number == 0)
15609 bgc_threads_sync_event.Set();
15613 bgc_threads_sync_event.Wait(INFINITE, FALSE);
15614 dprintf (2, ("bgc_threads_sync_event is signalled"));
15618 #endif //MULTIPLE_HEAPS
15620 //fix the allocation area so verify_heap can proceed.
15621 fix_allocation_contexts (FALSE);
15623 #endif //BACKGROUND_GC
15625 #ifdef BACKGROUND_GC
15626 assert (settings.concurrent == (uint32_t)(bgc_thread_id.IsCurrentThread()));
15627 #ifdef FEATURE_EVENT_TRACE
15628 if (bgc_heap_walk_for_etw_p && settings.concurrent)
15630 GCToEEInterface::DiagWalkBGCSurvivors(__this);
15632 #ifdef MULTIPLE_HEAPS
15633 bgc_t_join.join(this, gc_join_after_profiler_heap_walk);
15634 if (bgc_t_join.joined())
15636 bgc_t_join.restart();
15638 #endif // MULTIPLE_HEAPS
15640 #endif // FEATURE_EVENT_TRACE
15641 #endif //BACKGROUND_GC
15644 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
15645 verify_heap (FALSE);
15646 #endif // VERIFY_HEAP
15648 #ifdef BACKGROUND_GC
15649 if (settings.concurrent)
15651 repair_allocation_contexts (TRUE);
15653 #ifdef MULTIPLE_HEAPS
15654 bgc_t_join.join(this, gc_join_restart_ee_verify);
15655 if (bgc_t_join.joined())
15657 bgc_threads_sync_event.Reset();
15659 dprintf(2, ("Joining BGC threads to restart EE after verify heap"));
15660 bgc_t_join.restart();
15662 if (heap_number == 0)
15665 bgc_threads_sync_event.Set();
15669 bgc_threads_sync_event.Wait(INFINITE, FALSE);
15670 dprintf (2, ("bgc_threads_sync_event is signalled"));
15674 #endif //MULTIPLE_HEAPS
15676 disable_preemptive (cooperative_mode);
15678 #endif //BACKGROUND_GC
15680 #endif // defined(VERIFY_HEAP) || (defined(FEATURE_EVENT_TRACE) && defined(BACKGROUND_GC))
15682 #ifdef MULTIPLE_HEAPS
15683 if (!settings.concurrent)
15685 gc_t_join.join(this, gc_join_done);
15686 if (gc_t_join.joined ())
15688 gc_heap::internal_gc_done = false;
15690 //equalize the new desired size of the generations
15691 int limit = settings.condemned_generation;
15692 if (limit == max_generation)
15694 limit = max_generation+1;
15696 for (int gen = 0; gen <= limit; gen++)
15698 size_t total_desired = 0;
15700 for (int i = 0; i < gc_heap::n_heaps; i++)
15702 gc_heap* hp = gc_heap::g_heaps[i];
15703 dynamic_data* dd = hp->dynamic_data_of (gen);
15704 size_t temp_total_desired = total_desired + dd_desired_allocation (dd);
15705 if (temp_total_desired < total_desired)
15708 total_desired = (size_t)MAX_PTR;
15711 total_desired = temp_total_desired;
15714 size_t desired_per_heap = Align (total_desired/gc_heap::n_heaps,
15715 get_alignment_constant ((gen != (max_generation+1))));
15719 #if 1 //subsumed by the linear allocation model
15720 // to avoid spikes in mem usage due to short terms fluctuations in survivorship,
15721 // apply some smoothing.
15722 static size_t smoothed_desired_per_heap = 0;
15723 size_t smoothing = 3; // exponential smoothing factor
15724 if (smoothing > VolatileLoad(&settings.gc_index))
15725 smoothing = VolatileLoad(&settings.gc_index);
15726 smoothed_desired_per_heap = desired_per_heap / smoothing + ((smoothed_desired_per_heap / smoothing) * (smoothing-1));
15727 dprintf (1, ("sn = %Id n = %Id", smoothed_desired_per_heap, desired_per_heap));
15728 desired_per_heap = Align(smoothed_desired_per_heap, get_alignment_constant (true));
15731 // if desired_per_heap is close to min_gc_size, trim it
15732 // down to min_gc_size to stay in the cache
15733 gc_heap* hp = gc_heap::g_heaps[0];
15734 dynamic_data* dd = hp->dynamic_data_of (gen);
15735 size_t min_gc_size = dd_min_size(dd);
15736 // if min GC size larger than true on die cache, then don't bother
15737 // limiting the desired size
15738 if ((min_gc_size <= GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE)) &&
15739 desired_per_heap <= 2*min_gc_size)
15741 desired_per_heap = min_gc_size;
15744 desired_per_heap = joined_youngest_desired (desired_per_heap);
15745 dprintf (2, ("final gen0 new_alloc: %Id", desired_per_heap));
15748 gc_data_global.final_youngest_desired = desired_per_heap;
15750 #if 1 //subsumed by the linear allocation model
15751 if (gen == (max_generation + 1))
15753 // to avoid spikes in mem usage due to short terms fluctuations in survivorship,
15754 // apply some smoothing.
15755 static size_t smoothed_desired_per_heap_loh = 0;
15756 size_t smoothing = 3; // exponential smoothing factor
15757 size_t loh_count = dd_collection_count (dynamic_data_of (max_generation));
15758 if (smoothing > loh_count)
15759 smoothing = loh_count;
15760 smoothed_desired_per_heap_loh = desired_per_heap / smoothing + ((smoothed_desired_per_heap_loh / smoothing) * (smoothing-1));
15761 dprintf( 2, ("smoothed_desired_per_heap_loh = %Id desired_per_heap = %Id", smoothed_desired_per_heap_loh, desired_per_heap));
15762 desired_per_heap = Align(smoothed_desired_per_heap_loh, get_alignment_constant (false));
15765 for (int i = 0; i < gc_heap::n_heaps; i++)
15767 gc_heap* hp = gc_heap::g_heaps[i];
15768 dynamic_data* dd = hp->dynamic_data_of (gen);
15769 dd_desired_allocation (dd) = desired_per_heap;
15770 dd_gc_new_allocation (dd) = desired_per_heap;
15771 dd_new_allocation (dd) = desired_per_heap;
15775 hp->fgn_last_alloc = desired_per_heap;
15780 #ifdef FEATURE_LOH_COMPACTION
15781 BOOL all_heaps_compacted_p = TRUE;
15782 #endif //FEATURE_LOH_COMPACTION
15783 for (int i = 0; i < gc_heap::n_heaps; i++)
15785 gc_heap* hp = gc_heap::g_heaps[i];
15786 hp->decommit_ephemeral_segment_pages();
15787 hp->rearrange_large_heap_segments();
15788 #ifdef FEATURE_LOH_COMPACTION
15789 all_heaps_compacted_p &= hp->loh_compacted_p;
15790 #endif //FEATURE_LOH_COMPACTION
15793 #ifdef FEATURE_LOH_COMPACTION
15794 check_loh_compact_mode (all_heaps_compacted_p);
15795 #endif //FEATURE_LOH_COMPACTION
15799 gc_t_join.restart();
15801 alloc_context_count = 0;
15802 heap_select::mark_heap (heap_number);
15806 gc_data_global.final_youngest_desired =
15807 dd_desired_allocation (dynamic_data_of (0));
15809 check_loh_compact_mode (loh_compacted_p);
15811 decommit_ephemeral_segment_pages();
15814 if (!(settings.concurrent))
15816 rearrange_large_heap_segments();
15820 #ifdef BACKGROUND_GC
15821 recover_bgc_settings();
15822 #endif //BACKGROUND_GC
15823 #endif //MULTIPLE_HEAPS
15826 void gc_heap::save_data_for_no_gc()
15828 current_no_gc_region_info.saved_pause_mode = settings.pause_mode;
15829 #ifdef MULTIPLE_HEAPS
15830 // This is to affect heap balancing.
15831 for (int i = 0; i < n_heaps; i++)
15833 current_no_gc_region_info.saved_gen0_min_size = dd_min_size (g_heaps[i]->dynamic_data_of (0));
15834 dd_min_size (g_heaps[i]->dynamic_data_of (0)) = min_balance_threshold;
15835 current_no_gc_region_info.saved_gen3_min_size = dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1));
15836 dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1)) = 0;
15838 #endif //MULTIPLE_HEAPS
15841 void gc_heap::restore_data_for_no_gc()
15843 gc_heap::settings.pause_mode = current_no_gc_region_info.saved_pause_mode;
15844 #ifdef MULTIPLE_HEAPS
15845 for (int i = 0; i < n_heaps; i++)
15847 dd_min_size (g_heaps[i]->dynamic_data_of (0)) = current_no_gc_region_info.saved_gen0_min_size;
15848 dd_min_size (g_heaps[i]->dynamic_data_of (max_generation + 1)) = current_no_gc_region_info.saved_gen3_min_size;
15850 #endif //MULTIPLE_HEAPS
15853 start_no_gc_region_status gc_heap::prepare_for_no_gc_region (uint64_t total_size,
15854 BOOL loh_size_known,
15856 BOOL disallow_full_blocking)
15858 if (current_no_gc_region_info.started)
15860 return start_no_gc_in_progress;
15863 start_no_gc_region_status status = start_no_gc_success;
15865 save_data_for_no_gc();
15866 settings.pause_mode = pause_no_gc;
15867 current_no_gc_region_info.start_status = start_no_gc_success;
15869 uint64_t allocation_no_gc_loh = 0;
15870 uint64_t allocation_no_gc_soh = 0;
15871 assert(total_size != 0);
15872 if (loh_size_known)
15874 assert(loh_size != 0);
15875 assert(loh_size <= total_size);
15876 allocation_no_gc_loh = loh_size;
15877 allocation_no_gc_soh = total_size - loh_size;
15881 allocation_no_gc_soh = total_size;
15882 allocation_no_gc_loh = total_size;
15885 int soh_align_const = get_alignment_constant (TRUE);
15886 size_t max_soh_allocated = soh_segment_size - segment_info_size - eph_gen_starts_size;
15887 size_t size_per_heap = 0;
15888 const double scale_factor = 1.05;
15891 #ifdef MULTIPLE_HEAPS
15892 num_heaps = n_heaps;
15893 #endif // MULTIPLE_HEAPS
15895 uint64_t total_allowed_soh_allocation = max_soh_allocated * num_heaps;
15897 // In theory, the upper limit here is the physical memory of the machine, not
15898 // SIZE_T_MAX. This is not true today because total_physical_mem can be
15899 // larger than SIZE_T_MAX if running in wow64 on a machine with more than
15900 // 4GB of RAM. Once Local GC code divergence is resolved and code is flowing
15901 // more freely between branches, it would be good to clean this up to use
15902 // total_physical_mem instead of SIZE_T_MAX.
15903 assert(total_allowed_soh_allocation <= SIZE_T_MAX);
15904 uint64_t total_allowed_loh_allocation = SIZE_T_MAX;
15905 uint64_t total_allowed_soh_alloc_scaled = allocation_no_gc_soh > 0 ? static_cast<uint64_t>(total_allowed_soh_allocation / scale_factor) : 0;
15906 uint64_t total_allowed_loh_alloc_scaled = allocation_no_gc_loh > 0 ? static_cast<uint64_t>(total_allowed_loh_allocation / scale_factor) : 0;
15908 if (allocation_no_gc_soh > total_allowed_soh_alloc_scaled ||
15909 allocation_no_gc_loh > total_allowed_loh_alloc_scaled)
15911 status = start_no_gc_too_large;
15915 if (allocation_no_gc_soh > 0)
15917 allocation_no_gc_soh = static_cast<uint64_t>(allocation_no_gc_soh * scale_factor);
15918 allocation_no_gc_soh = min (allocation_no_gc_soh, total_allowed_soh_alloc_scaled);
15921 if (allocation_no_gc_loh > 0)
15923 allocation_no_gc_loh = static_cast<uint64_t>(allocation_no_gc_loh * scale_factor);
15924 allocation_no_gc_loh = min (allocation_no_gc_loh, total_allowed_loh_alloc_scaled);
15927 if (disallow_full_blocking)
15928 current_no_gc_region_info.minimal_gc_p = TRUE;
15930 if (allocation_no_gc_soh != 0)
15932 current_no_gc_region_info.soh_allocation_size = static_cast<size_t>(allocation_no_gc_soh);
15933 size_per_heap = current_no_gc_region_info.soh_allocation_size;
15934 #ifdef MULTIPLE_HEAPS
15935 size_per_heap /= n_heaps;
15936 for (int i = 0; i < n_heaps; i++)
15938 // due to heap balancing we need to allow some room before we even look to balance to another heap.
15939 g_heaps[i]->soh_allocation_no_gc = min (Align ((size_per_heap + min_balance_threshold), soh_align_const), max_soh_allocated);
15941 #else //MULTIPLE_HEAPS
15942 soh_allocation_no_gc = min (Align (size_per_heap, soh_align_const), max_soh_allocated);
15943 #endif //MULTIPLE_HEAPS
15946 if (allocation_no_gc_loh != 0)
15948 current_no_gc_region_info.loh_allocation_size = static_cast<size_t>(allocation_no_gc_loh);
15949 size_per_heap = current_no_gc_region_info.loh_allocation_size;
15950 #ifdef MULTIPLE_HEAPS
15951 size_per_heap /= n_heaps;
15952 for (int i = 0; i < n_heaps; i++)
15953 g_heaps[i]->loh_allocation_no_gc = Align (size_per_heap, get_alignment_constant (FALSE));
15954 #else //MULTIPLE_HEAPS
15955 loh_allocation_no_gc = Align (size_per_heap, get_alignment_constant (FALSE));
15956 #endif //MULTIPLE_HEAPS
15960 if (status != start_no_gc_success)
15961 restore_data_for_no_gc();
15965 void gc_heap::handle_failure_for_no_gc()
15967 gc_heap::restore_data_for_no_gc();
15968 // sets current_no_gc_region_info.started to FALSE here.
15969 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
15972 start_no_gc_region_status gc_heap::get_start_no_gc_region_status()
15974 return current_no_gc_region_info.start_status;
15977 void gc_heap::record_gcs_during_no_gc()
15979 if (current_no_gc_region_info.started)
15981 current_no_gc_region_info.num_gcs++;
15982 if (is_induced (settings.reason))
15983 current_no_gc_region_info.num_gcs_induced++;
15987 BOOL gc_heap::find_loh_free_for_no_gc()
15989 allocator* loh_allocator = generation_allocator (generation_of (max_generation + 1));
15990 size_t sz_list = loh_allocator->first_bucket_size();
15991 size_t size = loh_allocation_no_gc;
15992 for (unsigned int a_l_idx = 0; a_l_idx < loh_allocator->number_of_buckets(); a_l_idx++)
15994 if ((size < sz_list) || (a_l_idx == (loh_allocator->number_of_buckets()-1)))
15996 uint8_t* free_list = loh_allocator->alloc_list_head_of (a_l_idx);
15999 size_t free_list_size = unused_array_size(free_list);
16001 if (free_list_size > loh_allocation_no_gc)
16003 dprintf (3, ("free item %Ix(%Id) for no gc", (size_t)free_list, free_list_size));
16007 free_list = free_list_slot (free_list);
16010 sz_list = sz_list * 2;
16016 BOOL gc_heap::find_loh_space_for_no_gc()
16018 saved_loh_segment_no_gc = 0;
16020 if (find_loh_free_for_no_gc())
16023 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16027 size_t remaining = heap_segment_reserved (seg) - heap_segment_allocated (seg);
16028 if (remaining >= loh_allocation_no_gc)
16030 saved_loh_segment_no_gc = seg;
16033 seg = heap_segment_next (seg);
16036 if (!saved_loh_segment_no_gc && current_no_gc_region_info.minimal_gc_p)
16038 // If no full GC is allowed, we try to get a new seg right away.
16039 saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc)
16040 #ifdef MULTIPLE_HEAPS
16042 #endif //MULTIPLE_HEAPS
16046 return (saved_loh_segment_no_gc != 0);
16049 BOOL gc_heap::loh_allocated_for_no_gc()
16051 if (!saved_loh_segment_no_gc)
16054 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16057 if (seg == saved_loh_segment_no_gc)
16061 seg = heap_segment_next (seg);
16067 BOOL gc_heap::commit_loh_for_no_gc (heap_segment* seg)
16069 uint8_t* end_committed = heap_segment_allocated (seg) + loh_allocation_no_gc;
16070 assert (end_committed <= heap_segment_reserved (seg));
16071 return (grow_heap_segment (seg, end_committed));
16074 void gc_heap::thread_no_gc_loh_segments()
16076 #ifdef MULTIPLE_HEAPS
16077 for (int i = 0; i < n_heaps; i++)
16079 gc_heap* hp = g_heaps[i];
16080 if (hp->loh_allocated_for_no_gc())
16082 hp->thread_loh_segment (hp->saved_loh_segment_no_gc);
16083 hp->saved_loh_segment_no_gc = 0;
16086 #else //MULTIPLE_HEAPS
16087 if (loh_allocated_for_no_gc())
16089 thread_loh_segment (saved_loh_segment_no_gc);
16090 saved_loh_segment_no_gc = 0;
16092 #endif //MULTIPLE_HEAPS
16095 void gc_heap::set_loh_allocations_for_no_gc()
16097 if (current_no_gc_region_info.loh_allocation_size != 0)
16099 dynamic_data* dd = dynamic_data_of (max_generation + 1);
16100 dd_new_allocation (dd) = loh_allocation_no_gc;
16101 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
16105 void gc_heap::set_soh_allocations_for_no_gc()
16107 if (current_no_gc_region_info.soh_allocation_size != 0)
16109 dynamic_data* dd = dynamic_data_of (0);
16110 dd_new_allocation (dd) = soh_allocation_no_gc;
16111 dd_gc_new_allocation (dd) = dd_new_allocation (dd);
16112 #ifdef MULTIPLE_HEAPS
16113 alloc_context_count = 0;
16114 #endif //MULTIPLE_HEAPS
16118 void gc_heap::set_allocations_for_no_gc()
16120 #ifdef MULTIPLE_HEAPS
16121 for (int i = 0; i < n_heaps; i++)
16123 gc_heap* hp = g_heaps[i];
16124 hp->set_loh_allocations_for_no_gc();
16125 hp->set_soh_allocations_for_no_gc();
16127 #else //MULTIPLE_HEAPS
16128 set_loh_allocations_for_no_gc();
16129 set_soh_allocations_for_no_gc();
16130 #endif //MULTIPLE_HEAPS
16133 BOOL gc_heap::should_proceed_for_no_gc()
16135 BOOL gc_requested = FALSE;
16136 BOOL loh_full_gc_requested = FALSE;
16137 BOOL soh_full_gc_requested = FALSE;
16138 BOOL no_gc_requested = FALSE;
16139 BOOL get_new_loh_segments = FALSE;
16141 if (current_no_gc_region_info.soh_allocation_size)
16143 #ifdef MULTIPLE_HEAPS
16144 for (int i = 0; i < n_heaps; i++)
16146 gc_heap* hp = g_heaps[i];
16147 if ((size_t)(heap_segment_reserved (hp->ephemeral_heap_segment) - hp->alloc_allocated) < hp->soh_allocation_no_gc)
16149 gc_requested = TRUE;
16153 #else //MULTIPLE_HEAPS
16154 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated) < soh_allocation_no_gc)
16155 gc_requested = TRUE;
16156 #endif //MULTIPLE_HEAPS
16160 #ifdef MULTIPLE_HEAPS
16161 for (int i = 0; i < n_heaps; i++)
16163 gc_heap* hp = g_heaps[i];
16164 if (!(hp->grow_heap_segment (hp->ephemeral_heap_segment, (hp->alloc_allocated + hp->soh_allocation_no_gc))))
16166 soh_full_gc_requested = TRUE;
16170 #else //MULTIPLE_HEAPS
16171 if (!grow_heap_segment (ephemeral_heap_segment, (alloc_allocated + soh_allocation_no_gc)))
16172 soh_full_gc_requested = TRUE;
16173 #endif //MULTIPLE_HEAPS
16177 if (!current_no_gc_region_info.minimal_gc_p && gc_requested)
16179 soh_full_gc_requested = TRUE;
16182 no_gc_requested = !(soh_full_gc_requested || gc_requested);
16184 if (soh_full_gc_requested && current_no_gc_region_info.minimal_gc_p)
16186 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16190 if (!soh_full_gc_requested && current_no_gc_region_info.loh_allocation_size)
16192 // Check to see if we have enough reserved space.
16193 #ifdef MULTIPLE_HEAPS
16194 for (int i = 0; i < n_heaps; i++)
16196 gc_heap* hp = g_heaps[i];
16197 if (!hp->find_loh_space_for_no_gc())
16199 loh_full_gc_requested = TRUE;
16203 #else //MULTIPLE_HEAPS
16204 if (!find_loh_space_for_no_gc())
16205 loh_full_gc_requested = TRUE;
16206 #endif //MULTIPLE_HEAPS
16208 // Check to see if we have committed space.
16209 if (!loh_full_gc_requested)
16211 #ifdef MULTIPLE_HEAPS
16212 for (int i = 0; i < n_heaps; i++)
16214 gc_heap* hp = g_heaps[i];
16215 if (hp->saved_loh_segment_no_gc &&!hp->commit_loh_for_no_gc (hp->saved_loh_segment_no_gc))
16217 loh_full_gc_requested = TRUE;
16221 #else //MULTIPLE_HEAPS
16222 if (saved_loh_segment_no_gc && !commit_loh_for_no_gc (saved_loh_segment_no_gc))
16223 loh_full_gc_requested = TRUE;
16224 #endif //MULTIPLE_HEAPS
16228 if (loh_full_gc_requested || soh_full_gc_requested)
16230 if (current_no_gc_region_info.minimal_gc_p)
16231 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16234 no_gc_requested = !(loh_full_gc_requested || soh_full_gc_requested || gc_requested);
16236 if (current_no_gc_region_info.start_status == start_no_gc_success)
16238 if (no_gc_requested)
16239 set_allocations_for_no_gc();
16244 if ((current_no_gc_region_info.start_status == start_no_gc_success) && !no_gc_requested)
16248 // We are done with starting the no_gc_region.
16249 current_no_gc_region_info.started = TRUE;
16254 end_no_gc_region_status gc_heap::end_no_gc_region()
16256 dprintf (1, ("end no gc called"));
16258 end_no_gc_region_status status = end_no_gc_success;
16260 if (!(current_no_gc_region_info.started))
16261 status = end_no_gc_not_in_progress;
16262 if (current_no_gc_region_info.num_gcs_induced)
16263 status = end_no_gc_induced;
16264 else if (current_no_gc_region_info.num_gcs)
16265 status = end_no_gc_alloc_exceeded;
16267 if (settings.pause_mode == pause_no_gc)
16268 restore_data_for_no_gc();
16270 // sets current_no_gc_region_info.started to FALSE here.
16271 memset (¤t_no_gc_region_info, 0, sizeof (current_no_gc_region_info));
16277 void gc_heap::update_collection_counts ()
16279 dynamic_data* dd0 = dynamic_data_of (0);
16280 dd_gc_clock (dd0) += 1;
16282 size_t now = GetHighPrecisionTimeStamp();
16284 for (int i = 0; i <= settings.condemned_generation;i++)
16286 dynamic_data* dd = dynamic_data_of (i);
16287 dd_collection_count (dd)++;
16288 //this is needed by the linear allocation model
16289 if (i == max_generation)
16290 dd_collection_count (dynamic_data_of (max_generation+1))++;
16291 dd_gc_clock (dd) = dd_gc_clock (dd0);
16292 dd_time_clock (dd) = now;
16296 #ifdef HEAP_ANALYZE
16298 BOOL AnalyzeSurvivorsRequested(int condemnedGeneration)
16300 #ifndef BUILD_AS_STANDALONE
16301 // Is the list active?
16302 GcNotifications gn(g_pGcNotificationTable);
16305 GcEvtArgs gea = { GC_MARK_END, { (1<<condemnedGeneration) } };
16306 if (gn.GetNotification(gea) != 0)
16311 #endif // BUILD_AS_STANDALONE
16315 void DACNotifyGcMarkEnd(int condemnedGeneration)
16317 #ifndef BUILD_AS_STANDALONE
16318 // Is the list active?
16319 GcNotifications gn(g_pGcNotificationTable);
16322 GcEvtArgs gea = { GC_MARK_END, { (1<<condemnedGeneration) } };
16323 if (gn.GetNotification(gea) != 0)
16325 DACNotify::DoGCNotification(gea);
16328 #endif // BUILD_AS_STANDALONE
16330 #endif // HEAP_ANALYZE
16332 BOOL gc_heap::expand_soh_with_minimal_gc()
16334 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment)) >= soh_allocation_no_gc)
16337 heap_segment* new_seg = soh_get_segment_to_expand();
16340 if (g_gc_card_table != card_table)
16341 copy_brick_card_table();
16343 settings.promotion = TRUE;
16344 settings.demotion = FALSE;
16345 ephemeral_promotion = TRUE;
16346 int condemned_gen_number = max_generation - 1;
16348 generation* gen = 0;
16349 int align_const = get_alignment_constant (TRUE);
16351 for (int i = 0; i <= condemned_gen_number; i++)
16353 gen = generation_of (i);
16354 saved_ephemeral_plan_start[i] = generation_allocation_start (gen);
16355 saved_ephemeral_plan_start_size[i] = Align (size (generation_allocation_start (gen)), align_const);
16358 // We do need to clear the bricks here as we are converting a bunch of ephemeral objects to gen2
16359 // and need to make sure that there are no left over bricks from the previous GCs for the space
16360 // we just used for gen0 allocation. We will need to go through the bricks for these objects for
16361 // ephemeral GCs later.
16362 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
16363 b < brick_of (align_on_brick (heap_segment_allocated (ephemeral_heap_segment)));
16369 size_t ephemeral_size = (heap_segment_allocated (ephemeral_heap_segment) -
16370 generation_allocation_start (generation_of (max_generation - 1)));
16371 heap_segment_next (ephemeral_heap_segment) = new_seg;
16372 ephemeral_heap_segment = new_seg;
16373 uint8_t* start = heap_segment_mem (ephemeral_heap_segment);
16375 for (int i = condemned_gen_number; i >= 0; i--)
16377 gen = generation_of (i);
16378 size_t gen_start_size = Align (min_obj_size);
16379 make_generation (generation_table[i], ephemeral_heap_segment, start, 0);
16380 generation_plan_allocation_start (gen) = start;
16381 generation_plan_allocation_start_size (gen) = gen_start_size;
16382 start += gen_start_size;
16384 heap_segment_used (ephemeral_heap_segment) = start - plug_skew;
16385 heap_segment_plan_allocated (ephemeral_heap_segment) = start;
16387 fix_generation_bounds (condemned_gen_number, generation_of (0));
16389 dd_gc_new_allocation (dynamic_data_of (max_generation)) -= ephemeral_size;
16390 dd_new_allocation (dynamic_data_of (max_generation)) = dd_gc_new_allocation (dynamic_data_of (max_generation));
16392 adjust_ephemeral_limits();
16399 // Only to be done on the thread that calls restart in a join for server GC
16400 // and reset the oom status per heap.
16401 void gc_heap::check_and_set_no_gc_oom()
16403 #ifdef MULTIPLE_HEAPS
16404 for (int i = 0; i < n_heaps; i++)
16406 gc_heap* hp = g_heaps[i];
16407 if (hp->no_gc_oom_p)
16409 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16410 hp->no_gc_oom_p = false;
16416 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16417 no_gc_oom_p = false;
16419 #endif //MULTIPLE_HEAPS
16422 void gc_heap::allocate_for_no_gc_after_gc()
16424 if (current_no_gc_region_info.minimal_gc_p)
16425 repair_allocation_contexts (TRUE);
16427 no_gc_oom_p = false;
16429 if (current_no_gc_region_info.start_status != start_no_gc_no_memory)
16431 if (current_no_gc_region_info.soh_allocation_size != 0)
16433 if (((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment)) < soh_allocation_no_gc) ||
16434 (!grow_heap_segment (ephemeral_heap_segment, (heap_segment_allocated (ephemeral_heap_segment) + soh_allocation_no_gc))))
16436 no_gc_oom_p = true;
16439 #ifdef MULTIPLE_HEAPS
16440 gc_t_join.join(this, gc_join_after_commit_soh_no_gc);
16441 if (gc_t_join.joined())
16443 #endif //MULTIPLE_HEAPS
16445 check_and_set_no_gc_oom();
16447 #ifdef MULTIPLE_HEAPS
16448 gc_t_join.restart();
16450 #endif //MULTIPLE_HEAPS
16453 if ((current_no_gc_region_info.start_status == start_no_gc_success) &&
16454 !(current_no_gc_region_info.minimal_gc_p) &&
16455 (current_no_gc_region_info.loh_allocation_size != 0))
16457 gc_policy = policy_compact;
16458 saved_loh_segment_no_gc = 0;
16460 if (!find_loh_free_for_no_gc())
16462 heap_segment* seg = generation_allocation_segment (generation_of (max_generation + 1));
16463 BOOL found_seg_p = FALSE;
16466 if ((size_t)(heap_segment_reserved (seg) - heap_segment_allocated (seg)) >= loh_allocation_no_gc)
16468 found_seg_p = TRUE;
16469 if (!commit_loh_for_no_gc (seg))
16471 no_gc_oom_p = true;
16475 seg = heap_segment_next (seg);
16479 gc_policy = policy_expand;
16482 #ifdef MULTIPLE_HEAPS
16483 gc_t_join.join(this, gc_join_expand_loh_no_gc);
16484 if (gc_t_join.joined())
16486 check_and_set_no_gc_oom();
16488 if (current_no_gc_region_info.start_status == start_no_gc_success)
16490 for (int i = 0; i < n_heaps; i++)
16492 gc_heap* hp = g_heaps[i];
16493 if (hp->gc_policy == policy_expand)
16495 hp->saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc), hp);
16496 if (!(hp->saved_loh_segment_no_gc))
16498 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16505 gc_t_join.restart();
16507 #else //MULTIPLE_HEAPS
16508 check_and_set_no_gc_oom();
16510 if ((current_no_gc_region_info.start_status == start_no_gc_success) && (gc_policy == policy_expand))
16512 saved_loh_segment_no_gc = get_segment_for_loh (get_large_seg_size (loh_allocation_no_gc));
16513 if (!saved_loh_segment_no_gc)
16514 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16516 #endif //MULTIPLE_HEAPS
16518 if ((current_no_gc_region_info.start_status == start_no_gc_success) && saved_loh_segment_no_gc)
16520 if (!commit_loh_for_no_gc (saved_loh_segment_no_gc))
16522 no_gc_oom_p = true;
16528 #ifdef MULTIPLE_HEAPS
16529 gc_t_join.join(this, gc_join_final_no_gc);
16530 if (gc_t_join.joined())
16532 #endif //MULTIPLE_HEAPS
16534 check_and_set_no_gc_oom();
16536 if (current_no_gc_region_info.start_status == start_no_gc_success)
16538 set_allocations_for_no_gc();
16539 current_no_gc_region_info.started = TRUE;
16542 #ifdef MULTIPLE_HEAPS
16543 gc_t_join.restart();
16545 #endif //MULTIPLE_HEAPS
16548 void gc_heap::init_records()
16550 memset (&gc_data_per_heap, 0, sizeof (gc_data_per_heap));
16551 gc_data_per_heap.heap_index = heap_number;
16552 if (heap_number == 0)
16553 memset (&gc_data_global, 0, sizeof (gc_data_global));
16555 #ifdef GC_CONFIG_DRIVEN
16556 memset (interesting_data_per_gc, 0, sizeof (interesting_data_per_gc));
16557 #endif //GC_CONFIG_DRIVEN
16560 int gc_heap::garbage_collect (int n)
16562 //reset the number of alloc contexts
16563 alloc_contexts_used = 0;
16565 fix_allocation_contexts (TRUE);
16566 #ifdef MULTIPLE_HEAPS
16568 gc_t_join.start_ts(this);
16569 #endif //JOIN_STATS
16570 clear_gen0_bricks();
16571 #endif //MULTIPLE_HEAPS
16573 if ((settings.pause_mode == pause_no_gc) && current_no_gc_region_info.minimal_gc_p)
16575 #ifdef MULTIPLE_HEAPS
16576 gc_t_join.join(this, gc_join_minimal_gc);
16577 if (gc_t_join.joined())
16579 #endif //MULTIPLE_HEAPS
16581 #ifdef MULTIPLE_HEAPS
16582 // this is serialized because we need to get a segment
16583 for (int i = 0; i < n_heaps; i++)
16585 if (!(g_heaps[i]->expand_soh_with_minimal_gc()))
16586 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16589 if (!expand_soh_with_minimal_gc())
16590 current_no_gc_region_info.start_status = start_no_gc_no_memory;
16591 #endif //MULTIPLE_HEAPS
16593 update_collection_counts_for_no_gc();
16595 #ifdef MULTIPLE_HEAPS
16596 gc_t_join.restart();
16598 #endif //MULTIPLE_HEAPS
16604 memset (&fgm_result, 0, sizeof (fgm_result));
16606 settings.reason = gc_trigger_reason;
16607 verify_pinned_queue_p = FALSE;
16609 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
16610 num_pinned_objects = 0;
16611 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
16614 if (settings.reason == reason_gcstress)
16616 settings.reason = reason_induced;
16617 settings.stress_induced = TRUE;
16619 #endif // STRESS_HEAP
16621 #ifdef MULTIPLE_HEAPS
16622 //align all heaps on the max generation to condemn
16623 dprintf (3, ("Joining for max generation to condemn"));
16624 condemned_generation_num = generation_to_condemn (n,
16625 &blocking_collection,
16626 &elevation_requested,
16628 gc_t_join.join(this, gc_join_generation_determined);
16629 if (gc_t_join.joined())
16630 #endif //MULTIPLE_HEAPS
16632 #ifdef MULTIPLE_HEAPS
16633 #if !defined(SEG_MAPPING_TABLE) && !defined(FEATURE_BASICFREEZE)
16634 //delete old slots from the segment table
16635 seg_table->delete_old_slots();
16636 #endif //!SEG_MAPPING_TABLE && !FEATURE_BASICFREEZE
16637 for (int i = 0; i < n_heaps; i++)
16639 //copy the card and brick tables
16640 if (g_gc_card_table != g_heaps[i]->card_table)
16642 g_heaps[i]->copy_brick_card_table();
16645 g_heaps[i]->rearrange_large_heap_segments();
16646 if (!recursive_gc_sync::background_running_p())
16648 g_heaps[i]->rearrange_small_heap_segments();
16651 #else //MULTIPLE_HEAPS
16652 #ifdef BACKGROUND_GC
16653 //delete old slots from the segment table
16654 #if !defined(SEG_MAPPING_TABLE) && !defined(FEATURE_BASICFREEZE)
16655 seg_table->delete_old_slots();
16656 #endif //!SEG_MAPPING_TABLE && !FEATURE_BASICFREEZE
16657 rearrange_large_heap_segments();
16658 if (!recursive_gc_sync::background_running_p())
16660 rearrange_small_heap_segments();
16662 #endif //BACKGROUND_GC
16663 // check for card table growth
16664 if (g_gc_card_table != card_table)
16665 copy_brick_card_table();
16667 #endif //MULTIPLE_HEAPS
16669 BOOL should_evaluate_elevation = FALSE;
16670 BOOL should_do_blocking_collection = FALSE;
16672 #ifdef MULTIPLE_HEAPS
16673 int gen_max = condemned_generation_num;
16674 for (int i = 0; i < n_heaps; i++)
16676 if (gen_max < g_heaps[i]->condemned_generation_num)
16677 gen_max = g_heaps[i]->condemned_generation_num;
16678 if ((!should_evaluate_elevation) && (g_heaps[i]->elevation_requested))
16679 should_evaluate_elevation = TRUE;
16680 if ((!should_do_blocking_collection) && (g_heaps[i]->blocking_collection))
16681 should_do_blocking_collection = TRUE;
16684 settings.condemned_generation = gen_max;
16685 #else //MULTIPLE_HEAPS
16686 settings.condemned_generation = generation_to_condemn (n,
16687 &blocking_collection,
16688 &elevation_requested,
16690 should_evaluate_elevation = elevation_requested;
16691 should_do_blocking_collection = blocking_collection;
16692 #endif //MULTIPLE_HEAPS
16694 settings.condemned_generation = joined_generation_to_condemn (
16695 should_evaluate_elevation,
16696 settings.condemned_generation,
16697 &should_do_blocking_collection
16701 STRESS_LOG1(LF_GCROOTS|LF_GC|LF_GCALLOC, LL_INFO10,
16702 "condemned generation num: %d\n", settings.condemned_generation);
16704 record_gcs_during_no_gc();
16706 if (settings.condemned_generation > 1)
16707 settings.promotion = TRUE;
16709 #ifdef HEAP_ANALYZE
16710 // At this point we've decided what generation is condemned
16711 // See if we've been requested to analyze survivors after the mark phase
16712 if (AnalyzeSurvivorsRequested(settings.condemned_generation))
16714 heap_analyze_enabled = TRUE;
16716 #endif // HEAP_ANALYZE
16718 GCToEEInterface::DiagGCStart(settings.condemned_generation, settings.reason == reason_induced);
16720 #ifdef BACKGROUND_GC
16721 if ((settings.condemned_generation == max_generation) &&
16722 (recursive_gc_sync::background_running_p()))
16724 //TODO BACKGROUND_GC If we just wait for the end of gc, it won't work
16725 // because we have to collect 0 and 1 properly
16726 // in particular, the allocation contexts are gone.
16727 // For now, it is simpler to collect max_generation-1
16728 settings.condemned_generation = max_generation - 1;
16729 dprintf (GTC_LOG, ("bgc - 1 instead of 2"));
16732 if ((settings.condemned_generation == max_generation) &&
16733 (should_do_blocking_collection == FALSE) &&
16734 gc_can_use_concurrent &&
16735 !temp_disable_concurrent_p &&
16736 ((settings.pause_mode == pause_interactive) || (settings.pause_mode == pause_sustained_low_latency)))
16738 keep_bgc_threads_p = TRUE;
16739 c_write (settings.concurrent, TRUE);
16741 #endif //BACKGROUND_GC
16743 settings.gc_index = (uint32_t)dd_collection_count (dynamic_data_of (0)) + 1;
16745 // Call the EE for start of GC work
16746 // just one thread for MP GC
16747 GCToEEInterface::GcStartWork (settings.condemned_generation,
16750 // TODO: we could fire an ETW event to say this GC as a concurrent GC but later on due to not being able to
16751 // create threads or whatever, this could be a non concurrent GC. Maybe for concurrent GC we should fire
16752 // it in do_background_gc and if it failed to be a CGC we fire it in gc1... in other words, this should be
16756 #ifdef MULTIPLE_HEAPS
16757 gc_start_event.Reset();
16758 //start all threads on the roots.
16759 dprintf(3, ("Starting all gc threads for gc"));
16760 gc_t_join.restart();
16761 #endif //MULTIPLE_HEAPS
16765 int gen_num_for_data = max_generation + 1;
16766 for (int i = 0; i <= gen_num_for_data; i++)
16768 gc_data_per_heap.gen_data[i].size_before = generation_size (i);
16769 generation* gen = generation_of (i);
16770 gc_data_per_heap.gen_data[i].free_list_space_before = generation_free_list_space (gen);
16771 gc_data_per_heap.gen_data[i].free_obj_space_before = generation_free_obj_space (gen);
16774 descr_generations (TRUE);
16775 // descr_card_table();
16778 if ((GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC) &&
16779 !(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_POST_GC_ONLY))
16781 verify_heap (TRUE);
16783 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_BARRIERCHECK)
16784 checkGCWriteBarrier();
16786 #endif // VERIFY_HEAP
16788 #ifdef BACKGROUND_GC
16789 if (settings.concurrent)
16791 // We need to save the settings because we'll need to restore it after each FGC.
16792 assert (settings.condemned_generation == max_generation);
16793 settings.compaction = FALSE;
16794 saved_bgc_settings = settings;
16796 #ifdef MULTIPLE_HEAPS
16797 if (heap_number == 0)
16799 for (int i = 0; i < n_heaps; i++)
16801 prepare_bgc_thread (g_heaps[i]);
16803 dprintf (2, ("setting bgc_threads_sync_event"));
16804 bgc_threads_sync_event.Set();
16808 bgc_threads_sync_event.Wait(INFINITE, FALSE);
16809 dprintf (2, ("bgc_threads_sync_event is signalled"));
16812 prepare_bgc_thread(0);
16813 #endif //MULTIPLE_HEAPS
16815 #ifdef MULTIPLE_HEAPS
16816 gc_t_join.join(this, gc_join_start_bgc);
16817 if (gc_t_join.joined())
16818 #endif //MULTIPLE_HEAPS
16820 do_concurrent_p = TRUE;
16821 do_ephemeral_gc_p = FALSE;
16822 #ifdef MULTIPLE_HEAPS
16823 dprintf(2, ("Joined to perform a background GC"));
16825 for (int i = 0; i < n_heaps; i++)
16827 gc_heap* hp = g_heaps[i];
16828 if (!(hp->bgc_thread) || !hp->commit_mark_array_bgc_init (hp->mark_array))
16830 do_concurrent_p = FALSE;
16835 hp->background_saved_lowest_address = hp->lowest_address;
16836 hp->background_saved_highest_address = hp->highest_address;
16840 do_concurrent_p = (!!bgc_thread && commit_mark_array_bgc_init (mark_array));
16841 if (do_concurrent_p)
16843 background_saved_lowest_address = lowest_address;
16844 background_saved_highest_address = highest_address;
16846 #endif //MULTIPLE_HEAPS
16848 if (do_concurrent_p)
16850 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
16851 SoftwareWriteWatch::EnableForGCHeap();
16852 #endif //FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
16854 #ifdef MULTIPLE_HEAPS
16855 for (int i = 0; i < n_heaps; i++)
16856 g_heaps[i]->current_bgc_state = bgc_initialized;
16858 current_bgc_state = bgc_initialized;
16859 #endif //MULTIPLE_HEAPS
16861 int gen = check_for_ephemeral_alloc();
16862 // always do a gen1 GC before we start BGC.
16863 // This is temporary for testing purpose.
16864 //int gen = max_generation - 1;
16865 dont_restart_ee_p = TRUE;
16868 // If we decide to not do a GC before the BGC we need to
16869 // restore the gen0 alloc context.
16870 #ifdef MULTIPLE_HEAPS
16871 for (int i = 0; i < n_heaps; i++)
16873 generation_allocation_pointer (g_heaps[i]->generation_of (0)) = 0;
16874 generation_allocation_limit (g_heaps[i]->generation_of (0)) = 0;
16877 generation_allocation_pointer (youngest_generation) = 0;
16878 generation_allocation_limit (youngest_generation) = 0;
16879 #endif //MULTIPLE_HEAPS
16883 do_ephemeral_gc_p = TRUE;
16885 settings.init_mechanisms();
16886 settings.condemned_generation = gen;
16887 settings.gc_index = (size_t)dd_collection_count (dynamic_data_of (0)) + 2;
16890 // TODO BACKGROUND_GC need to add the profiling stuff here.
16891 dprintf (GTC_LOG, ("doing gen%d before doing a bgc", gen));
16894 //clear the cards so they don't bleed in gen 1 during collection
16895 // shouldn't this always be done at the beginning of any GC?
16896 //clear_card_for_addresses (
16897 // generation_allocation_start (generation_of (0)),
16898 // heap_segment_allocated (ephemeral_heap_segment));
16900 if (!do_ephemeral_gc_p)
16902 do_background_gc();
16907 settings.compaction = TRUE;
16908 c_write (settings.concurrent, FALSE);
16911 #ifdef MULTIPLE_HEAPS
16912 gc_t_join.restart();
16913 #endif //MULTIPLE_HEAPS
16916 if (do_concurrent_p)
16918 // At this point we are sure we'll be starting a BGC, so save its per heap data here.
16919 // global data is only calculated at the end of the GC so we don't need to worry about
16920 // FGCs overwriting it.
16921 memset (&bgc_data_per_heap, 0, sizeof (bgc_data_per_heap));
16922 memcpy (&bgc_data_per_heap, &gc_data_per_heap, sizeof(gc_data_per_heap));
16924 if (do_ephemeral_gc_p)
16926 dprintf (2, ("GC threads running, doing gen%d GC", settings.condemned_generation));
16928 gen_to_condemn_reasons.init();
16929 gen_to_condemn_reasons.set_condition (gen_before_bgc);
16930 gc_data_per_heap.gen_to_condemn_reasons.init (&gen_to_condemn_reasons);
16932 #ifdef MULTIPLE_HEAPS
16933 gc_t_join.join(this, gc_join_bgc_after_ephemeral);
16934 if (gc_t_join.joined())
16935 #endif //MULTIPLE_HEAPS
16937 #ifdef MULTIPLE_HEAPS
16939 #endif //MULTIPLE_HEAPS
16940 settings = saved_bgc_settings;
16941 assert (settings.concurrent);
16943 do_background_gc();
16945 #ifdef MULTIPLE_HEAPS
16946 gc_t_join.restart();
16947 #endif //MULTIPLE_HEAPS
16953 dprintf (2, ("couldn't create BGC threads, reverting to doing a blocking GC"));
16958 #endif //BACKGROUND_GC
16962 #ifndef MULTIPLE_HEAPS
16963 allocation_running_time = (size_t)GCToOSInterface::GetLowPrecisionTimeStamp();
16964 allocation_running_amount = dd_new_allocation (dynamic_data_of (0));
16965 fgn_last_alloc = dd_new_allocation (dynamic_data_of (0));
16966 #endif //MULTIPLE_HEAPS
16969 if (settings.pause_mode == pause_no_gc)
16970 allocate_for_no_gc_after_gc();
16972 int gn = settings.condemned_generation;
16976 #define mark_stack_empty_p() (mark_stack_base == mark_stack_tos)
16979 size_t& gc_heap::promoted_bytes(int thread)
16981 #ifdef MULTIPLE_HEAPS
16982 return g_promoted [thread*16];
16983 #else //MULTIPLE_HEAPS
16984 UNREFERENCED_PARAMETER(thread);
16986 #endif //MULTIPLE_HEAPS
16989 #ifdef INTERIOR_POINTERS
16990 heap_segment* gc_heap::find_segment (uint8_t* interior, BOOL small_segment_only_p)
16992 #ifdef SEG_MAPPING_TABLE
16993 heap_segment* seg = seg_mapping_table_segment_of (interior);
16996 if (small_segment_only_p && heap_segment_loh_p (seg))
17000 #else //SEG_MAPPING_TABLE
17001 #ifdef MULTIPLE_HEAPS
17002 for (int i = 0; i < gc_heap::n_heaps; i++)
17004 gc_heap* h = gc_heap::g_heaps [i];
17005 hs = h->find_segment_per_heap (o, small_segment_only_p);
17013 gc_heap* h = pGenGCHeap;
17014 hs = h->find_segment_per_heap (o, small_segment_only_p);
17016 #endif //MULTIPLE_HEAPS
17017 #endif //SEG_MAPPING_TABLE
17020 heap_segment* gc_heap::find_segment_per_heap (uint8_t* interior, BOOL small_segment_only_p)
17022 #ifdef SEG_MAPPING_TABLE
17023 return find_segment (interior, small_segment_only_p);
17024 #else //SEG_MAPPING_TABLE
17025 if (in_range_for_segment (interior, ephemeral_heap_segment))
17027 return ephemeral_heap_segment;
17031 heap_segment* found_seg = 0;
17034 heap_segment* seg = generation_start_segment (generation_of (max_generation));
17037 if (in_range_for_segment (interior, seg))
17040 goto end_find_segment;
17043 } while ((seg = heap_segment_next (seg)) != 0);
17045 if (!small_segment_only_p)
17047 #ifdef BACKGROUND_GC
17049 ptrdiff_t delta = 0;
17050 heap_segment* seg = segment_of (interior, delta);
17051 if (seg && in_range_for_segment (interior, seg))
17055 goto end_find_segment;
17057 #else //BACKGROUND_GC
17058 heap_segment* seg = generation_start_segment (generation_of (max_generation+1));
17061 if (in_range_for_segment(interior, seg))
17064 goto end_find_segment;
17067 } while ((seg = heap_segment_next (seg)) != 0);
17068 #endif //BACKGROUND_GC
17074 #endif //SEG_MAPPING_TABLE
17076 #endif //INTERIOR_POINTERS
17078 #if !defined(_DEBUG) && !defined(__GNUC__)
17079 inline // This causes link errors if global optimization is off
17080 #endif //!_DEBUG && !__GNUC__
17081 gc_heap* gc_heap::heap_of (uint8_t* o)
17083 #ifdef MULTIPLE_HEAPS
17085 return g_heaps [0];
17086 #ifdef SEG_MAPPING_TABLE
17087 gc_heap* hp = seg_mapping_table_heap_of (o);
17088 return (hp ? hp : g_heaps[0]);
17089 #else //SEG_MAPPING_TABLE
17090 ptrdiff_t delta = 0;
17091 heap_segment* seg = segment_of (o, delta);
17092 return (seg ? heap_segment_heap (seg) : g_heaps [0]);
17093 #endif //SEG_MAPPING_TABLE
17094 #else //MULTIPLE_HEAPS
17095 UNREFERENCED_PARAMETER(o);
17097 #endif //MULTIPLE_HEAPS
17101 gc_heap* gc_heap::heap_of_gc (uint8_t* o)
17103 #ifdef MULTIPLE_HEAPS
17105 return g_heaps [0];
17106 #ifdef SEG_MAPPING_TABLE
17107 gc_heap* hp = seg_mapping_table_heap_of_gc (o);
17108 return (hp ? hp : g_heaps[0]);
17109 #else //SEG_MAPPING_TABLE
17110 ptrdiff_t delta = 0;
17111 heap_segment* seg = segment_of (o, delta);
17112 return (seg ? heap_segment_heap (seg) : g_heaps [0]);
17113 #endif //SEG_MAPPING_TABLE
17114 #else //MULTIPLE_HEAPS
17115 UNREFERENCED_PARAMETER(o);
17117 #endif //MULTIPLE_HEAPS
17120 #ifdef INTERIOR_POINTERS
17121 // will find all heap objects (large and small)
17122 uint8_t* gc_heap::find_object (uint8_t* interior, uint8_t* low)
17124 if (!gen0_bricks_cleared)
17126 #ifdef MULTIPLE_HEAPS
17127 assert (!"Should have already been done in server GC");
17128 #endif //MULTIPLE_HEAPS
17129 gen0_bricks_cleared = TRUE;
17130 //initialize brick table for gen 0
17131 for (size_t b = brick_of (generation_allocation_start (generation_of (0)));
17132 b < brick_of (align_on_brick
17133 (heap_segment_allocated (ephemeral_heap_segment)));
17139 #ifdef FFIND_OBJECT
17140 //indicate that in the future this needs to be done during allocation
17141 #ifdef MULTIPLE_HEAPS
17142 gen0_must_clear_bricks = FFIND_DECAY*gc_heap::n_heaps;
17144 gen0_must_clear_bricks = FFIND_DECAY;
17145 #endif //MULTIPLE_HEAPS
17146 #endif //FFIND_OBJECT
17148 int brick_entry = get_brick_entry(brick_of (interior));
17149 if (brick_entry == 0)
17151 // this is a pointer to a large object
17152 heap_segment* seg = find_segment_per_heap (interior, FALSE);
17154 #ifdef FEATURE_CONSERVATIVE_GC
17155 && (GCConfig::GetConservativeGC() || interior <= heap_segment_allocated(seg))
17159 // If interior falls within the first free object at the beginning of a generation,
17160 // we don't have brick entry for it, and we may incorrectly treat it as on large object heap.
17161 int align_const = get_alignment_constant (heap_segment_read_only_p (seg)
17162 #ifdef FEATURE_CONSERVATIVE_GC
17163 || (GCConfig::GetConservativeGC() && !heap_segment_loh_p (seg))
17166 //int align_const = get_alignment_constant (heap_segment_read_only_p (seg));
17167 assert (interior < heap_segment_allocated (seg));
17169 uint8_t* o = heap_segment_mem (seg);
17170 while (o < heap_segment_allocated (seg))
17172 uint8_t* next_o = o + Align (size (o), align_const);
17173 assert (next_o > o);
17174 if ((o <= interior) && (interior < next_o))
17185 else if (interior >= low)
17187 heap_segment* seg = find_segment_per_heap (interior, TRUE);
17190 #ifdef FEATURE_CONSERVATIVE_GC
17191 if (interior >= heap_segment_allocated (seg))
17194 assert (interior < heap_segment_allocated (seg));
17196 uint8_t* o = find_first_object (interior, heap_segment_mem (seg));
17207 gc_heap::find_object_for_relocation (uint8_t* interior, uint8_t* low, uint8_t* high)
17209 uint8_t* old_address = interior;
17210 if (!((old_address >= low) && (old_address < high)))
17213 size_t brick = brick_of (old_address);
17214 int brick_entry = brick_table [ brick ];
17215 if (brick_entry != 0)
17219 while (brick_entry < 0)
17221 brick = (brick + brick_entry);
17222 brick_entry = brick_table [ brick ];
17224 uint8_t* old_loc = old_address;
17225 uint8_t* node = tree_search ((brick_address (brick) + brick_entry-1),
17227 if (node <= old_loc)
17232 brick_entry = brick_table [ brick ];
17238 //find the object by going along the plug
17240 while (o <= interior)
17242 uint8_t* next_o = o + Align (size (o));
17243 assert (next_o > o);
17244 if (next_o > interior)
17250 assert ((o <= interior) && ((o + Align (size (o))) > interior));
17255 // this is a pointer to a large object
17256 heap_segment* seg = find_segment_per_heap (interior, FALSE);
17259 assert (interior < heap_segment_allocated (seg));
17261 uint8_t* o = heap_segment_mem (seg);
17262 while (o < heap_segment_allocated (seg))
17264 uint8_t* next_o = o + Align (size (o));
17265 assert (next_o > o);
17266 if ((o < interior) && (interior < next_o))
17278 #else //INTERIOR_POINTERS
17280 uint8_t* gc_heap::find_object (uint8_t* o, uint8_t* low)
17284 #endif //INTERIOR_POINTERS
17287 #ifdef GC_CONFIG_DRIVEN
17288 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;} else {mark_list_index++;} if (slow > o) slow = o; if (shigh < o) shigh = o;}
17290 #define m_boundary(o) {if (mark_list_index <= mark_list_end) {*mark_list_index = o;mark_list_index++;}if (slow > o) slow = o; if (shigh < o) shigh = o;}
17291 #endif //GC_CONFIG_DRIVEN
17293 #define m_boundary(o) {if (slow > o) slow = o; if (shigh < o) shigh = o;}
17296 #define m_boundary_fullgc(o) {if (slow > o) slow = o; if (shigh < o) shigh = o;}
17298 #define method_table(o) ((CObjectHeader*)(o))->GetMethodTable()
17301 BOOL gc_heap::gc_mark1 (uint8_t* o)
17303 BOOL marked = !marked (o);
17305 dprintf (3, ("*%Ix*, newly marked: %d", (size_t)o, marked));
17310 BOOL gc_heap::gc_mark (uint8_t* o, uint8_t* low, uint8_t* high)
17312 BOOL marked = FALSE;
17313 if ((o >= low) && (o < high))
17314 marked = gc_mark1 (o);
17315 #ifdef MULTIPLE_HEAPS
17319 gc_heap* hp = heap_of_gc (o);
17321 if ((o >= hp->gc_low) && (o < hp->gc_high))
17322 marked = gc_mark1 (o);
17325 snoop_stat.objects_checked_count++;
17329 snoop_stat.objects_marked_count++;
17333 snoop_stat.zero_ref_count++;
17336 #endif //SNOOP_STATS
17337 #endif //MULTIPLE_HEAPS
17341 #ifdef BACKGROUND_GC
17344 BOOL gc_heap::background_marked (uint8_t* o)
17346 return mark_array_marked (o);
17349 BOOL gc_heap::background_mark1 (uint8_t* o)
17351 BOOL to_mark = !mark_array_marked (o);
17353 dprintf (3, ("b*%Ix*b(%d)", (size_t)o, (to_mark ? 1 : 0)));
17356 mark_array_set_marked (o);
17357 dprintf (4, ("n*%Ix*n", (size_t)o));
17364 // TODO: we could consider filtering out NULL's here instead of going to
17365 // look for it on other heaps
17367 BOOL gc_heap::background_mark (uint8_t* o, uint8_t* low, uint8_t* high)
17369 BOOL marked = FALSE;
17370 if ((o >= low) && (o < high))
17371 marked = background_mark1 (o);
17372 #ifdef MULTIPLE_HEAPS
17376 gc_heap* hp = heap_of (o);
17378 if ((o >= hp->background_saved_lowest_address) && (o < hp->background_saved_highest_address))
17379 marked = background_mark1 (o);
17381 #endif //MULTIPLE_HEAPS
17385 #endif //BACKGROUND_GC
17388 uint8_t* gc_heap::next_end (heap_segment* seg, uint8_t* f)
17390 if (seg == ephemeral_heap_segment)
17393 return heap_segment_allocated (seg);
17396 #define new_start() {if (ppstop <= start) {break;} else {parm = start}}
17397 #define ignore_start 0
17398 #define use_start 1
17400 #define go_through_object(mt,o,size,parm,start,start_useful,limit,exp) \
17402 CGCDesc* map = CGCDesc::GetCGCDescFromMT((MethodTable*)(mt)); \
17403 CGCDescSeries* cur = map->GetHighestSeries(); \
17404 ptrdiff_t cnt = (ptrdiff_t) map->GetNumSeries(); \
17408 CGCDescSeries* last = map->GetLowestSeries(); \
17409 uint8_t** parm = 0; \
17412 assert (parm <= (uint8_t**)((o) + cur->GetSeriesOffset())); \
17413 parm = (uint8_t**)((o) + cur->GetSeriesOffset()); \
17414 uint8_t** ppstop = \
17415 (uint8_t**)((uint8_t*)parm + cur->GetSeriesSize() + (size));\
17416 if (!start_useful || (uint8_t*)ppstop > (start)) \
17418 if (start_useful && (uint8_t*)parm < (start)) parm = (uint8_t**)(start);\
17419 while (parm < ppstop) \
17427 } while (cur >= last); \
17431 /* Handle the repeating case - array of valuetypes */ \
17432 uint8_t** parm = (uint8_t**)((o) + cur->startoffset); \
17433 if (start_useful && start > (uint8_t*)parm) \
17435 ptrdiff_t cs = mt->RawGetComponentSize(); \
17436 parm = (uint8_t**)((uint8_t*)parm + (((start) - (uint8_t*)parm)/cs)*cs); \
17438 while ((uint8_t*)parm < ((o)+(size)-plug_skew)) \
17440 for (ptrdiff_t __i = 0; __i > cnt; __i--) \
17442 HALF_SIZE_T skip = cur->val_serie[__i].skip; \
17443 HALF_SIZE_T nptrs = cur->val_serie[__i].nptrs; \
17444 uint8_t** ppstop = parm + nptrs; \
17445 if (!start_useful || (uint8_t*)ppstop > (start)) \
17447 if (start_useful && (uint8_t*)parm < (start)) parm = (uint8_t**)(start); \
17452 } while (parm < ppstop); \
17454 parm = (uint8_t**)((uint8_t*)ppstop + skip); \
17460 #define go_through_object_nostart(mt,o,size,parm,exp) {go_through_object(mt,o,size,parm,o,ignore_start,(o + size),exp); }
17462 // 1 thing to note about this macro:
17463 // 1) you can use *parm safely but in general you don't want to use parm
17464 // because for the collectible types it's not an address on the managed heap.
17465 #ifndef COLLECTIBLE_CLASS
17466 #define go_through_object_cl(mt,o,size,parm,exp) \
17468 if (header(o)->ContainsPointers()) \
17470 go_through_object_nostart(mt,o,size,parm,exp); \
17473 #else //COLLECTIBLE_CLASS
17474 #define go_through_object_cl(mt,o,size,parm,exp) \
17476 if (header(o)->Collectible()) \
17478 uint8_t* class_obj = get_class_object (o); \
17479 uint8_t** parm = &class_obj; \
17480 do {exp} while (false); \
17482 if (header(o)->ContainsPointers()) \
17484 go_through_object_nostart(mt,o,size,parm,exp); \
17487 #endif //COLLECTIBLE_CLASS
17489 // This starts a plug. But mark_stack_tos isn't increased until set_pinned_info is called.
17490 void gc_heap::enque_pinned_plug (uint8_t* plug,
17491 BOOL save_pre_plug_info_p,
17492 uint8_t* last_object_in_last_plug)
17494 if (mark_stack_array_length <= mark_stack_tos)
17496 if (!grow_mark_stack (mark_stack_array, mark_stack_array_length, MARK_STACK_INITIAL_LENGTH))
17498 // we don't want to continue here due to security
17499 // risks. This happens very rarely and fixing it in the
17500 // way so that we can continue is a bit involved and will
17501 // not be done in Dev10.
17502 GCToEEInterface::HandleFatalError(CORINFO_EXCEPTION_GC);
17506 dprintf (3, ("enqueuing P #%Id(%Ix): %Ix. oldest: %Id, LO: %Ix, pre: %d",
17507 mark_stack_tos, &mark_stack_array[mark_stack_tos], plug, mark_stack_bos, last_object_in_last_plug, (save_pre_plug_info_p ? 1 : 0)));
17508 mark& m = mark_stack_array[mark_stack_tos];
17510 // Must be set now because if we have a short object we'll need the value of saved_pre_p.
17511 m.saved_pre_p = save_pre_plug_info_p;
17513 if (save_pre_plug_info_p)
17516 BOOL is_padded = is_plug_padded (last_object_in_last_plug);
17518 clear_plug_padded (last_object_in_last_plug);
17519 #endif //SHORT_PLUGS
17520 memcpy (&(m.saved_pre_plug), &(((plug_and_gap*)plug)[-1]), sizeof (gap_reloc_pair));
17523 set_plug_padded (last_object_in_last_plug);
17524 #endif //SHORT_PLUGS
17526 memcpy (&(m.saved_pre_plug_reloc), &(((plug_and_gap*)plug)[-1]), sizeof (gap_reloc_pair));
17528 // If the last object in the last plug is too short, it requires special handling.
17529 size_t last_obj_size = plug - last_object_in_last_plug;
17530 if (last_obj_size < min_pre_pin_obj_size)
17532 record_interesting_data_point (idp_pre_short);
17535 record_interesting_data_point (idp_pre_short_padded);
17536 #endif //SHORT_PLUGS
17537 dprintf (3, ("encountered a short object %Ix right before pinned plug %Ix!",
17538 last_object_in_last_plug, plug));
17539 // Need to set the short bit regardless of having refs or not because we need to
17540 // indicate that this object is not walkable.
17543 #ifdef COLLECTIBLE_CLASS
17544 if (is_collectible (last_object_in_last_plug))
17546 m.set_pre_short_collectible();
17548 #endif //COLLECTIBLE_CLASS
17550 if (contain_pointers (last_object_in_last_plug))
17552 dprintf (3, ("short object: %Ix(%Ix)", last_object_in_last_plug, last_obj_size));
17554 go_through_object_nostart (method_table(last_object_in_last_plug), last_object_in_last_plug, last_obj_size, pval,
17556 size_t gap_offset = (((size_t)pval - (size_t)(plug - sizeof (gap_reloc_pair) - plug_skew))) / sizeof (uint8_t*);
17557 dprintf (3, ("member: %Ix->%Ix, %Id ptrs from beginning of gap", (uint8_t*)pval, *pval, gap_offset));
17558 m.set_pre_short_bit (gap_offset);
17565 m.saved_post_p = FALSE;
17568 void gc_heap::save_post_plug_info (uint8_t* last_pinned_plug, uint8_t* last_object_in_last_plug, uint8_t* post_plug)
17570 UNREFERENCED_PARAMETER(last_pinned_plug);
17572 mark& m = mark_stack_array[mark_stack_tos - 1];
17573 assert (last_pinned_plug == m.first);
17574 m.saved_post_plug_info_start = (uint8_t*)&(((plug_and_gap*)post_plug)[-1]);
17577 BOOL is_padded = is_plug_padded (last_object_in_last_plug);
17579 clear_plug_padded (last_object_in_last_plug);
17580 #endif //SHORT_PLUGS
17581 memcpy (&(m.saved_post_plug), m.saved_post_plug_info_start, sizeof (gap_reloc_pair));
17584 set_plug_padded (last_object_in_last_plug);
17585 #endif //SHORT_PLUGS
17587 memcpy (&(m.saved_post_plug_reloc), m.saved_post_plug_info_start, sizeof (gap_reloc_pair));
17589 // This is important - we need to clear all bits here except the last one.
17590 m.saved_post_p = TRUE;
17593 m.saved_post_plug_debug.gap = 1;
17596 dprintf (3, ("PP %Ix has NP %Ix right after", last_pinned_plug, post_plug));
17598 size_t last_obj_size = post_plug - last_object_in_last_plug;
17599 if (last_obj_size < min_pre_pin_obj_size)
17601 dprintf (3, ("PP %Ix last obj %Ix is too short", last_pinned_plug, last_object_in_last_plug));
17602 record_interesting_data_point (idp_post_short);
17605 record_interesting_data_point (idp_post_short_padded);
17606 #endif //SHORT_PLUGS
17607 m.set_post_short();
17608 verify_pinned_queue_p = TRUE;
17610 #ifdef COLLECTIBLE_CLASS
17611 if (is_collectible (last_object_in_last_plug))
17613 m.set_post_short_collectible();
17615 #endif //COLLECTIBLE_CLASS
17617 if (contain_pointers (last_object_in_last_plug))
17619 dprintf (3, ("short object: %Ix(%Ix)", last_object_in_last_plug, last_obj_size));
17621 // TODO: since we won't be able to walk this object in relocation, we still need to
17622 // take care of collectible assemblies here.
17623 go_through_object_nostart (method_table(last_object_in_last_plug), last_object_in_last_plug, last_obj_size, pval,
17625 size_t gap_offset = (((size_t)pval - (size_t)(post_plug - sizeof (gap_reloc_pair) - plug_skew))) / sizeof (uint8_t*);
17626 dprintf (3, ("member: %Ix->%Ix, %Id ptrs from beginning of gap", (uint8_t*)pval, *pval, gap_offset));
17627 m.set_post_short_bit (gap_offset);
17636 __declspec(naked) void __fastcall Prefetch(void* addr)
17644 inline void Prefetch (void* addr)
17646 UNREFERENCED_PARAMETER(addr);
17651 VOLATILE(uint8_t*)& gc_heap::ref_mark_stack (gc_heap* hp, int index)
17653 return ((VOLATILE(uint8_t*)*)(hp->mark_stack_array))[index];
17656 #endif //MH_SC_MARK
17660 #define partial_object 3
17662 uint8_t* ref_from_slot (uint8_t* r)
17664 return (uint8_t*)((size_t)r & ~(stolen | partial));
17667 BOOL stolen_p (uint8_t* r)
17669 return (((size_t)r&2) && !((size_t)r&1));
17672 BOOL ready_p (uint8_t* r)
17674 return ((size_t)r != 1);
17677 BOOL partial_p (uint8_t* r)
17679 return (((size_t)r&1) && !((size_t)r&2));
17682 BOOL straight_ref_p (uint8_t* r)
17684 return (!stolen_p (r) && !partial_p (r));
17687 BOOL partial_object_p (uint8_t* r)
17689 return (((size_t)r & partial_object) == partial_object);
17692 BOOL ref_p (uint8_t* r)
17694 return (straight_ref_p (r) || partial_object_p (r));
17697 void gc_heap::mark_object_simple1 (uint8_t* oo, uint8_t* start THREAD_NUMBER_DCL)
17699 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_tos = (SERVER_SC_MARK_VOLATILE(uint8_t*)*)mark_stack_array;
17700 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_limit = (SERVER_SC_MARK_VOLATILE(uint8_t*)*)&mark_stack_array[mark_stack_array_length];
17701 SERVER_SC_MARK_VOLATILE(uint8_t*)* mark_stack_base = mark_stack_tos;
17702 #ifdef SORT_MARK_STACK
17703 SERVER_SC_MARK_VOLATILE(uint8_t*)* sorted_tos = mark_stack_base;
17704 #endif //SORT_MARK_STACK
17706 // If we are doing a full GC we don't use mark list anyway so use m_boundary_fullgc that doesn't
17707 // update mark list.
17708 BOOL full_p = (settings.condemned_generation == max_generation);
17710 assert ((start >= oo) && (start < oo+size(oo)));
17713 *mark_stack_tos = oo;
17714 #endif //!MH_SC_MARK
17718 #ifdef MULTIPLE_HEAPS
17719 #else //MULTIPLE_HEAPS
17720 const int thread = 0;
17721 #endif //MULTIPLE_HEAPS
17723 if (oo && ((size_t)oo != 4))
17731 else if (!partial_p (oo) && ((s = size (oo)) < (partial_size_th*sizeof (uint8_t*))))
17733 BOOL overflow_p = FALSE;
17735 if (mark_stack_tos + (s) /sizeof (uint8_t*) >= (mark_stack_limit - 1))
17737 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
17738 if (mark_stack_tos + CGCDesc::GetNumPointers(method_table(oo), s, num_components) >= (mark_stack_limit - 1))
17744 if (overflow_p == FALSE)
17746 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
17748 go_through_object_cl (method_table(oo), oo, s, ppslot,
17750 uint8_t* o = *ppslot;
17752 if (gc_mark (o, gc_low, gc_high))
17756 m_boundary_fullgc (o);
17762 size_t obj_size = size (o);
17763 promoted_bytes (thread) += obj_size;
17764 if (contain_pointers_or_collectible (o))
17766 *(mark_stack_tos++) = o;
17774 dprintf(3,("mark stack overflow for object %Ix ", (size_t)oo));
17775 min_overflow_address = min (min_overflow_address, oo);
17776 max_overflow_address = max (max_overflow_address, oo);
17781 if (partial_p (oo))
17783 start = ref_from_slot (oo);
17784 oo = ref_from_slot (*(--mark_stack_tos));
17785 dprintf (4, ("oo: %Ix, start: %Ix\n", (size_t)oo, (size_t)start));
17786 assert ((oo < start) && (start < (oo + size (oo))));
17788 #ifdef COLLECTIBLE_CLASS
17791 // If there's a class object, push it now. We are guaranteed to have the slot since
17792 // we just popped one object off.
17793 if (is_collectible (oo))
17795 uint8_t* class_obj = get_class_object (oo);
17796 if (gc_mark (class_obj, gc_low, gc_high))
17800 m_boundary_fullgc (class_obj);
17804 m_boundary (class_obj);
17807 size_t obj_size = size (class_obj);
17808 promoted_bytes (thread) += obj_size;
17809 *(mark_stack_tos++) = class_obj;
17813 #endif //COLLECTIBLE_CLASS
17817 BOOL overflow_p = FALSE;
17819 if (mark_stack_tos + (num_partial_refs + 2) >= mark_stack_limit)
17823 if (overflow_p == FALSE)
17825 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
17827 //push the object and its current
17828 SERVER_SC_MARK_VOLATILE(uint8_t*)* place = ++mark_stack_tos;
17832 *(place) = (uint8_t*)partial;
17833 #endif //MH_SC_MARK
17834 int i = num_partial_refs;
17835 uint8_t* ref_to_continue = 0;
17837 go_through_object (method_table(oo), oo, s, ppslot,
17838 start, use_start, (oo + s),
17840 uint8_t* o = *ppslot;
17842 if (gc_mark (o, gc_low, gc_high))
17846 m_boundary_fullgc (o);
17852 size_t obj_size = size (o);
17853 promoted_bytes (thread) += obj_size;
17854 if (contain_pointers_or_collectible (o))
17856 *(mark_stack_tos++) = o;
17859 ref_to_continue = (uint8_t*)((size_t)(ppslot+1) | partial);
17868 //we are finished with this object
17869 assert (ref_to_continue == 0);
17871 assert ((*(place-1)) == (uint8_t*)0);
17874 #endif //MH_SC_MARK
17876 // shouldn't we decrease tos by 2 here??
17879 if (ref_to_continue)
17883 assert ((*(place-1)) == (uint8_t*)0);
17884 *(place-1) = (uint8_t*)((size_t)oo | partial_object);
17885 assert (((*place) == (uint8_t*)1) || ((*place) == (uint8_t*)2));
17886 #endif //MH_SC_MARK
17887 *place = ref_to_continue;
17892 dprintf(3,("mark stack overflow for object %Ix ", (size_t)oo));
17893 min_overflow_address = min (min_overflow_address, oo);
17894 max_overflow_address = max (max_overflow_address, oo);
17897 #ifdef SORT_MARK_STACK
17898 if (mark_stack_tos > sorted_tos + mark_stack_array_length/8)
17900 rqsort1 (sorted_tos, mark_stack_tos-1);
17901 sorted_tos = mark_stack_tos-1;
17903 #endif //SORT_MARK_STACK
17906 if (!(mark_stack_empty_p()))
17908 oo = *(--mark_stack_tos);
17911 #ifdef SORT_MARK_STACK
17912 sorted_tos = min ((size_t)sorted_tos, (size_t)mark_stack_tos);
17913 #endif //SORT_MARK_STACK
17921 BOOL same_numa_node_p (int hn1, int hn2)
17923 return (heap_select::find_numa_node_from_heap_no (hn1) == heap_select::find_numa_node_from_heap_no (hn2));
17926 int find_next_buddy_heap (int this_heap_number, int current_buddy, int n_heaps)
17928 int hn = (current_buddy+1)%n_heaps;
17929 while (hn != current_buddy)
17931 if ((this_heap_number != hn) && (same_numa_node_p (this_heap_number, hn)))
17933 hn = (hn+1)%n_heaps;
17935 return current_buddy;
17939 gc_heap::mark_steal()
17941 mark_stack_busy() = 0;
17942 //clear the mark stack in the snooping range
17943 for (int i = 0; i < max_snoop_level; i++)
17945 ((VOLATILE(uint8_t*)*)(mark_stack_array))[i] = 0;
17948 //pick the next heap as our buddy
17949 int thpn = find_next_buddy_heap (heap_number, heap_number, n_heaps);
17952 dprintf (SNOOP_LOG, ("(GC%d)heap%d: start snooping %d", settings.gc_index, heap_number, (heap_number+1)%n_heaps));
17953 uint32_t begin_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
17954 #endif //SNOOP_STATS
17956 int idle_loop_count = 0;
17957 int first_not_ready_level = 0;
17961 gc_heap* hp = g_heaps [thpn];
17962 int level = first_not_ready_level;
17963 first_not_ready_level = 0;
17965 while (check_next_mark_stack (hp) && (level < (max_snoop_level-1)))
17967 idle_loop_count = 0;
17969 snoop_stat.busy_count++;
17970 dprintf (SNOOP_LOG, ("heap%d: looking at next heap level %d stack contents: %Ix",
17971 heap_number, level, (int)((uint8_t**)(hp->mark_stack_array))[level]));
17972 #endif //SNOOP_STATS
17974 uint8_t* o = ref_mark_stack (hp, level);
17976 uint8_t* start = o;
17979 mark_stack_busy() = 1;
17981 BOOL success = TRUE;
17982 uint8_t* next = (ref_mark_stack (hp, level+1));
17985 if (((size_t)o > 4) && !partial_object_p (o))
17987 //this is a normal object, not a partial mark tuple
17988 //success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level), 0, o)==o);
17989 success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level), (uint8_t*)4, o)==o);
17991 snoop_stat.interlocked_count++;
17993 snoop_stat.normal_count++;
17994 #endif //SNOOP_STATS
17998 //it is a stolen entry, or beginning/ending of a partial mark
18001 snoop_stat.stolen_or_pm_count++;
18002 #endif //SNOOP_STATS
18006 else if (stolen_p (next))
18008 //ignore the stolen guy and go to the next level
18012 snoop_stat.stolen_entry_count++;
18013 #endif //SNOOP_STATS
18017 assert (partial_p (next));
18018 start = ref_from_slot (next);
18019 //re-read the object
18020 o = ref_from_slot (ref_mark_stack (hp, level));
18024 success = (Interlocked::CompareExchangePointer (&ref_mark_stack (hp, level+1), (uint8_t*)stolen, next)==next);
18026 snoop_stat.interlocked_count++;
18029 snoop_stat.partial_mark_parent_count++;
18031 #endif //SNOOP_STATS
18035 // stack is not ready, or o is completely different from the last time we read from this stack level.
18036 // go up 2 levels to steal children or totally unrelated objects.
18038 if (first_not_ready_level == 0)
18040 first_not_ready_level = level;
18044 snoop_stat.pm_not_ready_count++;
18045 #endif //SNOOP_STATS
18052 dprintf (SNOOP_LOG, ("heap%d: marking %Ix from %d [%d] tl:%dms",
18053 heap_number, (size_t)o, (heap_number+1)%n_heaps, level,
18054 (GCToOSInterface::GetLowPrecisionTimeStamp()-begin_tick)));
18055 uint32_t start_tick = GCToOSInterface::GetLowPrecisionTimeStamp();
18056 #endif //SNOOP_STATS
18058 mark_object_simple1 (o, start, heap_number);
18061 dprintf (SNOOP_LOG, ("heap%d: done marking %Ix from %d [%d] %dms tl:%dms",
18062 heap_number, (size_t)o, (heap_number+1)%n_heaps, level,
18063 (GCToOSInterface::GetLowPrecisionTimeStamp()-start_tick),(GCToOSInterface::GetLowPrecisionTimeStamp()-begin_tick)));
18064 #endif //SNOOP_STATS
18066 mark_stack_busy() = 0;
18068 //clear the mark stack in snooping range
18069 for (int i = 0; i < max_snoop_level; i++)
18071 if (((uint8_t**)mark_stack_array)[i] != 0)
18073 ((VOLATILE(uint8_t*)*)(mark_stack_array))[i] = 0;
18075 snoop_stat.stack_bottom_clear_count++;
18076 #endif //SNOOP_STATS
18082 mark_stack_busy() = 0;
18086 //slot is either partial or stolen
18090 if ((first_not_ready_level != 0) && hp->mark_stack_busy())
18094 if (!hp->mark_stack_busy())
18096 first_not_ready_level = 0;
18099 if ((idle_loop_count % (6) )==1)
18102 snoop_stat.switch_to_thread_count++;
18103 #endif //SNOOP_STATS
18104 GCToOSInterface::Sleep(1);
18106 int free_count = 1;
18108 snoop_stat.stack_idle_count++;
18109 //dprintf (SNOOP_LOG, ("heap%d: counting idle threads", heap_number));
18110 #endif //SNOOP_STATS
18111 for (int hpn = (heap_number+1)%n_heaps; hpn != heap_number;)
18113 if (!((g_heaps [hpn])->mark_stack_busy()))
18117 dprintf (SNOOP_LOG, ("heap%d: %d idle", heap_number, free_count));
18118 #endif //SNOOP_STATS
18120 else if (same_numa_node_p (hpn, heap_number) || ((idle_loop_count%1000))==999)
18125 hpn = (hpn+1)%n_heaps;
18128 if (free_count == n_heaps)
18137 BOOL gc_heap::check_next_mark_stack (gc_heap* next_heap)
18140 snoop_stat.check_level_count++;
18141 #endif //SNOOP_STATS
18142 return (next_heap->mark_stack_busy()>=1);
18144 #endif //MH_SC_MARK
18147 void gc_heap::print_snoop_stat()
18149 dprintf (1234, ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s",
18150 "heap", "check", "zero", "mark", "stole", "pstack", "nstack", "nonsk"));
18151 dprintf (1234, ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d",
18152 snoop_stat.heap_index,
18153 snoop_stat.objects_checked_count,
18154 snoop_stat.zero_ref_count,
18155 snoop_stat.objects_marked_count,
18156 snoop_stat.stolen_stack_count,
18157 snoop_stat.partial_stack_count,
18158 snoop_stat.normal_stack_count,
18159 snoop_stat.non_stack_count));
18160 dprintf (1234, ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s",
18161 "heap", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "clear"));
18162 dprintf (1234, ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
18163 snoop_stat.heap_index,
18164 snoop_stat.check_level_count,
18165 snoop_stat.busy_count,
18166 snoop_stat.interlocked_count,
18167 snoop_stat.partial_mark_parent_count,
18168 snoop_stat.stolen_or_pm_count,
18169 snoop_stat.stolen_entry_count,
18170 snoop_stat.pm_not_ready_count,
18171 snoop_stat.normal_count,
18172 snoop_stat.stack_bottom_clear_count));
18174 printf ("\n%4s | %8s | %8s | %8s | %8s | %8s\n",
18175 "heap", "check", "zero", "mark", "idle", "switch");
18176 printf ("%4d | %8d | %8d | %8d | %8d | %8d\n",
18177 snoop_stat.heap_index,
18178 snoop_stat.objects_checked_count,
18179 snoop_stat.zero_ref_count,
18180 snoop_stat.objects_marked_count,
18181 snoop_stat.stack_idle_count,
18182 snoop_stat.switch_to_thread_count);
18183 printf ("%4s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s\n",
18184 "heap", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "normal", "clear");
18185 printf ("%4d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
18186 snoop_stat.heap_index,
18187 snoop_stat.check_level_count,
18188 snoop_stat.busy_count,
18189 snoop_stat.interlocked_count,
18190 snoop_stat.partial_mark_parent_count,
18191 snoop_stat.stolen_or_pm_count,
18192 snoop_stat.stolen_entry_count,
18193 snoop_stat.pm_not_ready_count,
18194 snoop_stat.normal_count,
18195 snoop_stat.stack_bottom_clear_count);
18197 #endif //SNOOP_STATS
18199 #ifdef HEAP_ANALYZE
18201 gc_heap::ha_mark_object_simple (uint8_t** po THREAD_NUMBER_DCL)
18203 if (!internal_root_array)
18205 internal_root_array = new (nothrow) uint8_t* [internal_root_array_length];
18206 if (!internal_root_array)
18208 heap_analyze_success = FALSE;
18212 if (heap_analyze_success && (internal_root_array_length <= internal_root_array_index))
18214 size_t new_size = 2*internal_root_array_length;
18216 uint64_t available_physical = 0;
18217 get_memory_info (NULL, &available_physical);
18218 if (new_size > (size_t)(available_physical / 10))
18220 heap_analyze_success = FALSE;
18224 uint8_t** tmp = new (nothrow) uint8_t* [new_size];
18227 memcpy (tmp, internal_root_array,
18228 internal_root_array_length*sizeof (uint8_t*));
18229 delete[] internal_root_array;
18230 internal_root_array = tmp;
18231 internal_root_array_length = new_size;
18235 heap_analyze_success = FALSE;
18240 if (heap_analyze_success)
18242 PREFIX_ASSUME(internal_root_array_index < internal_root_array_length);
18244 uint8_t* ref = (uint8_t*)po;
18245 if (!current_obj ||
18246 !((ref >= current_obj) && (ref < (current_obj + current_obj_size))))
18248 gc_heap* hp = gc_heap::heap_of (ref);
18249 current_obj = hp->find_object (ref, hp->lowest_address);
18250 current_obj_size = size (current_obj);
18252 internal_root_array[internal_root_array_index] = current_obj;
18253 internal_root_array_index++;
18257 mark_object_simple (po THREAD_NUMBER_ARG);
18259 #endif //HEAP_ANALYZE
18261 //this method assumes that *po is in the [low. high[ range
18263 gc_heap::mark_object_simple (uint8_t** po THREAD_NUMBER_DCL)
18266 #ifdef MULTIPLE_HEAPS
18267 #else //MULTIPLE_HEAPS
18268 const int thread = 0;
18269 #endif //MULTIPLE_HEAPS
18272 snoop_stat.objects_checked_count++;
18273 #endif //SNOOP_STATS
18278 size_t s = size (o);
18279 promoted_bytes (thread) += s;
18281 go_through_object_cl (method_table(o), o, s, poo,
18283 uint8_t* oo = *poo;
18284 if (gc_mark (oo, gc_low, gc_high))
18287 size_t obj_size = size (oo);
18288 promoted_bytes (thread) += obj_size;
18290 if (contain_pointers_or_collectible (oo))
18291 mark_object_simple1 (oo, oo THREAD_NUMBER_ARG);
18301 uint8_t* gc_heap::mark_object (uint8_t* o THREAD_NUMBER_DCL)
18303 if ((o >= gc_low) && (o < gc_high))
18304 mark_object_simple (&o THREAD_NUMBER_ARG);
18305 #ifdef MULTIPLE_HEAPS
18309 gc_heap* hp = heap_of (o);
18311 if ((o >= hp->gc_low) && (o < hp->gc_high))
18312 mark_object_simple (&o THREAD_NUMBER_ARG);
18314 #endif //MULTIPLE_HEAPS
18319 #ifdef BACKGROUND_GC
18321 void gc_heap::background_mark_simple1 (uint8_t* oo THREAD_NUMBER_DCL)
18323 uint8_t** mark_stack_limit = &background_mark_stack_array[background_mark_stack_array_length];
18325 #ifdef SORT_MARK_STACK
18326 uint8_t** sorted_tos = background_mark_stack_array;
18327 #endif //SORT_MARK_STACK
18329 background_mark_stack_tos = background_mark_stack_array;
18333 #ifdef MULTIPLE_HEAPS
18334 #else //MULTIPLE_HEAPS
18335 const int thread = 0;
18336 #endif //MULTIPLE_HEAPS
18340 if ((((size_t)oo & 1) == 0) && ((s = size (oo)) < (partial_size_th*sizeof (uint8_t*))))
18342 BOOL overflow_p = FALSE;
18344 if (background_mark_stack_tos + (s) /sizeof (uint8_t*) >= (mark_stack_limit - 1))
18346 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
18347 size_t num_pointers = CGCDesc::GetNumPointers(method_table(oo), s, num_components);
18348 if (background_mark_stack_tos + num_pointers >= (mark_stack_limit - 1))
18350 dprintf (2, ("h%d: %Id left, obj (mt: %Ix) %Id ptrs",
18352 (size_t)(mark_stack_limit - 1 - background_mark_stack_tos),
18356 bgc_overflow_count++;
18361 if (overflow_p == FALSE)
18363 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18365 go_through_object_cl (method_table(oo), oo, s, ppslot,
18367 uint8_t* o = *ppslot;
18369 if (background_mark (o,
18370 background_saved_lowest_address,
18371 background_saved_highest_address))
18374 size_t obj_size = size (o);
18375 bpromoted_bytes (thread) += obj_size;
18376 if (contain_pointers_or_collectible (o))
18378 *(background_mark_stack_tos++) = o;
18387 dprintf (3,("mark stack overflow for object %Ix ", (size_t)oo));
18388 background_min_overflow_address = min (background_min_overflow_address, oo);
18389 background_max_overflow_address = max (background_max_overflow_address, oo);
18394 uint8_t* start = oo;
18395 if ((size_t)oo & 1)
18397 oo = (uint8_t*)((size_t)oo & ~1);
18398 start = *(--background_mark_stack_tos);
18399 dprintf (4, ("oo: %Ix, start: %Ix\n", (size_t)oo, (size_t)start));
18401 #ifdef COLLECTIBLE_CLASS
18404 // If there's a class object, push it now. We are guaranteed to have the slot since
18405 // we just popped one object off.
18406 if (is_collectible (oo))
18408 uint8_t* class_obj = get_class_object (oo);
18409 if (background_mark (class_obj,
18410 background_saved_lowest_address,
18411 background_saved_highest_address))
18413 size_t obj_size = size (class_obj);
18414 bpromoted_bytes (thread) += obj_size;
18416 *(background_mark_stack_tos++) = class_obj;
18420 #endif //COLLECTIBLE_CLASS
18424 BOOL overflow_p = FALSE;
18426 if (background_mark_stack_tos + (num_partial_refs + 2) >= mark_stack_limit)
18428 size_t num_components = ((method_table(oo))->HasComponentSize() ? ((CObjectHeader*)oo)->GetNumComponents() : 0);
18429 size_t num_pointers = CGCDesc::GetNumPointers(method_table(oo), s, num_components);
18431 dprintf (2, ("h%d: PM: %Id left, obj %Ix (mt: %Ix) start: %Ix, total: %Id",
18433 (size_t)(mark_stack_limit - background_mark_stack_tos),
18439 bgc_overflow_count++;
18442 if (overflow_p == FALSE)
18444 dprintf(3,("pushing mark for %Ix ", (size_t)oo));
18446 //push the object and its current
18447 uint8_t** place = background_mark_stack_tos++;
18449 *(background_mark_stack_tos++) = (uint8_t*)((size_t)oo | 1);
18451 int i = num_partial_refs;
18453 go_through_object (method_table(oo), oo, s, ppslot,
18454 start, use_start, (oo + s),
18456 uint8_t* o = *ppslot;
18459 if (background_mark (o,
18460 background_saved_lowest_address,
18461 background_saved_highest_address))
18464 size_t obj_size = size (o);
18465 bpromoted_bytes (thread) += obj_size;
18466 if (contain_pointers_or_collectible (o))
18468 *(background_mark_stack_tos++) = o;
18472 *place = (uint8_t*)(ppslot+1);
18481 //we are finished with this object
18489 dprintf (3,("mark stack overflow for object %Ix ", (size_t)oo));
18490 background_min_overflow_address = min (background_min_overflow_address, oo);
18491 background_max_overflow_address = max (background_max_overflow_address, oo);
18495 #ifdef SORT_MARK_STACK
18496 if (background_mark_stack_tos > sorted_tos + mark_stack_array_length/8)
18498 rqsort1 (sorted_tos, background_mark_stack_tos-1);
18499 sorted_tos = background_mark_stack_tos-1;
18501 #endif //SORT_MARK_STACK
18505 if (!(background_mark_stack_tos == background_mark_stack_array))
18507 oo = *(--background_mark_stack_tos);
18509 #ifdef SORT_MARK_STACK
18510 sorted_tos = (uint8_t**)min ((size_t)sorted_tos, (size_t)background_mark_stack_tos);
18511 #endif //SORT_MARK_STACK
18517 assert (background_mark_stack_tos == background_mark_stack_array);
18522 //this version is different than the foreground GC because
18523 //it can't keep pointers to the inside of an object
18524 //while calling background_mark_simple1. The object could be moved
18525 //by an intervening foreground gc.
18526 //this method assumes that *po is in the [low. high[ range
18528 gc_heap::background_mark_simple (uint8_t* o THREAD_NUMBER_DCL)
18530 #ifdef MULTIPLE_HEAPS
18531 #else //MULTIPLE_HEAPS
18532 const int thread = 0;
18533 #endif //MULTIPLE_HEAPS
18535 dprintf (3, ("bmarking %Ix", o));
18537 if (background_mark1 (o))
18540 size_t s = size (o);
18541 bpromoted_bytes (thread) += s;
18543 if (contain_pointers_or_collectible (o))
18545 background_mark_simple1 (o THREAD_NUMBER_ARG);
18552 uint8_t* gc_heap::background_mark_object (uint8_t* o THREAD_NUMBER_DCL)
18554 if ((o >= background_saved_lowest_address) && (o < background_saved_highest_address))
18556 background_mark_simple (o THREAD_NUMBER_ARG);
18562 dprintf (3, ("or-%Ix", o));
18568 void gc_heap::background_verify_mark (Object*& object, ScanContext* sc, uint32_t flags)
18570 UNREFERENCED_PARAMETER(sc);
18572 assert (settings.concurrent);
18573 uint8_t* o = (uint8_t*)object;
18575 gc_heap* hp = gc_heap::heap_of (o);
18576 #ifdef INTERIOR_POINTERS
18577 if (flags & GC_CALL_INTERIOR)
18579 o = hp->find_object (o, background_saved_lowest_address);
18581 #endif //INTERIOR_POINTERS
18583 if (!background_object_marked (o, FALSE))
18589 void gc_heap::background_promote (Object** ppObject, ScanContext* sc, uint32_t flags)
18591 UNREFERENCED_PARAMETER(sc);
18592 //in order to save space on the array, mark the object,
18593 //knowing that it will be visited later
18594 assert (settings.concurrent);
18596 THREAD_NUMBER_FROM_CONTEXT;
18597 #ifndef MULTIPLE_HEAPS
18598 const int thread = 0;
18599 #endif //!MULTIPLE_HEAPS
18601 uint8_t* o = (uint8_t*)*ppObject;
18606 #ifdef DEBUG_DestroyedHandleValue
18607 // we can race with destroy handle during concurrent scan
18608 if (o == (uint8_t*)DEBUG_DestroyedHandleValue)
18610 #endif //DEBUG_DestroyedHandleValue
18614 gc_heap* hp = gc_heap::heap_of (o);
18616 if ((o < hp->background_saved_lowest_address) || (o >= hp->background_saved_highest_address))
18621 #ifdef INTERIOR_POINTERS
18622 if (flags & GC_CALL_INTERIOR)
18624 o = hp->find_object (o, hp->background_saved_lowest_address);
18628 #endif //INTERIOR_POINTERS
18630 #ifdef FEATURE_CONSERVATIVE_GC
18631 // For conservative GC, a value on stack may point to middle of a free object.
18632 // In this case, we don't need to promote the pointer.
18633 if (GCConfig::GetConservativeGC() && ((CObjectHeader*)o)->IsFree())
18637 #endif //FEATURE_CONSERVATIVE_GC
18640 ((CObjectHeader*)o)->Validate();
18643 dprintf (BGC_LOG, ("Background Promote %Ix", (size_t)o));
18645 //needs to be called before the marking because it is possible for a foreground
18646 //gc to take place during the mark and move the object
18647 STRESS_LOG3(LF_GC|LF_GCROOTS, LL_INFO1000000, " GCHeap::Promote: Promote GC Root *%p = %p MT = %pT", ppObject, o, o ? ((Object*) o)->GetGCSafeMethodTable() : NULL);
18649 hpt->background_mark_simple (o THREAD_NUMBER_ARG);
18652 //used by the ephemeral collection to scan the local background structures
18653 //containing references.
18655 gc_heap::scan_background_roots (promote_func* fn, int hn, ScanContext *pSC)
18661 pSC->thread_number = hn;
18663 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
18664 pSC->pCurrentDomain = 0;
18667 BOOL relocate_p = (fn == &GCHeap::Relocate);
18669 dprintf (3, ("Scanning background mark list"));
18672 size_t mark_list_finger = 0;
18673 while (mark_list_finger < c_mark_list_index)
18675 uint8_t** o = &c_mark_list [mark_list_finger];
18678 // We may not be able to calculate the size during relocate as POPO
18679 // may have written over the object.
18680 size_t s = size (*o);
18681 assert (Align (s) >= Align (min_obj_size));
18682 dprintf(3,("background root %Ix", (size_t)*o));
18684 (*fn) ((Object**)o, pSC, 0);
18685 mark_list_finger++;
18688 //scan the mark stack
18689 dprintf (3, ("Scanning background mark stack"));
18691 uint8_t** finger = background_mark_stack_array;
18692 while (finger < background_mark_stack_tos)
18694 if ((finger + 1) < background_mark_stack_tos)
18696 // We need to check for the partial mark case here.
18697 uint8_t* parent_obj = *(finger + 1);
18698 if ((size_t)parent_obj & 1)
18700 uint8_t* place = *finger;
18701 size_t place_offset = 0;
18702 uint8_t* real_parent_obj = (uint8_t*)((size_t)parent_obj & ~1);
18706 *(finger + 1) = real_parent_obj;
18707 place_offset = place - real_parent_obj;
18708 dprintf(3,("relocating background root %Ix", (size_t)real_parent_obj));
18709 (*fn) ((Object**)(finger + 1), pSC, 0);
18710 real_parent_obj = *(finger + 1);
18711 *finger = real_parent_obj + place_offset;
18712 *(finger + 1) = (uint8_t*)((size_t)real_parent_obj | 1);
18713 dprintf(3,("roots changed to %Ix, %Ix", *finger, *(finger + 1)));
18717 uint8_t** temp = &real_parent_obj;
18718 dprintf(3,("marking background root %Ix", (size_t)real_parent_obj));
18719 (*fn) ((Object**)temp, pSC, 0);
18726 dprintf(3,("background root %Ix", (size_t)*finger));
18727 (*fn) ((Object**)finger, pSC, 0);
18733 void gc_heap::background_mark_through_object (uint8_t* oo THREAD_NUMBER_DCL)
18735 if (contain_pointers (oo))
18737 size_t total_refs = 0;
18738 size_t s = size (oo);
18739 go_through_object_nostart (method_table(oo), oo, s, po,
18743 background_mark_object (o THREAD_NUMBER_ARG);
18747 dprintf (3,("Background marking through %Ix went through %Id refs",
18753 uint8_t* gc_heap::background_seg_end (heap_segment* seg, BOOL concurrent_p)
18755 if (concurrent_p && (seg == saved_overflow_ephemeral_seg))
18757 // for now we stop at where gen1 started when we started processing
18758 return background_min_soh_overflow_address;
18762 return heap_segment_allocated (seg);
18766 uint8_t* gc_heap::background_first_overflow (uint8_t* min_add,
18769 BOOL small_object_p)
18773 if (small_object_p)
18775 if (in_range_for_segment (min_add, seg))
18777 // min_add was the beginning of gen1 when we did the concurrent
18778 // overflow. Now we could be in a situation where min_add is
18779 // actually the same as allocated for that segment (because
18780 // we expanded heap), in which case we can not call
18781 // find first on this address or we will AV.
18782 if (min_add >= heap_segment_allocated (seg))
18788 if (concurrent_p &&
18789 ((seg == saved_overflow_ephemeral_seg) && (min_add >= background_min_soh_overflow_address)))
18791 return background_min_soh_overflow_address;
18795 o = find_first_object (min_add, heap_segment_mem (seg));
18802 o = max (heap_segment_mem (seg), min_add);
18806 void gc_heap::background_process_mark_overflow_internal (int condemned_gen_number,
18807 uint8_t* min_add, uint8_t* max_add,
18812 current_bgc_state = bgc_overflow_soh;
18815 size_t total_marked_objects = 0;
18817 #ifdef MULTIPLE_HEAPS
18818 int thread = heap_number;
18819 #endif //MULTIPLE_HEAPS
18821 exclusive_sync* loh_alloc_lock = 0;
18823 dprintf (2,("Processing Mark overflow [%Ix %Ix]", (size_t)min_add, (size_t)max_add));
18824 #ifdef MULTIPLE_HEAPS
18825 // We don't have each heap scan all heaps concurrently because we are worried about
18826 // multiple threads calling things like find_first_object.
18827 int h_start = (concurrent_p ? heap_number : 0);
18828 int h_end = (concurrent_p ? (heap_number + 1) : n_heaps);
18829 for (int hi = h_start; hi < h_end; hi++)
18831 gc_heap* hp = (concurrent_p ? this : g_heaps [(heap_number + hi) % n_heaps]);
18837 #endif //MULTIPLE_HEAPS
18838 BOOL small_object_segments = TRUE;
18839 int align_const = get_alignment_constant (small_object_segments);
18840 generation* gen = hp->generation_of (condemned_gen_number);
18841 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
18842 PREFIX_ASSUME(seg != NULL);
18843 loh_alloc_lock = hp->bgc_alloc_lock;
18845 uint8_t* o = hp->background_first_overflow (min_add,
18848 small_object_segments);
18852 while ((o < hp->background_seg_end (seg, concurrent_p)) && (o <= max_add))
18854 dprintf (3, ("considering %Ix", (size_t)o));
18858 if (concurrent_p && !small_object_segments)
18860 loh_alloc_lock->bgc_mark_set (o);
18862 if (((CObjectHeader*)o)->IsFree())
18864 s = unused_array_size (o);
18876 if (background_object_marked (o, FALSE) && contain_pointers_or_collectible (o))
18878 total_marked_objects++;
18879 go_through_object_cl (method_table(o), o, s, poo,
18880 uint8_t* oo = *poo;
18881 background_mark_object (oo THREAD_NUMBER_ARG);
18885 if (concurrent_p && !small_object_segments)
18887 loh_alloc_lock->bgc_mark_done ();
18890 o = o + Align (s, align_const);
18898 dprintf (2, ("went through overflow objects in segment %Ix (%d) (so far %Id marked)",
18899 heap_segment_mem (seg), (small_object_segments ? 0 : 1), total_marked_objects));
18901 if ((concurrent_p && (seg == hp->saved_overflow_ephemeral_seg)) ||
18902 (seg = heap_segment_next_in_range (seg)) == 0)
18904 if (small_object_segments)
18908 current_bgc_state = bgc_overflow_loh;
18911 dprintf (2, ("h%d: SOH: ov-mo: %Id", heap_number, total_marked_objects));
18912 fire_overflow_event (min_add, max_add, total_marked_objects, !small_object_segments);
18913 concurrent_print_time_delta (concurrent_p ? "Cov SOH" : "Nov SOH");
18914 total_marked_objects = 0;
18915 small_object_segments = FALSE;
18916 align_const = get_alignment_constant (small_object_segments);
18917 seg = heap_segment_in_range (generation_start_segment (hp->generation_of (max_generation+1)));
18919 PREFIX_ASSUME(seg != NULL);
18921 o = max (heap_segment_mem (seg), min_add);
18926 dprintf (GTC_LOG, ("h%d: LOH: ov-mo: %Id", heap_number, total_marked_objects));
18927 fire_overflow_event (min_add, max_add, total_marked_objects, !small_object_segments);
18933 o = hp->background_first_overflow (min_add,
18936 small_object_segments);
18943 BOOL gc_heap::background_process_mark_overflow (BOOL concurrent_p)
18945 BOOL grow_mark_array_p = TRUE;
18949 assert (!processed_soh_overflow_p);
18951 if ((background_max_overflow_address != 0) &&
18952 (background_min_overflow_address != MAX_PTR))
18954 // We have overflow to process but we know we can't process the ephemeral generations
18955 // now (we actually could process till the current gen1 start but since we are going to
18956 // make overflow per segment, for now I'll just stop at the saved gen1 start.
18957 saved_overflow_ephemeral_seg = ephemeral_heap_segment;
18958 background_max_soh_overflow_address = heap_segment_reserved (saved_overflow_ephemeral_seg);
18959 background_min_soh_overflow_address = generation_allocation_start (generation_of (max_generation-1));
18964 assert ((saved_overflow_ephemeral_seg == 0) ||
18965 ((background_max_soh_overflow_address != 0) &&
18966 (background_min_soh_overflow_address != MAX_PTR)));
18968 if (!processed_soh_overflow_p)
18970 // if there was no more overflow we just need to process what we didn't process
18971 // on the saved ephemeral segment.
18972 if ((background_max_overflow_address == 0) && (background_min_overflow_address == MAX_PTR))
18974 dprintf (2, ("final processing mark overflow - no more overflow since last time"));
18975 grow_mark_array_p = FALSE;
18978 background_min_overflow_address = min (background_min_overflow_address,
18979 background_min_soh_overflow_address);
18980 background_max_overflow_address = max (background_max_overflow_address,
18981 background_max_soh_overflow_address);
18982 processed_soh_overflow_p = TRUE;
18986 BOOL overflow_p = FALSE;
18988 if ((! ((background_max_overflow_address == 0)) ||
18989 ! ((background_min_overflow_address == MAX_PTR))))
18993 if (grow_mark_array_p)
18995 // Try to grow the array.
18996 size_t new_size = max (MARK_STACK_INITIAL_LENGTH, 2*background_mark_stack_array_length);
18998 if ((new_size * sizeof(mark)) > 100*1024)
19000 size_t new_max_size = (get_total_heap_size() / 10) / sizeof(mark);
19002 new_size = min(new_max_size, new_size);
19005 if ((background_mark_stack_array_length < new_size) &&
19006 ((new_size - background_mark_stack_array_length) > (background_mark_stack_array_length / 2)))
19008 dprintf (2, ("h%d: ov grow to %Id", heap_number, new_size));
19010 uint8_t** tmp = new (nothrow) uint8_t* [new_size];
19013 delete background_mark_stack_array;
19014 background_mark_stack_array = tmp;
19015 background_mark_stack_array_length = new_size;
19016 background_mark_stack_tos = background_mark_stack_array;
19022 grow_mark_array_p = TRUE;
19025 uint8_t* min_add = background_min_overflow_address;
19026 uint8_t* max_add = background_max_overflow_address;
19028 background_max_overflow_address = 0;
19029 background_min_overflow_address = MAX_PTR;
19031 background_process_mark_overflow_internal (max_generation, min_add, max_add, concurrent_p);
19041 #endif //BACKGROUND_GC
19044 void gc_heap::mark_through_object (uint8_t* oo, BOOL mark_class_object_p THREAD_NUMBER_DCL)
19046 #ifndef COLLECTIBLE_CLASS
19047 UNREFERENCED_PARAMETER(mark_class_object_p);
19048 BOOL to_mark_class_object = FALSE;
19049 #else //COLLECTIBLE_CLASS
19050 BOOL to_mark_class_object = (mark_class_object_p && (is_collectible(oo)));
19051 #endif //COLLECTIBLE_CLASS
19052 if (contain_pointers (oo) || to_mark_class_object)
19054 dprintf(3,( "Marking through %Ix", (size_t)oo));
19055 size_t s = size (oo);
19057 #ifdef COLLECTIBLE_CLASS
19058 if (to_mark_class_object)
19060 uint8_t* class_obj = get_class_object (oo);
19061 mark_object (class_obj THREAD_NUMBER_ARG);
19063 #endif //COLLECTIBLE_CLASS
19065 if (contain_pointers (oo))
19067 go_through_object_nostart (method_table(oo), oo, s, po,
19069 mark_object (o THREAD_NUMBER_ARG);
19075 size_t gc_heap::get_total_heap_size()
19077 size_t total_heap_size = 0;
19079 #ifdef MULTIPLE_HEAPS
19082 for (hn = 0; hn < gc_heap::n_heaps; hn++)
19084 gc_heap* hp2 = gc_heap::g_heaps [hn];
19085 total_heap_size += hp2->generation_size (max_generation + 1) + hp2->generation_sizes (hp2->generation_of (max_generation));
19088 total_heap_size = generation_size (max_generation + 1) + generation_sizes (generation_of (max_generation));
19089 #endif //MULTIPLE_HEAPS
19091 return total_heap_size;
19094 size_t gc_heap::get_total_fragmentation()
19096 size_t total_fragmentation = 0;
19098 #ifdef MULTIPLE_HEAPS
19099 for (int i = 0; i < gc_heap::n_heaps; i++)
19101 gc_heap* hp = gc_heap::g_heaps[i];
19102 #else //MULTIPLE_HEAPS
19104 gc_heap* hp = pGenGCHeap;
19105 #endif //MULTIPLE_HEAPS
19106 for (int i = 0; i <= (max_generation + 1); i++)
19108 generation* gen = hp->generation_of (i);
19109 total_fragmentation += (generation_free_list_space (gen) + generation_free_obj_space (gen));
19113 return total_fragmentation;
19116 size_t gc_heap::committed_size()
19118 generation* gen = generation_of (max_generation);
19119 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
19120 size_t total_committed = 0;
19124 total_committed += heap_segment_committed (seg) - (uint8_t*)seg;
19126 seg = heap_segment_next (seg);
19129 if (gen != large_object_generation)
19131 gen = generation_of (max_generation + 1);
19132 seg = generation_start_segment (gen);
19139 return total_committed;
19142 size_t gc_heap::get_total_committed_size()
19144 size_t total_committed = 0;
19146 #ifdef MULTIPLE_HEAPS
19149 for (hn = 0; hn < gc_heap::n_heaps; hn++)
19151 gc_heap* hp = gc_heap::g_heaps [hn];
19152 total_committed += hp->committed_size();
19155 total_committed = committed_size();
19156 #endif //MULTIPLE_HEAPS
19158 return total_committed;
19161 void gc_heap::get_memory_info (uint32_t* memory_load,
19162 uint64_t* available_physical,
19163 uint64_t* available_page_file)
19165 GCToOSInterface::GetMemoryStatus(memory_load, available_physical, available_page_file);
19168 void fire_mark_event (int heap_num, int root_type, size_t bytes_marked)
19170 dprintf (DT_LOG_0, ("-----------[%d]mark %d: %Id", heap_num, root_type, bytes_marked));
19171 FIRE_EVENT(GCMarkWithType, heap_num, root_type, bytes_marked);
19174 //returns TRUE is an overflow happened.
19175 BOOL gc_heap::process_mark_overflow(int condemned_gen_number)
19177 size_t last_promoted_bytes = promoted_bytes (heap_number);
19178 BOOL overflow_p = FALSE;
19180 if ((! (max_overflow_address == 0) ||
19181 ! (min_overflow_address == MAX_PTR)))
19184 // Try to grow the array.
19186 max (MARK_STACK_INITIAL_LENGTH, 2*mark_stack_array_length);
19188 if ((new_size * sizeof(mark)) > 100*1024)
19190 size_t new_max_size = (get_total_heap_size() / 10) / sizeof(mark);
19192 new_size = min(new_max_size, new_size);
19195 if ((mark_stack_array_length < new_size) &&
19196 ((new_size - mark_stack_array_length) > (mark_stack_array_length / 2)))
19198 mark* tmp = new (nothrow) mark [new_size];
19201 delete mark_stack_array;
19202 mark_stack_array = tmp;
19203 mark_stack_array_length = new_size;
19207 uint8_t* min_add = min_overflow_address;
19208 uint8_t* max_add = max_overflow_address;
19209 max_overflow_address = 0;
19210 min_overflow_address = MAX_PTR;
19211 process_mark_overflow_internal (condemned_gen_number, min_add, max_add);
19215 size_t current_promoted_bytes = promoted_bytes (heap_number);
19217 if (current_promoted_bytes != last_promoted_bytes)
19218 fire_mark_event (heap_number, ETW::GC_ROOT_OVERFLOW, (current_promoted_bytes - last_promoted_bytes));
19222 void gc_heap::process_mark_overflow_internal (int condemned_gen_number,
19223 uint8_t* min_add, uint8_t* max_add)
19225 #ifdef MULTIPLE_HEAPS
19226 int thread = heap_number;
19227 #endif //MULTIPLE_HEAPS
19228 BOOL full_p = (condemned_gen_number == max_generation);
19230 dprintf(3,("Processing Mark overflow [%Ix %Ix]", (size_t)min_add, (size_t)max_add));
19231 #ifdef MULTIPLE_HEAPS
19232 for (int hi = 0; hi < n_heaps; hi++)
19234 gc_heap* hp = g_heaps [(heap_number + hi) % n_heaps];
19240 #endif //MULTIPLE_HEAPS
19241 BOOL small_object_segments = TRUE;
19242 int align_const = get_alignment_constant (small_object_segments);
19243 generation* gen = hp->generation_of (condemned_gen_number);
19244 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
19246 PREFIX_ASSUME(seg != NULL);
19247 uint8_t* o = max (heap_segment_mem (seg), min_add);
19250 uint8_t* end = heap_segment_allocated (seg);
19252 while ((o < end) && (o <= max_add))
19254 assert ((min_add <= o) && (max_add >= o));
19255 dprintf (3, ("considering %Ix", (size_t)o));
19258 mark_through_object (o, TRUE THREAD_NUMBER_ARG);
19261 o = o + Align (size (o), align_const);
19264 if (( seg = heap_segment_next_in_range (seg)) == 0)
19266 if (small_object_segments && full_p)
19268 small_object_segments = FALSE;
19269 align_const = get_alignment_constant (small_object_segments);
19270 seg = heap_segment_in_range (generation_start_segment (hp->generation_of (max_generation+1)));
19272 PREFIX_ASSUME(seg != NULL);
19274 o = max (heap_segment_mem (seg), min_add);
19284 o = max (heap_segment_mem (seg), min_add);
19291 // Scanning for promotion for dependent handles need special handling. Because the primary holds a strong
19292 // reference to the secondary (when the primary itself is reachable) and this can cause a cascading series of
19293 // promotions (the secondary of one handle is or promotes the primary of another) we might need to perform the
19294 // promotion scan multiple times.
19295 // This helper encapsulates the logic to complete all dependent handle promotions when running a server GC. It
19296 // also has the effect of processing any mark stack overflow.
19298 #ifdef MULTIPLE_HEAPS
19299 // When multiple heaps are enabled we have must utilize a more complex algorithm in order to keep all the GC
19300 // worker threads synchronized. The algorithms are sufficiently divergent that we have different
19301 // implementations based on whether MULTIPLE_HEAPS is defined or not.
19303 // Define some static variables used for synchronization in the method below. These should really be defined
19304 // locally but MSVC complains when the VOLATILE macro is expanded into an instantiation of the Volatile class.
19306 // A note about the synchronization used within this method. Communication between the worker threads is
19307 // achieved via two shared booleans (defined below). These both act as latches that are transitioned only from
19308 // false -> true by unsynchronized code. They are only read or reset to false by a single thread under the
19309 // protection of a join.
19310 static VOLATILE(BOOL) s_fUnpromotedHandles = FALSE;
19311 static VOLATILE(BOOL) s_fUnscannedPromotions = FALSE;
19312 static VOLATILE(BOOL) s_fScanRequired;
19313 void gc_heap::scan_dependent_handles (int condemned_gen_number, ScanContext *sc, BOOL initial_scan_p)
19315 // Whenever we call this method there may have been preceding object promotions. So set
19316 // s_fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
19317 // based on the how the scanning proceeded).
19318 s_fUnscannedPromotions = TRUE;
19320 // We don't know how many times we need to loop yet. In particular we can't base the loop condition on
19321 // the state of this thread's portion of the dependent handle table. That's because promotions on other
19322 // threads could cause handle promotions to become necessary here. Even if there are definitely no more
19323 // promotions possible in this thread's handles, we still have to stay in lock-step with those worker
19324 // threads that haven't finished yet (each GC worker thread has to join exactly the same number of times
19325 // as all the others or they'll get out of step).
19328 // The various worker threads are all currently racing in this code. We need to work out if at least
19329 // one of them think they have work to do this cycle. Each thread needs to rescan its portion of the
19330 // dependent handle table when both of the following conditions apply:
19331 // 1) At least one (arbitrary) object might have been promoted since the last scan (because if this
19332 // object happens to correspond to a primary in one of our handles we might potentially have to
19333 // promote the associated secondary).
19334 // 2) The table for this thread has at least one handle with a secondary that isn't promoted yet.
19336 // The first condition is represented by s_fUnscannedPromotions. This is always non-zero for the first
19337 // iteration of this loop (see comment above) and in subsequent cycles each thread updates this
19338 // whenever a mark stack overflow occurs or scanning their dependent handles results in a secondary
19339 // being promoted. This value is cleared back to zero in a synchronized fashion in the join that
19340 // follows below. Note that we can't read this outside of the join since on any iteration apart from
19341 // the first threads will be racing between reading this value and completing their previous
19342 // iteration's table scan.
19344 // The second condition is tracked by the dependent handle code itself on a per worker thread basis
19345 // (and updated by the GcDhReScan() method). We call GcDhUnpromotedHandlesExist() on each thread to
19346 // determine the local value and collect the results into the s_fUnpromotedHandles variable in what is
19347 // effectively an OR operation. As per s_fUnscannedPromotions we can't read the final result until
19348 // we're safely joined.
19349 if (GCScan::GcDhUnpromotedHandlesExist(sc))
19350 s_fUnpromotedHandles = TRUE;
19352 // Synchronize all the threads so we can read our state variables safely. The shared variable
19353 // s_fScanRequired, indicating whether we should scan the tables or terminate the loop, will be set by
19354 // a single thread inside the join.
19355 gc_t_join.join(this, gc_join_scan_dependent_handles);
19356 if (gc_t_join.joined())
19358 // We're synchronized so it's safe to read our shared state variables. We update another shared
19359 // variable to indicate to all threads whether we'll be scanning for another cycle or terminating
19360 // the loop. We scan if there has been at least one object promotion since last time and at least
19361 // one thread has a dependent handle table with a potential handle promotion possible.
19362 s_fScanRequired = s_fUnscannedPromotions && s_fUnpromotedHandles;
19364 // Reset our shared state variables (ready to be set again on this scan or with a good initial
19365 // value for the next call if we're terminating the loop).
19366 s_fUnscannedPromotions = FALSE;
19367 s_fUnpromotedHandles = FALSE;
19369 if (!s_fScanRequired)
19371 // We're terminating the loop. Perform any last operations that require single threaded access.
19372 if (!initial_scan_p)
19374 // On the second invocation we reconcile all mark overflow ranges across the heaps. This can help
19375 // load balance if some of the heaps have an abnormally large workload.
19376 uint8_t* all_heaps_max = 0;
19377 uint8_t* all_heaps_min = MAX_PTR;
19379 for (i = 0; i < n_heaps; i++)
19381 if (all_heaps_max < g_heaps[i]->max_overflow_address)
19382 all_heaps_max = g_heaps[i]->max_overflow_address;
19383 if (all_heaps_min > g_heaps[i]->min_overflow_address)
19384 all_heaps_min = g_heaps[i]->min_overflow_address;
19386 for (i = 0; i < n_heaps; i++)
19388 g_heaps[i]->max_overflow_address = all_heaps_max;
19389 g_heaps[i]->min_overflow_address = all_heaps_min;
19394 // Restart all the workers.
19395 dprintf(3, ("Starting all gc thread mark stack overflow processing"));
19396 gc_t_join.restart();
19399 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
19400 // being visible. If there really was an overflow (process_mark_overflow returns true) then set the
19401 // global flag indicating that at least one object promotion may have occurred (the usual comment
19402 // about races applies). (Note it's OK to set this flag even if we're about to terminate the loop and
19403 // exit the method since we unconditionally set this variable on method entry anyway).
19404 if (process_mark_overflow(condemned_gen_number))
19405 s_fUnscannedPromotions = TRUE;
19407 // If we decided that no scan was required we can terminate the loop now.
19408 if (!s_fScanRequired)
19411 // Otherwise we must join with the other workers to ensure that all mark stack overflows have been
19412 // processed before we start scanning dependent handle tables (if overflows remain while we scan we
19413 // could miss noting the promotion of some primary objects).
19414 gc_t_join.join(this, gc_join_rescan_dependent_handles);
19415 if (gc_t_join.joined())
19417 // Restart all the workers.
19418 dprintf(3, ("Starting all gc thread for dependent handle promotion"));
19419 gc_t_join.restart();
19422 // If the portion of the dependent handle table managed by this worker has handles that could still be
19423 // promoted perform a rescan. If the rescan resulted in at least one promotion note this fact since it
19424 // could require a rescan of handles on this or other workers.
19425 if (GCScan::GcDhUnpromotedHandlesExist(sc))
19426 if (GCScan::GcDhReScan(sc))
19427 s_fUnscannedPromotions = TRUE;
19430 #else //MULTIPLE_HEAPS
19431 // Non-multiple heap version of scan_dependent_handles: much simpler without the need to keep multiple worker
19432 // threads synchronized.
19433 void gc_heap::scan_dependent_handles (int condemned_gen_number, ScanContext *sc, BOOL initial_scan_p)
19435 UNREFERENCED_PARAMETER(initial_scan_p);
19437 // Whenever we call this method there may have been preceding object promotions. So set
19438 // fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
19439 // based on the how the scanning proceeded).
19440 bool fUnscannedPromotions = true;
19442 // Loop until there are either no more dependent handles that can have their secondary promoted or we've
19443 // managed to perform a scan without promoting anything new.
19444 while (GCScan::GcDhUnpromotedHandlesExist(sc) && fUnscannedPromotions)
19446 // On each iteration of the loop start with the assumption that no further objects have been promoted.
19447 fUnscannedPromotions = false;
19449 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
19450 // being visible. If there was an overflow (process_mark_overflow returned true) then additional
19451 // objects now appear to be promoted and we should set the flag.
19452 if (process_mark_overflow(condemned_gen_number))
19453 fUnscannedPromotions = true;
19455 // Perform the scan and set the flag if any promotions resulted.
19456 if (GCScan::GcDhReScan(sc))
19457 fUnscannedPromotions = true;
19460 // Process any mark stack overflow that may have resulted from scanning handles (or if we didn't need to
19461 // scan any handles at all this is the processing of overflows that may have occurred prior to this method
19463 process_mark_overflow(condemned_gen_number);
19465 #endif //MULTIPLE_HEAPS
19467 void gc_heap::mark_phase (int condemned_gen_number, BOOL mark_only_p)
19469 assert (settings.concurrent == FALSE);
19472 sc.thread_number = heap_number;
19473 sc.promotion = TRUE;
19474 sc.concurrent = FALSE;
19476 dprintf(2,("---- Mark Phase condemning %d ----", condemned_gen_number));
19477 BOOL full_p = (condemned_gen_number == max_generation);
19482 start = GetCycleCount32();
19485 int gen_to_init = condemned_gen_number;
19486 if (condemned_gen_number == max_generation)
19488 gen_to_init = max_generation + 1;
19490 for (int gen_idx = 0; gen_idx <= gen_to_init; gen_idx++)
19492 dynamic_data* dd = dynamic_data_of (gen_idx);
19493 dd_begin_data_size (dd) = generation_size (gen_idx) -
19494 dd_fragmentation (dd) -
19495 Align (size (generation_allocation_start (generation_of (gen_idx))));
19496 dprintf (2, ("begin data size for gen%d is %Id", gen_idx, dd_begin_data_size (dd)));
19497 dd_survived_size (dd) = 0;
19498 dd_pinned_survived_size (dd) = 0;
19499 dd_artificial_pinned_survived_size (dd) = 0;
19500 dd_added_pinned_size (dd) = 0;
19502 dd_padding_size (dd) = 0;
19503 #endif //SHORT_PLUGS
19504 #if defined (RESPECT_LARGE_ALIGNMENT) || defined (FEATURE_STRUCTALIGN)
19505 dd_num_npinned_plugs (dd) = 0;
19506 #endif //RESPECT_LARGE_ALIGNMENT || FEATURE_STRUCTALIGN
19509 #ifdef FFIND_OBJECT
19510 if (gen0_must_clear_bricks > 0)
19511 gen0_must_clear_bricks--;
19512 #endif //FFIND_OBJECT
19514 size_t last_promoted_bytes = 0;
19516 promoted_bytes (heap_number) = 0;
19517 reset_mark_stack();
19520 memset (&snoop_stat, 0, sizeof(snoop_stat));
19521 snoop_stat.heap_index = heap_number;
19522 #endif //SNOOP_STATS
19527 //initialize the mark stack
19528 for (int i = 0; i < max_snoop_level; i++)
19530 ((uint8_t**)(mark_stack_array))[i] = 0;
19533 mark_stack_busy() = 1;
19535 #endif //MH_SC_MARK
19537 static uint32_t num_sizedrefs = 0;
19540 static BOOL do_mark_steal_p = FALSE;
19541 #endif //MH_SC_MARK
19543 #ifdef MULTIPLE_HEAPS
19544 gc_t_join.join(this, gc_join_begin_mark_phase);
19545 if (gc_t_join.joined())
19547 #endif //MULTIPLE_HEAPS
19549 num_sizedrefs = GCToEEInterface::GetTotalNumSizedRefHandles();
19551 #ifdef MULTIPLE_HEAPS
19556 size_t total_heap_size = get_total_heap_size();
19558 if (total_heap_size > (100 * 1024 * 1024))
19560 do_mark_steal_p = TRUE;
19564 do_mark_steal_p = FALSE;
19569 do_mark_steal_p = FALSE;
19571 #endif //MH_SC_MARK
19573 gc_t_join.restart();
19575 #endif //MULTIPLE_HEAPS
19580 //set up the mark lists from g_mark_list
19581 assert (g_mark_list);
19582 #ifdef MULTIPLE_HEAPS
19583 mark_list = &g_mark_list [heap_number*mark_list_size];
19585 mark_list = g_mark_list;
19586 #endif //MULTIPLE_HEAPS
19587 //dont use the mark list for full gc
19588 //because multiple segments are more complex to handle and the list
19589 //is likely to overflow
19590 if (condemned_gen_number != max_generation)
19591 mark_list_end = &mark_list [mark_list_size-1];
19593 mark_list_end = &mark_list [0];
19594 mark_list_index = &mark_list [0];
19597 shigh = (uint8_t*) 0;
19600 //%type% category = quote (mark);
19602 if ((condemned_gen_number == max_generation) && (num_sizedrefs > 0))
19604 GCScan::GcScanSizedRefs(GCHeap::Promote, condemned_gen_number, max_generation, &sc);
19605 fire_mark_event (heap_number, ETW::GC_ROOT_SIZEDREF, (promoted_bytes (heap_number) - last_promoted_bytes));
19606 last_promoted_bytes = promoted_bytes (heap_number);
19608 #ifdef MULTIPLE_HEAPS
19609 gc_t_join.join(this, gc_join_scan_sizedref_done);
19610 if (gc_t_join.joined())
19612 dprintf(3, ("Done with marking all sized refs. Starting all gc thread for marking other strong roots"));
19613 gc_t_join.restart();
19615 #endif //MULTIPLE_HEAPS
19618 dprintf(3,("Marking Roots"));
19620 GCScan::GcScanRoots(GCHeap::Promote,
19621 condemned_gen_number, max_generation,
19624 fire_mark_event (heap_number, ETW::GC_ROOT_STACK, (promoted_bytes (heap_number) - last_promoted_bytes));
19625 last_promoted_bytes = promoted_bytes (heap_number);
19627 #ifdef BACKGROUND_GC
19628 if (recursive_gc_sync::background_running_p())
19630 scan_background_roots (GCHeap::Promote, heap_number, &sc);
19632 #endif //BACKGROUND_GC
19634 #ifdef FEATURE_PREMORTEM_FINALIZATION
19635 dprintf(3, ("Marking finalization data"));
19636 finalize_queue->GcScanRoots(GCHeap::Promote, heap_number, 0);
19637 #endif // FEATURE_PREMORTEM_FINALIZATION
19639 fire_mark_event (heap_number, ETW::GC_ROOT_FQ, (promoted_bytes (heap_number) - last_promoted_bytes));
19640 last_promoted_bytes = promoted_bytes (heap_number);
19645 dprintf(3,("Marking handle table"));
19646 GCScan::GcScanHandles(GCHeap::Promote,
19647 condemned_gen_number, max_generation,
19649 fire_mark_event (heap_number, ETW::GC_ROOT_HANDLES, (promoted_bytes (heap_number) - last_promoted_bytes));
19650 last_promoted_bytes = promoted_bytes (heap_number);
19654 size_t promoted_before_cards = promoted_bytes (heap_number);
19657 dprintf (3, ("before cards: %Id", promoted_before_cards));
19661 #ifdef MULTIPLE_HEAPS
19662 if (gc_t_join.r_join(this, gc_r_join_update_card_bundle))
19664 #endif //MULTIPLE_HEAPS
19666 #ifndef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
19667 // If we are manually managing card bundles, every write to the card table should already be
19668 // accounted for in the card bundle table so there's nothing to update here.
19669 update_card_table_bundle();
19671 if (card_bundles_enabled())
19673 verify_card_bundles();
19676 #ifdef MULTIPLE_HEAPS
19677 gc_t_join.r_restart();
19679 #endif //MULTIPLE_HEAPS
19680 #endif //CARD_BUNDLE
19682 card_fn mark_object_fn = &gc_heap::mark_object_simple;
19683 #ifdef HEAP_ANALYZE
19684 heap_analyze_success = TRUE;
19685 if (heap_analyze_enabled)
19687 internal_root_array_index = 0;
19689 current_obj_size = 0;
19690 mark_object_fn = &gc_heap::ha_mark_object_simple;
19692 #endif //HEAP_ANALYZE
19694 dprintf(3,("Marking cross generation pointers"));
19695 mark_through_cards_for_segments (mark_object_fn, FALSE);
19697 dprintf(3,("Marking cross generation pointers for large objects"));
19698 mark_through_cards_for_large_objects (mark_object_fn, FALSE);
19700 dprintf (3, ("marked by cards: %Id",
19701 (promoted_bytes (heap_number) - promoted_before_cards)));
19702 fire_mark_event (heap_number, ETW::GC_ROOT_OLDER, (promoted_bytes (heap_number) - last_promoted_bytes));
19703 last_promoted_bytes = promoted_bytes (heap_number);
19708 if (do_mark_steal_p)
19712 #endif //MH_SC_MARK
19714 // Dependent handles need to be scanned with a special algorithm (see the header comment on
19715 // scan_dependent_handles for more detail). We perform an initial scan without synchronizing with other
19716 // worker threads or processing any mark stack overflow. This is not guaranteed to complete the operation
19717 // but in a common case (where there are no dependent handles that are due to be collected) it allows us
19718 // to optimize away further scans. The call to scan_dependent_handles is what will cycle through more
19719 // iterations if required and will also perform processing of any mark stack overflow once the dependent
19720 // handle table has been fully promoted.
19721 GCScan::GcDhInitialScan(GCHeap::Promote, condemned_gen_number, max_generation, &sc);
19722 scan_dependent_handles(condemned_gen_number, &sc, true);
19724 #ifdef MULTIPLE_HEAPS
19725 dprintf(3, ("Joining for short weak handle scan"));
19726 gc_t_join.join(this, gc_join_null_dead_short_weak);
19727 if (gc_t_join.joined())
19728 #endif //MULTIPLE_HEAPS
19730 #ifdef HEAP_ANALYZE
19731 heap_analyze_enabled = FALSE;
19732 DACNotifyGcMarkEnd(condemned_gen_number);
19733 #endif // HEAP_ANALYZE
19734 GCToEEInterface::AfterGcScanRoots (condemned_gen_number, max_generation, &sc);
19736 #ifdef MULTIPLE_HEAPS
19739 // we used r_join and need to reinitialize states for it here.
19740 gc_t_join.r_init();
19743 //start all threads on the roots.
19744 dprintf(3, ("Starting all gc thread for short weak handle scan"));
19745 gc_t_join.restart();
19746 #endif //MULTIPLE_HEAPS
19750 // null out the target of short weakref that were not promoted.
19751 GCScan::GcShortWeakPtrScan(GCHeap::Promote, condemned_gen_number, max_generation,&sc);
19753 // MTHTS: keep by single thread
19754 #ifdef MULTIPLE_HEAPS
19755 dprintf(3, ("Joining for finalization"));
19756 gc_t_join.join(this, gc_join_scan_finalization);
19757 if (gc_t_join.joined())
19758 #endif //MULTIPLE_HEAPS
19761 #ifdef MULTIPLE_HEAPS
19762 //start all threads on the roots.
19763 dprintf(3, ("Starting all gc thread for Finalization"));
19764 gc_t_join.restart();
19765 #endif //MULTIPLE_HEAPS
19768 //Handle finalization.
19769 size_t promoted_bytes_live = promoted_bytes (heap_number);
19771 #ifdef FEATURE_PREMORTEM_FINALIZATION
19772 dprintf (3, ("Finalize marking"));
19773 finalize_queue->ScanForFinalization (GCHeap::Promote, condemned_gen_number, mark_only_p, __this);
19775 GCToEEInterface::DiagWalkFReachableObjects(__this);
19776 #endif // FEATURE_PREMORTEM_FINALIZATION
19778 // Scan dependent handles again to promote any secondaries associated with primaries that were promoted
19779 // for finalization. As before scan_dependent_handles will also process any mark stack overflow.
19780 scan_dependent_handles(condemned_gen_number, &sc, false);
19782 #ifdef MULTIPLE_HEAPS
19783 dprintf(3, ("Joining for weak pointer deletion"));
19784 gc_t_join.join(this, gc_join_null_dead_long_weak);
19785 if (gc_t_join.joined())
19787 //start all threads on the roots.
19788 dprintf(3, ("Starting all gc thread for weak pointer deletion"));
19789 gc_t_join.restart();
19791 #endif //MULTIPLE_HEAPS
19793 // null out the target of long weakref that were not promoted.
19794 GCScan::GcWeakPtrScan (GCHeap::Promote, condemned_gen_number, max_generation, &sc);
19796 // MTHTS: keep by single thread
19797 #ifdef MULTIPLE_HEAPS
19799 #ifdef PARALLEL_MARK_LIST_SORT
19800 // unsigned long start = GetCycleCount32();
19802 // printf("sort_mark_list took %u cycles\n", GetCycleCount32() - start);
19803 #endif //PARALLEL_MARK_LIST_SORT
19806 dprintf (3, ("Joining for sync block cache entry scanning"));
19807 gc_t_join.join(this, gc_join_null_dead_syncblk);
19808 if (gc_t_join.joined())
19809 #endif //MULTIPLE_HEAPS
19811 // scan for deleted entries in the syncblk cache
19812 GCScan::GcWeakPtrScanBySingleThread (condemned_gen_number, max_generation, &sc);
19814 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
19817 size_t promoted_all_heaps = 0;
19818 #ifdef MULTIPLE_HEAPS
19819 for (int i = 0; i < n_heaps; i++)
19821 promoted_all_heaps += promoted_bytes (i);
19824 promoted_all_heaps = promoted_bytes (heap_number);
19825 #endif //MULTIPLE_HEAPS
19826 SystemDomain::RecordTotalSurvivedBytes (promoted_all_heaps);
19828 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
19830 #ifdef MULTIPLE_HEAPS
19833 #ifndef PARALLEL_MARK_LIST_SORT
19834 //compact g_mark_list and sort it.
19835 combine_mark_lists();
19836 #endif //PARALLEL_MARK_LIST_SORT
19839 //decide on promotion
19840 if (!settings.promotion)
19843 for (int n = 0; n <= condemned_gen_number;n++)
19845 m += (size_t)(dd_min_size (dynamic_data_of (n))*(n+1)*0.1);
19848 for (int i = 0; i < n_heaps; i++)
19850 dynamic_data* dd = g_heaps[i]->dynamic_data_of (min (condemned_gen_number +1,
19852 size_t older_gen_size = (dd_current_size (dd) +
19853 (dd_desired_allocation (dd) -
19854 dd_new_allocation (dd)));
19856 if ((m > (older_gen_size)) ||
19857 (promoted_bytes (i) > m))
19859 settings.promotion = TRUE;
19865 if (do_mark_steal_p)
19867 size_t objects_checked_count = 0;
19868 size_t zero_ref_count = 0;
19869 size_t objects_marked_count = 0;
19870 size_t check_level_count = 0;
19871 size_t busy_count = 0;
19872 size_t interlocked_count = 0;
19873 size_t partial_mark_parent_count = 0;
19874 size_t stolen_or_pm_count = 0;
19875 size_t stolen_entry_count = 0;
19876 size_t pm_not_ready_count = 0;
19877 size_t normal_count = 0;
19878 size_t stack_bottom_clear_count = 0;
19880 for (int i = 0; i < n_heaps; i++)
19882 gc_heap* hp = g_heaps[i];
19883 hp->print_snoop_stat();
19884 objects_checked_count += hp->snoop_stat.objects_checked_count;
19885 zero_ref_count += hp->snoop_stat.zero_ref_count;
19886 objects_marked_count += hp->snoop_stat.objects_marked_count;
19887 check_level_count += hp->snoop_stat.check_level_count;
19888 busy_count += hp->snoop_stat.busy_count;
19889 interlocked_count += hp->snoop_stat.interlocked_count;
19890 partial_mark_parent_count += hp->snoop_stat.partial_mark_parent_count;
19891 stolen_or_pm_count += hp->snoop_stat.stolen_or_pm_count;
19892 stolen_entry_count += hp->snoop_stat.stolen_entry_count;
19893 pm_not_ready_count += hp->snoop_stat.pm_not_ready_count;
19894 normal_count += hp->snoop_stat.normal_count;
19895 stack_bottom_clear_count += hp->snoop_stat.stack_bottom_clear_count;
19900 printf ("-------total stats-------\n");
19901 printf ("%8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s | %8s\n",
19902 "checked", "zero", "marked", "level", "busy", "xchg", "pmparent", "s_pm", "stolen", "nready", "normal", "clear");
19903 printf ("%8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d | %8d\n",
19904 objects_checked_count,
19906 objects_marked_count,
19910 partial_mark_parent_count,
19911 stolen_or_pm_count,
19912 stolen_entry_count,
19913 pm_not_ready_count,
19915 stack_bottom_clear_count);
19917 #endif //SNOOP_STATS
19919 //start all threads.
19920 dprintf(3, ("Starting all threads for end of mark phase"));
19921 gc_t_join.restart();
19922 #else //MULTIPLE_HEAPS
19924 //decide on promotion
19925 if (!settings.promotion)
19928 for (int n = 0; n <= condemned_gen_number;n++)
19930 m += (size_t)(dd_min_size (dynamic_data_of (n))*(n+1)*0.06);
19932 dynamic_data* dd = dynamic_data_of (min (condemned_gen_number +1,
19934 size_t older_gen_size = (dd_current_size (dd) +
19935 (dd_desired_allocation (dd) -
19936 dd_new_allocation (dd)));
19938 dprintf (2, ("promotion threshold: %Id, promoted bytes: %Id size n+1: %Id",
19939 m, promoted_bytes (heap_number), older_gen_size));
19941 if ((m > older_gen_size) ||
19942 (promoted_bytes (heap_number) > m))
19944 settings.promotion = TRUE;
19948 #endif //MULTIPLE_HEAPS
19951 #ifdef MULTIPLE_HEAPS
19953 #ifdef PARALLEL_MARK_LIST_SORT
19954 // start = GetCycleCount32();
19955 merge_mark_lists();
19956 // printf("merge_mark_lists took %u cycles\n", GetCycleCount32() - start);
19957 #endif //PARALLEL_MARK_LIST_SORT
19959 #endif //MULTIPLE_HEAPS
19961 #ifdef BACKGROUND_GC
19962 total_promoted_bytes = promoted_bytes (heap_number);
19963 #endif //BACKGROUND_GC
19965 promoted_bytes (heap_number) -= promoted_bytes_live;
19968 finish = GetCycleCount32();
19969 mark_time = finish - start;
19972 dprintf(2,("---- End of mark phase ----"));
19976 void gc_heap::pin_object (uint8_t* o, uint8_t** ppObject, uint8_t* low, uint8_t* high)
19978 dprintf (3, ("Pinning %Ix", (size_t)o));
19979 if ((o >= low) && (o < high))
19981 dprintf(3,("^%Ix^", (size_t)o));
19984 #ifdef FEATURE_EVENT_TRACE
19985 if(EVENT_ENABLED(PinObjectAtGCTime))
19987 fire_etw_pin_object_event(o, ppObject);
19989 #endif // FEATURE_EVENT_TRACE
19991 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
19992 num_pinned_objects++;
19993 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
19997 #if defined(ENABLE_PERF_COUNTERS) || defined(FEATURE_EVENT_TRACE)
19998 size_t gc_heap::get_total_pinned_objects()
20000 #ifdef MULTIPLE_HEAPS
20001 size_t total_num_pinned_objects = 0;
20002 for (int i = 0; i < gc_heap::n_heaps; i++)
20004 gc_heap* hp = gc_heap::g_heaps[i];
20005 total_num_pinned_objects += hp->num_pinned_objects;
20007 return total_num_pinned_objects;
20008 #else //MULTIPLE_HEAPS
20009 return num_pinned_objects;
20010 #endif //MULTIPLE_HEAPS
20012 #endif //ENABLE_PERF_COUNTERS || FEATURE_EVENT_TRACE
20014 void gc_heap::reset_mark_stack ()
20016 reset_pinned_queue();
20017 max_overflow_address = 0;
20018 min_overflow_address = MAX_PTR;
20021 #ifdef FEATURE_STRUCTALIGN
20023 // The word with left child, right child, and align info is laid out as follows:
20025 // | upper short word | lower short word |
20026 // |<------------> <----->|<------------> <----->|
20027 // | left child info hi| right child info lo|
20028 // x86: | 10 bits 6 bits| 10 bits 6 bits|
20030 // where left/right child are signed values and concat(info hi, info lo) is unsigned.
20032 // The "align info" encodes two numbers: the required alignment (a power of two)
20033 // and the misalignment (the number of machine words the destination address needs
20034 // to be adjusted by to provide alignment - so this number is always smaller than
20035 // the required alignment). Thus, the two can be represented as the "logical or"
20036 // of the two numbers. Note that the actual pad is computed from the misalignment
20037 // by adding the alignment iff the misalignment is non-zero and less than min_obj_size.
20040 // The number of bits in a brick.
20041 #if defined (_TARGET_AMD64_)
20042 #define brick_bits (12)
20044 #define brick_bits (11)
20045 #endif //_TARGET_AMD64_
20046 C_ASSERT(brick_size == (1 << brick_bits));
20048 // The number of bits needed to represent the offset to a child node.
20049 // "brick_bits + 1" allows us to represent a signed offset within a brick.
20050 #define child_bits (brick_bits + 1 - LOG2_PTRSIZE)
20052 // The number of bits in each of the pad hi, pad lo fields.
20053 #define pad_bits (sizeof(short) * 8 - child_bits)
20055 #define child_from_short(w) (((signed short)(w) / (1 << (pad_bits - LOG2_PTRSIZE))) & ~((1 << LOG2_PTRSIZE) - 1))
20056 #define pad_mask ((1 << pad_bits) - 1)
20057 #define pad_from_short(w) ((size_t)(w) & pad_mask)
20058 #else // FEATURE_STRUCTALIGN
20059 #define child_from_short(w) (w)
20060 #endif // FEATURE_STRUCTALIGN
20063 short node_left_child(uint8_t* node)
20065 return child_from_short(((plug_and_pair*)node)[-1].m_pair.left);
20069 void set_node_left_child(uint8_t* node, ptrdiff_t val)
20071 assert (val > -(ptrdiff_t)brick_size);
20072 assert (val < (ptrdiff_t)brick_size);
20073 assert (Aligned (val));
20074 #ifdef FEATURE_STRUCTALIGN
20075 size_t pad = pad_from_short(((plug_and_pair*)node)[-1].m_pair.left);
20076 ((plug_and_pair*)node)[-1].m_pair.left = ((short)val << (pad_bits - LOG2_PTRSIZE)) | (short)pad;
20077 #else // FEATURE_STRUCTALIGN
20078 ((plug_and_pair*)node)[-1].m_pair.left = (short)val;
20079 #endif // FEATURE_STRUCTALIGN
20080 assert (node_left_child (node) == val);
20084 short node_right_child(uint8_t* node)
20086 return child_from_short(((plug_and_pair*)node)[-1].m_pair.right);
20090 void set_node_right_child(uint8_t* node, ptrdiff_t val)
20092 assert (val > -(ptrdiff_t)brick_size);
20093 assert (val < (ptrdiff_t)brick_size);
20094 assert (Aligned (val));
20095 #ifdef FEATURE_STRUCTALIGN
20096 size_t pad = pad_from_short(((plug_and_pair*)node)[-1].m_pair.right);
20097 ((plug_and_pair*)node)[-1].m_pair.right = ((short)val << (pad_bits - LOG2_PTRSIZE)) | (short)pad;
20098 #else // FEATURE_STRUCTALIGN
20099 ((plug_and_pair*)node)[-1].m_pair.right = (short)val;
20100 #endif // FEATURE_STRUCTALIGN
20101 assert (node_right_child (node) == val);
20104 #ifdef FEATURE_STRUCTALIGN
20105 void node_aligninfo (uint8_t* node, int& requiredAlignment, ptrdiff_t& pad)
20107 // Extract the single-number aligninfo from the fields.
20108 short left = ((plug_and_pair*)node)[-1].m_pair.left;
20109 short right = ((plug_and_pair*)node)[-1].m_pair.right;
20110 ptrdiff_t pad_shifted = (pad_from_short(left) << pad_bits) | pad_from_short(right);
20111 ptrdiff_t aligninfo = pad_shifted * DATA_ALIGNMENT;
20113 // Replicate the topmost bit into all lower bits.
20114 ptrdiff_t x = aligninfo;
20120 // Clear all bits but the highest.
20121 requiredAlignment = (int)(x ^ (x >> 1));
20122 pad = aligninfo - requiredAlignment;
20123 pad += AdjustmentForMinPadSize(pad, requiredAlignment);
20127 ptrdiff_t node_alignpad (uint8_t* node)
20129 int requiredAlignment;
20130 ptrdiff_t alignpad;
20131 node_aligninfo (node, requiredAlignment, alignpad);
20135 void clear_node_aligninfo (uint8_t* node)
20137 ((plug_and_pair*)node)[-1].m_pair.left &= ~0 << pad_bits;
20138 ((plug_and_pair*)node)[-1].m_pair.right &= ~0 << pad_bits;
20141 void set_node_aligninfo (uint8_t* node, int requiredAlignment, ptrdiff_t pad)
20143 // Encode the alignment requirement and alignment offset as a single number
20144 // as described above.
20145 ptrdiff_t aligninfo = (size_t)requiredAlignment + (pad & (requiredAlignment-1));
20146 assert (Aligned (aligninfo));
20147 ptrdiff_t aligninfo_shifted = aligninfo / DATA_ALIGNMENT;
20148 assert (aligninfo_shifted < (1 << (pad_bits + pad_bits)));
20150 ptrdiff_t hi = aligninfo_shifted >> pad_bits;
20151 assert (pad_from_short(((plug_and_gap*)node)[-1].m_pair.left) == 0);
20152 ((plug_and_pair*)node)[-1].m_pair.left |= hi;
20154 ptrdiff_t lo = aligninfo_shifted & pad_mask;
20155 assert (pad_from_short(((plug_and_gap*)node)[-1].m_pair.right) == 0);
20156 ((plug_and_pair*)node)[-1].m_pair.right |= lo;
20159 int requiredAlignment2;
20161 node_aligninfo (node, requiredAlignment2, pad2);
20162 assert (requiredAlignment == requiredAlignment2);
20163 assert (pad == pad2);
20166 #endif // FEATURE_STRUCTALIGN
20169 void loh_set_node_relocation_distance(uint8_t* node, ptrdiff_t val)
20171 ptrdiff_t* place = &(((loh_obj_and_pad*)node)[-1].reloc);
20176 ptrdiff_t loh_node_relocation_distance(uint8_t* node)
20178 return (((loh_obj_and_pad*)node)[-1].reloc);
20182 ptrdiff_t node_relocation_distance (uint8_t* node)
20184 return (((plug_and_reloc*)(node))[-1].reloc & ~3);
20188 void set_node_relocation_distance(uint8_t* node, ptrdiff_t val)
20190 assert (val == (val & ~3));
20191 ptrdiff_t* place = &(((plug_and_reloc*)node)[-1].reloc);
20192 //clear the left bit and the relocation field
20198 #define node_left_p(node) (((plug_and_reloc*)(node))[-1].reloc & 2)
20200 #define set_node_left(node) ((plug_and_reloc*)(node))[-1].reloc |= 2;
20202 #ifndef FEATURE_STRUCTALIGN
20203 void set_node_realigned(uint8_t* node)
20205 ((plug_and_reloc*)(node))[-1].reloc |= 1;
20208 void clear_node_realigned(uint8_t* node)
20210 #ifdef RESPECT_LARGE_ALIGNMENT
20211 ((plug_and_reloc*)(node))[-1].reloc &= ~1;
20213 UNREFERENCED_PARAMETER(node);
20214 #endif //RESPECT_LARGE_ALIGNMENT
20216 #endif // FEATURE_STRUCTALIGN
20219 size_t node_gap_size (uint8_t* node)
20221 return ((plug_and_gap *)node)[-1].gap;
20224 void set_gap_size (uint8_t* node, size_t size)
20226 assert (Aligned (size));
20228 // clear the 2 uint32_t used by the node.
20229 ((plug_and_gap *)node)[-1].reloc = 0;
20230 ((plug_and_gap *)node)[-1].lr =0;
20231 ((plug_and_gap *)node)[-1].gap = size;
20233 assert ((size == 0 )||(size >= sizeof(plug_and_reloc)));
20237 uint8_t* gc_heap::insert_node (uint8_t* new_node, size_t sequence_number,
20238 uint8_t* tree, uint8_t* last_node)
20240 dprintf (3, ("IN: %Ix(%Ix), T: %Ix(%Ix), L: %Ix(%Ix) [%Ix]",
20241 (size_t)new_node, brick_of(new_node),
20242 (size_t)tree, brick_of(tree),
20243 (size_t)last_node, brick_of(last_node),
20245 if (power_of_two_p (sequence_number))
20247 set_node_left_child (new_node, (tree - new_node));
20248 dprintf (3, ("NT: %Ix, LC->%Ix", (size_t)new_node, (tree - new_node)));
20253 if (oddp (sequence_number))
20255 set_node_right_child (last_node, (new_node - last_node));
20256 dprintf (3, ("%Ix RC->%Ix", last_node, (new_node - last_node)));
20260 uint8_t* earlier_node = tree;
20261 size_t imax = logcount(sequence_number) - 2;
20262 for (size_t i = 0; i != imax; i++)
20264 earlier_node = earlier_node + node_right_child (earlier_node);
20266 int tmp_offset = node_right_child (earlier_node);
20267 assert (tmp_offset); // should never be empty
20268 set_node_left_child (new_node, ((earlier_node + tmp_offset ) - new_node));
20269 set_node_right_child (earlier_node, (new_node - earlier_node));
20271 dprintf (3, ("%Ix LC->%Ix, %Ix RC->%Ix",
20272 new_node, ((earlier_node + tmp_offset ) - new_node),
20273 earlier_node, (new_node - earlier_node)));
20279 size_t gc_heap::update_brick_table (uint8_t* tree, size_t current_brick,
20280 uint8_t* x, uint8_t* plug_end)
20282 dprintf (3, ("tree: %Ix, current b: %Ix, x: %Ix, plug_end: %Ix",
20283 tree, current_brick, x, plug_end));
20287 dprintf (3, ("b- %Ix->%Ix pointing to tree %Ix",
20288 current_brick, (size_t)(tree - brick_address (current_brick)), tree));
20289 set_brick (current_brick, (tree - brick_address (current_brick)));
20293 dprintf (3, ("b- %Ix->-1", current_brick));
20294 set_brick (current_brick, -1);
20296 size_t b = 1 + current_brick;
20297 ptrdiff_t offset = 0;
20298 size_t last_br = brick_of (plug_end-1);
20299 current_brick = brick_of (x-1);
20300 dprintf (3, ("ubt: %Ix->%Ix]->%Ix]", b, last_br, current_brick));
20301 while (b <= current_brick)
20305 set_brick (b, --offset);
20313 return brick_of (x);
20316 void gc_heap::plan_generation_start (generation* gen, generation* consing_gen, uint8_t* next_plug_to_allocate)
20319 // We should never demote big plugs to gen0.
20320 if (gen == youngest_generation)
20322 heap_segment* seg = ephemeral_heap_segment;
20323 size_t mark_stack_large_bos = mark_stack_bos;
20324 size_t large_plug_pos = 0;
20325 while (mark_stack_large_bos < mark_stack_tos)
20327 if (mark_stack_array[mark_stack_large_bos].len > demotion_plug_len_th)
20329 while (mark_stack_bos <= mark_stack_large_bos)
20331 size_t entry = deque_pinned_plug();
20332 size_t len = pinned_len (pinned_plug_of (entry));
20333 uint8_t* plug = pinned_plug (pinned_plug_of(entry));
20334 if (len > demotion_plug_len_th)
20336 dprintf (2, ("ps(%d): S %Ix (%Id)(%Ix)", gen->gen_num, plug, len, (plug+len)));
20338 pinned_len (pinned_plug_of (entry)) = plug - generation_allocation_pointer (consing_gen);
20339 assert(mark_stack_array[entry].len == 0 ||
20340 mark_stack_array[entry].len >= Align(min_obj_size));
20341 generation_allocation_pointer (consing_gen) = plug + len;
20342 generation_allocation_limit (consing_gen) = heap_segment_plan_allocated (seg);
20343 set_allocator_next_pin (consing_gen);
20347 mark_stack_large_bos++;
20352 generation_plan_allocation_start (gen) =
20353 allocate_in_condemned_generations (consing_gen, Align (min_obj_size), -1);
20354 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
20355 size_t allocation_left = (size_t)(generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
20356 if (next_plug_to_allocate)
20358 size_t dist_to_next_plug = (size_t)(next_plug_to_allocate - generation_allocation_pointer (consing_gen));
20359 if (allocation_left > dist_to_next_plug)
20361 allocation_left = dist_to_next_plug;
20364 if (allocation_left < Align (min_obj_size))
20366 generation_plan_allocation_start_size (gen) += allocation_left;
20367 generation_allocation_pointer (consing_gen) += allocation_left;
20370 dprintf (1, ("plan alloc gen%d(%Ix) start at %Ix (ptr: %Ix, limit: %Ix, next: %Ix)", gen->gen_num,
20371 generation_plan_allocation_start (gen),
20372 generation_plan_allocation_start_size (gen),
20373 generation_allocation_pointer (consing_gen), generation_allocation_limit (consing_gen),
20374 next_plug_to_allocate));
20377 void gc_heap::realloc_plan_generation_start (generation* gen, generation* consing_gen)
20379 BOOL adjacentp = FALSE;
20381 generation_plan_allocation_start (gen) =
20382 allocate_in_expanded_heap (consing_gen, Align(min_obj_size), adjacentp, 0,
20385 #endif //SHORT_PLUGS
20386 FALSE, -1 REQD_ALIGN_AND_OFFSET_ARG);
20388 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
20389 size_t allocation_left = (size_t)(generation_allocation_limit (consing_gen) - generation_allocation_pointer (consing_gen));
20390 if ((allocation_left < Align (min_obj_size)) &&
20391 (generation_allocation_limit (consing_gen)!=heap_segment_plan_allocated (generation_allocation_segment (consing_gen))))
20393 generation_plan_allocation_start_size (gen) += allocation_left;
20394 generation_allocation_pointer (consing_gen) += allocation_left;
20397 dprintf (1, ("plan re-alloc gen%d start at %Ix (ptr: %Ix, limit: %Ix)", gen->gen_num,
20398 generation_plan_allocation_start (consing_gen),
20399 generation_allocation_pointer (consing_gen),
20400 generation_allocation_limit (consing_gen)));
20403 void gc_heap::plan_generation_starts (generation*& consing_gen)
20405 //make sure that every generation has a planned allocation start
20406 int gen_number = settings.condemned_generation;
20407 while (gen_number >= 0)
20409 if (gen_number < max_generation)
20411 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
20413 generation* gen = generation_of (gen_number);
20414 if (0 == generation_plan_allocation_start (gen))
20416 plan_generation_start (gen, consing_gen, 0);
20417 assert (generation_plan_allocation_start (gen));
20421 // now we know the planned allocation size
20422 heap_segment_plan_allocated (ephemeral_heap_segment) =
20423 generation_allocation_pointer (consing_gen);
20426 void gc_heap::advance_pins_for_demotion (generation* gen)
20428 uint8_t* original_youngest_start = generation_allocation_start (youngest_generation);
20429 heap_segment* seg = ephemeral_heap_segment;
20431 if ((!(pinned_plug_que_empty_p())))
20433 size_t gen1_pinned_promoted = generation_pinned_allocation_compact_size (generation_of (max_generation));
20434 size_t gen1_pins_left = dd_pinned_survived_size (dynamic_data_of (max_generation - 1)) - gen1_pinned_promoted;
20435 size_t total_space_to_skip = last_gen1_pin_end - generation_allocation_pointer (gen);
20436 float pin_frag_ratio = (float)gen1_pins_left / (float)total_space_to_skip;
20437 float pin_surv_ratio = (float)gen1_pins_left / (float)(dd_survived_size (dynamic_data_of (max_generation - 1)));
20438 if ((pin_frag_ratio > 0.15) && (pin_surv_ratio > 0.30))
20440 while (!pinned_plug_que_empty_p() &&
20441 (pinned_plug (oldest_pin()) < original_youngest_start))
20443 size_t entry = deque_pinned_plug();
20444 size_t len = pinned_len (pinned_plug_of (entry));
20445 uint8_t* plug = pinned_plug (pinned_plug_of(entry));
20446 pinned_len (pinned_plug_of (entry)) = plug - generation_allocation_pointer (gen);
20447 assert(mark_stack_array[entry].len == 0 ||
20448 mark_stack_array[entry].len >= Align(min_obj_size));
20449 generation_allocation_pointer (gen) = plug + len;
20450 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20451 set_allocator_next_pin (gen);
20453 //Add the size of the pinned plug to the right pinned allocations
20454 //find out which gen this pinned plug came from
20455 int frgn = object_gennum (plug);
20456 if ((frgn != (int)max_generation) && settings.promotion)
20458 int togn = object_gennum_plan (plug);
20459 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
20462 generation_pinned_allocation_compact_size (generation_of (togn)) += len;
20466 dprintf (2, ("skipping gap %d, pin %Ix (%Id)",
20467 pinned_len (pinned_plug_of (entry)), plug, len));
20470 dprintf (2, ("ad_p_d: PL: %Id, SL: %Id, pfr: %d, psr: %d",
20471 gen1_pins_left, total_space_to_skip, (int)(pin_frag_ratio*100), (int)(pin_surv_ratio*100)));
20475 void gc_heap::process_ephemeral_boundaries (uint8_t* x,
20476 int& active_new_gen_number,
20477 int& active_old_gen_number,
20478 generation*& consing_gen,
20479 BOOL& allocate_in_condemned)
20482 if ((active_old_gen_number > 0) &&
20483 (x >= generation_allocation_start (generation_of (active_old_gen_number - 1))))
20485 dprintf (1, ("crossing gen%d, x is %Ix", active_old_gen_number - 1, x));
20487 if (!pinned_plug_que_empty_p())
20489 dprintf (1, ("oldest pin: %Ix(%Id)",
20490 pinned_plug (oldest_pin()),
20491 (x - pinned_plug (oldest_pin()))));
20494 if (active_old_gen_number <= (settings.promotion ? (max_generation - 1) : max_generation))
20496 active_new_gen_number--;
20499 active_old_gen_number--;
20500 assert ((!settings.promotion) || (active_new_gen_number>0));
20502 if (active_new_gen_number == (max_generation - 1))
20504 #ifdef FREE_USAGE_STATS
20505 if (settings.condemned_generation == max_generation)
20507 // We need to do this before we skip the rest of the pinned plugs.
20508 generation* gen_2 = generation_of (max_generation);
20509 generation* gen_1 = generation_of (max_generation - 1);
20511 size_t total_num_pinned_free_spaces_left = 0;
20513 // We are about to allocate gen1, check to see how efficient fitting in gen2 pinned free spaces is.
20514 for (int j = 0; j < NUM_GEN_POWER2; j++)
20516 dprintf (1, ("[h%d][#%Id]2^%d: current: %Id, S: 2: %Id, 1: %Id(%Id)",
20520 gen_2->gen_current_pinned_free_spaces[j],
20521 gen_2->gen_plugs[j], gen_1->gen_plugs[j],
20522 (gen_2->gen_plugs[j] + gen_1->gen_plugs[j])));
20524 total_num_pinned_free_spaces_left += gen_2->gen_current_pinned_free_spaces[j];
20527 float pinned_free_list_efficiency = 0;
20528 size_t total_pinned_free_space = generation_allocated_in_pinned_free (gen_2) + generation_pinned_free_obj_space (gen_2);
20529 if (total_pinned_free_space != 0)
20531 pinned_free_list_efficiency = (float)(generation_allocated_in_pinned_free (gen_2)) / (float)total_pinned_free_space;
20534 dprintf (1, ("[h%d] gen2 allocated %Id bytes with %Id bytes pinned free spaces (effi: %d%%), %Id (%Id) left",
20536 generation_allocated_in_pinned_free (gen_2),
20537 total_pinned_free_space,
20538 (int)(pinned_free_list_efficiency * 100),
20539 generation_pinned_free_obj_space (gen_2),
20540 total_num_pinned_free_spaces_left));
20542 #endif //FREE_USAGE_STATS
20544 //Go past all of the pinned plugs for this generation.
20545 while (!pinned_plug_que_empty_p() &&
20546 (!in_range_for_segment ((pinned_plug (oldest_pin())), ephemeral_heap_segment)))
20548 size_t entry = deque_pinned_plug();
20549 mark* m = pinned_plug_of (entry);
20550 uint8_t* plug = pinned_plug (m);
20551 size_t len = pinned_len (m);
20552 // detect pinned block in different segment (later) than
20553 // allocation segment, skip those until the oldest pin is in the ephemeral seg.
20554 // adjust the allocation segment along the way (at the end it will
20555 // be the ephemeral segment.
20556 heap_segment* nseg = heap_segment_in_range (generation_allocation_segment (consing_gen));
20558 PREFIX_ASSUME(nseg != NULL);
20560 while (!((plug >= generation_allocation_pointer (consing_gen))&&
20561 (plug < heap_segment_allocated (nseg))))
20563 //adjust the end of the segment to be the end of the plug
20564 assert (generation_allocation_pointer (consing_gen)>=
20565 heap_segment_mem (nseg));
20566 assert (generation_allocation_pointer (consing_gen)<=
20567 heap_segment_committed (nseg));
20569 heap_segment_plan_allocated (nseg) =
20570 generation_allocation_pointer (consing_gen);
20571 //switch allocation segment
20572 nseg = heap_segment_next_rw (nseg);
20573 generation_allocation_segment (consing_gen) = nseg;
20574 //reset the allocation pointer and limits
20575 generation_allocation_pointer (consing_gen) =
20576 heap_segment_mem (nseg);
20578 set_new_pin_info (m, generation_allocation_pointer (consing_gen));
20579 assert(pinned_len(m) == 0 || pinned_len(m) >= Align(min_obj_size));
20580 generation_allocation_pointer (consing_gen) = plug + len;
20581 generation_allocation_limit (consing_gen) =
20582 generation_allocation_pointer (consing_gen);
20584 allocate_in_condemned = TRUE;
20585 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
20588 if (active_new_gen_number != max_generation)
20590 if (active_new_gen_number == (max_generation - 1))
20592 maxgen_pinned_compact_before_advance = generation_pinned_allocation_compact_size (generation_of (max_generation));
20593 if (!demote_gen1_p)
20594 advance_pins_for_demotion (consing_gen);
20597 plan_generation_start (generation_of (active_new_gen_number), consing_gen, x);
20599 dprintf (1, ("process eph: allocated gen%d start at %Ix",
20600 active_new_gen_number,
20601 generation_plan_allocation_start (generation_of (active_new_gen_number))));
20603 if ((demotion_low == MAX_PTR) && !pinned_plug_que_empty_p())
20605 uint8_t* pplug = pinned_plug (oldest_pin());
20606 if (object_gennum (pplug) > 0)
20608 demotion_low = pplug;
20609 dprintf (3, ("process eph: dlow->%Ix", demotion_low));
20613 assert (generation_plan_allocation_start (generation_of (active_new_gen_number)));
20621 void gc_heap::seg_clear_mark_bits (heap_segment* seg)
20623 uint8_t* o = heap_segment_mem (seg);
20624 while (o < heap_segment_allocated (seg))
20630 o = o + Align (size (o));
20634 #ifdef FEATURE_BASICFREEZE
20635 void gc_heap::sweep_ro_segments (heap_segment* start_seg)
20637 //go through all of the segment in range and reset the mark bit
20638 //TODO works only on small object segments
20640 heap_segment* seg = start_seg;
20644 if (heap_segment_read_only_p (seg) &&
20645 heap_segment_in_range_p (seg))
20647 #ifdef BACKGROUND_GC
20648 if (settings.concurrent)
20650 seg_clear_mark_array_bits_soh (seg);
20654 seg_clear_mark_bits (seg);
20656 #else //BACKGROUND_GC
20659 if(gc_can_use_concurrent)
20661 clear_mark_array (max (heap_segment_mem (seg), lowest_address),
20662 min (heap_segment_allocated (seg), highest_address),
20663 FALSE); // read_only segments need the mark clear
20666 seg_clear_mark_bits (seg);
20667 #endif //MARK_ARRAY
20669 #endif //BACKGROUND_GC
20671 seg = heap_segment_next (seg);
20674 #endif // FEATURE_BASICFREEZE
20676 #ifdef FEATURE_LOH_COMPACTION
20678 BOOL gc_heap::loh_pinned_plug_que_empty_p()
20680 return (loh_pinned_queue_bos == loh_pinned_queue_tos);
20683 void gc_heap::loh_set_allocator_next_pin()
20685 if (!(loh_pinned_plug_que_empty_p()))
20687 mark* oldest_entry = loh_oldest_pin();
20688 uint8_t* plug = pinned_plug (oldest_entry);
20689 generation* gen = large_object_generation;
20690 if ((plug >= generation_allocation_pointer (gen)) &&
20691 (plug < generation_allocation_limit (gen)))
20693 generation_allocation_limit (gen) = pinned_plug (oldest_entry);
20696 assert (!((plug < generation_allocation_pointer (gen)) &&
20697 (plug >= heap_segment_mem (generation_allocation_segment (gen)))));
20701 size_t gc_heap::loh_deque_pinned_plug ()
20703 size_t m = loh_pinned_queue_bos;
20704 loh_pinned_queue_bos++;
20709 mark* gc_heap::loh_pinned_plug_of (size_t bos)
20711 return &loh_pinned_queue[bos];
20715 mark* gc_heap::loh_oldest_pin()
20717 return loh_pinned_plug_of (loh_pinned_queue_bos);
20720 // If we can't grow the queue, then don't compact.
20721 BOOL gc_heap::loh_enque_pinned_plug (uint8_t* plug, size_t len)
20723 assert(len >= Align(min_obj_size, get_alignment_constant (FALSE)));
20725 if (loh_pinned_queue_length <= loh_pinned_queue_tos)
20727 if (!grow_mark_stack (loh_pinned_queue, loh_pinned_queue_length, LOH_PIN_QUEUE_LENGTH))
20732 dprintf (3, (" P: %Ix(%Id)", plug, len));
20733 mark& m = loh_pinned_queue[loh_pinned_queue_tos];
20736 loh_pinned_queue_tos++;
20737 loh_set_allocator_next_pin();
20742 BOOL gc_heap::loh_size_fit_p (size_t size, uint8_t* alloc_pointer, uint8_t* alloc_limit)
20744 dprintf (1235, ("trying to fit %Id(%Id) between %Ix and %Ix (%Id)",
20746 (2* AlignQword (loh_padding_obj_size) + size),
20749 (alloc_limit - alloc_pointer)));
20751 return ((alloc_pointer + 2* AlignQword (loh_padding_obj_size) + size) <= alloc_limit);
20754 uint8_t* gc_heap::loh_allocate_in_condemned (uint8_t* old_loc, size_t size)
20756 UNREFERENCED_PARAMETER(old_loc);
20758 generation* gen = large_object_generation;
20759 dprintf (1235, ("E: p:%Ix, l:%Ix, s: %Id",
20760 generation_allocation_pointer (gen),
20761 generation_allocation_limit (gen),
20766 heap_segment* seg = generation_allocation_segment (gen);
20767 if (!(loh_size_fit_p (size, generation_allocation_pointer (gen), generation_allocation_limit (gen))))
20769 if ((!(loh_pinned_plug_que_empty_p()) &&
20770 (generation_allocation_limit (gen) ==
20771 pinned_plug (loh_oldest_pin()))))
20773 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
20774 size_t len = pinned_len (m);
20775 uint8_t* plug = pinned_plug (m);
20776 dprintf (1235, ("AIC: %Ix->%Ix(%Id)", generation_allocation_pointer (gen), plug, plug - generation_allocation_pointer (gen)));
20777 pinned_len (m) = plug - generation_allocation_pointer (gen);
20778 generation_allocation_pointer (gen) = plug + len;
20780 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20781 loh_set_allocator_next_pin();
20782 dprintf (1235, ("s: p: %Ix, l: %Ix (%Id)",
20783 generation_allocation_pointer (gen),
20784 generation_allocation_limit (gen),
20785 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20790 if (generation_allocation_limit (gen) != heap_segment_plan_allocated (seg))
20792 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20793 dprintf (1235, ("l->pa(%Ix)", generation_allocation_limit (gen)));
20797 if (heap_segment_plan_allocated (seg) != heap_segment_committed (seg))
20799 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
20800 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20801 dprintf (1235, ("l->c(%Ix)", generation_allocation_limit (gen)));
20805 if (loh_size_fit_p (size, generation_allocation_pointer (gen), heap_segment_reserved (seg)) &&
20806 (grow_heap_segment (seg, (generation_allocation_pointer (gen) + size + 2* AlignQword (loh_padding_obj_size)))))
20808 dprintf (1235, ("growing seg from %Ix to %Ix\n", heap_segment_committed (seg),
20809 (generation_allocation_pointer (gen) + size)));
20811 heap_segment_plan_allocated (seg) = heap_segment_committed (seg);
20812 generation_allocation_limit (gen) = heap_segment_plan_allocated (seg);
20814 dprintf (1235, ("g: p: %Ix, l: %Ix (%Id)",
20815 generation_allocation_pointer (gen),
20816 generation_allocation_limit (gen),
20817 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20821 heap_segment* next_seg = heap_segment_next (seg);
20822 assert (generation_allocation_pointer (gen)>=
20823 heap_segment_mem (seg));
20824 // Verify that all pinned plugs for this segment are consumed
20825 if (!loh_pinned_plug_que_empty_p() &&
20826 ((pinned_plug (loh_oldest_pin()) <
20827 heap_segment_allocated (seg)) &&
20828 (pinned_plug (loh_oldest_pin()) >=
20829 generation_allocation_pointer (gen))))
20831 LOG((LF_GC, LL_INFO10, "remaining pinned plug %Ix while leaving segment on allocation",
20832 pinned_plug (loh_oldest_pin())));
20833 dprintf (1236, ("queue empty: %d", loh_pinned_plug_que_empty_p()));
20836 assert (generation_allocation_pointer (gen)>=
20837 heap_segment_mem (seg));
20838 assert (generation_allocation_pointer (gen)<=
20839 heap_segment_committed (seg));
20840 heap_segment_plan_allocated (seg) = generation_allocation_pointer (gen);
20844 // for LOH do we want to try starting from the first LOH every time though?
20845 generation_allocation_segment (gen) = next_seg;
20846 generation_allocation_pointer (gen) = heap_segment_mem (next_seg);
20847 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
20849 dprintf (1235, ("n: p: %Ix, l: %Ix (%Id)",
20850 generation_allocation_pointer (gen),
20851 generation_allocation_limit (gen),
20852 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20856 dprintf (1, ("We ran out of space compacting, shouldn't happen"));
20862 loh_set_allocator_next_pin();
20864 dprintf (1235, ("r: p: %Ix, l: %Ix (%Id)",
20865 generation_allocation_pointer (gen),
20866 generation_allocation_limit (gen),
20867 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20874 assert (generation_allocation_pointer (gen)>=
20875 heap_segment_mem (generation_allocation_segment (gen)));
20876 uint8_t* result = generation_allocation_pointer (gen);
20877 size_t loh_pad = AlignQword (loh_padding_obj_size);
20879 generation_allocation_pointer (gen) += size + loh_pad;
20880 assert (generation_allocation_pointer (gen) <= generation_allocation_limit (gen));
20882 dprintf (1235, ("p: %Ix, l: %Ix (%Id)",
20883 generation_allocation_pointer (gen),
20884 generation_allocation_limit (gen),
20885 (generation_allocation_limit (gen) - generation_allocation_pointer (gen))));
20887 assert (result + loh_pad);
20888 return result + loh_pad;
20892 BOOL gc_heap::should_compact_loh()
20894 return (loh_compaction_always_p || (loh_compaction_mode != loh_compaction_default));
20898 void gc_heap::check_loh_compact_mode (BOOL all_heaps_compacted_p)
20900 if (settings.loh_compaction && (loh_compaction_mode == loh_compaction_once))
20902 if (all_heaps_compacted_p)
20904 // If the compaction mode says to compact once and we are going to compact LOH,
20905 // we need to revert it back to no compaction.
20906 loh_compaction_mode = loh_compaction_default;
20911 BOOL gc_heap::plan_loh()
20913 if (!loh_pinned_queue)
20915 loh_pinned_queue = new (nothrow) (mark [LOH_PIN_QUEUE_LENGTH]);
20916 if (!loh_pinned_queue)
20918 dprintf (1, ("Cannot allocate the LOH pinned queue (%Id bytes), no compaction",
20919 LOH_PIN_QUEUE_LENGTH * sizeof (mark)));
20923 loh_pinned_queue_length = LOH_PIN_QUEUE_LENGTH;
20926 if (heap_number == 0)
20927 loh_pinned_queue_decay = LOH_PIN_DECAY;
20929 loh_pinned_queue_tos = 0;
20930 loh_pinned_queue_bos = 0;
20932 generation* gen = large_object_generation;
20933 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
20934 PREFIX_ASSUME(start_seg != NULL);
20935 heap_segment* seg = start_seg;
20936 uint8_t* o = generation_allocation_start (gen);
20938 dprintf (1235, ("before GC LOH size: %Id, free list: %Id, free obj: %Id\n",
20939 generation_size (max_generation + 1),
20940 generation_free_list_space (gen),
20941 generation_free_obj_space (gen)));
20945 heap_segment_plan_allocated (seg) = heap_segment_mem (seg);
20946 seg = heap_segment_next (seg);
20951 //Skip the generation gap object
20952 o = o + AlignQword (size (o));
20953 // We don't need to ever realloc gen3 start so don't touch it.
20954 heap_segment_plan_allocated (seg) = o;
20955 generation_allocation_pointer (gen) = o;
20956 generation_allocation_limit (gen) = generation_allocation_pointer (gen);
20957 generation_allocation_segment (gen) = start_seg;
20959 uint8_t* free_space_start = o;
20960 uint8_t* free_space_end = o;
20961 uint8_t* new_address = 0;
20965 if (o >= heap_segment_allocated (seg))
20967 seg = heap_segment_next (seg);
20973 o = heap_segment_mem (seg);
20978 free_space_end = o;
20979 size_t size = AlignQword (size (o));
20980 dprintf (1235, ("%Ix(%Id) M", o, size));
20984 // We don't clear the pinned bit yet so we can check in
20985 // compact phase how big a free object we should allocate
20986 // in front of the pinned object. We use the reloc address
20987 // field to store this.
20988 if (!loh_enque_pinned_plug (o, size))
20996 new_address = loh_allocate_in_condemned (o, size);
20999 loh_set_node_relocation_distance (o, (new_address - o));
21000 dprintf (1235, ("lobj %Ix-%Ix -> %Ix-%Ix (%Id)", o, (o + size), new_address, (new_address + size), (new_address - o)));
21003 free_space_start = o;
21004 if (o < heap_segment_allocated (seg))
21006 assert (!marked (o));
21011 while (o < heap_segment_allocated (seg) && !marked (o))
21013 dprintf (1235, ("%Ix(%Id) F (%d)", o, AlignQword (size (o)), ((method_table (o) == g_gc_pFreeObjectMethodTable) ? 1 : 0)));
21014 o = o + AlignQword (size (o));
21019 while (!loh_pinned_plug_que_empty_p())
21021 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
21022 size_t len = pinned_len (m);
21023 uint8_t* plug = pinned_plug (m);
21025 // detect pinned block in different segment (later) than
21026 // allocation segment
21027 heap_segment* nseg = heap_segment_rw (generation_allocation_segment (gen));
21029 while ((plug < generation_allocation_pointer (gen)) ||
21030 (plug >= heap_segment_allocated (nseg)))
21032 assert ((plug < heap_segment_mem (nseg)) ||
21033 (plug > heap_segment_reserved (nseg)));
21034 //adjust the end of the segment to be the end of the plug
21035 assert (generation_allocation_pointer (gen)>=
21036 heap_segment_mem (nseg));
21037 assert (generation_allocation_pointer (gen)<=
21038 heap_segment_committed (nseg));
21040 heap_segment_plan_allocated (nseg) =
21041 generation_allocation_pointer (gen);
21042 //switch allocation segment
21043 nseg = heap_segment_next_rw (nseg);
21044 generation_allocation_segment (gen) = nseg;
21045 //reset the allocation pointer and limits
21046 generation_allocation_pointer (gen) =
21047 heap_segment_mem (nseg);
21050 dprintf (1235, ("SP: %Ix->%Ix(%Id)", generation_allocation_pointer (gen), plug, plug - generation_allocation_pointer (gen)));
21051 pinned_len (m) = plug - generation_allocation_pointer (gen);
21052 generation_allocation_pointer (gen) = plug + len;
21055 heap_segment_plan_allocated (generation_allocation_segment (gen)) = generation_allocation_pointer (gen);
21056 generation_allocation_pointer (gen) = 0;
21057 generation_allocation_limit (gen) = 0;
21062 void gc_heap::compact_loh()
21064 assert (should_compact_loh());
21066 generation* gen = large_object_generation;
21067 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
21068 PREFIX_ASSUME(start_seg != NULL);
21069 heap_segment* seg = start_seg;
21070 heap_segment* prev_seg = 0;
21071 uint8_t* o = generation_allocation_start (gen);
21073 //Skip the generation gap object
21074 o = o + AlignQword (size (o));
21075 // We don't need to ever realloc gen3 start so don't touch it.
21076 uint8_t* free_space_start = o;
21077 uint8_t* free_space_end = o;
21078 generation_allocator (gen)->clear();
21079 generation_free_list_space (gen) = 0;
21080 generation_free_obj_space (gen) = 0;
21082 loh_pinned_queue_bos = 0;
21086 if (o >= heap_segment_allocated (seg))
21088 heap_segment* next_seg = heap_segment_next (seg);
21090 if ((heap_segment_plan_allocated (seg) == heap_segment_mem (seg)) &&
21091 (seg != start_seg) && !heap_segment_read_only_p (seg))
21093 dprintf (3, ("Preparing empty large segment %Ix", (size_t)seg));
21095 heap_segment_next (prev_seg) = next_seg;
21096 heap_segment_next (seg) = freeable_large_heap_segment;
21097 freeable_large_heap_segment = seg;
21101 if (!heap_segment_read_only_p (seg))
21103 // We grew the segment to accommodate allocations.
21104 if (heap_segment_plan_allocated (seg) > heap_segment_allocated (seg))
21106 if ((heap_segment_plan_allocated (seg) - plug_skew) > heap_segment_used (seg))
21108 heap_segment_used (seg) = heap_segment_plan_allocated (seg) - plug_skew;
21112 heap_segment_allocated (seg) = heap_segment_plan_allocated (seg);
21113 dprintf (3, ("Trimming seg to %Ix[", heap_segment_allocated (seg)));
21114 decommit_heap_segment_pages (seg, 0);
21115 dprintf (1236, ("CLOH: seg: %Ix, alloc: %Ix, used: %Ix, committed: %Ix",
21117 heap_segment_allocated (seg),
21118 heap_segment_used (seg),
21119 heap_segment_committed (seg)));
21120 //heap_segment_used (seg) = heap_segment_allocated (seg) - plug_skew;
21121 dprintf (1236, ("CLOH: used is set to %Ix", heap_segment_used (seg)));
21131 o = heap_segment_mem (seg);
21137 free_space_end = o;
21138 size_t size = AlignQword (size (o));
21141 uint8_t* reloc = o;
21146 // We are relying on the fact the pinned objects are always looked at in the same order
21147 // in plan phase and in compact phase.
21148 mark* m = loh_pinned_plug_of (loh_deque_pinned_plug());
21149 uint8_t* plug = pinned_plug (m);
21150 assert (plug == o);
21152 loh_pad = pinned_len (m);
21157 loh_pad = AlignQword (loh_padding_obj_size);
21159 reloc += loh_node_relocation_distance (o);
21160 gcmemcopy (reloc, o, size, TRUE);
21163 thread_gap ((reloc - loh_pad), loh_pad, gen);
21166 free_space_start = o;
21167 if (o < heap_segment_allocated (seg))
21169 assert (!marked (o));
21174 while (o < heap_segment_allocated (seg) && !marked (o))
21176 o = o + AlignQword (size (o));
21181 assert (loh_pinned_plug_que_empty_p());
21183 dprintf (1235, ("after GC LOH size: %Id, free list: %Id, free obj: %Id\n\n",
21184 generation_size (max_generation + 1),
21185 generation_free_list_space (gen),
21186 generation_free_obj_space (gen)));
21189 void gc_heap::relocate_in_loh_compact()
21191 generation* gen = large_object_generation;
21192 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
21193 uint8_t* o = generation_allocation_start (gen);
21195 //Skip the generation gap object
21196 o = o + AlignQword (size (o));
21198 relocate_args args;
21200 args.high = gc_high;
21201 args.last_plug = 0;
21205 if (o >= heap_segment_allocated (seg))
21207 seg = heap_segment_next (seg);
21213 o = heap_segment_mem (seg);
21218 size_t size = AlignQword (size (o));
21220 check_class_object_demotion (o);
21221 if (contain_pointers (o))
21223 go_through_object_nostart (method_table (o), o, size(o), pval,
21225 reloc_survivor_helper (pval);
21230 if (o < heap_segment_allocated (seg))
21232 assert (!marked (o));
21237 while (o < heap_segment_allocated (seg) && !marked (o))
21239 o = o + AlignQword (size (o));
21244 dprintf (1235, ("after GC LOH size: %Id, free list: %Id, free obj: %Id\n\n",
21245 generation_size (max_generation + 1),
21246 generation_free_list_space (gen),
21247 generation_free_obj_space (gen)));
21250 void gc_heap::walk_relocation_for_loh (void* profiling_context, record_surv_fn fn)
21252 generation* gen = large_object_generation;
21253 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
21254 uint8_t* o = generation_allocation_start (gen);
21256 //Skip the generation gap object
21257 o = o + AlignQword (size (o));
21261 if (o >= heap_segment_allocated (seg))
21263 seg = heap_segment_next (seg);
21269 o = heap_segment_mem (seg);
21274 size_t size = AlignQword (size (o));
21276 ptrdiff_t reloc = loh_node_relocation_distance (o);
21278 STRESS_LOG_PLUG_MOVE(o, (o + size), -reloc);
21280 fn (o, (o + size), reloc, profiling_context, !!settings.compaction, false);
21283 if (o < heap_segment_allocated (seg))
21285 assert (!marked (o));
21290 while (o < heap_segment_allocated (seg) && !marked (o))
21292 o = o + AlignQword (size (o));
21298 BOOL gc_heap::loh_object_p (uint8_t* o)
21300 #ifdef MULTIPLE_HEAPS
21301 gc_heap* hp = gc_heap::g_heaps [0];
21302 int brick_entry = hp->brick_table[hp->brick_of (o)];
21303 #else //MULTIPLE_HEAPS
21304 int brick_entry = brick_table[brick_of (o)];
21305 #endif //MULTIPLE_HEAPS
21307 return (brick_entry == 0);
21309 #endif //FEATURE_LOH_COMPACTION
21311 void gc_heap::convert_to_pinned_plug (BOOL& last_npinned_plug_p,
21312 BOOL& last_pinned_plug_p,
21313 BOOL& pinned_plug_p,
21315 size_t& artificial_pinned_size)
21317 last_npinned_plug_p = FALSE;
21318 last_pinned_plug_p = TRUE;
21319 pinned_plug_p = TRUE;
21320 artificial_pinned_size = ps;
21323 // Because we have the artificial pinning, we can't guarantee that pinned and npinned
21324 // plugs are always interleaved.
21325 void gc_heap::store_plug_gap_info (uint8_t* plug_start,
21327 BOOL& last_npinned_plug_p,
21328 BOOL& last_pinned_plug_p,
21329 uint8_t*& last_pinned_plug,
21330 BOOL& pinned_plug_p,
21331 uint8_t* last_object_in_last_plug,
21332 BOOL& merge_with_last_pin_p,
21333 // this is only for verification purpose
21334 size_t last_plug_len)
21336 UNREFERENCED_PARAMETER(last_plug_len);
21338 if (!last_npinned_plug_p && !last_pinned_plug_p)
21340 //dprintf (3, ("last full plug end: %Ix, full plug start: %Ix", plug_end, plug_start));
21341 dprintf (3, ("Free: %Ix", (plug_start - plug_end)));
21342 assert ((plug_start == plug_end) || ((size_t)(plug_start - plug_end) >= Align (min_obj_size)));
21343 set_gap_size (plug_start, plug_start - plug_end);
21346 if (pinned (plug_start))
21348 BOOL save_pre_plug_info_p = FALSE;
21350 if (last_npinned_plug_p || last_pinned_plug_p)
21352 //if (last_plug_len == Align (min_obj_size))
21354 // dprintf (3, ("debugging only - last npinned plug is min, check to see if it's correct"));
21355 // GCToOSInterface::DebugBreak();
21357 save_pre_plug_info_p = TRUE;
21360 pinned_plug_p = TRUE;
21361 last_npinned_plug_p = FALSE;
21363 if (last_pinned_plug_p)
21365 dprintf (3, ("last plug %Ix was also pinned, should merge", last_pinned_plug));
21366 merge_with_last_pin_p = TRUE;
21370 last_pinned_plug_p = TRUE;
21371 last_pinned_plug = plug_start;
21373 enque_pinned_plug (last_pinned_plug, save_pre_plug_info_p, last_object_in_last_plug);
21375 if (save_pre_plug_info_p)
21377 set_gap_size (plug_start, sizeof (gap_reloc_pair));
21383 if (last_pinned_plug_p)
21385 //if (Align (last_plug_len) < min_pre_pin_obj_size)
21387 // dprintf (3, ("debugging only - last pinned plug is min, check to see if it's correct"));
21388 // GCToOSInterface::DebugBreak();
21391 save_post_plug_info (last_pinned_plug, last_object_in_last_plug, plug_start);
21392 set_gap_size (plug_start, sizeof (gap_reloc_pair));
21394 verify_pins_with_post_plug_info("after saving post plug info");
21396 last_npinned_plug_p = TRUE;
21397 last_pinned_plug_p = FALSE;
21401 void gc_heap::record_interesting_data_point (interesting_data_point idp)
21403 #ifdef GC_CONFIG_DRIVEN
21404 (interesting_data_per_gc[idp])++;
21406 UNREFERENCED_PARAMETER(idp);
21407 #endif //GC_CONFIG_DRIVEN
21411 #pragma warning(push)
21412 #pragma warning(disable:21000) // Suppress PREFast warning about overly large function
21414 void gc_heap::plan_phase (int condemned_gen_number)
21416 size_t old_gen2_allocated = 0;
21417 size_t old_gen2_size = 0;
21419 if (condemned_gen_number == (max_generation - 1))
21421 old_gen2_allocated = generation_free_list_allocated (generation_of (max_generation));
21422 old_gen2_size = generation_size (max_generation);
21425 assert (settings.concurrent == FALSE);
21427 // %type% category = quote (plan);
21431 start = GetCycleCount32();
21434 dprintf (2,("---- Plan Phase ---- Condemned generation %d, promotion: %d",
21435 condemned_gen_number, settings.promotion ? 1 : 0));
21437 generation* condemned_gen1 = generation_of (condemned_gen_number);
21440 BOOL use_mark_list = FALSE;
21441 uint8_t** mark_list_next = &mark_list[0];
21442 #ifdef GC_CONFIG_DRIVEN
21443 dprintf (3, ("total number of marked objects: %Id (%Id)",
21444 (mark_list_index - &mark_list[0]), ((mark_list_end - &mark_list[0]))));
21446 dprintf (3, ("mark_list length: %Id",
21447 (mark_list_index - &mark_list[0])));
21448 #endif //GC_CONFIG_DRIVEN
21450 if ((condemned_gen_number < max_generation) &&
21451 (mark_list_index <= mark_list_end)
21452 #ifdef BACKGROUND_GC
21453 && (!recursive_gc_sync::background_running_p())
21454 #endif //BACKGROUND_GC
21457 #ifndef MULTIPLE_HEAPS
21458 _sort (&mark_list[0], mark_list_index-1, 0);
21459 //printf ("using mark list at GC #%d", dd_collection_count (dynamic_data_of (0)));
21460 //verify_qsort_array (&mark_list[0], mark_list_index-1);
21461 #endif //!MULTIPLE_HEAPS
21462 use_mark_list = TRUE;
21463 get_gc_data_per_heap()->set_mechanism_bit (gc_mark_list_bit);
21467 dprintf (3, ("mark_list not used"));
21472 #ifdef FEATURE_BASICFREEZE
21473 if ((generation_start_segment (condemned_gen1) != ephemeral_heap_segment) &&
21474 ro_segments_in_range)
21476 sweep_ro_segments (generation_start_segment (condemned_gen1));
21478 #endif // FEATURE_BASICFREEZE
21480 #ifndef MULTIPLE_HEAPS
21481 if (shigh != (uint8_t*)0)
21483 heap_segment* seg = heap_segment_rw (generation_start_segment (condemned_gen1));
21485 PREFIX_ASSUME(seg != NULL);
21487 heap_segment* fseg = seg;
21490 if (slow > heap_segment_mem (seg) &&
21491 slow < heap_segment_reserved (seg))
21495 uint8_t* o = generation_allocation_start (condemned_gen1) +
21496 Align (size (generation_allocation_start (condemned_gen1)));
21499 assert ((slow - o) >= (int)Align (min_obj_size));
21500 #ifdef BACKGROUND_GC
21501 if (current_c_gc_state == c_gc_state_marking)
21503 bgc_clear_batch_mark_array_bits (o, slow);
21505 #endif //BACKGROUND_GC
21506 make_unused_array (o, slow - o);
21511 assert (condemned_gen_number == max_generation);
21512 make_unused_array (heap_segment_mem (seg),
21513 slow - heap_segment_mem (seg));
21516 if (in_range_for_segment (shigh, seg))
21518 #ifdef BACKGROUND_GC
21519 if (current_c_gc_state == c_gc_state_marking)
21521 bgc_clear_batch_mark_array_bits ((shigh + Align (size (shigh))), heap_segment_allocated (seg));
21523 #endif //BACKGROUND_GC
21524 heap_segment_allocated (seg) = shigh + Align (size (shigh));
21526 // test if the segment is in the range of [slow, shigh]
21527 if (!((heap_segment_reserved (seg) >= slow) &&
21528 (heap_segment_mem (seg) <= shigh)))
21530 // shorten it to minimum
21531 heap_segment_allocated (seg) = heap_segment_mem (seg);
21533 seg = heap_segment_next_rw (seg);
21538 heap_segment* seg = heap_segment_rw (generation_start_segment (condemned_gen1));
21540 PREFIX_ASSUME(seg != NULL);
21542 heap_segment* sseg = seg;
21545 // shorten it to minimum
21548 // no survivors make all generations look empty
21549 uint8_t* o = generation_allocation_start (condemned_gen1) +
21550 Align (size (generation_allocation_start (condemned_gen1)));
21551 #ifdef BACKGROUND_GC
21552 if (current_c_gc_state == c_gc_state_marking)
21554 bgc_clear_batch_mark_array_bits (o, heap_segment_allocated (seg));
21556 #endif //BACKGROUND_GC
21557 heap_segment_allocated (seg) = o;
21561 assert (condemned_gen_number == max_generation);
21562 #ifdef BACKGROUND_GC
21563 if (current_c_gc_state == c_gc_state_marking)
21565 bgc_clear_batch_mark_array_bits (heap_segment_mem (seg), heap_segment_allocated (seg));
21567 #endif //BACKGROUND_GC
21568 heap_segment_allocated (seg) = heap_segment_mem (seg);
21570 seg = heap_segment_next_rw (seg);
21574 #endif //MULTIPLE_HEAPS
21576 heap_segment* seg1 = heap_segment_rw (generation_start_segment (condemned_gen1));
21578 PREFIX_ASSUME(seg1 != NULL);
21580 uint8_t* end = heap_segment_allocated (seg1);
21581 uint8_t* first_condemned_address = generation_allocation_start (condemned_gen1);
21582 uint8_t* x = first_condemned_address;
21584 assert (!marked (x));
21585 uint8_t* plug_end = x;
21587 size_t sequence_number = 0;
21588 uint8_t* last_node = 0;
21589 size_t current_brick = brick_of (x);
21590 BOOL allocate_in_condemned = ((condemned_gen_number == max_generation)||
21591 (settings.promotion == FALSE));
21592 int active_old_gen_number = condemned_gen_number;
21593 int active_new_gen_number = (allocate_in_condemned ? condemned_gen_number:
21594 (1 + condemned_gen_number));
21595 generation* older_gen = 0;
21596 generation* consing_gen = condemned_gen1;
21597 alloc_list r_free_list [MAX_BUCKET_COUNT];
21599 size_t r_free_list_space = 0;
21600 size_t r_free_obj_space = 0;
21601 size_t r_older_gen_free_list_allocated = 0;
21602 size_t r_older_gen_condemned_allocated = 0;
21603 size_t r_older_gen_end_seg_allocated = 0;
21604 uint8_t* r_allocation_pointer = 0;
21605 uint8_t* r_allocation_limit = 0;
21606 uint8_t* r_allocation_start_region = 0;
21607 heap_segment* r_allocation_segment = 0;
21608 #ifdef FREE_USAGE_STATS
21609 size_t r_older_gen_free_space[NUM_GEN_POWER2];
21610 #endif //FREE_USAGE_STATS
21612 if ((condemned_gen_number < max_generation))
21614 older_gen = generation_of (min (max_generation, 1 + condemned_gen_number));
21615 generation_allocator (older_gen)->copy_to_alloc_list (r_free_list);
21617 r_free_list_space = generation_free_list_space (older_gen);
21618 r_free_obj_space = generation_free_obj_space (older_gen);
21619 #ifdef FREE_USAGE_STATS
21620 memcpy (r_older_gen_free_space, older_gen->gen_free_spaces, sizeof (r_older_gen_free_space));
21621 #endif //FREE_USAGE_STATS
21622 generation_allocate_end_seg_p (older_gen) = FALSE;
21623 r_older_gen_free_list_allocated = generation_free_list_allocated (older_gen);
21624 r_older_gen_condemned_allocated = generation_condemned_allocated (older_gen);
21625 r_older_gen_end_seg_allocated = generation_end_seg_allocated (older_gen);
21626 r_allocation_limit = generation_allocation_limit (older_gen);
21627 r_allocation_pointer = generation_allocation_pointer (older_gen);
21628 r_allocation_start_region = generation_allocation_context_start_region (older_gen);
21629 r_allocation_segment = generation_allocation_segment (older_gen);
21630 heap_segment* start_seg = heap_segment_rw (generation_start_segment (older_gen));
21632 PREFIX_ASSUME(start_seg != NULL);
21634 if (start_seg != ephemeral_heap_segment)
21636 assert (condemned_gen_number == (max_generation - 1));
21637 while (start_seg && (start_seg != ephemeral_heap_segment))
21639 assert (heap_segment_allocated (start_seg) >=
21640 heap_segment_mem (start_seg));
21641 assert (heap_segment_allocated (start_seg) <=
21642 heap_segment_reserved (start_seg));
21643 heap_segment_plan_allocated (start_seg) =
21644 heap_segment_allocated (start_seg);
21645 start_seg = heap_segment_next_rw (start_seg);
21650 //reset all of the segment allocated sizes
21652 heap_segment* seg2 = heap_segment_rw (generation_start_segment (condemned_gen1));
21654 PREFIX_ASSUME(seg2 != NULL);
21658 heap_segment_plan_allocated (seg2) =
21659 heap_segment_mem (seg2);
21660 seg2 = heap_segment_next_rw (seg2);
21663 int condemned_gn = condemned_gen_number;
21665 int bottom_gen = 0;
21666 init_free_and_plug();
21668 while (condemned_gn >= bottom_gen)
21670 generation* condemned_gen2 = generation_of (condemned_gn);
21671 generation_allocator (condemned_gen2)->clear();
21672 generation_free_list_space (condemned_gen2) = 0;
21673 generation_free_obj_space (condemned_gen2) = 0;
21674 generation_allocation_size (condemned_gen2) = 0;
21675 generation_condemned_allocated (condemned_gen2) = 0;
21676 generation_pinned_allocated (condemned_gen2) = 0;
21677 generation_free_list_allocated(condemned_gen2) = 0;
21678 generation_end_seg_allocated (condemned_gen2) = 0;
21679 generation_pinned_allocation_sweep_size (condemned_gen2) = 0;
21680 generation_pinned_allocation_compact_size (condemned_gen2) = 0;
21681 #ifdef FREE_USAGE_STATS
21682 generation_pinned_free_obj_space (condemned_gen2) = 0;
21683 generation_allocated_in_pinned_free (condemned_gen2) = 0;
21684 generation_allocated_since_last_pin (condemned_gen2) = 0;
21685 #endif //FREE_USAGE_STATS
21686 generation_plan_allocation_start (condemned_gen2) = 0;
21687 generation_allocation_segment (condemned_gen2) =
21688 heap_segment_rw (generation_start_segment (condemned_gen2));
21690 PREFIX_ASSUME(generation_allocation_segment(condemned_gen2) != NULL);
21692 if (generation_start_segment (condemned_gen2) != ephemeral_heap_segment)
21694 generation_allocation_pointer (condemned_gen2) =
21695 heap_segment_mem (generation_allocation_segment (condemned_gen2));
21699 generation_allocation_pointer (condemned_gen2) = generation_allocation_start (condemned_gen2);
21702 generation_allocation_limit (condemned_gen2) = generation_allocation_pointer (condemned_gen2);
21703 generation_allocation_context_start_region (condemned_gen2) = generation_allocation_pointer (condemned_gen2);
21708 BOOL allocate_first_generation_start = FALSE;
21710 if (allocate_in_condemned)
21712 allocate_first_generation_start = TRUE;
21715 dprintf(3,( " From %Ix to %Ix", (size_t)x, (size_t)end));
21717 demotion_low = MAX_PTR;
21718 demotion_high = heap_segment_allocated (ephemeral_heap_segment);
21720 // If we are doing a gen1 only because of cards, it means we should not demote any pinned plugs
21721 // from gen1. They should get promoted to gen2.
21722 demote_gen1_p = !(settings.promotion &&
21723 (settings.condemned_generation == (max_generation - 1)) &&
21724 gen_to_condemn_reasons.is_only_condition (gen_low_card_p));
21726 total_ephemeral_size = 0;
21728 print_free_and_plug ("BP");
21730 for (int gen_idx = 0; gen_idx <= max_generation; gen_idx++)
21732 generation* temp_gen = generation_of (gen_idx);
21734 dprintf (2, ("gen%d start %Ix, plan start %Ix",
21736 generation_allocation_start (temp_gen),
21737 generation_plan_allocation_start (temp_gen)));
21740 BOOL fire_pinned_plug_events_p = ETW_EVENT_ENABLED(MICROSOFT_WINDOWS_DOTNETRUNTIME_PRIVATE_PROVIDER_Context, PinPlugAtGCTime);
21741 size_t last_plug_len = 0;
21748 assert (heap_segment_allocated (seg1) == end);
21749 heap_segment_allocated (seg1) = plug_end;
21751 current_brick = update_brick_table (tree, current_brick, x, plug_end);
21752 dprintf (3, ("end of seg: new tree, sequence# 0"));
21753 sequence_number = 0;
21756 if (heap_segment_next_rw (seg1))
21758 seg1 = heap_segment_next_rw (seg1);
21759 end = heap_segment_allocated (seg1);
21760 plug_end = x = heap_segment_mem (seg1);
21761 current_brick = brick_of (x);
21762 dprintf(3,( " From %Ix to %Ix", (size_t)x, (size_t)end));
21771 BOOL last_npinned_plug_p = FALSE;
21772 BOOL last_pinned_plug_p = FALSE;
21774 // last_pinned_plug is the beginning of the last pinned plug. If we merge a plug into a pinned
21775 // plug we do not change the value of last_pinned_plug. This happens with artificially pinned plugs -
21776 // it can be merged with a previous pinned plug and a pinned plug after it can be merged with it.
21777 uint8_t* last_pinned_plug = 0;
21778 size_t num_pinned_plugs_in_plug = 0;
21780 uint8_t* last_object_in_plug = 0;
21782 while ((x < end) && marked (x))
21784 uint8_t* plug_start = x;
21785 uint8_t* saved_plug_end = plug_end;
21786 BOOL pinned_plug_p = FALSE;
21787 BOOL npin_before_pin_p = FALSE;
21788 BOOL saved_last_npinned_plug_p = last_npinned_plug_p;
21789 uint8_t* saved_last_object_in_plug = last_object_in_plug;
21790 BOOL merge_with_last_pin_p = FALSE;
21792 size_t added_pinning_size = 0;
21793 size_t artificial_pinned_size = 0;
21795 store_plug_gap_info (plug_start, plug_end, last_npinned_plug_p, last_pinned_plug_p,
21796 last_pinned_plug, pinned_plug_p, last_object_in_plug,
21797 merge_with_last_pin_p, last_plug_len);
21799 #ifdef FEATURE_STRUCTALIGN
21800 int requiredAlignment = ((CObjectHeader*)plug_start)->GetRequiredAlignment();
21801 size_t alignmentOffset = OBJECT_ALIGNMENT_OFFSET;
21802 #endif // FEATURE_STRUCTALIGN
21806 while ((xl < end) && marked (xl) && (pinned (xl) == pinned_plug_p))
21813 #ifdef FEATURE_STRUCTALIGN
21816 int obj_requiredAlignment = ((CObjectHeader*)xl)->GetRequiredAlignment();
21817 if (obj_requiredAlignment > requiredAlignment)
21819 requiredAlignment = obj_requiredAlignment;
21820 alignmentOffset = xl - plug_start + OBJECT_ALIGNMENT_OFFSET;
21823 #endif // FEATURE_STRUCTALIGN
21827 dprintf(4, ("+%Ix+", (size_t)xl));
21828 assert ((size (xl) > 0));
21829 assert ((size (xl) <= LARGE_OBJECT_SIZE));
21831 last_object_in_plug = xl;
21833 xl = xl + Align (size (xl));
21837 BOOL next_object_marked_p = ((xl < end) && marked (xl));
21841 // If it is pinned we need to extend to the next marked object as we can't use part of
21842 // a pinned object to make the artificial gap (unless the last 3 ptr sized words are all
21843 // references but for now I am just using the next non pinned object for that).
21844 if (next_object_marked_p)
21847 last_object_in_plug = xl;
21848 size_t extra_size = Align (size (xl));
21849 xl = xl + extra_size;
21850 added_pinning_size = extra_size;
21855 if (next_object_marked_p)
21856 npin_before_pin_p = TRUE;
21859 assert (xl <= end);
21862 dprintf (3, ( "%Ix[", (size_t)x));
21864 size_t ps = plug_end - plug_start;
21865 last_plug_len = ps;
21866 dprintf (3, ( "%Ix[(%Ix)", (size_t)x, ps));
21867 uint8_t* new_address = 0;
21869 if (!pinned_plug_p)
21871 if (allocate_in_condemned &&
21872 (settings.condemned_generation == max_generation) &&
21873 (ps > OS_PAGE_SIZE))
21875 ptrdiff_t reloc = plug_start - generation_allocation_pointer (consing_gen);
21876 //reloc should >=0 except when we relocate
21877 //across segments and the dest seg is higher then the src
21879 if ((ps > (8*OS_PAGE_SIZE)) &&
21881 ((size_t)reloc < (ps/16)))
21883 dprintf (3, ("Pinning %Ix; reloc would have been: %Ix",
21884 (size_t)plug_start, reloc));
21885 // The last plug couldn't have been a npinned plug or it would have
21886 // included this plug.
21887 assert (!saved_last_npinned_plug_p);
21889 if (last_pinned_plug)
21891 dprintf (3, ("artificially pinned plug merged with last pinned plug"));
21892 merge_with_last_pin_p = TRUE;
21896 enque_pinned_plug (plug_start, FALSE, 0);
21897 last_pinned_plug = plug_start;
21900 convert_to_pinned_plug (last_npinned_plug_p, last_pinned_plug_p, pinned_plug_p,
21901 ps, artificial_pinned_size);
21906 if (allocate_first_generation_start)
21908 allocate_first_generation_start = FALSE;
21909 plan_generation_start (condemned_gen1, consing_gen, plug_start);
21910 assert (generation_plan_allocation_start (condemned_gen1));
21913 if (seg1 == ephemeral_heap_segment)
21915 process_ephemeral_boundaries (plug_start, active_new_gen_number,
21916 active_old_gen_number,
21918 allocate_in_condemned);
21921 dprintf (3, ("adding %Id to gen%d surv", ps, active_old_gen_number));
21923 dynamic_data* dd_active_old = dynamic_data_of (active_old_gen_number);
21924 dd_survived_size (dd_active_old) += ps;
21926 BOOL convert_to_pinned_p = FALSE;
21928 if (!pinned_plug_p)
21930 #if defined (RESPECT_LARGE_ALIGNMENT) || defined (FEATURE_STRUCTALIGN)
21931 dd_num_npinned_plugs (dd_active_old)++;
21932 #endif //RESPECT_LARGE_ALIGNMENT || FEATURE_STRUCTALIGN
21934 add_gen_plug (active_old_gen_number, ps);
21936 if (allocate_in_condemned)
21938 verify_pins_with_post_plug_info("before aic");
21941 allocate_in_condemned_generations (consing_gen,
21943 active_old_gen_number,
21945 &convert_to_pinned_p,
21946 (npin_before_pin_p ? plug_end : 0),
21948 #endif //SHORT_PLUGS
21949 plug_start REQD_ALIGN_AND_OFFSET_ARG);
21950 verify_pins_with_post_plug_info("after aic");
21954 new_address = allocate_in_older_generation (older_gen, ps, active_old_gen_number, plug_start REQD_ALIGN_AND_OFFSET_ARG);
21956 if (new_address != 0)
21958 if (settings.condemned_generation == (max_generation - 1))
21960 dprintf (3, (" NA: %Ix-%Ix -> %Ix, %Ix (%Ix)",
21961 plug_start, plug_end,
21962 (size_t)new_address, (size_t)new_address + (plug_end - plug_start),
21963 (size_t)(plug_end - plug_start)));
21968 allocate_in_condemned = TRUE;
21970 new_address = allocate_in_condemned_generations (consing_gen, ps, active_old_gen_number,
21972 &convert_to_pinned_p,
21973 (npin_before_pin_p ? plug_end : 0),
21975 #endif //SHORT_PLUGS
21976 plug_start REQD_ALIGN_AND_OFFSET_ARG);
21980 if (convert_to_pinned_p)
21982 assert (last_npinned_plug_p != FALSE);
21983 assert (last_pinned_plug_p == FALSE);
21984 convert_to_pinned_plug (last_npinned_plug_p, last_pinned_plug_p, pinned_plug_p,
21985 ps, artificial_pinned_size);
21986 enque_pinned_plug (plug_start, FALSE, 0);
21987 last_pinned_plug = plug_start;
21993 //verify that we are at then end of the ephemeral segment
21994 assert (generation_allocation_segment (consing_gen) ==
21995 ephemeral_heap_segment);
21996 //verify that we are near the end
21997 assert ((generation_allocation_pointer (consing_gen) + Align (ps)) <
21998 heap_segment_allocated (ephemeral_heap_segment));
21999 assert ((generation_allocation_pointer (consing_gen) + Align (ps)) >
22000 (heap_segment_allocated (ephemeral_heap_segment) + Align (min_obj_size)));
22004 #ifdef SIMPLE_DPRINTF
22005 dprintf (3, ("(%Ix)[%Ix->%Ix, NA: [%Ix(%Id), %Ix[: %Ix(%d)",
22006 (size_t)(node_gap_size (plug_start)),
22007 plug_start, plug_end, (size_t)new_address, (size_t)(plug_start - new_address),
22008 (size_t)new_address + ps, ps,
22009 (is_plug_padded (plug_start) ? 1 : 0)));
22010 #endif //SIMPLE_DPRINTF
22013 if (is_plug_padded (plug_start))
22015 dprintf (3, ("%Ix was padded", plug_start));
22016 dd_padding_size (dd_active_old) += Align (min_obj_size);
22018 #endif //SHORT_PLUGS
22025 if (fire_pinned_plug_events_p)
22026 FireEtwPinPlugAtGCTime(plug_start, plug_end,
22027 (merge_with_last_pin_p ? 0 : (uint8_t*)node_gap_size (plug_start)),
22028 GetClrInstanceId());
22030 if (merge_with_last_pin_p)
22032 merge_with_last_pinned_plug (last_pinned_plug, ps);
22036 assert (last_pinned_plug == plug_start);
22037 set_pinned_info (plug_start, ps, consing_gen);
22040 new_address = plug_start;
22042 dprintf (3, ( "(%Ix)PP: [%Ix, %Ix[%Ix](m:%d)",
22043 (size_t)(node_gap_size (plug_start)), (size_t)plug_start,
22044 (size_t)plug_end, ps,
22045 (merge_with_last_pin_p ? 1 : 0)));
22047 dprintf (3, ("adding %Id to gen%d pinned surv", plug_end - plug_start, active_old_gen_number));
22048 dd_pinned_survived_size (dd_active_old) += plug_end - plug_start;
22049 dd_added_pinned_size (dd_active_old) += added_pinning_size;
22050 dd_artificial_pinned_survived_size (dd_active_old) += artificial_pinned_size;
22052 if (!demote_gen1_p && (active_old_gen_number == (max_generation - 1)))
22054 last_gen1_pin_end = plug_end;
22059 // detect forward allocation in the same segment
22060 assert (!((new_address > plug_start) &&
22061 (new_address < heap_segment_reserved (seg1))));
22064 if (!merge_with_last_pin_p)
22066 if (current_brick != brick_of (plug_start))
22068 current_brick = update_brick_table (tree, current_brick, plug_start, saved_plug_end);
22069 sequence_number = 0;
22073 set_node_relocation_distance (plug_start, (new_address - plug_start));
22074 if (last_node && (node_relocation_distance (last_node) ==
22075 (node_relocation_distance (plug_start) +
22076 node_gap_size (plug_start))))
22078 //dprintf(3,( " Lb"));
22079 dprintf (3, ("%Ix Lb", plug_start));
22080 set_node_left (plug_start);
22082 if (0 == sequence_number)
22084 dprintf (2, ("sn: 0, tree is set to %Ix", plug_start));
22088 verify_pins_with_post_plug_info("before insert node");
22090 tree = insert_node (plug_start, ++sequence_number, tree, last_node);
22091 dprintf (3, ("tree is %Ix (b: %Ix) after insert_node", tree, brick_of (tree)));
22092 last_node = plug_start;
22095 // If we detect if the last plug is pinned plug right before us, we should save this gap info
22096 if (!pinned_plug_p)
22098 if (mark_stack_tos > 0)
22100 mark& m = mark_stack_array[mark_stack_tos - 1];
22101 if (m.has_post_plug_info())
22103 uint8_t* post_plug_info_start = m.saved_post_plug_info_start;
22104 size_t* current_plug_gap_start = (size_t*)(plug_start - sizeof (plug_and_gap));
22105 if ((uint8_t*)current_plug_gap_start == post_plug_info_start)
22107 dprintf (3, ("Ginfo: %Ix, %Ix, %Ix",
22108 *current_plug_gap_start, *(current_plug_gap_start + 1),
22109 *(current_plug_gap_start + 2)));
22110 memcpy (&(m.saved_post_plug_debug), current_plug_gap_start, sizeof (gap_reloc_pair));
22117 verify_pins_with_post_plug_info("after insert node");
22121 if (num_pinned_plugs_in_plug > 1)
22123 dprintf (3, ("more than %Id pinned plugs in this plug", num_pinned_plugs_in_plug));
22130 while ((mark_list_next < mark_list_index) &&
22131 (*mark_list_next <= x))
22135 if ((mark_list_next < mark_list_index)
22136 #ifdef MULTIPLE_HEAPS
22137 && (*mark_list_next < end) //for multiple segments
22138 #endif //MULTIPLE_HEAPS
22140 x = *mark_list_next;
22148 #ifdef BACKGROUND_GC
22149 if (current_c_gc_state == c_gc_state_marking)
22151 assert (recursive_gc_sync::background_running_p());
22152 while ((xl < end) && !marked (xl))
22154 dprintf (4, ("-%Ix-", (size_t)xl));
22155 assert ((size (xl) > 0));
22156 background_object_marked (xl, TRUE);
22157 xl = xl + Align (size (xl));
22162 #endif //BACKGROUND_GC
22164 while ((xl < end) && !marked (xl))
22166 dprintf (4, ("-%Ix-", (size_t)xl));
22167 assert ((size (xl) > 0));
22168 xl = xl + Align (size (xl));
22172 assert (xl <= end);
22178 while (!pinned_plug_que_empty_p())
22180 if (settings.promotion)
22182 uint8_t* pplug = pinned_plug (oldest_pin());
22183 if (in_range_for_segment (pplug, ephemeral_heap_segment))
22185 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
22186 //allocate all of the generation gaps
22187 while (active_new_gen_number > 0)
22189 active_new_gen_number--;
22191 if (active_new_gen_number == (max_generation - 1))
22193 maxgen_pinned_compact_before_advance = generation_pinned_allocation_compact_size (generation_of (max_generation));
22194 if (!demote_gen1_p)
22195 advance_pins_for_demotion (consing_gen);
22198 generation* gen = generation_of (active_new_gen_number);
22199 plan_generation_start (gen, consing_gen, 0);
22201 if (demotion_low == MAX_PTR)
22203 demotion_low = pplug;
22204 dprintf (3, ("end plan: dlow->%Ix", demotion_low));
22207 dprintf (2, ("(%d)gen%d plan start: %Ix",
22208 heap_number, active_new_gen_number, (size_t)generation_plan_allocation_start (gen)));
22209 assert (generation_plan_allocation_start (gen));
22214 if (pinned_plug_que_empty_p())
22217 size_t entry = deque_pinned_plug();
22218 mark* m = pinned_plug_of (entry);
22219 uint8_t* plug = pinned_plug (m);
22220 size_t len = pinned_len (m);
22222 // detect pinned block in different segment (later) than
22223 // allocation segment
22224 heap_segment* nseg = heap_segment_rw (generation_allocation_segment (consing_gen));
22226 while ((plug < generation_allocation_pointer (consing_gen)) ||
22227 (plug >= heap_segment_allocated (nseg)))
22229 assert ((plug < heap_segment_mem (nseg)) ||
22230 (plug > heap_segment_reserved (nseg)));
22231 //adjust the end of the segment to be the end of the plug
22232 assert (generation_allocation_pointer (consing_gen)>=
22233 heap_segment_mem (nseg));
22234 assert (generation_allocation_pointer (consing_gen)<=
22235 heap_segment_committed (nseg));
22237 heap_segment_plan_allocated (nseg) =
22238 generation_allocation_pointer (consing_gen);
22239 //switch allocation segment
22240 nseg = heap_segment_next_rw (nseg);
22241 generation_allocation_segment (consing_gen) = nseg;
22242 //reset the allocation pointer and limits
22243 generation_allocation_pointer (consing_gen) =
22244 heap_segment_mem (nseg);
22247 set_new_pin_info (m, generation_allocation_pointer (consing_gen));
22248 dprintf (2, ("pin %Ix b: %Ix->%Ix", plug, brick_of (plug),
22249 (size_t)(brick_table[brick_of (plug)])));
22251 generation_allocation_pointer (consing_gen) = plug + len;
22252 generation_allocation_limit (consing_gen) =
22253 generation_allocation_pointer (consing_gen);
22254 //Add the size of the pinned plug to the right pinned allocations
22255 //find out which gen this pinned plug came from
22256 int frgn = object_gennum (plug);
22257 if ((frgn != (int)max_generation) && settings.promotion)
22259 generation_pinned_allocation_sweep_size ((generation_of (frgn +1))) += len;
22264 plan_generation_starts (consing_gen);
22265 print_free_and_plug ("AP");
22268 #ifdef SIMPLE_DPRINTF
22269 for (int gen_idx = 0; gen_idx <= max_generation; gen_idx++)
22271 generation* temp_gen = generation_of (gen_idx);
22272 dynamic_data* temp_dd = dynamic_data_of (gen_idx);
22274 int added_pinning_ratio = 0;
22275 int artificial_pinned_ratio = 0;
22277 if (dd_pinned_survived_size (temp_dd) != 0)
22279 added_pinning_ratio = (int)((float)dd_added_pinned_size (temp_dd) * 100 / (float)dd_pinned_survived_size (temp_dd));
22280 artificial_pinned_ratio = (int)((float)dd_artificial_pinned_survived_size (temp_dd) * 100 / (float)dd_pinned_survived_size (temp_dd));
22283 size_t padding_size =
22285 dd_padding_size (temp_dd);
22288 #endif //SHORT_PLUGS
22289 dprintf (1, ("gen%d: %Ix, %Ix(%Id), NON PIN alloc: %Id, pin com: %Id, sweep: %Id, surv: %Id, pinsurv: %Id(%d%% added, %d%% art), np surv: %Id, pad: %Id",
22291 generation_allocation_start (temp_gen),
22292 generation_plan_allocation_start (temp_gen),
22293 (size_t)(generation_plan_allocation_start (temp_gen) - generation_allocation_start (temp_gen)),
22294 generation_allocation_size (temp_gen),
22295 generation_pinned_allocation_compact_size (temp_gen),
22296 generation_pinned_allocation_sweep_size (temp_gen),
22297 dd_survived_size (temp_dd),
22298 dd_pinned_survived_size (temp_dd),
22299 added_pinning_ratio,
22300 artificial_pinned_ratio,
22301 (dd_survived_size (temp_dd) - dd_pinned_survived_size (temp_dd)),
22304 #endif //SIMPLE_DPRINTF
22307 if (settings.condemned_generation == (max_generation - 1 ))
22309 size_t plan_gen2_size = generation_plan_size (max_generation);
22310 size_t growth = plan_gen2_size - old_gen2_size;
22314 dprintf (1, ("gen2 grew %Id (end seg alloc: %Id, gen1 c alloc: %Id",
22315 growth, generation_end_seg_allocated (generation_of (max_generation)),
22316 generation_condemned_allocated (generation_of (max_generation - 1))));
22320 dprintf (1, ("gen2 shrank %Id (end seg alloc: %Id, gen1 c alloc: %Id",
22321 (old_gen2_size - plan_gen2_size), generation_end_seg_allocated (generation_of (max_generation)),
22322 generation_condemned_allocated (generation_of (max_generation - 1))));
22325 generation* older_gen = generation_of (settings.condemned_generation + 1);
22326 size_t rejected_free_space = generation_free_obj_space (older_gen) - r_free_obj_space;
22327 size_t free_list_allocated = generation_free_list_allocated (older_gen) - r_older_gen_free_list_allocated;
22328 size_t end_seg_allocated = generation_end_seg_allocated (older_gen) - r_older_gen_end_seg_allocated;
22329 size_t condemned_allocated = generation_condemned_allocated (older_gen) - r_older_gen_condemned_allocated;
22331 dprintf (1, ("older gen's free alloc: %Id->%Id, seg alloc: %Id->%Id, condemned alloc: %Id->%Id",
22332 r_older_gen_free_list_allocated, generation_free_list_allocated (older_gen),
22333 r_older_gen_end_seg_allocated, generation_end_seg_allocated (older_gen),
22334 r_older_gen_condemned_allocated, generation_condemned_allocated (older_gen)));
22336 dprintf (1, ("this GC did %Id free list alloc(%Id bytes free space rejected), %Id seg alloc and %Id condemned alloc, gen1 condemned alloc is %Id",
22337 free_list_allocated, rejected_free_space, end_seg_allocated,
22338 condemned_allocated, generation_condemned_allocated (generation_of (settings.condemned_generation))));
22340 maxgen_size_increase* maxgen_size_info = &(get_gc_data_per_heap()->maxgen_size_info);
22341 maxgen_size_info->free_list_allocated = free_list_allocated;
22342 maxgen_size_info->free_list_rejected = rejected_free_space;
22343 maxgen_size_info->end_seg_allocated = end_seg_allocated;
22344 maxgen_size_info->condemned_allocated = condemned_allocated;
22345 maxgen_size_info->pinned_allocated = maxgen_pinned_compact_before_advance;
22346 maxgen_size_info->pinned_allocated_advance = generation_pinned_allocation_compact_size (generation_of (max_generation)) - maxgen_pinned_compact_before_advance;
22348 #ifdef FREE_USAGE_STATS
22349 int free_list_efficiency = 0;
22350 if ((free_list_allocated + rejected_free_space) != 0)
22351 free_list_efficiency = (int)(((float) (free_list_allocated) / (float)(free_list_allocated + rejected_free_space)) * (float)100);
22353 int running_free_list_efficiency = (int)(generation_allocator_efficiency(older_gen)*100);
22355 dprintf (1, ("gen%d free list alloc effi: %d%%, current effi: %d%%",
22356 older_gen->gen_num,
22357 free_list_efficiency, running_free_list_efficiency));
22359 dprintf (1, ("gen2 free list change"));
22360 for (int j = 0; j < NUM_GEN_POWER2; j++)
22362 dprintf (1, ("[h%d][#%Id]: 2^%d: F: %Id->%Id(%Id), P: %Id",
22365 (j + 10), r_older_gen_free_space[j], older_gen->gen_free_spaces[j],
22366 (ptrdiff_t)(r_older_gen_free_space[j] - older_gen->gen_free_spaces[j]),
22367 (generation_of(max_generation - 1))->gen_plugs[j]));
22369 #endif //FREE_USAGE_STATS
22372 size_t fragmentation =
22373 generation_fragmentation (generation_of (condemned_gen_number),
22375 heap_segment_allocated (ephemeral_heap_segment));
22377 dprintf (2,("Fragmentation: %Id", fragmentation));
22378 dprintf (2,("---- End of Plan phase ----"));
22381 finish = GetCycleCount32();
22382 plan_time = finish - start;
22385 // We may update write barrier code. We assume here EE has been suspended if we are on a GC thread.
22386 assert(IsGCInProgress());
22388 BOOL should_expand = FALSE;
22389 BOOL should_compact= FALSE;
22390 ephemeral_promotion = FALSE;
22393 if ((!settings.concurrent) &&
22394 ((condemned_gen_number < max_generation) &&
22395 ((settings.gen0_reduction_count > 0) || (settings.entry_memory_load >= 95))))
22397 dprintf (2, ("gen0 reduction count is %d, condemning %d, mem load %d",
22398 settings.gen0_reduction_count,
22399 condemned_gen_number,
22400 settings.entry_memory_load));
22401 should_compact = TRUE;
22403 get_gc_data_per_heap()->set_mechanism (gc_heap_compact,
22404 ((settings.gen0_reduction_count > 0) ? compact_fragmented_gen0 : compact_high_mem_load));
22406 if ((condemned_gen_number >= (max_generation - 1)) &&
22407 dt_low_ephemeral_space_p (tuning_deciding_expansion))
22409 dprintf (2, ("Not enough space for all ephemeral generations with compaction"));
22410 should_expand = TRUE;
22416 should_compact = decide_on_compacting (condemned_gen_number, fragmentation, should_expand);
22421 #ifdef FEATURE_LOH_COMPACTION
22422 loh_compacted_p = FALSE;
22423 #endif //FEATURE_LOH_COMPACTION
22425 if (condemned_gen_number == max_generation)
22427 #ifdef FEATURE_LOH_COMPACTION
22428 if (settings.loh_compaction)
22432 should_compact = TRUE;
22433 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_loh_forced);
22434 loh_compacted_p = TRUE;
22439 if ((heap_number == 0) && (loh_pinned_queue))
22441 loh_pinned_queue_decay--;
22443 if (!loh_pinned_queue_decay)
22445 delete loh_pinned_queue;
22446 loh_pinned_queue = 0;
22451 if (!loh_compacted_p)
22452 #endif //FEATURE_LOH_COMPACTION
22454 GCToEEInterface::DiagWalkLOHSurvivors(__this);
22455 sweep_large_objects();
22460 settings.loh_compaction = FALSE;
22463 #ifdef MULTIPLE_HEAPS
22465 new_heap_segment = NULL;
22467 if (should_compact && should_expand)
22468 gc_policy = policy_expand;
22469 else if (should_compact)
22470 gc_policy = policy_compact;
22472 gc_policy = policy_sweep;
22474 //vote for result of should_compact
22475 dprintf (3, ("Joining for compaction decision"));
22476 gc_t_join.join(this, gc_join_decide_on_compaction);
22477 if (gc_t_join.joined())
22479 //safe place to delete large heap segments
22480 if (condemned_gen_number == max_generation)
22482 for (int i = 0; i < n_heaps; i++)
22484 g_heaps [i]->rearrange_large_heap_segments ();
22488 settings.demotion = FALSE;
22489 int pol_max = policy_sweep;
22490 #ifdef GC_CONFIG_DRIVEN
22491 BOOL is_compaction_mandatory = FALSE;
22492 #endif //GC_CONFIG_DRIVEN
22495 for (i = 0; i < n_heaps; i++)
22497 if (pol_max < g_heaps[i]->gc_policy)
22498 pol_max = policy_compact;
22499 // set the demotion flag is any of the heap has demotion
22500 if (g_heaps[i]->demotion_high >= g_heaps[i]->demotion_low)
22502 (g_heaps[i]->get_gc_data_per_heap())->set_mechanism_bit (gc_demotion_bit);
22503 settings.demotion = TRUE;
22506 #ifdef GC_CONFIG_DRIVEN
22507 if (!is_compaction_mandatory)
22509 int compact_reason = (g_heaps[i]->get_gc_data_per_heap())->get_mechanism (gc_heap_compact);
22510 if (compact_reason >= 0)
22512 if (gc_heap_compact_reason_mandatory_p[compact_reason])
22513 is_compaction_mandatory = TRUE;
22516 #endif //GC_CONFIG_DRIVEN
22519 #ifdef GC_CONFIG_DRIVEN
22520 if (!is_compaction_mandatory)
22522 // If compaction is not mandatory we can feel free to change it to a sweeping GC.
22523 // Note that we may want to change this to only checking every so often instead of every single GC.
22524 if (should_do_sweeping_gc (pol_max >= policy_compact))
22526 pol_max = policy_sweep;
22530 if (pol_max == policy_sweep)
22531 pol_max = policy_compact;
22534 #endif //GC_CONFIG_DRIVEN
22536 for (i = 0; i < n_heaps; i++)
22538 if (pol_max > g_heaps[i]->gc_policy)
22539 g_heaps[i]->gc_policy = pol_max;
22540 //get the segment while we are serialized
22541 if (g_heaps[i]->gc_policy == policy_expand)
22543 g_heaps[i]->new_heap_segment =
22544 g_heaps[i]->soh_get_segment_to_expand();
22545 if (!g_heaps[i]->new_heap_segment)
22547 set_expand_in_full_gc (condemned_gen_number);
22548 //we are out of memory, cancel the expansion
22549 g_heaps[i]->gc_policy = policy_compact;
22554 BOOL is_full_compacting_gc = FALSE;
22556 if ((gc_policy >= policy_compact) && (condemned_gen_number == max_generation))
22558 full_gc_counts[gc_type_compacting]++;
22559 is_full_compacting_gc = TRUE;
22562 for (i = 0; i < n_heaps; i++)
22564 //copy the card and brick tables
22565 if (g_gc_card_table!= g_heaps[i]->card_table)
22567 g_heaps[i]->copy_brick_card_table();
22570 if (is_full_compacting_gc)
22572 g_heaps[i]->loh_alloc_since_cg = 0;
22576 //start all threads on the roots.
22577 dprintf(3, ("Starting all gc threads after compaction decision"));
22578 gc_t_join.restart();
22581 //reset the local variable accordingly
22582 should_compact = (gc_policy >= policy_compact);
22583 should_expand = (gc_policy >= policy_expand);
22585 #else //MULTIPLE_HEAPS
22587 //safe place to delete large heap segments
22588 if (condemned_gen_number == max_generation)
22590 rearrange_large_heap_segments ();
22593 settings.demotion = ((demotion_high >= demotion_low) ? TRUE : FALSE);
22594 if (settings.demotion)
22595 get_gc_data_per_heap()->set_mechanism_bit (gc_demotion_bit);
22597 #ifdef GC_CONFIG_DRIVEN
22598 BOOL is_compaction_mandatory = FALSE;
22599 int compact_reason = get_gc_data_per_heap()->get_mechanism (gc_heap_compact);
22600 if (compact_reason >= 0)
22601 is_compaction_mandatory = gc_heap_compact_reason_mandatory_p[compact_reason];
22603 if (!is_compaction_mandatory)
22605 if (should_do_sweeping_gc (should_compact))
22606 should_compact = FALSE;
22608 should_compact = TRUE;
22610 #endif //GC_CONFIG_DRIVEN
22612 if (should_compact && (condemned_gen_number == max_generation))
22614 full_gc_counts[gc_type_compacting]++;
22615 loh_alloc_since_cg = 0;
22617 #endif //MULTIPLE_HEAPS
22619 if (should_compact)
22621 dprintf (2,( "**** Doing Compacting GC ****"));
22625 #ifndef MULTIPLE_HEAPS
22626 heap_segment* new_heap_segment = soh_get_segment_to_expand();
22627 #endif //!MULTIPLE_HEAPS
22628 if (new_heap_segment)
22630 consing_gen = expand_heap(condemned_gen_number,
22635 // If we couldn't get a new segment, or we were able to
22636 // reserve one but no space to commit, we couldn't
22638 if (ephemeral_heap_segment != new_heap_segment)
22640 set_expand_in_full_gc (condemned_gen_number);
22641 should_expand = FALSE;
22644 generation_allocation_limit (condemned_gen1) =
22645 generation_allocation_pointer (condemned_gen1);
22646 if ((condemned_gen_number < max_generation))
22648 generation_allocator (older_gen)->commit_alloc_list_changes();
22650 // Fix the allocation area of the older generation
22651 fix_older_allocation_area (older_gen);
22653 assert (generation_allocation_segment (consing_gen) ==
22654 ephemeral_heap_segment);
22656 GCToEEInterface::DiagWalkSurvivors(__this);
22658 relocate_phase (condemned_gen_number, first_condemned_address);
22659 compact_phase (condemned_gen_number, first_condemned_address,
22660 (!settings.demotion && settings.promotion));
22661 fix_generation_bounds (condemned_gen_number, consing_gen);
22662 assert (generation_allocation_limit (youngest_generation) ==
22663 generation_allocation_pointer (youngest_generation));
22664 if (condemned_gen_number >= (max_generation -1))
22666 #ifdef MULTIPLE_HEAPS
22667 // this needs be serialized just because we have one
22668 // segment_standby_list/seg_table for all heaps. We should make it at least
22669 // so that when hoarding is not on we don't need this join because
22670 // decommitting memory can take a long time.
22671 //must serialize on deleting segments
22672 gc_t_join.join(this, gc_join_rearrange_segs_compaction);
22673 if (gc_t_join.joined())
22675 for (int i = 0; i < n_heaps; i++)
22677 g_heaps[i]->rearrange_heap_segments(TRUE);
22679 gc_t_join.restart();
22682 rearrange_heap_segments(TRUE);
22683 #endif //MULTIPLE_HEAPS
22687 //fix the start_segment for the ephemeral generations
22688 for (int i = 0; i < max_generation; i++)
22690 generation* gen = generation_of (i);
22691 generation_start_segment (gen) = ephemeral_heap_segment;
22692 generation_allocation_segment (gen) = ephemeral_heap_segment;
22698 #ifdef FEATURE_PREMORTEM_FINALIZATION
22699 finalize_queue->UpdatePromotedGenerations (condemned_gen_number,
22700 (!settings.demotion && settings.promotion));
22701 #endif // FEATURE_PREMORTEM_FINALIZATION
22703 #ifdef MULTIPLE_HEAPS
22704 dprintf(3, ("Joining after end of compaction"));
22705 gc_t_join.join(this, gc_join_adjust_handle_age_compact);
22706 if (gc_t_join.joined())
22707 #endif //MULTIPLE_HEAPS
22709 #ifdef MULTIPLE_HEAPS
22710 //join all threads to make sure they are synchronized
22711 dprintf(3, ("Restarting after Promotion granted"));
22712 gc_t_join.restart();
22713 #endif //MULTIPLE_HEAPS
22717 sc.thread_number = heap_number;
22718 sc.promotion = FALSE;
22719 sc.concurrent = FALSE;
22720 // new generations bounds are set can call this guy
22721 if (settings.promotion && !settings.demotion)
22723 dprintf (2, ("Promoting EE roots for gen %d",
22724 condemned_gen_number));
22725 GCScan::GcPromotionsGranted(condemned_gen_number,
22726 max_generation, &sc);
22728 else if (settings.demotion)
22730 dprintf (2, ("Demoting EE roots for gen %d",
22731 condemned_gen_number));
22732 GCScan::GcDemote (condemned_gen_number, max_generation, &sc);
22737 gen0_big_free_spaces = 0;
22739 reset_pinned_queue_bos();
22740 unsigned int gen_number = min (max_generation, 1 + condemned_gen_number);
22741 generation* gen = generation_of (gen_number);
22742 uint8_t* low = generation_allocation_start (generation_of (gen_number-1));
22743 uint8_t* high = heap_segment_allocated (ephemeral_heap_segment);
22745 while (!pinned_plug_que_empty_p())
22747 mark* m = pinned_plug_of (deque_pinned_plug());
22748 size_t len = pinned_len (m);
22749 uint8_t* arr = (pinned_plug (m) - len);
22750 dprintf(3,("free [%Ix %Ix[ pin",
22751 (size_t)arr, (size_t)arr + len));
22754 assert (len >= Align (min_obj_size));
22755 make_unused_array (arr, len);
22756 // fix fully contained bricks + first one
22757 // if the array goes beyond the first brick
22758 size_t start_brick = brick_of (arr);
22759 size_t end_brick = brick_of (arr + len);
22760 if (end_brick != start_brick)
22763 ("Fixing bricks [%Ix, %Ix[ to point to unused array %Ix",
22764 start_brick, end_brick, (size_t)arr));
22765 set_brick (start_brick,
22766 arr - brick_address (start_brick));
22767 size_t brick = start_brick+1;
22768 while (brick < end_brick)
22770 set_brick (brick, start_brick - brick);
22775 //when we take an old segment to make the new
22776 //ephemeral segment. we can have a bunch of
22777 //pinned plugs out of order going to the new ephemeral seg
22778 //and then the next plugs go back to max_generation
22779 if ((heap_segment_mem (ephemeral_heap_segment) <= arr) &&
22780 (heap_segment_reserved (ephemeral_heap_segment) > arr))
22783 while ((low <= arr) && (high > arr))
22786 assert ((gen_number >= 1) || (demotion_low != MAX_PTR) ||
22787 settings.demotion || !settings.promotion);
22788 dprintf (3, ("new free list generation %d", gen_number));
22790 gen = generation_of (gen_number);
22791 if (gen_number >= 1)
22792 low = generation_allocation_start (generation_of (gen_number-1));
22799 dprintf (3, ("new free list generation %d", max_generation));
22800 gen_number = max_generation;
22801 gen = generation_of (gen_number);
22804 dprintf(3,("threading it into generation %d", gen_number));
22805 thread_gap (arr, len, gen);
22806 add_gen_free (gen_number, len);
22812 for (int x = 0; x <= max_generation; x++)
22814 assert (generation_allocation_start (generation_of (x)));
22818 if (!settings.demotion && settings.promotion)
22820 //clear card for generation 1. generation 0 is empty
22821 clear_card_for_addresses (
22822 generation_allocation_start (generation_of (1)),
22823 generation_allocation_start (generation_of (0)));
22825 if (settings.promotion && !settings.demotion)
22827 uint8_t* start = generation_allocation_start (youngest_generation);
22828 MAYBE_UNUSED_VAR(start);
22829 assert (heap_segment_allocated (ephemeral_heap_segment) ==
22830 (start + Align (size (start))));
22835 //force promotion for sweep
22836 settings.promotion = TRUE;
22837 settings.compaction = FALSE;
22840 sc.thread_number = heap_number;
22841 sc.promotion = FALSE;
22842 sc.concurrent = FALSE;
22844 dprintf (2, ("**** Doing Mark and Sweep GC****"));
22846 if ((condemned_gen_number < max_generation))
22848 generation_allocator (older_gen)->copy_from_alloc_list (r_free_list);
22849 generation_free_list_space (older_gen) = r_free_list_space;
22850 generation_free_obj_space (older_gen) = r_free_obj_space;
22851 generation_free_list_allocated (older_gen) = r_older_gen_free_list_allocated;
22852 generation_end_seg_allocated (older_gen) = r_older_gen_end_seg_allocated;
22853 generation_condemned_allocated (older_gen) = r_older_gen_condemned_allocated;
22854 generation_allocation_limit (older_gen) = r_allocation_limit;
22855 generation_allocation_pointer (older_gen) = r_allocation_pointer;
22856 generation_allocation_context_start_region (older_gen) = r_allocation_start_region;
22857 generation_allocation_segment (older_gen) = r_allocation_segment;
22860 if ((condemned_gen_number < max_generation))
22862 // Fix the allocation area of the older generation
22863 fix_older_allocation_area (older_gen);
22866 GCToEEInterface::DiagWalkSurvivors(__this);
22868 gen0_big_free_spaces = 0;
22869 make_free_lists (condemned_gen_number);
22870 recover_saved_pinned_info();
22872 #ifdef FEATURE_PREMORTEM_FINALIZATION
22873 finalize_queue->UpdatePromotedGenerations (condemned_gen_number, TRUE);
22874 #endif // FEATURE_PREMORTEM_FINALIZATION
22875 // MTHTS: leave single thread for HT processing on plan_phase
22876 #ifdef MULTIPLE_HEAPS
22877 dprintf(3, ("Joining after end of sweep"));
22878 gc_t_join.join(this, gc_join_adjust_handle_age_sweep);
22879 if (gc_t_join.joined())
22880 #endif //MULTIPLE_HEAPS
22882 GCScan::GcPromotionsGranted(condemned_gen_number,
22883 max_generation, &sc);
22884 if (condemned_gen_number >= (max_generation -1))
22886 #ifdef MULTIPLE_HEAPS
22887 for (int i = 0; i < n_heaps; i++)
22889 g_heaps[i]->rearrange_heap_segments(FALSE);
22892 rearrange_heap_segments(FALSE);
22893 #endif //MULTIPLE_HEAPS
22896 #ifdef MULTIPLE_HEAPS
22897 //join all threads to make sure they are synchronized
22898 dprintf(3, ("Restarting after Promotion granted"));
22899 gc_t_join.restart();
22900 #endif //MULTIPLE_HEAPS
22904 for (int x = 0; x <= max_generation; x++)
22906 assert (generation_allocation_start (generation_of (x)));
22910 //clear card for generation 1. generation 0 is empty
22911 clear_card_for_addresses (
22912 generation_allocation_start (generation_of (1)),
22913 generation_allocation_start (generation_of (0)));
22914 assert ((heap_segment_allocated (ephemeral_heap_segment) ==
22915 (generation_allocation_start (youngest_generation) +
22916 Align (min_obj_size))));
22919 //verify_partial();
22922 #pragma warning(pop)
22926 /*****************************
22927 Called after compact phase to fix all generation gaps
22928 ********************************/
22929 void gc_heap::fix_generation_bounds (int condemned_gen_number,
22930 generation* consing_gen)
22932 UNREFERENCED_PARAMETER(consing_gen);
22934 assert (generation_allocation_segment (consing_gen) ==
22935 ephemeral_heap_segment);
22937 //assign the planned allocation start to the generation
22938 int gen_number = condemned_gen_number;
22939 int bottom_gen = 0;
22941 while (gen_number >= bottom_gen)
22943 generation* gen = generation_of (gen_number);
22944 dprintf(3,("Fixing generation pointers for %Ix", gen_number));
22945 if ((gen_number < max_generation) && ephemeral_promotion)
22947 make_unused_array (saved_ephemeral_plan_start[gen_number],
22948 saved_ephemeral_plan_start_size[gen_number]);
22950 reset_allocation_pointers (gen, generation_plan_allocation_start (gen));
22951 make_unused_array (generation_allocation_start (gen), generation_plan_allocation_start_size (gen));
22952 dprintf(3,(" start %Ix", (size_t)generation_allocation_start (gen)));
22955 #ifdef MULTIPLE_HEAPS
22956 if (ephemeral_promotion)
22958 //we are creating a generation fault. set the cards.
22959 // and we are only doing this for multiple heaps because in the single heap scenario the
22960 // new ephemeral generations will be empty and there'll be no need to set cards for the
22961 // old ephemeral generations that got promoted into max_generation.
22962 ptrdiff_t delta = 0;
22963 #ifdef SEG_MAPPING_TABLE
22964 heap_segment* old_ephemeral_seg = seg_mapping_table_segment_of (saved_ephemeral_plan_start[max_generation-1]);
22965 #else //SEG_MAPPING_TABLE
22966 heap_segment* old_ephemeral_seg = segment_of (saved_ephemeral_plan_start[max_generation-1], delta);
22967 #endif //SEG_MAPPING_TABLE
22969 assert (in_range_for_segment (saved_ephemeral_plan_start[max_generation-1], old_ephemeral_seg));
22970 size_t end_card = card_of (align_on_card (heap_segment_plan_allocated (old_ephemeral_seg)));
22971 size_t card = card_of (saved_ephemeral_plan_start[max_generation-1]);
22972 while (card != end_card)
22978 #endif //MULTIPLE_HEAPS
22980 alloc_allocated = heap_segment_plan_allocated(ephemeral_heap_segment);
22981 //reset the allocated size
22982 uint8_t* start = generation_allocation_start (youngest_generation);
22983 MAYBE_UNUSED_VAR(start);
22984 if (settings.promotion && !settings.demotion)
22986 assert ((start + Align (size (start))) ==
22987 heap_segment_plan_allocated(ephemeral_heap_segment));
22990 heap_segment_allocated(ephemeral_heap_segment)=
22991 heap_segment_plan_allocated(ephemeral_heap_segment);
22995 uint8_t* gc_heap::generation_limit (int gen_number)
22997 if (settings.promotion)
22999 if (gen_number <= 1)
23000 return heap_segment_reserved (ephemeral_heap_segment);
23002 return generation_allocation_start (generation_of ((gen_number - 2)));
23006 if (gen_number <= 0)
23007 return heap_segment_reserved (ephemeral_heap_segment);
23009 return generation_allocation_start (generation_of ((gen_number - 1)));
23013 BOOL gc_heap::ensure_gap_allocation (int condemned_gen_number)
23015 uint8_t* start = heap_segment_allocated (ephemeral_heap_segment);
23016 size_t size = Align (min_obj_size)*(condemned_gen_number+1);
23017 assert ((start + size) <=
23018 heap_segment_reserved (ephemeral_heap_segment));
23019 if ((start + size) >
23020 heap_segment_committed (ephemeral_heap_segment))
23022 if (!grow_heap_segment (ephemeral_heap_segment, start + size))
23030 uint8_t* gc_heap::allocate_at_end (size_t size)
23032 uint8_t* start = heap_segment_allocated (ephemeral_heap_segment);
23033 size = Align (size);
23034 uint8_t* result = start;
23035 // only called to allocate a min obj so can't overflow here.
23036 assert ((start + size) <=
23037 heap_segment_reserved (ephemeral_heap_segment));
23038 //ensure_gap_allocation took care of it
23039 assert ((start + size) <=
23040 heap_segment_committed (ephemeral_heap_segment));
23041 heap_segment_allocated (ephemeral_heap_segment) += size;
23046 void gc_heap::make_free_lists (int condemned_gen_number)
23051 start = GetCycleCount32();
23054 //Promotion has to happen in sweep case.
23055 assert (settings.promotion);
23057 generation* condemned_gen = generation_of (condemned_gen_number);
23058 uint8_t* start_address = generation_allocation_start (condemned_gen);
23060 size_t current_brick = brick_of (start_address);
23061 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
23063 PREFIX_ASSUME(current_heap_segment != NULL);
23065 uint8_t* end_address = heap_segment_allocated (current_heap_segment);
23066 size_t end_brick = brick_of (end_address-1);
23067 make_free_args args;
23068 args.free_list_gen_number = min (max_generation, 1 + condemned_gen_number);
23069 args.current_gen_limit = (((condemned_gen_number == max_generation)) ?
23071 (generation_limit (args.free_list_gen_number)));
23072 args.free_list_gen = generation_of (args.free_list_gen_number);
23073 args.highest_plug = 0;
23075 if ((start_address < end_address) ||
23076 (condemned_gen_number == max_generation))
23080 if ((current_brick > end_brick))
23082 if (args.current_gen_limit == MAX_PTR)
23084 //We had an empty segment
23085 //need to allocate the generation start
23087 generation* gen = generation_of (max_generation);
23089 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
23091 PREFIX_ASSUME(start_seg != NULL);
23093 uint8_t* gap = heap_segment_mem (start_seg);
23095 generation_allocation_start (gen) = gap;
23096 heap_segment_allocated (start_seg) = gap + Align (min_obj_size);
23097 make_unused_array (gap, Align (min_obj_size));
23098 reset_allocation_pointers (gen, gap);
23099 dprintf (3, ("Start segment empty, fixing generation start of %d to: %Ix",
23100 max_generation, (size_t)gap));
23101 args.current_gen_limit = generation_limit (args.free_list_gen_number);
23103 if (heap_segment_next_rw (current_heap_segment))
23105 current_heap_segment = heap_segment_next_rw (current_heap_segment);
23106 current_brick = brick_of (heap_segment_mem (current_heap_segment));
23107 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
23117 int brick_entry = brick_table [ current_brick ];
23118 if ((brick_entry >= 0))
23120 make_free_list_in_brick (brick_address (current_brick) + brick_entry-1, &args);
23121 dprintf(3,("Fixing brick entry %Ix to %Ix",
23122 current_brick, (size_t)args.highest_plug));
23123 set_brick (current_brick,
23124 (args.highest_plug - brick_address (current_brick)));
23128 if ((brick_entry > -32768))
23132 ptrdiff_t offset = brick_of (args.highest_plug) - current_brick;
23133 if ((brick_entry != -32767) && (! ((offset == brick_entry))))
23135 assert ((brick_entry == -1));
23138 //init to -1 for faster find_first_object
23139 set_brick (current_brick, -1);
23147 int bottom_gen = 0;
23148 args.free_list_gen_number--;
23149 while (args.free_list_gen_number >= bottom_gen)
23152 generation* gen2 = generation_of (args.free_list_gen_number);
23153 gap = allocate_at_end (Align(min_obj_size));
23154 generation_allocation_start (gen2) = gap;
23155 reset_allocation_pointers (gen2, gap);
23156 dprintf(3,("Fixing generation start of %d to: %Ix",
23157 args.free_list_gen_number, (size_t)gap));
23158 PREFIX_ASSUME(gap != NULL);
23159 make_unused_array (gap, Align (min_obj_size));
23161 args.free_list_gen_number--;
23164 //reset the allocated size
23165 uint8_t* start2 = generation_allocation_start (youngest_generation);
23166 alloc_allocated = start2 + Align (size (start2));
23170 finish = GetCycleCount32();
23171 sweep_time = finish - start;
23175 void gc_heap::make_free_list_in_brick (uint8_t* tree, make_free_args* args)
23177 assert ((tree != NULL));
23179 int right_node = node_right_child (tree);
23180 int left_node = node_left_child (tree);
23181 args->highest_plug = 0;
23184 if (! (0 == left_node))
23186 make_free_list_in_brick (tree + left_node, args);
23190 uint8_t* plug = tree;
23191 size_t gap_size = node_gap_size (tree);
23192 uint8_t* gap = (plug - gap_size);
23193 dprintf (3,("Making free list %Ix len %d in %d",
23194 //dprintf (3,("F: %Ix len %Ix in %d",
23195 (size_t)gap, gap_size, args->free_list_gen_number));
23196 args->highest_plug = tree;
23198 if (is_plug_padded (plug))
23200 dprintf (3, ("%Ix padded", plug));
23201 clear_plug_padded (plug);
23203 #endif //SHORT_PLUGS
23206 if ((args->current_gen_limit == MAX_PTR) ||
23207 ((plug >= args->current_gen_limit) &&
23208 ephemeral_pointer_p (plug)))
23210 dprintf(3,(" Crossing Generation boundary at %Ix",
23211 (size_t)args->current_gen_limit));
23212 if (!(args->current_gen_limit == MAX_PTR))
23214 args->free_list_gen_number--;
23215 args->free_list_gen = generation_of (args->free_list_gen_number);
23217 dprintf(3,( " Fixing generation start of %d to: %Ix",
23218 args->free_list_gen_number, (size_t)gap));
23220 reset_allocation_pointers (args->free_list_gen, gap);
23221 args->current_gen_limit = generation_limit (args->free_list_gen_number);
23223 if ((gap_size >= (2*Align (min_obj_size))))
23225 dprintf(3,(" Splitting the gap in two %Id left",
23227 make_unused_array (gap, Align(min_obj_size));
23228 gap_size = (gap_size - Align(min_obj_size));
23229 gap = (gap + Align(min_obj_size));
23233 make_unused_array (gap, gap_size);
23240 thread_gap (gap, gap_size, args->free_list_gen);
23241 add_gen_free (args->free_list_gen->gen_num, gap_size);
23243 if (! (0 == right_node))
23245 make_free_list_in_brick (tree + right_node, args);
23251 void gc_heap::thread_gap (uint8_t* gap_start, size_t size, generation* gen)
23253 assert (generation_allocation_start (gen));
23256 if ((gen->gen_num == 0) && (size > CLR_SIZE))
23258 gen0_big_free_spaces += size;
23261 assert ((heap_segment_rw (generation_start_segment (gen))!=
23262 ephemeral_heap_segment) ||
23263 (gap_start > generation_allocation_start (gen)));
23264 // The beginning of a segment gap is not aligned
23265 assert (size >= Align (min_obj_size));
23266 make_unused_array (gap_start, size,
23267 (!settings.concurrent && (gen != youngest_generation)),
23268 (gen->gen_num == max_generation));
23269 dprintf (3, ("fr: [%Ix, %Ix[", (size_t)gap_start, (size_t)gap_start+size));
23271 if ((size >= min_free_list))
23273 generation_free_list_space (gen) += size;
23274 generation_allocator (gen)->thread_item (gap_start, size);
23278 generation_free_obj_space (gen) += size;
23283 void gc_heap::loh_thread_gap_front (uint8_t* gap_start, size_t size, generation* gen)
23285 assert (generation_allocation_start (gen));
23286 if (size >= min_free_list)
23288 generation_free_list_space (gen) += size;
23289 generation_allocator (gen)->thread_item_front (gap_start, size);
23293 void gc_heap::make_unused_array (uint8_t* x, size_t size, BOOL clearp, BOOL resetp)
23295 dprintf (3, ("Making unused array [%Ix, %Ix[",
23296 (size_t)x, (size_t)(x+size)));
23297 assert (size >= Align (min_obj_size));
23299 //#if defined (VERIFY_HEAP) && defined (BACKGROUND_GC)
23300 // check_batch_mark_array_bits (x, x+size);
23301 //#endif //VERIFY_HEAP && BACKGROUND_GC
23304 reset_memory (x, size);
23306 ((CObjectHeader*)x)->SetFree(size);
23311 #error "This won't work on big endian platforms"
23314 size_t size_as_object = (uint32_t)(size - free_object_base_size) + free_object_base_size;
23316 if (size_as_object < size)
23319 // If the size is more than 4GB, we need to create multiple objects because of
23320 // the Array::m_NumComponents is uint32_t and the high 32 bits of unused array
23321 // size is ignored in regular object size computation.
23323 uint8_t * tmp = x + size_as_object;
23324 size_t remaining_size = size - size_as_object;
23326 while (remaining_size > UINT32_MAX)
23328 // Make sure that there will be at least Align(min_obj_size) left
23329 size_t current_size = UINT32_MAX - get_alignment_constant (FALSE)
23330 - Align (min_obj_size, get_alignment_constant (FALSE));
23332 ((CObjectHeader*)tmp)->SetFree(current_size);
23334 remaining_size -= current_size;
23335 tmp += current_size;
23338 ((CObjectHeader*)tmp)->SetFree(remaining_size);
23343 clear_card_for_addresses (x, x + Align(size));
23346 // Clear memory set by make_unused_array.
23347 void gc_heap::clear_unused_array (uint8_t* x, size_t size)
23349 // Also clear the sync block
23350 *(((PTR_PTR)x)-1) = 0;
23352 ((CObjectHeader*)x)->UnsetFree();
23357 #error "This won't work on big endian platforms"
23360 // The memory could have been cleared in the meantime. We have to mirror the algorithm
23361 // from make_unused_array since we cannot depend on the object sizes in memory.
23362 size_t size_as_object = (uint32_t)(size - free_object_base_size) + free_object_base_size;
23364 if (size_as_object < size)
23366 uint8_t * tmp = x + size_as_object;
23367 size_t remaining_size = size - size_as_object;
23369 while (remaining_size > UINT32_MAX)
23371 size_t current_size = UINT32_MAX - get_alignment_constant (FALSE)
23372 - Align (min_obj_size, get_alignment_constant (FALSE));
23374 ((CObjectHeader*)tmp)->UnsetFree();
23376 remaining_size -= current_size;
23377 tmp += current_size;
23380 ((CObjectHeader*)tmp)->UnsetFree();
23383 UNREFERENCED_PARAMETER(size);
23388 uint8_t* tree_search (uint8_t* tree, uint8_t* old_address)
23390 uint8_t* candidate = 0;
23394 if (tree < old_address)
23396 if ((cn = node_right_child (tree)) != 0)
23398 assert (candidate < tree);
23401 Prefetch (tree - 8);
23407 else if (tree > old_address)
23409 if ((cn = node_left_child (tree)) != 0)
23412 Prefetch (tree - 8);
23420 if (tree <= old_address)
23422 else if (candidate)
23428 #ifdef FEATURE_BASICFREEZE
23429 bool gc_heap::frozen_object_p (Object* obj)
23431 heap_segment* pSegment = gc_heap::find_segment ((uint8_t*)obj, FALSE);
23432 _ASSERTE(pSegment);
23434 return heap_segment_read_only_p(pSegment);
23436 #endif // FEATURE_BASICFREEZE
23438 #ifdef FEATURE_REDHAWK
23439 // TODO: this was added on RH, we have not done perf runs to see if this is the right
23440 // thing to do for other versions of the CLR.
23442 #endif // FEATURE_REDHAWK
23443 void gc_heap::relocate_address (uint8_t** pold_address THREAD_NUMBER_DCL)
23445 uint8_t* old_address = *pold_address;
23446 if (!((old_address >= gc_low) && (old_address < gc_high)))
23447 #ifdef MULTIPLE_HEAPS
23449 UNREFERENCED_PARAMETER(thread);
23450 if (old_address == 0)
23452 gc_heap* hp = heap_of (old_address);
23453 if ((hp == this) ||
23454 !((old_address >= hp->gc_low) && (old_address < hp->gc_high)))
23457 #else //MULTIPLE_HEAPS
23459 #endif //MULTIPLE_HEAPS
23460 // delta translates old_address into address_gc (old_address);
23461 size_t brick = brick_of (old_address);
23462 int brick_entry = brick_table [ brick ];
23463 uint8_t* new_address = old_address;
23464 if (! ((brick_entry == 0)))
23468 while (brick_entry < 0)
23470 brick = (brick + brick_entry);
23471 brick_entry = brick_table [ brick ];
23473 uint8_t* old_loc = old_address;
23475 uint8_t* node = tree_search ((brick_address (brick) + brick_entry-1),
23477 if ((node <= old_loc))
23478 new_address = (old_address + node_relocation_distance (node));
23481 if (node_left_p (node))
23483 dprintf(3,(" L: %Ix", (size_t)node));
23484 new_address = (old_address +
23485 (node_relocation_distance (node) +
23486 node_gap_size (node)));
23491 brick_entry = brick_table [ brick ];
23497 *pold_address = new_address;
23501 #ifdef FEATURE_LOH_COMPACTION
23502 if (loh_compacted_p
23503 #ifdef FEATURE_BASICFREEZE
23504 && !frozen_object_p((Object*)old_address)
23505 #endif // FEATURE_BASICFREEZE
23508 *pold_address = old_address + loh_node_relocation_distance (old_address);
23511 #endif //FEATURE_LOH_COMPACTION
23513 *pold_address = new_address;
23518 gc_heap::check_class_object_demotion (uint8_t* obj)
23520 #ifdef COLLECTIBLE_CLASS
23521 if (is_collectible(obj))
23523 check_class_object_demotion_internal (obj);
23526 UNREFERENCED_PARAMETER(obj);
23527 #endif //COLLECTIBLE_CLASS
23530 #ifdef COLLECTIBLE_CLASS
23532 gc_heap::check_class_object_demotion_internal (uint8_t* obj)
23534 if (settings.demotion)
23536 #ifdef MULTIPLE_HEAPS
23537 // We set the card without checking the demotion range 'cause at this point
23538 // the handle that points to the loader allocator object may or may not have
23539 // been relocated by other GC threads.
23540 set_card (card_of (obj));
23543 uint8_t* class_obj = get_class_object (obj);
23544 dprintf (3, ("%Ix: got classobj %Ix", obj, class_obj));
23545 uint8_t* temp_class_obj = class_obj;
23546 uint8_t** temp = &temp_class_obj;
23547 relocate_address (temp THREAD_NUMBER_ARG);
23549 check_demotion_helper (temp, obj);
23550 #endif //MULTIPLE_HEAPS
23554 #endif //COLLECTIBLE_CLASS
23557 gc_heap::check_demotion_helper (uint8_t** pval, uint8_t* parent_obj)
23559 // detect if we are demoting an object
23560 if ((*pval < demotion_high) &&
23561 (*pval >= demotion_low))
23563 dprintf(3, ("setting card %Ix:%Ix",
23564 card_of((uint8_t*)pval),
23567 set_card (card_of (parent_obj));
23569 #ifdef MULTIPLE_HEAPS
23570 else if (settings.demotion)
23572 dprintf (4, ("Demotion active, computing heap_of object"));
23573 gc_heap* hp = heap_of (*pval);
23574 if ((*pval < hp->demotion_high) &&
23575 (*pval >= hp->demotion_low))
23577 dprintf(3, ("setting card %Ix:%Ix",
23578 card_of((uint8_t*)pval),
23581 set_card (card_of (parent_obj));
23584 #endif //MULTIPLE_HEAPS
23588 gc_heap::reloc_survivor_helper (uint8_t** pval)
23591 relocate_address (pval THREAD_NUMBER_ARG);
23593 check_demotion_helper (pval, (uint8_t*)pval);
23597 gc_heap::relocate_obj_helper (uint8_t* x, size_t s)
23600 if (contain_pointers (x))
23602 dprintf (3, ("$%Ix$", (size_t)x));
23604 go_through_object_nostart (method_table(x), x, s, pval,
23606 uint8_t* child = *pval;
23607 reloc_survivor_helper (pval);
23610 dprintf (3, ("%Ix->%Ix->%Ix", (uint8_t*)pval, child, *pval));
23615 check_class_object_demotion (x);
23619 void gc_heap::reloc_ref_in_shortened_obj (uint8_t** address_to_set_card, uint8_t** address_to_reloc)
23623 uint8_t* old_val = (address_to_reloc ? *address_to_reloc : 0);
23624 relocate_address (address_to_reloc THREAD_NUMBER_ARG);
23625 if (address_to_reloc)
23627 dprintf (3, ("SR %Ix: %Ix->%Ix", (uint8_t*)address_to_reloc, old_val, *address_to_reloc));
23630 //check_demotion_helper (current_saved_info_to_relocate, (uint8_t*)pval);
23631 uint8_t* relocated_addr = *address_to_reloc;
23632 if ((relocated_addr < demotion_high) &&
23633 (relocated_addr >= demotion_low))
23635 dprintf (3, ("set card for location %Ix(%Ix)",
23636 (size_t)address_to_set_card, card_of((uint8_t*)address_to_set_card)));
23638 set_card (card_of ((uint8_t*)address_to_set_card));
23640 #ifdef MULTIPLE_HEAPS
23641 else if (settings.demotion)
23643 gc_heap* hp = heap_of (relocated_addr);
23644 if ((relocated_addr < hp->demotion_high) &&
23645 (relocated_addr >= hp->demotion_low))
23647 dprintf (3, ("%Ix on h%d, set card for location %Ix(%Ix)",
23648 relocated_addr, hp->heap_number, (size_t)address_to_set_card, card_of((uint8_t*)address_to_set_card)));
23650 set_card (card_of ((uint8_t*)address_to_set_card));
23653 #endif //MULTIPLE_HEAPS
23656 void gc_heap::relocate_pre_plug_info (mark* pinned_plug_entry)
23659 uint8_t* plug = pinned_plug (pinned_plug_entry);
23660 uint8_t* pre_plug_start = plug - sizeof (plug_and_gap);
23661 // Note that we need to add one ptr size here otherwise we may not be able to find the relocated
23662 // address. Consider this scenario:
23663 // gen1 start | 3-ptr sized NP | PP
23665 // If we are asking for the reloc address of 0x10 we will AV in relocate_address because
23666 // the first plug we saw in the brick is 0x18 which means 0x10 will cause us to go back a brick
23667 // which is 0, and then we'll AV in tree_search when we try to do node_right_child (tree).
23668 pre_plug_start += sizeof (uint8_t*);
23669 uint8_t** old_address = &pre_plug_start;
23671 uint8_t* old_val = (old_address ? *old_address : 0);
23672 relocate_address (old_address THREAD_NUMBER_ARG);
23675 dprintf (3, ("PreR %Ix: %Ix->%Ix, set reloc: %Ix",
23676 (uint8_t*)old_address, old_val, *old_address, (pre_plug_start - sizeof (uint8_t*))));
23679 pinned_plug_entry->set_pre_plug_info_reloc_start (pre_plug_start - sizeof (uint8_t*));
23683 void gc_heap::relocate_shortened_obj_helper (uint8_t* x, size_t s, uint8_t* end, mark* pinned_plug_entry, BOOL is_pinned)
23686 uint8_t* plug = pinned_plug (pinned_plug_entry);
23690 //// Temporary - we just wanna make sure we are doing things right when padding is needed.
23691 //if ((x + s) < plug)
23693 // dprintf (3, ("obj %Ix needed padding: end %Ix is %d bytes from pinned obj %Ix",
23694 // x, (x + s), (plug- (x + s)), plug));
23695 // GCToOSInterface::DebugBreak();
23698 relocate_pre_plug_info (pinned_plug_entry);
23701 verify_pins_with_post_plug_info("after relocate_pre_plug_info");
23703 uint8_t* saved_plug_info_start = 0;
23704 uint8_t** saved_info_to_relocate = 0;
23708 saved_plug_info_start = (uint8_t*)(pinned_plug_entry->get_post_plug_info_start());
23709 saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_post_plug_reloc_info());
23713 saved_plug_info_start = (plug - sizeof (plug_and_gap));
23714 saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_pre_plug_reloc_info());
23717 uint8_t** current_saved_info_to_relocate = 0;
23718 uint8_t* child = 0;
23720 dprintf (3, ("x: %Ix, pp: %Ix, end: %Ix", x, plug, end));
23722 if (contain_pointers (x))
23724 dprintf (3,("$%Ix$", (size_t)x));
23726 go_through_object_nostart (method_table(x), x, s, pval,
23728 dprintf (3, ("obj %Ix, member: %Ix->%Ix", x, (uint8_t*)pval, *pval));
23730 if ((uint8_t*)pval >= end)
23732 current_saved_info_to_relocate = saved_info_to_relocate + ((uint8_t*)pval - saved_plug_info_start) / sizeof (uint8_t**);
23733 child = *current_saved_info_to_relocate;
23734 reloc_ref_in_shortened_obj (pval, current_saved_info_to_relocate);
23735 dprintf (3, ("last part: R-%Ix(saved: %Ix)->%Ix ->%Ix",
23736 (uint8_t*)pval, current_saved_info_to_relocate, child, *current_saved_info_to_relocate));
23740 reloc_survivor_helper (pval);
23745 check_class_object_demotion (x);
23748 void gc_heap::relocate_survivor_helper (uint8_t* plug, uint8_t* plug_end)
23751 while (x < plug_end)
23753 size_t s = size (x);
23754 uint8_t* next_obj = x + Align (s);
23755 Prefetch (next_obj);
23756 relocate_obj_helper (x, s);
23762 // if we expanded, right now we are not handling it as We are not saving the new reloc info.
23763 void gc_heap::verify_pins_with_post_plug_info (const char* msg)
23765 #if defined (_DEBUG) && defined (VERIFY_HEAP)
23766 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
23768 if (!verify_pinned_queue_p)
23771 if (settings.heap_expansion)
23774 for (size_t i = 0; i < mark_stack_tos; i++)
23776 mark& m = mark_stack_array[i];
23778 mark* pinned_plug_entry = pinned_plug_of(i);
23780 if (pinned_plug_entry->has_post_plug_info() &&
23781 pinned_plug_entry->post_short_p() &&
23782 (pinned_plug_entry->saved_post_plug_debug.gap != 1))
23784 uint8_t* next_obj = pinned_plug_entry->get_post_plug_info_start() + sizeof (plug_and_gap);
23785 // object after pin
23786 dprintf (3, ("OFP: %Ix, G: %Ix, R: %Ix, LC: %d, RC: %d",
23787 next_obj, node_gap_size (next_obj), node_relocation_distance (next_obj),
23788 (int)node_left_child (next_obj), (int)node_right_child (next_obj)));
23790 size_t* post_plug_debug = (size_t*)(&m.saved_post_plug_debug);
23792 if (node_gap_size (next_obj) != *post_plug_debug)
23794 dprintf (3, ("obj: %Ix gap should be %Ix but it is %Ix",
23795 next_obj, *post_plug_debug, (size_t)(node_gap_size (next_obj))));
23799 // can't do node_relocation_distance here as it clears the left bit.
23800 //if (node_relocation_distance (next_obj) != *post_plug_debug)
23801 if (*((size_t*)(next_obj - 3 * sizeof (size_t))) != *post_plug_debug)
23803 dprintf (3, ("obj: %Ix reloc should be %Ix but it is %Ix",
23804 next_obj, *post_plug_debug, (size_t)(node_relocation_distance (next_obj))));
23807 if (node_left_child (next_obj) > 0)
23809 dprintf (3, ("obj: %Ix, vLC: %d\n", next_obj, (int)(node_left_child (next_obj))));
23815 dprintf (3, ("%s verified", msg));
23817 #else // _DEBUG && VERIFY_HEAP
23818 UNREFERENCED_PARAMETER(msg);
23819 #endif // _DEBUG && VERIFY_HEAP
23822 #ifdef COLLECTIBLE_CLASS
23823 // We don't want to burn another ptr size space for pinned plugs to record this so just
23824 // set the card unconditionally for collectible objects if we are demoting.
23826 gc_heap::unconditional_set_card_collectible (uint8_t* obj)
23828 if (settings.demotion)
23830 set_card (card_of (obj));
23833 #endif //COLLECTIBLE_CLASS
23835 void gc_heap::relocate_shortened_survivor_helper (uint8_t* plug, uint8_t* plug_end, mark* pinned_plug_entry)
23838 uint8_t* p_plug = pinned_plug (pinned_plug_entry);
23839 BOOL is_pinned = (plug == p_plug);
23840 BOOL check_short_obj_p = (is_pinned ? pinned_plug_entry->post_short_p() : pinned_plug_entry->pre_short_p());
23842 plug_end += sizeof (gap_reloc_pair);
23844 //dprintf (3, ("%s %Ix is shortened, and last object %s overwritten", (is_pinned ? "PP" : "NP"), plug, (check_short_obj_p ? "is" : "is not")));
23845 dprintf (3, ("%s %Ix-%Ix short, LO: %s OW", (is_pinned ? "PP" : "NP"), plug, plug_end, (check_short_obj_p ? "is" : "is not")));
23847 verify_pins_with_post_plug_info("begin reloc short surv");
23849 while (x < plug_end)
23851 if (check_short_obj_p && ((plug_end - x) < min_pre_pin_obj_size))
23853 dprintf (3, ("last obj %Ix is short", x));
23857 #ifdef COLLECTIBLE_CLASS
23858 if (pinned_plug_entry->post_short_collectible_p())
23859 unconditional_set_card_collectible (x);
23860 #endif //COLLECTIBLE_CLASS
23862 // Relocate the saved references based on bits set.
23863 uint8_t** saved_plug_info_start = (uint8_t**)(pinned_plug_entry->get_post_plug_info_start());
23864 uint8_t** saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_post_plug_reloc_info());
23865 for (size_t i = 0; i < pinned_plug_entry->get_max_short_bits(); i++)
23867 if (pinned_plug_entry->post_short_bit_p (i))
23869 reloc_ref_in_shortened_obj ((saved_plug_info_start + i), (saved_info_to_relocate + i));
23875 #ifdef COLLECTIBLE_CLASS
23876 if (pinned_plug_entry->pre_short_collectible_p())
23877 unconditional_set_card_collectible (x);
23878 #endif //COLLECTIBLE_CLASS
23880 relocate_pre_plug_info (pinned_plug_entry);
23882 // Relocate the saved references based on bits set.
23883 uint8_t** saved_plug_info_start = (uint8_t**)(p_plug - sizeof (plug_and_gap));
23884 uint8_t** saved_info_to_relocate = (uint8_t**)(pinned_plug_entry->get_pre_plug_reloc_info());
23885 for (size_t i = 0; i < pinned_plug_entry->get_max_short_bits(); i++)
23887 if (pinned_plug_entry->pre_short_bit_p (i))
23889 reloc_ref_in_shortened_obj ((saved_plug_info_start + i), (saved_info_to_relocate + i));
23897 size_t s = size (x);
23898 uint8_t* next_obj = x + Align (s);
23899 Prefetch (next_obj);
23901 if (next_obj >= plug_end)
23903 dprintf (3, ("object %Ix is at the end of the plug %Ix->%Ix",
23904 next_obj, plug, plug_end));
23906 verify_pins_with_post_plug_info("before reloc short obj");
23908 relocate_shortened_obj_helper (x, s, (x + Align (s) - sizeof (plug_and_gap)), pinned_plug_entry, is_pinned);
23912 relocate_obj_helper (x, s);
23919 verify_pins_with_post_plug_info("end reloc short surv");
23922 void gc_heap::relocate_survivors_in_plug (uint8_t* plug, uint8_t* plug_end,
23923 BOOL check_last_object_p,
23924 mark* pinned_plug_entry)
23926 //dprintf(3,("Relocating pointers in Plug [%Ix,%Ix[", (size_t)plug, (size_t)plug_end));
23927 dprintf (3,("RP: [%Ix,%Ix[", (size_t)plug, (size_t)plug_end));
23929 if (check_last_object_p)
23931 relocate_shortened_survivor_helper (plug, plug_end, pinned_plug_entry);
23935 relocate_survivor_helper (plug, plug_end);
23939 void gc_heap::relocate_survivors_in_brick (uint8_t* tree, relocate_args* args)
23941 assert ((tree != NULL));
23943 dprintf (3, ("tree: %Ix, args->last_plug: %Ix, left: %Ix, right: %Ix, gap(t): %Ix",
23944 tree, args->last_plug,
23945 (tree + node_left_child (tree)),
23946 (tree + node_right_child (tree)),
23947 node_gap_size (tree)));
23949 if (node_left_child (tree))
23951 relocate_survivors_in_brick (tree + node_left_child (tree), args);
23954 uint8_t* plug = tree;
23955 BOOL has_post_plug_info_p = FALSE;
23956 BOOL has_pre_plug_info_p = FALSE;
23958 if (tree == oldest_pinned_plug)
23960 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
23961 &has_post_plug_info_p);
23962 assert (tree == pinned_plug (args->pinned_plug_entry));
23964 dprintf (3, ("tree is the oldest pin: %Ix", tree));
23966 if (args->last_plug)
23968 size_t gap_size = node_gap_size (tree);
23969 uint8_t* gap = (plug - gap_size);
23970 dprintf (3, ("tree: %Ix, gap: %Ix (%Ix)", tree, gap, gap_size));
23971 assert (gap_size >= Align (min_obj_size));
23972 uint8_t* last_plug_end = gap;
23974 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
23977 relocate_survivors_in_plug (args->last_plug, last_plug_end, check_last_object_p, args->pinned_plug_entry);
23982 assert (!has_pre_plug_info_p);
23985 args->last_plug = plug;
23986 args->is_shortened = has_post_plug_info_p;
23987 if (has_post_plug_info_p)
23989 dprintf (3, ("setting %Ix as shortened", plug));
23991 dprintf (3, ("last_plug: %Ix(shortened: %d)", plug, (args->is_shortened ? 1 : 0)));
23993 if (node_right_child (tree))
23995 relocate_survivors_in_brick (tree + node_right_child (tree), args);
24000 void gc_heap::update_oldest_pinned_plug()
24002 oldest_pinned_plug = (pinned_plug_que_empty_p() ? 0 : pinned_plug (oldest_pin()));
24005 void gc_heap::relocate_survivors (int condemned_gen_number,
24006 uint8_t* first_condemned_address)
24008 generation* condemned_gen = generation_of (condemned_gen_number);
24009 uint8_t* start_address = first_condemned_address;
24010 size_t current_brick = brick_of (start_address);
24011 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
24013 PREFIX_ASSUME(current_heap_segment != NULL);
24015 uint8_t* end_address = 0;
24017 reset_pinned_queue_bos();
24018 update_oldest_pinned_plug();
24020 end_address = heap_segment_allocated (current_heap_segment);
24022 size_t end_brick = brick_of (end_address - 1);
24023 relocate_args args;
24025 args.high = gc_high;
24026 args.is_shortened = FALSE;
24027 args.pinned_plug_entry = 0;
24028 args.last_plug = 0;
24031 if (current_brick > end_brick)
24033 if (args.last_plug)
24036 assert (!(args.is_shortened));
24037 relocate_survivors_in_plug (args.last_plug,
24038 heap_segment_allocated (current_heap_segment),
24040 args.pinned_plug_entry);
24043 args.last_plug = 0;
24046 if (heap_segment_next_rw (current_heap_segment))
24048 current_heap_segment = heap_segment_next_rw (current_heap_segment);
24049 current_brick = brick_of (heap_segment_mem (current_heap_segment));
24050 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24059 int brick_entry = brick_table [ current_brick ];
24061 if (brick_entry >= 0)
24063 relocate_survivors_in_brick (brick_address (current_brick) +
24072 void gc_heap::walk_plug (uint8_t* plug, size_t size, BOOL check_last_object_p, walk_relocate_args* args)
24074 if (check_last_object_p)
24076 size += sizeof (gap_reloc_pair);
24077 mark* entry = args->pinned_plug_entry;
24079 if (args->is_shortened)
24081 assert (entry->has_post_plug_info());
24082 entry->swap_post_plug_and_saved_for_profiler();
24086 assert (entry->has_pre_plug_info());
24087 entry->swap_pre_plug_and_saved_for_profiler();
24091 ptrdiff_t last_plug_relocation = node_relocation_distance (plug);
24092 STRESS_LOG_PLUG_MOVE(plug, (plug + size), -last_plug_relocation);
24093 ptrdiff_t reloc = settings.compaction ? last_plug_relocation : 0;
24095 (args->fn) (plug, (plug + size), reloc, args->profiling_context, !!settings.compaction, false);
24097 if (check_last_object_p)
24099 mark* entry = args->pinned_plug_entry;
24101 if (args->is_shortened)
24103 entry->swap_post_plug_and_saved_for_profiler();
24107 entry->swap_pre_plug_and_saved_for_profiler();
24112 void gc_heap::walk_relocation_in_brick (uint8_t* tree, walk_relocate_args* args)
24114 assert ((tree != NULL));
24115 if (node_left_child (tree))
24117 walk_relocation_in_brick (tree + node_left_child (tree), args);
24120 uint8_t* plug = tree;
24121 BOOL has_pre_plug_info_p = FALSE;
24122 BOOL has_post_plug_info_p = FALSE;
24124 if (tree == oldest_pinned_plug)
24126 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
24127 &has_post_plug_info_p);
24128 assert (tree == pinned_plug (args->pinned_plug_entry));
24131 if (args->last_plug != 0)
24133 size_t gap_size = node_gap_size (tree);
24134 uint8_t* gap = (plug - gap_size);
24135 uint8_t* last_plug_end = gap;
24136 size_t last_plug_size = (last_plug_end - args->last_plug);
24137 dprintf (3, ("tree: %Ix, last_plug: %Ix, gap: %Ix(%Ix), last_plug_end: %Ix, size: %Ix",
24138 tree, args->last_plug, gap, gap_size, last_plug_end, last_plug_size));
24140 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
24141 if (!check_last_object_p)
24143 assert (last_plug_size >= Align (min_obj_size));
24146 walk_plug (args->last_plug, last_plug_size, check_last_object_p, args);
24150 assert (!has_pre_plug_info_p);
24153 dprintf (3, ("set args last plug to plug: %Ix", plug));
24154 args->last_plug = plug;
24155 args->is_shortened = has_post_plug_info_p;
24157 if (node_right_child (tree))
24159 walk_relocation_in_brick (tree + node_right_child (tree), args);
24163 void gc_heap::walk_relocation (void* profiling_context, record_surv_fn fn)
24165 generation* condemned_gen = generation_of (settings.condemned_generation);
24166 uint8_t* start_address = generation_allocation_start (condemned_gen);
24167 size_t current_brick = brick_of (start_address);
24168 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
24170 PREFIX_ASSUME(current_heap_segment != NULL);
24172 reset_pinned_queue_bos();
24173 update_oldest_pinned_plug();
24174 size_t end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24175 walk_relocate_args args;
24176 args.is_shortened = FALSE;
24177 args.pinned_plug_entry = 0;
24178 args.last_plug = 0;
24179 args.profiling_context = profiling_context;
24184 if (current_brick > end_brick)
24186 if (args.last_plug)
24188 walk_plug (args.last_plug,
24189 (heap_segment_allocated (current_heap_segment) - args.last_plug),
24192 args.last_plug = 0;
24194 if (heap_segment_next_rw (current_heap_segment))
24196 current_heap_segment = heap_segment_next_rw (current_heap_segment);
24197 current_brick = brick_of (heap_segment_mem (current_heap_segment));
24198 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24207 int brick_entry = brick_table [ current_brick ];
24208 if (brick_entry >= 0)
24210 walk_relocation_in_brick (brick_address (current_brick) +
24219 void gc_heap::walk_survivors (record_surv_fn fn, void* context, walk_surv_type type)
24221 if (type == walk_for_gc)
24222 walk_survivors_relocation (context, fn);
24223 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24224 else if (type == walk_for_bgc)
24225 walk_survivors_for_bgc (context, fn);
24226 #endif //BACKGROUND_GC && FEATURE_EVENT_TRACE
24227 else if (type == walk_for_loh)
24228 walk_survivors_for_loh (context, fn);
24230 assert (!"unknown type!");
24233 #if defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24234 void gc_heap::walk_survivors_for_bgc (void* profiling_context, record_surv_fn fn)
24236 // This should only be called for BGCs
24237 assert(settings.concurrent);
24239 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
24241 BOOL small_object_segments = TRUE;
24242 int align_const = get_alignment_constant (small_object_segments);
24248 if (small_object_segments)
24250 //switch to large segment
24251 small_object_segments = FALSE;
24253 align_const = get_alignment_constant (small_object_segments);
24254 seg = heap_segment_rw (generation_start_segment (large_object_generation));
24256 PREFIX_ASSUME(seg != NULL);
24264 uint8_t* o = heap_segment_mem (seg);
24265 uint8_t* end = heap_segment_allocated (seg);
24269 if (method_table(o) == g_gc_pFreeObjectMethodTable)
24271 o += Align (size (o), align_const);
24275 // It's survived. Make a fake plug, starting at o,
24276 // and send the event
24278 uint8_t* plug_start = o;
24280 while (method_table(o) != g_gc_pFreeObjectMethodTable)
24282 o += Align (size (o), align_const);
24289 uint8_t* plug_end = o;
24293 0, // Reloc distance == 0 as this is non-compacting
24295 false, // Non-compacting
24299 seg = heap_segment_next (seg);
24302 #endif // defined(BACKGROUND_GC) && defined(FEATURE_EVENT_TRACE)
24304 void gc_heap::relocate_phase (int condemned_gen_number,
24305 uint8_t* first_condemned_address)
24308 sc.thread_number = heap_number;
24309 sc.promotion = FALSE;
24310 sc.concurrent = FALSE;
24316 start = GetCycleCount32();
24319 // %type% category = quote (relocate);
24320 dprintf (2,("---- Relocate phase -----"));
24322 #ifdef MULTIPLE_HEAPS
24323 //join all threads to make sure they are synchronized
24324 dprintf(3, ("Joining after end of plan"));
24325 gc_t_join.join(this, gc_join_begin_relocate_phase);
24326 if (gc_t_join.joined())
24327 #endif //MULTIPLE_HEAPS
24330 #ifdef MULTIPLE_HEAPS
24332 //join all threads to make sure they are synchronized
24333 dprintf(3, ("Restarting for relocation"));
24334 gc_t_join.restart();
24335 #endif //MULTIPLE_HEAPS
24338 dprintf(3,("Relocating roots"));
24339 GCScan::GcScanRoots(GCHeap::Relocate,
24340 condemned_gen_number, max_generation, &sc);
24342 verify_pins_with_post_plug_info("after reloc stack");
24344 #ifdef BACKGROUND_GC
24345 if (recursive_gc_sync::background_running_p())
24347 scan_background_roots (GCHeap::Relocate, heap_number, &sc);
24349 #endif //BACKGROUND_GC
24351 if (condemned_gen_number != max_generation)
24353 dprintf(3,("Relocating cross generation pointers"));
24354 mark_through_cards_for_segments (&gc_heap::relocate_address, TRUE);
24355 verify_pins_with_post_plug_info("after reloc cards");
24357 if (condemned_gen_number != max_generation)
24359 dprintf(3,("Relocating cross generation pointers for large objects"));
24360 mark_through_cards_for_large_objects (&gc_heap::relocate_address, TRUE);
24364 #ifdef FEATURE_LOH_COMPACTION
24365 if (loh_compacted_p)
24367 assert (settings.condemned_generation == max_generation);
24368 relocate_in_loh_compact();
24371 #endif //FEATURE_LOH_COMPACTION
24373 relocate_in_large_objects ();
24377 dprintf(3,("Relocating survivors"));
24378 relocate_survivors (condemned_gen_number,
24379 first_condemned_address);
24382 #ifdef FEATURE_PREMORTEM_FINALIZATION
24383 dprintf(3,("Relocating finalization data"));
24384 finalize_queue->RelocateFinalizationData (condemned_gen_number,
24386 #endif // FEATURE_PREMORTEM_FINALIZATION
24391 dprintf(3,("Relocating handle table"));
24392 GCScan::GcScanHandles(GCHeap::Relocate,
24393 condemned_gen_number, max_generation, &sc);
24396 #ifdef MULTIPLE_HEAPS
24397 //join all threads to make sure they are synchronized
24398 dprintf(3, ("Joining after end of relocation"));
24399 gc_t_join.join(this, gc_join_relocate_phase_done);
24401 #endif //MULTIPLE_HEAPS
24404 finish = GetCycleCount32();
24405 reloc_time = finish - start;
24408 dprintf(2,( "---- End of Relocate phase ----"));
24411 // This compares to see if tree is the current pinned plug and returns info
24412 // for this pinned plug. Also advances the pinned queue if that's the case.
24414 // We don't change the values of the plug info if tree is not the same as
24415 // the current pinned plug - the caller is responsible for setting the right
24416 // values to begin with.
24418 // POPO TODO: We are keeping this temporarily as this is also used by realloc
24419 // where it passes FALSE to deque_p, change it to use the same optimization
24420 // as relocate. Not as essential since realloc is already a slow path.
24421 mark* gc_heap::get_next_pinned_entry (uint8_t* tree,
24422 BOOL* has_pre_plug_info_p,
24423 BOOL* has_post_plug_info_p,
24426 if (!pinned_plug_que_empty_p())
24428 mark* oldest_entry = oldest_pin();
24429 uint8_t* oldest_plug = pinned_plug (oldest_entry);
24430 if (tree == oldest_plug)
24432 *has_pre_plug_info_p = oldest_entry->has_pre_plug_info();
24433 *has_post_plug_info_p = oldest_entry->has_post_plug_info();
24437 deque_pinned_plug();
24440 dprintf (3, ("found a pinned plug %Ix, pre: %d, post: %d",
24442 (*has_pre_plug_info_p ? 1 : 0),
24443 (*has_post_plug_info_p ? 1 : 0)));
24445 return oldest_entry;
24452 // This also deques the oldest entry and update the oldest plug
24453 mark* gc_heap::get_oldest_pinned_entry (BOOL* has_pre_plug_info_p,
24454 BOOL* has_post_plug_info_p)
24456 mark* oldest_entry = oldest_pin();
24457 *has_pre_plug_info_p = oldest_entry->has_pre_plug_info();
24458 *has_post_plug_info_p = oldest_entry->has_post_plug_info();
24460 deque_pinned_plug();
24461 update_oldest_pinned_plug();
24462 return oldest_entry;
24466 void gc_heap::copy_cards_range (uint8_t* dest, uint8_t* src, size_t len, BOOL copy_cards_p)
24469 copy_cards_for_addresses (dest, src, len);
24471 clear_card_for_addresses (dest, dest + len);
24474 // POPO TODO: We should actually just recover the artifically made gaps here..because when we copy
24475 // we always copy the earlier plugs first which means we won't need the gap sizes anymore. This way
24476 // we won't need to individually recover each overwritten part of plugs.
24478 void gc_heap::gcmemcopy (uint8_t* dest, uint8_t* src, size_t len, BOOL copy_cards_p)
24482 #ifdef BACKGROUND_GC
24483 if (current_c_gc_state == c_gc_state_marking)
24485 //TODO: should look to see whether we should consider changing this
24486 // to copy a consecutive region of the mark array instead.
24487 copy_mark_bits_for_addresses (dest, src, len);
24489 #endif //BACKGROUND_GC
24490 //dprintf(3,(" Memcopy [%Ix->%Ix, %Ix->%Ix[", (size_t)src, (size_t)dest, (size_t)src+len, (size_t)dest+len));
24491 dprintf(3,(" mc: [%Ix->%Ix, %Ix->%Ix[", (size_t)src, (size_t)dest, (size_t)src+len, (size_t)dest+len));
24492 memcopy (dest - plug_skew, src - plug_skew, len);
24493 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
24494 if (SoftwareWriteWatch::IsEnabledForGCHeap())
24496 // The ranges [src - plug_kew .. src[ and [src + len - plug_skew .. src + len[ are ObjHeaders, which don't have GC
24497 // references, and are not relevant for write watch. The latter range actually corresponds to the ObjHeader for the
24498 // object at (src + len), so it can be ignored anyway.
24499 SoftwareWriteWatch::SetDirtyRegion(dest, len - plug_skew);
24501 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
24502 copy_cards_range (dest, src, len, copy_cards_p);
24506 void gc_heap::compact_plug (uint8_t* plug, size_t size, BOOL check_last_object_p, compact_args* args)
24509 uint8_t* reloc_plug = plug + args->last_plug_relocation;
24511 if (check_last_object_p)
24513 size += sizeof (gap_reloc_pair);
24514 mark* entry = args->pinned_plug_entry;
24516 if (args->is_shortened)
24518 assert (entry->has_post_plug_info());
24519 entry->swap_post_plug_and_saved();
24523 assert (entry->has_pre_plug_info());
24524 entry->swap_pre_plug_and_saved();
24528 int old_brick_entry = brick_table [brick_of (plug)];
24530 assert (node_relocation_distance (plug) == args->last_plug_relocation);
24532 #ifdef FEATURE_STRUCTALIGN
24533 ptrdiff_t alignpad = node_alignpad(plug);
24536 make_unused_array (reloc_plug - alignpad, alignpad);
24537 if (brick_of (reloc_plug - alignpad) != brick_of (reloc_plug))
24539 // The alignment padding is straddling one or more bricks;
24540 // it has to be the last "object" of its first brick.
24541 fix_brick_to_highest (reloc_plug - alignpad, reloc_plug);
24544 #else // FEATURE_STRUCTALIGN
24545 size_t unused_arr_size = 0;
24546 BOOL already_padded_p = FALSE;
24548 if (is_plug_padded (plug))
24550 already_padded_p = TRUE;
24551 clear_plug_padded (plug);
24552 unused_arr_size = Align (min_obj_size);
24554 #endif //SHORT_PLUGS
24555 if (node_realigned (plug))
24557 unused_arr_size += switch_alignment_size (already_padded_p);
24560 if (unused_arr_size != 0)
24562 make_unused_array (reloc_plug - unused_arr_size, unused_arr_size);
24564 if (brick_of (reloc_plug - unused_arr_size) != brick_of (reloc_plug))
24566 dprintf (3, ("fix B for padding: %Id: %Ix->%Ix",
24567 unused_arr_size, (reloc_plug - unused_arr_size), reloc_plug));
24568 // The alignment padding is straddling one or more bricks;
24569 // it has to be the last "object" of its first brick.
24570 fix_brick_to_highest (reloc_plug - unused_arr_size, reloc_plug);
24573 #endif // FEATURE_STRUCTALIGN
24576 if (is_plug_padded (plug))
24578 make_unused_array (reloc_plug - Align (min_obj_size), Align (min_obj_size));
24580 if (brick_of (reloc_plug - Align (min_obj_size)) != brick_of (reloc_plug))
24582 // The alignment padding is straddling one or more bricks;
24583 // it has to be the last "object" of its first brick.
24584 fix_brick_to_highest (reloc_plug - Align (min_obj_size), reloc_plug);
24587 #endif //SHORT_PLUGS
24589 gcmemcopy (reloc_plug, plug, size, args->copy_cards_p);
24591 if (args->check_gennum_p)
24593 int src_gennum = args->src_gennum;
24594 if (src_gennum == -1)
24596 src_gennum = object_gennum (plug);
24599 int dest_gennum = object_gennum_plan (reloc_plug);
24601 if (src_gennum < dest_gennum)
24603 generation_allocation_size (generation_of (dest_gennum)) += size;
24607 size_t current_reloc_brick = args->current_compacted_brick;
24609 if (brick_of (reloc_plug) != current_reloc_brick)
24611 dprintf (3, ("last reloc B: %Ix, current reloc B: %Ix",
24612 current_reloc_brick, brick_of (reloc_plug)));
24614 if (args->before_last_plug)
24616 dprintf (3,(" fixing last brick %Ix to point to last plug %Ix(%Ix)",
24617 current_reloc_brick,
24618 args->before_last_plug,
24619 (args->before_last_plug - brick_address (current_reloc_brick))));
24622 set_brick (current_reloc_brick,
24623 args->before_last_plug - brick_address (current_reloc_brick));
24626 current_reloc_brick = brick_of (reloc_plug);
24628 size_t end_brick = brick_of (reloc_plug + size-1);
24629 if (end_brick != current_reloc_brick)
24631 // The plug is straddling one or more bricks
24632 // It has to be the last plug of its first brick
24633 dprintf (3,("plug spanning multiple bricks, fixing first brick %Ix to %Ix(%Ix)",
24634 current_reloc_brick, (size_t)reloc_plug,
24635 (reloc_plug - brick_address (current_reloc_brick))));
24638 set_brick (current_reloc_brick,
24639 reloc_plug - brick_address (current_reloc_brick));
24641 // update all intervening brick
24642 size_t brick = current_reloc_brick + 1;
24643 dprintf (3,("setting intervening bricks %Ix->%Ix to -1",
24644 brick, (end_brick - 1)));
24645 while (brick < end_brick)
24647 set_brick (brick, -1);
24650 // code last brick offset as a plug address
24651 args->before_last_plug = brick_address (end_brick) -1;
24652 current_reloc_brick = end_brick;
24653 dprintf (3, ("setting before last to %Ix, last brick to %Ix",
24654 args->before_last_plug, current_reloc_brick));
24658 dprintf (3, ("still in the same brick: %Ix", end_brick));
24659 args->before_last_plug = reloc_plug;
24661 args->current_compacted_brick = current_reloc_brick;
24663 if (check_last_object_p)
24665 mark* entry = args->pinned_plug_entry;
24667 if (args->is_shortened)
24669 entry->swap_post_plug_and_saved();
24673 entry->swap_pre_plug_and_saved();
24678 void gc_heap::compact_in_brick (uint8_t* tree, compact_args* args)
24680 assert (tree != NULL);
24681 int left_node = node_left_child (tree);
24682 int right_node = node_right_child (tree);
24683 ptrdiff_t relocation = node_relocation_distance (tree);
24689 dprintf (3, ("B: L: %d->%Ix", left_node, (tree + left_node)));
24690 compact_in_brick ((tree + left_node), args);
24693 uint8_t* plug = tree;
24694 BOOL has_pre_plug_info_p = FALSE;
24695 BOOL has_post_plug_info_p = FALSE;
24697 if (tree == oldest_pinned_plug)
24699 args->pinned_plug_entry = get_oldest_pinned_entry (&has_pre_plug_info_p,
24700 &has_post_plug_info_p);
24701 assert (tree == pinned_plug (args->pinned_plug_entry));
24704 if (args->last_plug != 0)
24706 size_t gap_size = node_gap_size (tree);
24707 uint8_t* gap = (plug - gap_size);
24708 uint8_t* last_plug_end = gap;
24709 size_t last_plug_size = (last_plug_end - args->last_plug);
24710 dprintf (3, ("tree: %Ix, last_plug: %Ix, gap: %Ix(%Ix), last_plug_end: %Ix, size: %Ix",
24711 tree, args->last_plug, gap, gap_size, last_plug_end, last_plug_size));
24713 BOOL check_last_object_p = (args->is_shortened || has_pre_plug_info_p);
24714 if (!check_last_object_p)
24716 assert (last_plug_size >= Align (min_obj_size));
24719 compact_plug (args->last_plug, last_plug_size, check_last_object_p, args);
24723 assert (!has_pre_plug_info_p);
24726 dprintf (3, ("set args last plug to plug: %Ix, reloc: %Ix", plug, relocation));
24727 args->last_plug = plug;
24728 args->last_plug_relocation = relocation;
24729 args->is_shortened = has_post_plug_info_p;
24733 dprintf (3, ("B: R: %d->%Ix", right_node, (tree + right_node)));
24734 compact_in_brick ((tree + right_node), args);
24738 void gc_heap::recover_saved_pinned_info()
24740 reset_pinned_queue_bos();
24742 while (!(pinned_plug_que_empty_p()))
24744 mark* oldest_entry = oldest_pin();
24745 oldest_entry->recover_plug_info();
24746 #ifdef GC_CONFIG_DRIVEN
24747 if (oldest_entry->has_pre_plug_info() && oldest_entry->has_post_plug_info())
24748 record_interesting_data_point (idp_pre_and_post_pin);
24749 else if (oldest_entry->has_pre_plug_info())
24750 record_interesting_data_point (idp_pre_pin);
24751 else if (oldest_entry->has_post_plug_info())
24752 record_interesting_data_point (idp_post_pin);
24753 #endif //GC_CONFIG_DRIVEN
24755 deque_pinned_plug();
24759 void gc_heap::compact_phase (int condemned_gen_number,
24760 uint8_t* first_condemned_address,
24763 // %type% category = quote (compact);
24767 start = GetCycleCount32();
24769 generation* condemned_gen = generation_of (condemned_gen_number);
24770 uint8_t* start_address = first_condemned_address;
24771 size_t current_brick = brick_of (start_address);
24772 heap_segment* current_heap_segment = heap_segment_rw (generation_start_segment (condemned_gen));
24774 PREFIX_ASSUME(current_heap_segment != NULL);
24776 reset_pinned_queue_bos();
24777 update_oldest_pinned_plug();
24779 BOOL reused_seg = expand_reused_seg_p();
24782 for (int i = 1; i <= max_generation; i++)
24784 generation_allocation_size (generation_of (i)) = 0;
24788 uint8_t* end_address = heap_segment_allocated (current_heap_segment);
24790 size_t end_brick = brick_of (end_address-1);
24792 args.last_plug = 0;
24793 args.before_last_plug = 0;
24794 args.current_compacted_brick = ~((size_t)1);
24795 args.is_shortened = FALSE;
24796 args.pinned_plug_entry = 0;
24797 args.copy_cards_p = (condemned_gen_number >= 1) || !clear_cards;
24798 args.check_gennum_p = reused_seg;
24799 if (args.check_gennum_p)
24801 args.src_gennum = ((current_heap_segment == ephemeral_heap_segment) ? -1 : 2);
24804 dprintf (2,("---- Compact Phase: %Ix(%Ix)----",
24805 first_condemned_address, brick_of (first_condemned_address)));
24807 #ifdef MULTIPLE_HEAPS
24809 if (gc_t_join.joined())
24811 #endif //MULTIPLE_HEAPS
24813 #ifdef MULTIPLE_HEAPS
24814 dprintf(3, ("Restarting for compaction"));
24815 gc_t_join.restart();
24817 #endif //MULTIPLE_HEAPS
24819 reset_pinned_queue_bos();
24821 #ifdef FEATURE_LOH_COMPACTION
24822 if (loh_compacted_p)
24826 #endif //FEATURE_LOH_COMPACTION
24828 if ((start_address < end_address) ||
24829 (condemned_gen_number == max_generation))
24833 if (current_brick > end_brick)
24835 if (args.last_plug != 0)
24837 dprintf (3, ("compacting last plug: %Ix", args.last_plug))
24838 compact_plug (args.last_plug,
24839 (heap_segment_allocated (current_heap_segment) - args.last_plug),
24844 if (heap_segment_next_rw (current_heap_segment))
24846 current_heap_segment = heap_segment_next_rw (current_heap_segment);
24847 current_brick = brick_of (heap_segment_mem (current_heap_segment));
24848 end_brick = brick_of (heap_segment_allocated (current_heap_segment)-1);
24849 args.last_plug = 0;
24850 if (args.check_gennum_p)
24852 args.src_gennum = ((current_heap_segment == ephemeral_heap_segment) ? -1 : 2);
24858 if (args.before_last_plug !=0)
24860 dprintf (3, ("Fixing last brick %Ix to point to plug %Ix",
24861 args.current_compacted_brick, (size_t)args.before_last_plug));
24862 assert (args.current_compacted_brick != ~1u);
24863 set_brick (args.current_compacted_brick,
24864 args.before_last_plug - brick_address (args.current_compacted_brick));
24870 int brick_entry = brick_table [ current_brick ];
24871 dprintf (3, ("B: %Ix(%Ix)->%Ix",
24872 current_brick, (size_t)brick_entry, (brick_address (current_brick) + brick_entry - 1)));
24874 if (brick_entry >= 0)
24876 compact_in_brick ((brick_address (current_brick) + brick_entry -1),
24885 recover_saved_pinned_info();
24888 finish = GetCycleCount32();
24889 compact_time = finish - start;
24892 concurrent_print_time_delta ("compact end");
24894 dprintf(2,("---- End of Compact phase ----"));
24897 #ifdef MULTIPLE_HEAPS
24900 #pragma warning(push)
24901 #pragma warning(disable:4702) // C4702: unreachable code: gc_thread_function may not return
24903 void gc_heap::gc_thread_stub (void* arg)
24905 gc_heap* heap = (gc_heap*)arg;
24906 if (!gc_thread_no_affinitize_p)
24908 GCThreadAffinity affinity;
24909 affinity.Group = GCThreadAffinity::None;
24910 affinity.Processor = GCThreadAffinity::None;
24912 // We are about to set affinity for GC threads. It is a good place to set up NUMA and
24913 // CPU groups because the process mask, processor number, and group number are all
24914 // readily available.
24915 if (GCToOSInterface::CanEnableGCCPUGroups())
24916 set_thread_group_affinity_for_heap(heap->heap_number, &affinity);
24918 set_thread_affinity_mask_for_heap(heap->heap_number, &affinity);
24920 if (!GCToOSInterface::SetThreadAffinity(&affinity))
24922 dprintf(1, ("Failed to set thread affinity for server GC thread"));
24926 // server GC threads run at a higher priority than normal.
24927 GCToOSInterface::BoostThreadPriority();
24928 _alloca (256*heap->heap_number);
24929 heap->gc_thread_function();
24932 #pragma warning(pop)
24935 #endif //MULTIPLE_HEAPS
24937 #ifdef BACKGROUND_GC
24940 #pragma warning(push)
24941 #pragma warning(disable:4702) // C4702: unreachable code: gc_thread_function may not return
24943 void gc_heap::bgc_thread_stub (void* arg)
24945 gc_heap* heap = (gc_heap*)arg;
24946 heap->bgc_thread = GCToEEInterface::GetThread();
24947 assert(heap->bgc_thread != nullptr);
24948 heap->bgc_thread_function();
24951 #pragma warning(pop)
24954 #endif //BACKGROUND_GC
24956 /*------------------ Background GC ----------------------------*/
24958 #ifdef BACKGROUND_GC
24960 void gc_heap::background_drain_mark_list (int thread)
24962 UNREFERENCED_PARAMETER(thread);
24964 size_t saved_c_mark_list_index = c_mark_list_index;
24966 if (saved_c_mark_list_index)
24968 concurrent_print_time_delta ("SML");
24970 while (c_mark_list_index != 0)
24972 size_t current_index = c_mark_list_index - 1;
24973 uint8_t* o = c_mark_list [current_index];
24974 background_mark_object (o THREAD_NUMBER_ARG);
24975 c_mark_list_index--;
24977 if (saved_c_mark_list_index)
24980 concurrent_print_time_delta ("EML");
24983 fire_drain_mark_list_event (saved_c_mark_list_index);
24987 // The background GC version of scan_dependent_handles (see that method for a more in-depth comment).
24988 #ifdef MULTIPLE_HEAPS
24989 // Since we only scan dependent handles while we are stopped we'll never interfere with FGCs scanning
24990 // them. So we can use the same static variables.
24991 void gc_heap::background_scan_dependent_handles (ScanContext *sc)
24993 // Whenever we call this method there may have been preceding object promotions. So set
24994 // s_fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
24995 // based on the how the scanning proceeded).
24996 s_fUnscannedPromotions = TRUE;
24998 // We don't know how many times we need to loop yet. In particular we can't base the loop condition on
24999 // the state of this thread's portion of the dependent handle table. That's because promotions on other
25000 // threads could cause handle promotions to become necessary here. Even if there are definitely no more
25001 // promotions possible in this thread's handles, we still have to stay in lock-step with those worker
25002 // threads that haven't finished yet (each GC worker thread has to join exactly the same number of times
25003 // as all the others or they'll get out of step).
25006 // The various worker threads are all currently racing in this code. We need to work out if at least
25007 // one of them think they have work to do this cycle. Each thread needs to rescan its portion of the
25008 // dependent handle table when both of the following conditions apply:
25009 // 1) At least one (arbitrary) object might have been promoted since the last scan (because if this
25010 // object happens to correspond to a primary in one of our handles we might potentially have to
25011 // promote the associated secondary).
25012 // 2) The table for this thread has at least one handle with a secondary that isn't promoted yet.
25014 // The first condition is represented by s_fUnscannedPromotions. This is always non-zero for the first
25015 // iteration of this loop (see comment above) and in subsequent cycles each thread updates this
25016 // whenever a mark stack overflow occurs or scanning their dependent handles results in a secondary
25017 // being promoted. This value is cleared back to zero in a synchronized fashion in the join that
25018 // follows below. Note that we can't read this outside of the join since on any iteration apart from
25019 // the first threads will be racing between reading this value and completing their previous
25020 // iteration's table scan.
25022 // The second condition is tracked by the dependent handle code itself on a per worker thread basis
25023 // (and updated by the GcDhReScan() method). We call GcDhUnpromotedHandlesExist() on each thread to
25024 // determine the local value and collect the results into the s_fUnpromotedHandles variable in what is
25025 // effectively an OR operation. As per s_fUnscannedPromotions we can't read the final result until
25026 // we're safely joined.
25027 if (GCScan::GcDhUnpromotedHandlesExist(sc))
25028 s_fUnpromotedHandles = TRUE;
25030 // Synchronize all the threads so we can read our state variables safely. The following shared
25031 // variable (indicating whether we should scan the tables or terminate the loop) will be set by a
25032 // single thread inside the join.
25033 bgc_t_join.join(this, gc_join_scan_dependent_handles);
25034 if (bgc_t_join.joined())
25036 // We're synchronized so it's safe to read our shared state variables. We update another shared
25037 // variable to indicate to all threads whether we'll be scanning for another cycle or terminating
25038 // the loop. We scan if there has been at least one object promotion since last time and at least
25039 // one thread has a dependent handle table with a potential handle promotion possible.
25040 s_fScanRequired = s_fUnscannedPromotions && s_fUnpromotedHandles;
25042 // Reset our shared state variables (ready to be set again on this scan or with a good initial
25043 // value for the next call if we're terminating the loop).
25044 s_fUnscannedPromotions = FALSE;
25045 s_fUnpromotedHandles = FALSE;
25047 if (!s_fScanRequired)
25049 uint8_t* all_heaps_max = 0;
25050 uint8_t* all_heaps_min = MAX_PTR;
25052 for (i = 0; i < n_heaps; i++)
25054 if (all_heaps_max < g_heaps[i]->background_max_overflow_address)
25055 all_heaps_max = g_heaps[i]->background_max_overflow_address;
25056 if (all_heaps_min > g_heaps[i]->background_min_overflow_address)
25057 all_heaps_min = g_heaps[i]->background_min_overflow_address;
25059 for (i = 0; i < n_heaps; i++)
25061 g_heaps[i]->background_max_overflow_address = all_heaps_max;
25062 g_heaps[i]->background_min_overflow_address = all_heaps_min;
25066 // Restart all the workers.
25067 dprintf(2, ("Starting all gc thread mark stack overflow processing"));
25068 bgc_t_join.restart();
25071 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
25072 // being visible. If there really was an overflow (process_mark_overflow returns true) then set the
25073 // global flag indicating that at least one object promotion may have occurred (the usual comment
25074 // about races applies). (Note it's OK to set this flag even if we're about to terminate the loop and
25075 // exit the method since we unconditionally set this variable on method entry anyway).
25076 if (background_process_mark_overflow (sc->concurrent))
25077 s_fUnscannedPromotions = TRUE;
25079 // If we decided that no scan was required we can terminate the loop now.
25080 if (!s_fScanRequired)
25083 // Otherwise we must join with the other workers to ensure that all mark stack overflows have been
25084 // processed before we start scanning dependent handle tables (if overflows remain while we scan we
25085 // could miss noting the promotion of some primary objects).
25086 bgc_t_join.join(this, gc_join_rescan_dependent_handles);
25087 if (bgc_t_join.joined())
25089 // Restart all the workers.
25090 dprintf(3, ("Starting all gc thread for dependent handle promotion"));
25091 bgc_t_join.restart();
25094 // If the portion of the dependent handle table managed by this worker has handles that could still be
25095 // promoted perform a rescan. If the rescan resulted in at least one promotion note this fact since it
25096 // could require a rescan of handles on this or other workers.
25097 if (GCScan::GcDhUnpromotedHandlesExist(sc))
25098 if (GCScan::GcDhReScan(sc))
25099 s_fUnscannedPromotions = TRUE;
25103 void gc_heap::background_scan_dependent_handles (ScanContext *sc)
25105 // Whenever we call this method there may have been preceding object promotions. So set
25106 // fUnscannedPromotions unconditionally (during further iterations of the scanning loop this will be set
25107 // based on the how the scanning proceeded).
25108 bool fUnscannedPromotions = true;
25110 // Scan dependent handles repeatedly until there are no further promotions that can be made or we made a
25111 // scan without performing any new promotions.
25112 while (GCScan::GcDhUnpromotedHandlesExist(sc) && fUnscannedPromotions)
25114 // On each iteration of the loop start with the assumption that no further objects have been promoted.
25115 fUnscannedPromotions = false;
25117 // Handle any mark stack overflow: scanning dependent handles relies on all previous object promotions
25118 // being visible. If there was an overflow (background_process_mark_overflow returned true) then
25119 // additional objects now appear to be promoted and we should set the flag.
25120 if (background_process_mark_overflow (sc->concurrent))
25121 fUnscannedPromotions = true;
25123 // Perform the scan and set the flag if any promotions resulted.
25124 if (GCScan::GcDhReScan (sc))
25125 fUnscannedPromotions = true;
25128 // Perform a last processing of any overflowed mark stack.
25129 background_process_mark_overflow (sc->concurrent);
25131 #endif //MULTIPLE_HEAPS
25133 void gc_heap::recover_bgc_settings()
25135 if ((settings.condemned_generation < max_generation) && recursive_gc_sync::background_running_p())
25137 dprintf (2, ("restoring bgc settings"));
25138 settings = saved_bgc_settings;
25139 GCHeap::GcCondemnedGeneration = gc_heap::settings.condemned_generation;
25143 void gc_heap::allow_fgc()
25145 assert (bgc_thread == GCToEEInterface::GetThread());
25146 bool bToggleGC = false;
25148 if (g_fSuspensionPending > 0)
25150 bToggleGC = GCToEEInterface::EnablePreemptiveGC();
25153 GCToEEInterface::DisablePreemptiveGC();
25158 BOOL gc_heap::should_commit_mark_array()
25160 return (recursive_gc_sync::background_running_p() || (current_bgc_state == bgc_initialized));
25163 void gc_heap::clear_commit_flag()
25165 generation* gen = generation_of (max_generation);
25166 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
25171 if (gen != large_object_generation)
25173 gen = large_object_generation;
25174 seg = heap_segment_in_range (generation_start_segment (gen));
25182 if (seg->flags & heap_segment_flags_ma_committed)
25184 seg->flags &= ~heap_segment_flags_ma_committed;
25187 if (seg->flags & heap_segment_flags_ma_pcommitted)
25189 seg->flags &= ~heap_segment_flags_ma_pcommitted;
25192 seg = heap_segment_next (seg);
25196 void gc_heap::clear_commit_flag_global()
25198 #ifdef MULTIPLE_HEAPS
25199 for (int i = 0; i < n_heaps; i++)
25201 g_heaps[i]->clear_commit_flag();
25204 clear_commit_flag();
25205 #endif //MULTIPLE_HEAPS
25208 void gc_heap::verify_mark_array_cleared (uint8_t* begin, uint8_t* end, uint32_t* mark_array_addr)
25211 size_t markw = mark_word_of (begin);
25212 size_t markw_end = mark_word_of (end);
25214 while (markw < markw_end)
25216 if (mark_array_addr[markw])
25218 dprintf (1, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
25219 markw, mark_array_addr[markw], mark_word_address (markw)));
25225 UNREFERENCED_PARAMETER(begin);
25226 UNREFERENCED_PARAMETER(end);
25227 UNREFERENCED_PARAMETER(mark_array_addr);
25231 void gc_heap::verify_mark_array_cleared (heap_segment* seg, uint32_t* mark_array_addr)
25233 verify_mark_array_cleared (heap_segment_mem (seg), heap_segment_reserved (seg), mark_array_addr);
25236 BOOL gc_heap::commit_mark_array_new_seg (gc_heap* hp,
25238 uint32_t* new_card_table,
25239 uint8_t* new_lowest_address)
25241 UNREFERENCED_PARAMETER(hp); // compiler bug? -- this *is*, indeed, referenced
25243 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
25244 uint8_t* end = heap_segment_reserved (seg);
25246 uint8_t* lowest = hp->background_saved_lowest_address;
25247 uint8_t* highest = hp->background_saved_highest_address;
25249 uint8_t* commit_start = NULL;
25250 uint8_t* commit_end = NULL;
25251 size_t commit_flag = 0;
25253 if ((highest >= start) &&
25256 if ((start >= lowest) && (end <= highest))
25258 dprintf (GC_TABLE_LOG, ("completely in bgc range: seg %Ix-%Ix, bgc: %Ix-%Ix",
25259 start, end, lowest, highest));
25260 commit_flag = heap_segment_flags_ma_committed;
25264 dprintf (GC_TABLE_LOG, ("partially in bgc range: seg %Ix-%Ix, bgc: %Ix-%Ix",
25265 start, end, lowest, highest));
25266 commit_flag = heap_segment_flags_ma_pcommitted;
25269 commit_start = max (lowest, start);
25270 commit_end = min (highest, end);
25272 if (!commit_mark_array_by_range (commit_start, commit_end, hp->mark_array))
25277 if (new_card_table == 0)
25279 new_card_table = g_gc_card_table;
25282 if (hp->card_table != new_card_table)
25284 if (new_lowest_address == 0)
25286 new_lowest_address = g_gc_lowest_address;
25289 uint32_t* ct = &new_card_table[card_word (gcard_of (new_lowest_address))];
25290 uint32_t* ma = (uint32_t*)((uint8_t*)card_table_mark_array (ct) - size_mark_array_of (0, new_lowest_address));
25292 dprintf (GC_TABLE_LOG, ("table realloc-ed: %Ix->%Ix, MA: %Ix->%Ix",
25293 hp->card_table, new_card_table,
25294 hp->mark_array, ma));
25296 if (!commit_mark_array_by_range (commit_start, commit_end, ma))
25302 seg->flags |= commit_flag;
25308 BOOL gc_heap::commit_mark_array_by_range (uint8_t* begin, uint8_t* end, uint32_t* mark_array_addr)
25310 size_t beg_word = mark_word_of (begin);
25311 size_t end_word = mark_word_of (align_on_mark_word (end));
25312 uint8_t* commit_start = align_lower_page ((uint8_t*)&mark_array_addr[beg_word]);
25313 uint8_t* commit_end = align_on_page ((uint8_t*)&mark_array_addr[end_word]);
25314 size_t size = (size_t)(commit_end - commit_start);
25316 #ifdef SIMPLE_DPRINTF
25317 dprintf (GC_TABLE_LOG, ("range: %Ix->%Ix mark word: %Ix->%Ix(%Id), mark array: %Ix->%Ix(%Id), commit %Ix->%Ix(%Id)",
25319 beg_word, end_word,
25320 (end_word - beg_word) * sizeof (uint32_t),
25321 &mark_array_addr[beg_word],
25322 &mark_array_addr[end_word],
25323 (size_t)(&mark_array_addr[end_word] - &mark_array_addr[beg_word]),
25324 commit_start, commit_end,
25326 #endif //SIMPLE_DPRINTF
25328 if (GCToOSInterface::VirtualCommit (commit_start, size))
25330 // We can only verify the mark array is cleared from begin to end, the first and the last
25331 // page aren't necessarily all cleared 'cause they could be used by other segments or
25333 verify_mark_array_cleared (begin, end, mark_array_addr);
25338 dprintf (GC_TABLE_LOG, ("failed to commit %Id bytes", (end_word - beg_word) * sizeof (uint32_t)));
25343 BOOL gc_heap::commit_mark_array_with_check (heap_segment* seg, uint32_t* new_mark_array_addr)
25345 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
25346 uint8_t* end = heap_segment_reserved (seg);
25348 #ifdef MULTIPLE_HEAPS
25349 uint8_t* lowest = heap_segment_heap (seg)->background_saved_lowest_address;
25350 uint8_t* highest = heap_segment_heap (seg)->background_saved_highest_address;
25352 uint8_t* lowest = background_saved_lowest_address;
25353 uint8_t* highest = background_saved_highest_address;
25354 #endif //MULTIPLE_HEAPS
25356 if ((highest >= start) &&
25359 start = max (lowest, start);
25360 end = min (highest, end);
25361 if (!commit_mark_array_by_range (start, end, new_mark_array_addr))
25370 BOOL gc_heap::commit_mark_array_by_seg (heap_segment* seg, uint32_t* mark_array_addr)
25372 dprintf (GC_TABLE_LOG, ("seg: %Ix->%Ix; MA: %Ix",
25374 heap_segment_reserved (seg),
25376 uint8_t* start = (heap_segment_read_only_p (seg) ? heap_segment_mem (seg) : (uint8_t*)seg);
25378 return commit_mark_array_by_range (start, heap_segment_reserved (seg), mark_array_addr);
25381 BOOL gc_heap::commit_mark_array_bgc_init (uint32_t* mark_array_addr)
25383 UNREFERENCED_PARAMETER(mark_array_addr);
25385 dprintf (GC_TABLE_LOG, ("BGC init commit: lowest: %Ix, highest: %Ix, mark_array: %Ix",
25386 lowest_address, highest_address, mark_array));
25388 generation* gen = generation_of (max_generation);
25389 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
25394 if (gen != large_object_generation)
25396 gen = large_object_generation;
25397 seg = heap_segment_in_range (generation_start_segment (gen));
25405 dprintf (GC_TABLE_LOG, ("seg: %Ix, flags: %Id", seg, seg->flags));
25407 if (!(seg->flags & heap_segment_flags_ma_committed))
25409 // For ro segments they could always be only partially in range so we'd
25410 // be calling this at the beginning of every BGC. We are not making this
25411 // more efficient right now - ro segments are currently only used by redhawk.
25412 if (heap_segment_read_only_p (seg))
25414 if ((heap_segment_mem (seg) >= lowest_address) &&
25415 (heap_segment_reserved (seg) <= highest_address))
25417 if (commit_mark_array_by_seg (seg, mark_array))
25419 seg->flags |= heap_segment_flags_ma_committed;
25428 uint8_t* start = max (lowest_address, heap_segment_mem (seg));
25429 uint8_t* end = min (highest_address, heap_segment_reserved (seg));
25430 if (commit_mark_array_by_range (start, end, mark_array))
25432 seg->flags |= heap_segment_flags_ma_pcommitted;
25442 // For normal segments they are by design completely in range so just
25443 // commit the whole mark array for each seg.
25444 if (commit_mark_array_by_seg (seg, mark_array))
25446 if (seg->flags & heap_segment_flags_ma_pcommitted)
25448 seg->flags &= ~heap_segment_flags_ma_pcommitted;
25450 seg->flags |= heap_segment_flags_ma_committed;
25459 seg = heap_segment_next (seg);
25465 // This function doesn't check the commit flag since it's for a new array -
25466 // the mark_array flag for these segments will remain the same.
25467 BOOL gc_heap::commit_new_mark_array (uint32_t* new_mark_array_addr)
25469 dprintf (GC_TABLE_LOG, ("commiting existing segs on MA %Ix", new_mark_array_addr));
25470 generation* gen = generation_of (max_generation);
25471 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
25476 if (gen != large_object_generation)
25478 gen = large_object_generation;
25479 seg = heap_segment_in_range (generation_start_segment (gen));
25487 if (!commit_mark_array_with_check (seg, new_mark_array_addr))
25492 seg = heap_segment_next (seg);
25495 #ifdef MULTIPLE_HEAPS
25496 if (new_heap_segment)
25498 if (!commit_mark_array_with_check (new_heap_segment, new_mark_array_addr))
25503 #endif //MULTIPLE_HEAPS
25508 BOOL gc_heap::commit_new_mark_array_global (uint32_t* new_mark_array)
25510 #ifdef MULTIPLE_HEAPS
25511 for (int i = 0; i < n_heaps; i++)
25513 if (!g_heaps[i]->commit_new_mark_array (new_mark_array))
25519 if (!commit_new_mark_array (new_mark_array))
25523 #endif //MULTIPLE_HEAPS
25528 void gc_heap::decommit_mark_array_by_seg (heap_segment* seg)
25530 // if BGC is disabled (the finalize watchdog does this at shutdown), the mark array could have
25531 // been set to NULL.
25532 if (mark_array == NULL)
25537 dprintf (GC_TABLE_LOG, ("decommitting seg %Ix(%Ix), MA: %Ix", seg, seg->flags, mark_array));
25539 size_t flags = seg->flags;
25541 if ((flags & heap_segment_flags_ma_committed) ||
25542 (flags & heap_segment_flags_ma_pcommitted))
25544 uint8_t* start = (heap_segment_read_only_p(seg) ? heap_segment_mem(seg) : (uint8_t*)seg);
25545 uint8_t* end = heap_segment_reserved (seg);
25547 if (flags & heap_segment_flags_ma_pcommitted)
25549 start = max (lowest_address, start);
25550 end = min (highest_address, end);
25553 size_t beg_word = mark_word_of (start);
25554 size_t end_word = mark_word_of (align_on_mark_word (end));
25555 uint8_t* decommit_start = align_on_page ((uint8_t*)&mark_array[beg_word]);
25556 uint8_t* decommit_end = align_lower_page ((uint8_t*)&mark_array[end_word]);
25557 size_t size = (size_t)(decommit_end - decommit_start);
25559 #ifdef SIMPLE_DPRINTF
25560 dprintf (GC_TABLE_LOG, ("seg: %Ix mark word: %Ix->%Ix(%Id), mark array: %Ix->%Ix(%Id), decommit %Ix->%Ix(%Id)",
25562 beg_word, end_word,
25563 (end_word - beg_word) * sizeof (uint32_t),
25564 &mark_array[beg_word],
25565 &mark_array[end_word],
25566 (size_t)(&mark_array[end_word] - &mark_array[beg_word]),
25567 decommit_start, decommit_end,
25569 #endif //SIMPLE_DPRINTF
25571 if (decommit_start < decommit_end)
25573 if (!GCToOSInterface::VirtualDecommit (decommit_start, size))
25575 dprintf (GC_TABLE_LOG, ("GCToOSInterface::VirtualDecommit on %Ix for %Id bytes failed",
25576 decommit_start, size));
25577 assert (!"decommit failed");
25581 dprintf (GC_TABLE_LOG, ("decommited [%Ix for address [%Ix", beg_word, seg));
25585 void gc_heap::background_mark_phase ()
25587 verify_mark_array_cleared();
25590 sc.thread_number = heap_number;
25591 sc.promotion = TRUE;
25592 sc.concurrent = FALSE;
25595 BOOL cooperative_mode = TRUE;
25596 #ifndef MULTIPLE_HEAPS
25597 const int thread = heap_number;
25598 #endif //!MULTIPLE_HEAPS
25600 dprintf(2,("-(GC%d)BMark-", VolatileLoad(&settings.gc_index)));
25602 assert (settings.concurrent);
25607 start = GetCycleCount32();
25610 #ifdef FFIND_OBJECT
25611 if (gen0_must_clear_bricks > 0)
25612 gen0_must_clear_bricks--;
25613 #endif //FFIND_OBJECT
25615 background_soh_alloc_count = 0;
25616 background_loh_alloc_count = 0;
25617 bgc_overflow_count = 0;
25619 bpromoted_bytes (heap_number) = 0;
25620 static uint32_t num_sizedrefs = 0;
25622 background_min_overflow_address = MAX_PTR;
25623 background_max_overflow_address = 0;
25624 background_min_soh_overflow_address = MAX_PTR;
25625 background_max_soh_overflow_address = 0;
25626 processed_soh_overflow_p = FALSE;
25629 //set up the mark lists from g_mark_list
25630 assert (g_mark_list);
25631 mark_list = g_mark_list;
25632 //dont use the mark list for full gc
25633 //because multiple segments are more complex to handle and the list
25634 //is likely to overflow
25635 mark_list_end = &mark_list [0];
25636 mark_list_index = &mark_list [0];
25638 c_mark_list_index = 0;
25640 shigh = (uint8_t*) 0;
25643 generation* gen = generation_of (max_generation);
25645 dprintf(3,("BGC: stack marking"));
25646 sc.concurrent = TRUE;
25648 GCScan::GcScanRoots(background_promote_callback,
25649 max_generation, max_generation,
25654 dprintf(3,("BGC: finalization marking"));
25655 finalize_queue->GcScanRoots(background_promote_callback, heap_number, 0);
25658 size_t total_loh_size = generation_size (max_generation + 1);
25659 bgc_begin_loh_size = total_loh_size;
25660 bgc_alloc_spin_loh = 0;
25661 bgc_loh_size_increased = 0;
25662 bgc_loh_allocated_in_free = 0;
25663 size_t total_soh_size = generation_sizes (generation_of (max_generation));
25665 dprintf (GTC_LOG, ("BM: h%d: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
25668 //concurrent_print_time_delta ("copying stack roots");
25669 concurrent_print_time_delta ("CS");
25671 FIRE_EVENT(BGC1stNonConEnd);
25673 expanded_in_fgc = FALSE;
25674 saved_overflow_ephemeral_seg = 0;
25675 current_bgc_state = bgc_reset_ww;
25677 // we don't need a join here - just whichever thread that gets here
25678 // first can change the states and call restart_vm.
25679 // this is not true - we can't let the EE run when we are scanning stack.
25680 // since we now allow reset ww to run concurrently and have a join for it,
25681 // we can do restart ee on the 1st thread that got here. Make sure we handle the
25682 // sizedref handles correctly.
25683 #ifdef MULTIPLE_HEAPS
25684 bgc_t_join.join(this, gc_join_restart_ee);
25685 if (bgc_t_join.joined())
25686 #endif //MULTIPLE_HEAPS
25688 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25689 // Resetting write watch for software write watch is pretty fast, much faster than for hardware write watch. Reset
25690 // can be done while the runtime is suspended or after the runtime is restarted, the preference was to reset while
25691 // the runtime is suspended. The reset for hardware write watch is done after the runtime is restarted below.
25693 concurrent_print_time_delta ("CRWW begin");
25695 #ifdef MULTIPLE_HEAPS
25696 for (int i = 0; i < n_heaps; i++)
25698 g_heaps[i]->reset_write_watch (FALSE);
25701 reset_write_watch (FALSE);
25702 #endif //MULTIPLE_HEAPS
25704 concurrent_print_time_delta ("CRWW");
25705 #endif //WRITE_WATCH
25706 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25708 num_sizedrefs = GCToEEInterface::GetTotalNumSizedRefHandles();
25710 // this c_write is not really necessary because restart_vm
25711 // has an instruction that will flush the cpu cache (interlocked
25712 // or whatever) but we don't want to rely on that.
25713 dprintf (BGC_LOG, ("setting cm_in_progress"));
25714 c_write (cm_in_progress, TRUE);
25716 //restart all thread, doing the marking from the array
25717 assert (dont_restart_ee_p);
25718 dont_restart_ee_p = FALSE;
25721 GCToOSInterface::YieldThread (0);
25722 #ifdef MULTIPLE_HEAPS
25723 dprintf(3, ("Starting all gc threads for gc"));
25724 bgc_t_join.restart();
25725 #endif //MULTIPLE_HEAPS
25728 #ifdef MULTIPLE_HEAPS
25729 bgc_t_join.join(this, gc_join_after_reset);
25730 if (bgc_t_join.joined())
25731 #endif //MULTIPLE_HEAPS
25733 disable_preemptive (true);
25735 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25736 // When software write watch is enabled, resetting write watch is done while the runtime is suspended above. The
25737 // post-reset call to revisit_written_pages is only necessary for concurrent reset_write_watch, to discard dirtied
25738 // pages during the concurrent reset.
25741 concurrent_print_time_delta ("CRWW begin");
25743 #ifdef MULTIPLE_HEAPS
25744 for (int i = 0; i < n_heaps; i++)
25746 g_heaps[i]->reset_write_watch (TRUE);
25749 reset_write_watch (TRUE);
25750 #endif //MULTIPLE_HEAPS
25752 concurrent_print_time_delta ("CRWW");
25753 #endif //WRITE_WATCH
25755 #ifdef MULTIPLE_HEAPS
25756 for (int i = 0; i < n_heaps; i++)
25758 g_heaps[i]->revisit_written_pages (TRUE, TRUE);
25761 revisit_written_pages (TRUE, TRUE);
25762 #endif //MULTIPLE_HEAPS
25764 concurrent_print_time_delta ("CRW");
25765 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25767 #ifdef MULTIPLE_HEAPS
25768 for (int i = 0; i < n_heaps; i++)
25770 g_heaps[i]->current_bgc_state = bgc_mark_handles;
25773 current_bgc_state = bgc_mark_handles;
25774 #endif //MULTIPLE_HEAPS
25776 current_c_gc_state = c_gc_state_marking;
25778 enable_preemptive ();
25780 #ifdef MULTIPLE_HEAPS
25781 dprintf(3, ("Joining BGC threads after resetting writewatch"));
25782 bgc_t_join.restart();
25783 #endif //MULTIPLE_HEAPS
25786 disable_preemptive (true);
25788 if (num_sizedrefs > 0)
25790 GCScan::GcScanSizedRefs(background_promote, max_generation, max_generation, &sc);
25792 enable_preemptive ();
25794 #ifdef MULTIPLE_HEAPS
25795 bgc_t_join.join(this, gc_join_scan_sizedref_done);
25796 if (bgc_t_join.joined())
25798 dprintf(3, ("Done with marking all sized refs. Starting all bgc thread for marking other strong roots"));
25799 bgc_t_join.restart();
25801 #endif //MULTIPLE_HEAPS
25803 disable_preemptive (true);
25806 dprintf (3,("BGC: handle table marking"));
25807 GCScan::GcScanHandles(background_promote,
25808 max_generation, max_generation,
25810 //concurrent_print_time_delta ("concurrent marking handle table");
25811 concurrent_print_time_delta ("CRH");
25813 current_bgc_state = bgc_mark_stack;
25814 dprintf (2,("concurrent draining mark list"));
25815 background_drain_mark_list (thread);
25816 //concurrent_print_time_delta ("concurrent marking stack roots");
25817 concurrent_print_time_delta ("CRS");
25819 dprintf (2,("concurrent revisiting dirtied pages"));
25820 revisit_written_pages (TRUE);
25821 revisit_written_pages (TRUE);
25822 //concurrent_print_time_delta ("concurrent marking dirtied pages on LOH");
25823 concurrent_print_time_delta ("CRre");
25825 enable_preemptive ();
25827 #ifdef MULTIPLE_HEAPS
25828 bgc_t_join.join(this, gc_join_concurrent_overflow);
25829 if (bgc_t_join.joined())
25831 uint8_t* all_heaps_max = 0;
25832 uint8_t* all_heaps_min = MAX_PTR;
25834 for (i = 0; i < n_heaps; i++)
25836 dprintf (3, ("heap %d overflow max is %Ix, min is %Ix",
25838 g_heaps[i]->background_max_overflow_address,
25839 g_heaps[i]->background_min_overflow_address));
25840 if (all_heaps_max < g_heaps[i]->background_max_overflow_address)
25841 all_heaps_max = g_heaps[i]->background_max_overflow_address;
25842 if (all_heaps_min > g_heaps[i]->background_min_overflow_address)
25843 all_heaps_min = g_heaps[i]->background_min_overflow_address;
25845 for (i = 0; i < n_heaps; i++)
25847 g_heaps[i]->background_max_overflow_address = all_heaps_max;
25848 g_heaps[i]->background_min_overflow_address = all_heaps_min;
25850 dprintf(3, ("Starting all bgc threads after updating the overflow info"));
25851 bgc_t_join.restart();
25853 #endif //MULTIPLE_HEAPS
25855 disable_preemptive (true);
25857 dprintf (2, ("before CRov count: %d", bgc_overflow_count));
25858 bgc_overflow_count = 0;
25859 background_process_mark_overflow (TRUE);
25860 dprintf (2, ("after CRov count: %d", bgc_overflow_count));
25861 bgc_overflow_count = 0;
25862 //concurrent_print_time_delta ("concurrent processing mark overflow");
25863 concurrent_print_time_delta ("CRov");
25865 // Stop all threads, crawl all stacks and revisit changed pages.
25866 FIRE_EVENT(BGC1stConEnd);
25868 dprintf (2, ("Stopping the EE"));
25870 enable_preemptive ();
25872 #ifdef MULTIPLE_HEAPS
25873 bgc_t_join.join(this, gc_join_suspend_ee);
25874 if (bgc_t_join.joined())
25876 bgc_threads_sync_event.Reset();
25878 dprintf(3, ("Joining BGC threads for non concurrent final marking"));
25879 bgc_t_join.restart();
25881 #endif //MULTIPLE_HEAPS
25883 if (heap_number == 0)
25885 enter_spin_lock (&gc_lock);
25889 bgc_threads_sync_event.Set();
25893 bgc_threads_sync_event.Wait(INFINITE, FALSE);
25894 dprintf (2, ("bgc_threads_sync_event is signalled"));
25897 assert (settings.concurrent);
25898 assert (settings.condemned_generation == max_generation);
25900 dprintf (2, ("clearing cm_in_progress"));
25901 c_write (cm_in_progress, FALSE);
25903 bgc_alloc_lock->check();
25905 current_bgc_state = bgc_final_marking;
25907 //concurrent_print_time_delta ("concurrent marking ended");
25908 concurrent_print_time_delta ("CR");
25910 FIRE_EVENT(BGC2ndNonConBegin);
25912 mark_absorb_new_alloc();
25914 // We need a join here 'cause find_object would complain if the gen0
25915 // bricks of another heap haven't been fixed up. So we need to make sure
25916 // that every heap's gen0 bricks are fixed up before we proceed.
25917 #ifdef MULTIPLE_HEAPS
25918 bgc_t_join.join(this, gc_join_after_absorb);
25919 if (bgc_t_join.joined())
25921 dprintf(3, ("Joining BGC threads after absorb"));
25922 bgc_t_join.restart();
25924 #endif //MULTIPLE_HEAPS
25926 // give VM a chance to do work
25927 GCToEEInterface::GcBeforeBGCSweepWork();
25929 //reset the flag, indicating that the EE no longer expect concurrent
25931 sc.concurrent = FALSE;
25933 total_loh_size = generation_size (max_generation + 1);
25934 total_soh_size = generation_sizes (generation_of (max_generation));
25936 dprintf (GTC_LOG, ("FM: h%d: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
25938 dprintf (2, ("nonconcurrent marking stack roots"));
25939 GCScan::GcScanRoots(background_promote,
25940 max_generation, max_generation,
25942 //concurrent_print_time_delta ("nonconcurrent marking stack roots");
25943 concurrent_print_time_delta ("NRS");
25945 // finalize_queue->EnterFinalizeLock();
25946 finalize_queue->GcScanRoots(background_promote, heap_number, 0);
25947 // finalize_queue->LeaveFinalizeLock();
25949 dprintf (2, ("nonconcurrent marking handle table"));
25950 GCScan::GcScanHandles(background_promote,
25951 max_generation, max_generation,
25953 //concurrent_print_time_delta ("nonconcurrent marking handle table");
25954 concurrent_print_time_delta ("NRH");
25956 dprintf (2,("---- (GC%d)final going through written pages ----", VolatileLoad(&settings.gc_index)));
25957 revisit_written_pages (FALSE);
25958 //concurrent_print_time_delta ("nonconcurrent revisit dirtied pages on LOH");
25959 concurrent_print_time_delta ("NRre LOH");
25961 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25962 #ifdef MULTIPLE_HEAPS
25963 bgc_t_join.join(this, gc_join_disable_software_write_watch);
25964 if (bgc_t_join.joined())
25965 #endif // MULTIPLE_HEAPS
25967 // The runtime is suspended, and we will be doing a final query of dirty pages, so pause tracking written pages to
25968 // avoid further perf penalty after the runtime is restarted
25969 SoftwareWriteWatch::DisableForGCHeap();
25971 #ifdef MULTIPLE_HEAPS
25972 dprintf(3, ("Restarting BGC threads after disabling software write watch"));
25973 bgc_t_join.restart();
25974 #endif // MULTIPLE_HEAPS
25976 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
25978 dprintf (2, ("before NR 1st Hov count: %d", bgc_overflow_count));
25979 bgc_overflow_count = 0;
25981 // Dependent handles need to be scanned with a special algorithm (see the header comment on
25982 // scan_dependent_handles for more detail). We perform an initial scan without processing any mark
25983 // stack overflow. This is not guaranteed to complete the operation but in a common case (where there
25984 // are no dependent handles that are due to be collected) it allows us to optimize away further scans.
25985 // The call to background_scan_dependent_handles is what will cycle through more iterations if
25986 // required and will also perform processing of any mark stack overflow once the dependent handle
25987 // table has been fully promoted.
25988 dprintf (2, ("1st dependent handle scan and process mark overflow"));
25989 GCScan::GcDhInitialScan(background_promote, max_generation, max_generation, &sc);
25990 background_scan_dependent_handles (&sc);
25991 //concurrent_print_time_delta ("1st nonconcurrent dependent handle scan and process mark overflow");
25992 concurrent_print_time_delta ("NR 1st Hov");
25994 dprintf (2, ("after NR 1st Hov count: %d", bgc_overflow_count));
25995 bgc_overflow_count = 0;
25997 #ifdef MULTIPLE_HEAPS
25998 bgc_t_join.join(this, gc_join_null_dead_short_weak);
25999 if (bgc_t_join.joined())
26000 #endif //MULTIPLE_HEAPS
26002 GCToEEInterface::AfterGcScanRoots (max_generation, max_generation, &sc);
26004 #ifdef MULTIPLE_HEAPS
26005 dprintf(3, ("Joining BGC threads for short weak handle scan"));
26006 bgc_t_join.restart();
26007 #endif //MULTIPLE_HEAPS
26010 // null out the target of short weakref that were not promoted.
26011 GCScan::GcShortWeakPtrScan(background_promote, max_generation, max_generation,&sc);
26013 //concurrent_print_time_delta ("bgc GcShortWeakPtrScan");
26014 concurrent_print_time_delta ("NR GcShortWeakPtrScan");
26018 #ifdef MULTIPLE_HEAPS
26019 bgc_t_join.join(this, gc_join_scan_finalization);
26020 if (bgc_t_join.joined())
26022 dprintf(3, ("Joining BGC threads for finalization"));
26023 bgc_t_join.restart();
26025 #endif //MULTIPLE_HEAPS
26027 //Handle finalization.
26028 dprintf(3,("Marking finalization data"));
26029 //concurrent_print_time_delta ("bgc joined to mark finalization");
26030 concurrent_print_time_delta ("NRj");
26032 // finalize_queue->EnterFinalizeLock();
26033 finalize_queue->ScanForFinalization (background_promote, max_generation, FALSE, __this);
26034 // finalize_queue->LeaveFinalizeLock();
26036 concurrent_print_time_delta ("NRF");
26039 dprintf (2, ("before NR 2nd Hov count: %d", bgc_overflow_count));
26040 bgc_overflow_count = 0;
26042 // Scan dependent handles again to promote any secondaries associated with primaries that were promoted
26043 // for finalization. As before background_scan_dependent_handles will also process any mark stack
26045 dprintf (2, ("2nd dependent handle scan and process mark overflow"));
26046 background_scan_dependent_handles (&sc);
26047 //concurrent_print_time_delta ("2nd nonconcurrent dependent handle scan and process mark overflow");
26048 concurrent_print_time_delta ("NR 2nd Hov");
26050 #ifdef MULTIPLE_HEAPS
26051 bgc_t_join.join(this, gc_join_null_dead_long_weak);
26052 if (bgc_t_join.joined())
26054 dprintf(2, ("Joining BGC threads for weak pointer deletion"));
26055 bgc_t_join.restart();
26057 #endif //MULTIPLE_HEAPS
26059 // null out the target of long weakref that were not promoted.
26060 GCScan::GcWeakPtrScan (background_promote, max_generation, max_generation, &sc);
26061 concurrent_print_time_delta ("NR GcWeakPtrScan");
26063 #ifdef MULTIPLE_HEAPS
26064 bgc_t_join.join(this, gc_join_null_dead_syncblk);
26065 if (bgc_t_join.joined())
26066 #endif //MULTIPLE_HEAPS
26068 dprintf (2, ("calling GcWeakPtrScanBySingleThread"));
26069 // scan for deleted entries in the syncblk cache
26070 GCScan::GcWeakPtrScanBySingleThread (max_generation, max_generation, &sc);
26071 concurrent_print_time_delta ("NR GcWeakPtrScanBySingleThread");
26072 #ifdef MULTIPLE_HEAPS
26073 dprintf(2, ("Starting BGC threads for end of background mark phase"));
26074 bgc_t_join.restart();
26075 #endif //MULTIPLE_HEAPS
26078 gen0_bricks_cleared = FALSE;
26080 dprintf (2, ("end of bgc mark: loh: %d, soh: %d",
26081 generation_size (max_generation + 1),
26082 generation_sizes (generation_of (max_generation))));
26084 for (int gen_idx = max_generation; gen_idx <= (max_generation + 1); gen_idx++)
26086 generation* gen = generation_of (gen_idx);
26087 dynamic_data* dd = dynamic_data_of (gen_idx);
26088 dd_begin_data_size (dd) = generation_size (gen_idx) -
26089 (generation_free_list_space (gen) + generation_free_obj_space (gen)) -
26090 Align (size (generation_allocation_start (gen)));
26091 dd_survived_size (dd) = 0;
26092 dd_pinned_survived_size (dd) = 0;
26093 dd_artificial_pinned_survived_size (dd) = 0;
26094 dd_added_pinned_size (dd) = 0;
26097 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
26098 PREFIX_ASSUME(seg != NULL);
26102 seg->flags &= ~heap_segment_flags_swept;
26104 if (heap_segment_allocated (seg) == heap_segment_mem (seg))
26106 // This can't happen...
26110 if (seg == ephemeral_heap_segment)
26112 heap_segment_background_allocated (seg) = generation_allocation_start (generation_of (max_generation - 1));
26116 heap_segment_background_allocated (seg) = heap_segment_allocated (seg);
26119 dprintf (2, ("seg %Ix background allocated is %Ix",
26120 heap_segment_mem (seg),
26121 heap_segment_background_allocated (seg)));
26122 seg = heap_segment_next_rw (seg);
26125 // We need to void alloc contexts here 'cause while background_ephemeral_sweep is running
26126 // we can't let the user code consume the left over parts in these alloc contexts.
26127 repair_allocation_contexts (FALSE);
26130 finish = GetCycleCount32();
26131 mark_time = finish - start;
26134 dprintf (2, ("end of bgc mark: gen2 free list space: %d, free obj space: %d",
26135 generation_free_list_space (generation_of (max_generation)),
26136 generation_free_obj_space (generation_of (max_generation))));
26138 dprintf(2,("---- (GC%d)End of background mark phase ----", VolatileLoad(&settings.gc_index)));
26142 gc_heap::suspend_EE ()
26144 dprintf (2, ("suspend_EE"));
26145 #ifdef MULTIPLE_HEAPS
26146 gc_heap* hp = gc_heap::g_heaps[0];
26147 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26149 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26150 #endif //MULTIPLE_HEAPS
26153 #ifdef MULTIPLE_HEAPS
26155 gc_heap::bgc_suspend_EE ()
26157 for (int i = 0; i < n_heaps; i++)
26159 gc_heap::g_heaps[i]->reset_gc_done();
26162 dprintf (2, ("bgc_suspend_EE"));
26163 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26165 gc_started = FALSE;
26166 for (int i = 0; i < n_heaps; i++)
26168 gc_heap::g_heaps[i]->set_gc_done();
26173 gc_heap::bgc_suspend_EE ()
26177 dprintf (2, ("bgc_suspend_EE"));
26178 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC_PREP);
26179 gc_started = FALSE;
26182 #endif //MULTIPLE_HEAPS
26185 gc_heap::restart_EE ()
26187 dprintf (2, ("restart_EE"));
26188 #ifdef MULTIPLE_HEAPS
26189 GCToEEInterface::RestartEE(FALSE);
26191 GCToEEInterface::RestartEE(FALSE);
26192 #endif //MULTIPLE_HEAPS
26195 inline uint8_t* gc_heap::high_page ( heap_segment* seg, BOOL concurrent_p)
26199 uint8_t* end = ((seg == ephemeral_heap_segment) ?
26200 generation_allocation_start (generation_of (max_generation-1)) :
26201 heap_segment_allocated (seg));
26202 return align_lower_page (end);
26206 return heap_segment_allocated (seg);
26210 void gc_heap::revisit_written_page (uint8_t* page,
26214 uint8_t*& last_page,
26215 uint8_t*& last_object,
26216 BOOL large_objects_p,
26217 size_t& num_marked_objects)
26219 UNREFERENCED_PARAMETER(seg);
26221 uint8_t* start_address = page;
26223 int align_const = get_alignment_constant (!large_objects_p);
26224 uint8_t* high_address = end;
26225 uint8_t* current_lowest_address = background_saved_lowest_address;
26226 uint8_t* current_highest_address = background_saved_highest_address;
26227 BOOL no_more_loop_p = FALSE;
26230 #ifndef MULTIPLE_HEAPS
26231 const int thread = heap_number;
26232 #endif //!MULTIPLE_HEAPS
26234 if (large_objects_p)
26240 if (((last_page + WRITE_WATCH_UNIT_SIZE) == page)
26241 || (start_address <= last_object))
26247 o = find_first_object (start_address, last_object);
26248 // We can visit the same object again, but on a different page.
26249 assert (o >= last_object);
26253 dprintf (3,("page %Ix start: %Ix, %Ix[ ",
26254 (size_t)page, (size_t)o,
26255 (size_t)(min (high_address, page + WRITE_WATCH_UNIT_SIZE))));
26257 while (o < (min (high_address, page + WRITE_WATCH_UNIT_SIZE)))
26261 if (concurrent_p && large_objects_p)
26263 bgc_alloc_lock->bgc_mark_set (o);
26265 if (((CObjectHeader*)o)->IsFree())
26267 s = unused_array_size (o);
26279 dprintf (3,("Considering object %Ix(%s)", (size_t)o, (background_object_marked (o, FALSE) ? "bm" : "nbm")));
26281 assert (Align (s) >= Align (min_obj_size));
26283 uint8_t* next_o = o + Align (s, align_const);
26285 if (next_o >= start_address)
26287 #ifdef MULTIPLE_HEAPS
26290 // We set last_object here for SVR BGC here because SVR BGC has more than
26291 // one GC thread. When we have more than one GC thread we would run into this
26292 // situation if we skipped unmarked objects:
26293 // bgc thread 1 calls GWW, and detect object X not marked so it would skip it
26295 // bgc thread 2 marks X and all its current children.
26296 // user thread comes along and dirties more (and later) pages in X.
26297 // bgc thread 1 calls GWW again and gets those later pages but it will not mark anything
26298 // on them because it had already skipped X. We need to detect that this object is now
26299 // marked and mark the children on the dirtied pages.
26300 // In the future if we have less BGC threads than we have heaps we should add
26301 // the check to the number of BGC threads.
26304 #endif //MULTIPLE_HEAPS
26306 if (contain_pointers (o) &&
26307 (!((o >= current_lowest_address) && (o < current_highest_address)) ||
26308 background_marked (o)))
26310 dprintf (3, ("going through %Ix", (size_t)o));
26311 go_through_object (method_table(o), o, s, poo, start_address, use_start, (o + s),
26312 if ((uint8_t*)poo >= min (high_address, page + WRITE_WATCH_UNIT_SIZE))
26314 no_more_loop_p = TRUE;
26317 uint8_t* oo = *poo;
26319 num_marked_objects++;
26320 background_mark_object (oo THREAD_NUMBER_ARG);
26325 #ifndef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP // see comment below
26327 #endif // !FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26328 ((CObjectHeader*)o)->IsFree() &&
26329 (next_o > min (high_address, page + WRITE_WATCH_UNIT_SIZE)))
26331 // We need to not skip the object here because of this corner scenario:
26332 // A large object was being allocated during BGC mark so we first made it
26333 // into a free object, then cleared its memory. In this loop we would detect
26334 // that it's a free object which normally we would skip. But by the next time
26335 // we call GetWriteWatch we could still be on this object and the object had
26336 // been made into a valid object and some of its memory was changed. We need
26337 // to be sure to process those written pages so we can't skip the object just
26340 // Similarly, when using software write watch, don't advance last_object when
26341 // the current object is a free object that spans beyond the current page or
26342 // high_address. Software write watch acquires gc_lock before the concurrent
26343 // GetWriteWatch() call during revisit_written_pages(). A foreground GC may
26344 // happen at that point and allocate from this free region, so when
26345 // revisit_written_pages() continues, it cannot skip now-valid objects in this
26347 no_more_loop_p = TRUE;
26352 if (concurrent_p && large_objects_p)
26354 bgc_alloc_lock->bgc_mark_done ();
26356 if (no_more_loop_p)
26363 #ifdef MULTIPLE_HEAPS
26366 assert (last_object < (min (high_address, page + WRITE_WATCH_UNIT_SIZE)));
26369 #endif //MULTIPLE_HEAPS
26374 dprintf (3,("Last object: %Ix", (size_t)last_object));
26375 last_page = align_write_watch_lower_page (o);
26378 // When reset_only_p is TRUE, we should only reset pages that are in range
26379 // because we need to consider the segments or part of segments that were
26380 // allocated out of range all live.
26381 void gc_heap::revisit_written_pages (BOOL concurrent_p, BOOL reset_only_p)
26384 if (concurrent_p && !reset_only_p)
26386 current_bgc_state = bgc_revisit_soh;
26389 size_t total_dirtied_pages = 0;
26390 size_t total_marked_objects = 0;
26392 heap_segment* seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
26394 PREFIX_ASSUME(seg != NULL);
26396 bool reset_watch_state = !!concurrent_p;
26397 bool is_runtime_suspended = !concurrent_p;
26398 BOOL small_object_segments = TRUE;
26399 int align_const = get_alignment_constant (small_object_segments);
26405 if (small_object_segments)
26407 //switch to large segment
26408 if (concurrent_p && !reset_only_p)
26410 current_bgc_state = bgc_revisit_loh;
26415 dprintf (GTC_LOG, ("h%d: SOH: dp:%Id; mo: %Id", heap_number, total_dirtied_pages, total_marked_objects));
26416 fire_revisit_event (total_dirtied_pages, total_marked_objects, !small_object_segments);
26417 concurrent_print_time_delta (concurrent_p ? "CR SOH" : "NR SOH");
26418 total_dirtied_pages = 0;
26419 total_marked_objects = 0;
26422 small_object_segments = FALSE;
26423 //concurrent_print_time_delta (concurrent_p ? "concurrent marking dirtied pages on SOH" : "nonconcurrent marking dirtied pages on SOH");
26425 dprintf (3, ("now revisiting large object segments"));
26426 align_const = get_alignment_constant (small_object_segments);
26427 seg = heap_segment_rw (generation_start_segment (large_object_generation));
26429 PREFIX_ASSUME(seg != NULL);
26437 dprintf (GTC_LOG, ("h%d: tdp: %Id", heap_number, total_dirtied_pages));
26441 dprintf (GTC_LOG, ("h%d: LOH: dp:%Id; mo: %Id", heap_number, total_dirtied_pages, total_marked_objects));
26442 fire_revisit_event (total_dirtied_pages, total_marked_objects, !small_object_segments);
26447 uint8_t* base_address = (uint8_t*)heap_segment_mem (seg);
26448 //we need to truncate to the base of the page because
26449 //some newly allocated could exist beyond heap_segment_allocated
26450 //and if we reset the last page write watch status,
26451 // they wouldn't be guaranteed to be visited -> gc hole.
26452 uintptr_t bcount = array_size;
26453 uint8_t* last_page = 0;
26454 uint8_t* last_object = heap_segment_mem (seg);
26455 uint8_t* high_address = 0;
26457 BOOL skip_seg_p = FALSE;
26461 if ((heap_segment_mem (seg) >= background_saved_lowest_address) ||
26462 (heap_segment_reserved (seg) <= background_saved_highest_address))
26464 dprintf (3, ("h%d: sseg: %Ix(-%Ix)", heap_number,
26465 heap_segment_mem (seg), heap_segment_reserved (seg)));
26472 dprintf (3, ("looking at seg %Ix", (size_t)last_object));
26476 base_address = max (base_address, background_saved_lowest_address);
26477 dprintf (3, ("h%d: reset only starting %Ix", heap_number, base_address));
26480 dprintf (3, ("h%d: starting: %Ix, seg %Ix-%Ix", heap_number, base_address,
26481 heap_segment_mem (seg), heap_segment_reserved (seg)));
26488 high_address = ((seg == ephemeral_heap_segment) ? alloc_allocated : heap_segment_allocated (seg));
26489 high_address = min (high_address, background_saved_highest_address);
26493 high_address = high_page (seg, concurrent_p);
26496 if ((base_address < high_address) &&
26497 (bcount >= array_size))
26499 ptrdiff_t region_size = high_address - base_address;
26500 dprintf (3, ("h%d: gw: [%Ix(%Id)", heap_number, (size_t)base_address, (size_t)region_size));
26502 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26503 // When the runtime is not suspended, it's possible for the table to be resized concurrently with the scan
26504 // for dirty pages below. Prevent that by synchronizing with grow_brick_card_tables(). When the runtime is
26505 // suspended, it's ok to scan for dirty pages concurrently from multiple background GC threads for disjoint
26507 if (!is_runtime_suspended)
26509 enter_spin_lock(&gc_lock);
26511 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26513 get_write_watch_for_gc_heap (reset_watch_state, base_address, region_size,
26514 (void**)background_written_addresses,
26515 &bcount, is_runtime_suspended);
26517 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26518 if (!is_runtime_suspended)
26520 leave_spin_lock(&gc_lock);
26522 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
26526 total_dirtied_pages += bcount;
26528 dprintf (3, ("Found %d pages [%Ix, %Ix[",
26529 bcount, (size_t)base_address, (size_t)high_address));
26534 for (unsigned i = 0; i < bcount; i++)
26536 uint8_t* page = (uint8_t*)background_written_addresses[i];
26537 dprintf (3, ("looking at page %d at %Ix(h: %Ix)", i,
26538 (size_t)page, (size_t)high_address));
26539 if (page < high_address)
26541 //search for marked objects in the page
26542 revisit_written_page (page, high_address, concurrent_p,
26543 seg, last_page, last_object,
26544 !small_object_segments,
26545 total_marked_objects);
26549 dprintf (3, ("page %d at %Ix is >= %Ix!", i, (size_t)page, (size_t)high_address));
26550 assert (!"page shouldn't have exceeded limit");
26555 if (bcount >= array_size){
26556 base_address = background_written_addresses [array_size-1] + WRITE_WATCH_UNIT_SIZE;
26557 bcount = array_size;
26567 seg = heap_segment_next_rw (seg);
26570 #endif //WRITE_WATCH
26573 void gc_heap::background_grow_c_mark_list()
26575 assert (c_mark_list_index >= c_mark_list_length);
26576 BOOL should_drain_p = FALSE;
26578 #ifndef MULTIPLE_HEAPS
26579 const int thread = heap_number;
26580 #endif //!MULTIPLE_HEAPS
26582 dprintf (2, ("stack copy buffer overflow"));
26583 uint8_t** new_c_mark_list = 0;
26586 if (c_mark_list_length >= (SIZE_T_MAX / (2 * sizeof (uint8_t*))))
26588 should_drain_p = TRUE;
26592 new_c_mark_list = new (nothrow) uint8_t*[c_mark_list_length*2];
26593 if (new_c_mark_list == 0)
26595 should_drain_p = TRUE;
26599 if (should_drain_p)
26602 dprintf (2, ("No more memory for the stacks copy, draining.."));
26603 //drain the list by marking its elements
26604 background_drain_mark_list (thread);
26608 assert (new_c_mark_list);
26609 memcpy (new_c_mark_list, c_mark_list, c_mark_list_length*sizeof(uint8_t*));
26610 c_mark_list_length = c_mark_list_length*2;
26611 delete c_mark_list;
26612 c_mark_list = new_c_mark_list;
26616 void gc_heap::background_promote_callback (Object** ppObject, ScanContext* sc,
26619 UNREFERENCED_PARAMETER(sc);
26620 //in order to save space on the array, mark the object,
26621 //knowing that it will be visited later
26622 assert (settings.concurrent);
26624 THREAD_NUMBER_FROM_CONTEXT;
26625 #ifndef MULTIPLE_HEAPS
26626 const int thread = 0;
26627 #endif //!MULTIPLE_HEAPS
26629 uint8_t* o = (uint8_t*)*ppObject;
26636 gc_heap* hp = gc_heap::heap_of (o);
26638 if ((o < hp->background_saved_lowest_address) || (o >= hp->background_saved_highest_address))
26643 #ifdef INTERIOR_POINTERS
26644 if (flags & GC_CALL_INTERIOR)
26646 o = hp->find_object (o, hp->background_saved_lowest_address);
26650 #endif //INTERIOR_POINTERS
26652 #ifdef FEATURE_CONSERVATIVE_GC
26653 // For conservative GC, a value on stack may point to middle of a free object.
26654 // In this case, we don't need to promote the pointer.
26655 if (GCConfig::GetConservativeGC() && ((CObjectHeader*)o)->IsFree())
26659 #endif //FEATURE_CONSERVATIVE_GC
26662 ((CObjectHeader*)o)->Validate();
26665 dprintf (3, ("Concurrent Background Promote %Ix", (size_t)o));
26666 if (o && (size (o) > LARGE_OBJECT_SIZE))
26668 dprintf (3, ("Brc %Ix", (size_t)o));
26671 if (hpt->c_mark_list_index >= hpt->c_mark_list_length)
26673 hpt->background_grow_c_mark_list();
26675 dprintf (3, ("pushing %08x into mark_list", (size_t)o));
26676 hpt->c_mark_list [hpt->c_mark_list_index++] = o;
26678 STRESS_LOG3(LF_GC|LF_GCROOTS, LL_INFO1000000, " GCHeap::Background Promote: Promote GC Root *%p = %p MT = %pT", ppObject, o, o ? ((Object*) o)->GetGCSafeMethodTable() : NULL);
26681 void gc_heap::mark_absorb_new_alloc()
26683 fix_allocation_contexts (FALSE);
26685 gen0_bricks_cleared = FALSE;
26687 clear_gen0_bricks();
26690 BOOL gc_heap::prepare_bgc_thread(gc_heap* gh)
26692 BOOL success = FALSE;
26693 BOOL thread_created = FALSE;
26694 dprintf (2, ("Preparing gc thread"));
26695 gh->bgc_threads_timeout_cs.Enter();
26696 if (!(gh->bgc_thread_running))
26698 dprintf (2, ("GC thread not runnning"));
26699 if ((gh->bgc_thread == 0) && create_bgc_thread(gh))
26702 thread_created = TRUE;
26707 dprintf (3, ("GC thread already running"));
26710 gh->bgc_threads_timeout_cs.Leave();
26713 FIRE_EVENT(GCCreateConcurrentThread_V1);
26718 BOOL gc_heap::create_bgc_thread(gc_heap* gh)
26720 assert (background_gc_done_event.IsValid());
26722 //dprintf (2, ("Creating BGC thread"));
26724 gh->bgc_thread_running = GCToEEInterface::CreateThread(gh->bgc_thread_stub, gh, true, ".NET Background GC");
26725 return gh->bgc_thread_running;
26728 BOOL gc_heap::create_bgc_threads_support (int number_of_heaps)
26731 dprintf (3, ("Creating concurrent GC thread for the first time"));
26732 if (!background_gc_done_event.CreateManualEventNoThrow(TRUE))
26736 if (!bgc_threads_sync_event.CreateManualEventNoThrow(FALSE))
26740 if (!ee_proceed_event.CreateAutoEventNoThrow(FALSE))
26744 if (!bgc_start_event.CreateManualEventNoThrow(FALSE))
26749 #ifdef MULTIPLE_HEAPS
26750 bgc_t_join.init (number_of_heaps, join_flavor_bgc);
26752 UNREFERENCED_PARAMETER(number_of_heaps);
26753 #endif //MULTIPLE_HEAPS
26761 if (background_gc_done_event.IsValid())
26763 background_gc_done_event.CloseEvent();
26765 if (bgc_threads_sync_event.IsValid())
26767 bgc_threads_sync_event.CloseEvent();
26769 if (ee_proceed_event.IsValid())
26771 ee_proceed_event.CloseEvent();
26773 if (bgc_start_event.IsValid())
26775 bgc_start_event.CloseEvent();
26782 BOOL gc_heap::create_bgc_thread_support()
26787 if (!gc_lh_block_event.CreateManualEventNoThrow(FALSE))
26792 //needs to have room for enough smallest objects fitting on a page
26793 parr = new (nothrow) uint8_t*[1 + OS_PAGE_SIZE / MIN_OBJECT_SIZE];
26799 make_c_mark_list (parr);
26807 if (gc_lh_block_event.IsValid())
26809 gc_lh_block_event.CloseEvent();
26816 int gc_heap::check_for_ephemeral_alloc()
26818 int gen = ((settings.reason == reason_oos_soh) ? (max_generation - 1) : -1);
26822 #ifdef MULTIPLE_HEAPS
26823 for (int heap_index = 0; heap_index < n_heaps; heap_index++)
26824 #endif //MULTIPLE_HEAPS
26826 for (int i = 0; i <= (max_generation - 1); i++)
26828 #ifdef MULTIPLE_HEAPS
26829 if (g_heaps[heap_index]->get_new_allocation (i) <= 0)
26831 if (get_new_allocation (i) <= 0)
26832 #endif //MULTIPLE_HEAPS
26834 gen = max (gen, i);
26845 // Wait for gc to finish sequential part
26846 void gc_heap::wait_to_proceed()
26848 assert (background_gc_done_event.IsValid());
26849 assert (bgc_start_event.IsValid());
26851 user_thread_wait(&ee_proceed_event, FALSE);
26854 // Start a new concurrent gc
26855 void gc_heap::start_c_gc()
26857 assert (background_gc_done_event.IsValid());
26858 assert (bgc_start_event.IsValid());
26860 //Need to make sure that the gc thread is in the right place.
26861 background_gc_done_event.Wait(INFINITE, FALSE);
26862 background_gc_done_event.Reset();
26863 bgc_start_event.Set();
26866 void gc_heap::do_background_gc()
26868 dprintf (2, ("starting a BGC"));
26869 #ifdef MULTIPLE_HEAPS
26870 for (int i = 0; i < n_heaps; i++)
26872 g_heaps[i]->init_background_gc();
26875 init_background_gc();
26876 #endif //MULTIPLE_HEAPS
26877 //start the background gc
26880 //wait until we get restarted by the BGC.
26884 void gc_heap::kill_gc_thread()
26886 //assert (settings.concurrent == FALSE);
26888 // We are doing a two-stage shutdown now.
26889 // In the first stage, we do minimum work, and call ExitProcess at the end.
26890 // In the secodn stage, we have the Loader lock and only one thread is
26891 // alive. Hence we do not need to kill gc thread.
26892 background_gc_done_event.CloseEvent();
26893 gc_lh_block_event.CloseEvent();
26894 bgc_start_event.CloseEvent();
26895 bgc_threads_timeout_cs.Destroy();
26897 recursive_gc_sync::shutdown();
26900 void gc_heap::bgc_thread_function()
26902 assert (background_gc_done_event.IsValid());
26903 assert (bgc_start_event.IsValid());
26905 dprintf (3, ("gc_thread thread starting..."));
26907 BOOL do_exit = FALSE;
26909 bool cooperative_mode = true;
26910 bgc_thread_id.SetToCurrentThread();
26911 dprintf (1, ("bgc_thread_id is set to %x", (uint32_t)GCToOSInterface::GetCurrentThreadIdForLogging()));
26914 // Wait for work to do...
26915 dprintf (3, ("bgc thread: waiting..."));
26917 cooperative_mode = enable_preemptive ();
26918 //current_thread->m_fPreemptiveGCDisabled = 0;
26920 uint32_t result = bgc_start_event.Wait(
26922 #ifdef MULTIPLE_HEAPS
26926 #endif //MULTIPLE_HEAPS
26928 #ifdef MULTIPLE_HEAPS
26932 #endif //MULTIPLE_HEAPS
26935 dprintf (2, ("gc thread: finished waiting"));
26937 // not calling disable_preemptive here 'cause we
26938 // can't wait for GC complete here - RestartEE will be called
26939 // when we've done the init work.
26941 if (result == WAIT_TIMEOUT)
26943 // Should join the bgc threads and terminate all of them
26945 dprintf (1, ("GC thread timeout"));
26946 bgc_threads_timeout_cs.Enter();
26947 if (!keep_bgc_threads_p)
26949 dprintf (2, ("GC thread exiting"));
26950 bgc_thread_running = FALSE;
26952 bgc_thread_id.Clear();
26955 bgc_threads_timeout_cs.Leave();
26960 dprintf (3, ("GC thread needed, not exiting"));
26964 // if we signal the thread with no concurrent work to do -> exit
26965 if (!settings.concurrent)
26967 dprintf (3, ("no concurrent GC needed, exiting"));
26973 recursive_gc_sync::begin_background();
26974 dprintf (2, ("beginning of bgc: gen2 FL: %d, FO: %d, frag: %d",
26975 generation_free_list_space (generation_of (max_generation)),
26976 generation_free_obj_space (generation_of (max_generation)),
26977 dd_fragmentation (dynamic_data_of (max_generation))));
26981 current_bgc_state = bgc_not_in_process;
26984 //trace_gc = FALSE;
26987 enable_preemptive ();
26988 #ifdef MULTIPLE_HEAPS
26989 bgc_t_join.join(this, gc_join_done);
26990 if (bgc_t_join.joined())
26991 #endif //MULTIPLE_HEAPS
26993 enter_spin_lock (&gc_lock);
26994 dprintf (SPINLOCK_LOG, ("bgc Egc"));
26996 bgc_start_event.Reset();
26998 #ifdef MULTIPLE_HEAPS
26999 for (int gen = max_generation; gen <= (max_generation + 1); gen++)
27001 size_t desired_per_heap = 0;
27002 size_t total_desired = 0;
27005 for (int i = 0; i < n_heaps; i++)
27008 dd = hp->dynamic_data_of (gen);
27009 size_t temp_total_desired = total_desired + dd_desired_allocation (dd);
27010 if (temp_total_desired < total_desired)
27013 total_desired = (size_t)MAX_PTR;
27016 total_desired = temp_total_desired;
27019 desired_per_heap = Align ((total_desired/n_heaps), get_alignment_constant (FALSE));
27021 for (int i = 0; i < n_heaps; i++)
27023 hp = gc_heap::g_heaps[i];
27024 dd = hp->dynamic_data_of (gen);
27025 dd_desired_allocation (dd) = desired_per_heap;
27026 dd_gc_new_allocation (dd) = desired_per_heap;
27027 dd_new_allocation (dd) = desired_per_heap;
27030 #endif //MULTIPLE_HEAPS
27031 #ifdef MULTIPLE_HEAPS
27033 #endif //MULTIPLE_HEAPS
27035 c_write (settings.concurrent, FALSE);
27036 recursive_gc_sync::end_background();
27037 keep_bgc_threads_p = FALSE;
27038 background_gc_done_event.Set();
27040 dprintf (SPINLOCK_LOG, ("bgc Lgc"));
27041 leave_spin_lock (&gc_lock);
27042 #ifdef MULTIPLE_HEAPS
27043 dprintf(1, ("End of BGC - starting all BGC threads"));
27044 bgc_t_join.restart();
27045 #endif //MULTIPLE_HEAPS
27047 // We can't disable preempt here because there might've been a GC already
27048 // started and decided to do a BGC and waiting for a BGC thread to restart
27049 // vm. That GC will be waiting in wait_to_proceed and we are waiting for it
27050 // to restart the VM so we deadlock.
27051 //gc_heap::disable_preemptive (current_thread, TRUE);
27054 FIRE_EVENT(GCTerminateConcurrentThread_V1);
27056 dprintf (3, ("bgc_thread thread exiting"));
27060 #endif //BACKGROUND_GC
27062 //Clear the cards [start_card, end_card[
27063 void gc_heap::clear_cards (size_t start_card, size_t end_card)
27065 if (start_card < end_card)
27067 size_t start_word = card_word (start_card);
27068 size_t end_word = card_word (end_card);
27069 if (start_word < end_word)
27071 // Figure out the bit positions of the cards within their words
27072 unsigned bits = card_bit (start_card);
27073 card_table [start_word] &= lowbits (~0, bits);
27074 for (size_t i = start_word+1; i < end_word; i++)
27075 card_table [i] = 0;
27076 bits = card_bit (end_card);
27077 // Don't write beyond end_card (and possibly uncommitted card table space).
27080 card_table [end_word] &= highbits (~0, bits);
27085 // If the start and end cards are in the same word, just clear the appropriate card
27086 // bits in that word.
27087 card_table [start_word] &= (lowbits (~0, card_bit (start_card)) |
27088 highbits (~0, card_bit (end_card)));
27090 #ifdef VERYSLOWDEBUG
27091 size_t card = start_card;
27092 while (card < end_card)
27094 assert (! (card_set_p (card)));
27097 #endif //VERYSLOWDEBUG
27098 dprintf (3,("Cleared cards [%Ix:%Ix, %Ix:%Ix[",
27099 start_card, (size_t)card_address (start_card),
27100 end_card, (size_t)card_address (end_card)));
27104 void gc_heap::clear_card_for_addresses (uint8_t* start_address, uint8_t* end_address)
27106 size_t start_card = card_of (align_on_card (start_address));
27107 size_t end_card = card_of (align_lower_card (end_address));
27108 clear_cards (start_card, end_card);
27111 // copy [srccard, ...[ to [dst_card, end_card[
27112 // This will set the same bit twice. Can be optimized.
27114 void gc_heap::copy_cards (size_t dst_card,
27119 // If the range is empty, this function is a no-op - with the subtlety that
27120 // either of the accesses card_table[srcwrd] or card_table[dstwrd] could be
27121 // outside the committed region. To avoid the access, leave early.
27122 if (!(dst_card < end_card))
27125 unsigned int srcbit = card_bit (src_card);
27126 unsigned int dstbit = card_bit (dst_card);
27127 size_t srcwrd = card_word (src_card);
27128 size_t dstwrd = card_word (dst_card);
27129 unsigned int srctmp = card_table[srcwrd];
27130 unsigned int dsttmp = card_table[dstwrd];
27132 for (size_t card = dst_card; card < end_card; card++)
27134 if (srctmp & (1 << srcbit))
27135 dsttmp |= 1 << dstbit;
27137 dsttmp &= ~(1 << dstbit);
27139 if (!(++srcbit % 32))
27141 srctmp = card_table[++srcwrd];
27147 if (srctmp & (1 << srcbit))
27148 dsttmp |= 1 << dstbit;
27151 if (!(++dstbit % 32))
27153 card_table[dstwrd] = dsttmp;
27155 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27158 card_bundle_set(cardw_card_bundle(dstwrd));
27163 dsttmp = card_table[dstwrd];
27168 card_table[dstwrd] = dsttmp;
27170 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27173 card_bundle_set(cardw_card_bundle(dstwrd));
27178 void gc_heap::copy_cards_for_addresses (uint8_t* dest, uint8_t* src, size_t len)
27180 ptrdiff_t relocation_distance = src - dest;
27181 size_t start_dest_card = card_of (align_on_card (dest));
27182 size_t end_dest_card = card_of (dest + len - 1);
27183 size_t dest_card = start_dest_card;
27184 size_t src_card = card_of (card_address (dest_card)+relocation_distance);
27185 dprintf (3,("Copying cards [%Ix:%Ix->%Ix:%Ix, ",
27186 src_card, (size_t)src, dest_card, (size_t)dest));
27187 dprintf (3,(" %Ix->%Ix:%Ix[",
27188 (size_t)src+len, end_dest_card, (size_t)dest+len));
27190 dprintf (3, ("dest: %Ix, src: %Ix, len: %Ix, reloc: %Ix, align_on_card(dest) is %Ix",
27191 dest, src, len, relocation_distance, (align_on_card (dest))));
27193 dprintf (3, ("start_dest_card: %Ix (address: %Ix), end_dest_card: %Ix(addr: %Ix), card_of (dest): %Ix",
27194 start_dest_card, card_address (start_dest_card), end_dest_card, card_address (end_dest_card), card_of (dest)));
27196 //First card has two boundaries
27197 if (start_dest_card != card_of (dest))
27199 if ((card_of (card_address (start_dest_card) + relocation_distance) <= card_of (src + len - 1))&&
27200 card_set_p (card_of (card_address (start_dest_card) + relocation_distance)))
27202 dprintf (3, ("card_address (start_dest_card) + reloc is %Ix, card: %Ix(set), src+len-1: %Ix, card: %Ix",
27203 (card_address (start_dest_card) + relocation_distance),
27204 card_of (card_address (start_dest_card) + relocation_distance),
27206 card_of (src + len - 1)));
27208 dprintf (3, ("setting card: %Ix", card_of (dest)));
27209 set_card (card_of (dest));
27213 if (card_set_p (card_of (src)))
27214 set_card (card_of (dest));
27217 copy_cards (dest_card, src_card, end_dest_card,
27218 ((dest - align_lower_card (dest)) != (src - align_lower_card (src))));
27220 //Last card has two boundaries.
27221 if ((card_of (card_address (end_dest_card) + relocation_distance) >= card_of (src)) &&
27222 card_set_p (card_of (card_address (end_dest_card) + relocation_distance)))
27224 dprintf (3, ("card_address (end_dest_card) + reloc is %Ix, card: %Ix(set), src: %Ix, card: %Ix",
27225 (card_address (end_dest_card) + relocation_distance),
27226 card_of (card_address (end_dest_card) + relocation_distance),
27230 dprintf (3, ("setting card: %Ix", end_dest_card));
27231 set_card (end_dest_card);
27234 if (card_set_p (card_of (src + len - 1)))
27235 set_card (end_dest_card);
27237 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
27238 card_bundles_set(cardw_card_bundle(card_word(card_of(dest))), cardw_card_bundle(align_cardw_on_bundle(card_word(end_dest_card))));
27242 #ifdef BACKGROUND_GC
27243 // this does not need the Interlocked version of mark_array_set_marked.
27244 void gc_heap::copy_mark_bits_for_addresses (uint8_t* dest, uint8_t* src, size_t len)
27246 dprintf (3, ("Copying mark_bits for addresses [%Ix->%Ix, %Ix->%Ix[",
27247 (size_t)src, (size_t)dest,
27248 (size_t)src+len, (size_t)dest+len));
27250 uint8_t* src_o = src;
27252 uint8_t* src_end = src + len;
27253 int align_const = get_alignment_constant (TRUE);
27254 ptrdiff_t reloc = dest - src;
27256 while (src_o < src_end)
27258 uint8_t* next_o = src_o + Align (size (src_o), align_const);
27260 if (background_object_marked (src_o, TRUE))
27262 dest_o = src_o + reloc;
27264 //if (background_object_marked (dest_o, FALSE))
27266 // dprintf (3, ("*%Ix shouldn't have already been marked!", (size_t)(dest_o)));
27267 // FATAL_GC_ERROR();
27270 background_mark (dest_o,
27271 background_saved_lowest_address,
27272 background_saved_highest_address);
27273 dprintf (3, ("bc*%Ix*bc, b*%Ix*b", (size_t)src_o, (size_t)(dest_o)));
27279 #endif //BACKGROUND_GC
27281 void gc_heap::fix_brick_to_highest (uint8_t* o, uint8_t* next_o)
27283 size_t new_current_brick = brick_of (o);
27284 set_brick (new_current_brick,
27285 (o - brick_address (new_current_brick)));
27286 size_t b = 1 + new_current_brick;
27287 size_t limit = brick_of (next_o);
27288 //dprintf(3,(" fixing brick %Ix to point to object %Ix, till %Ix(%Ix)",
27289 dprintf(3,("b:%Ix->%Ix-%Ix",
27290 new_current_brick, (size_t)o, (size_t)next_o));
27293 set_brick (b,(new_current_brick - b));
27298 // start can not be >= heap_segment_allocated for the segment.
27299 uint8_t* gc_heap::find_first_object (uint8_t* start, uint8_t* first_object)
27301 size_t brick = brick_of (start);
27303 //last_object == null -> no search shortcut needed
27304 if ((brick == brick_of (first_object) || (start <= first_object)))
27310 ptrdiff_t min_brick = (ptrdiff_t)brick_of (first_object);
27311 ptrdiff_t prev_brick = (ptrdiff_t)brick - 1;
27312 int brick_entry = 0;
27315 if (prev_brick < min_brick)
27319 if ((brick_entry = get_brick_entry(prev_brick)) >= 0)
27323 assert (! ((brick_entry == 0)));
27324 prev_brick = (brick_entry + prev_brick);
27327 o = ((prev_brick < min_brick) ? first_object :
27328 brick_address (prev_brick) + brick_entry - 1);
27329 assert (o <= start);
27332 assert (Align (size (o)) >= Align (min_obj_size));
27333 uint8_t* next_o = o + Align (size (o));
27334 size_t curr_cl = (size_t)next_o / brick_size;
27335 size_t min_cl = (size_t)first_object / brick_size;
27337 //dprintf (3,( "Looking for intersection with %Ix from %Ix", (size_t)start, (size_t)o));
27339 unsigned int n_o = 1;
27342 uint8_t* next_b = min (align_lower_brick (next_o) + brick_size, start+1);
27344 while (next_o <= start)
27352 assert (Align (size (o)) >= Align (min_obj_size));
27353 next_o = o + Align (size (o));
27355 }while (next_o < next_b);
27357 if (((size_t)next_o / brick_size) != curr_cl)
27359 if (curr_cl >= min_cl)
27361 fix_brick_to_highest (o, next_o);
27363 curr_cl = (size_t) next_o / brick_size;
27365 next_b = min (align_lower_brick (next_o) + brick_size, start+1);
27368 size_t bo = brick_of (o);
27369 //dprintf (3, ("Looked at %Id objects, fixing brick [%Ix-[%Ix",
27370 dprintf (3, ("%Id o, [%Ix-[%Ix",
27374 set_brick (bo, (o - brick_address(bo)));
27389 // Find the first non-zero card word between cardw and cardw_end.
27390 // The index of the word we find is returned in cardw.
27391 BOOL gc_heap::find_card_dword (size_t& cardw, size_t cardw_end)
27393 dprintf (3, ("gc: %d, find_card_dword cardw: %Ix, cardw_end: %Ix",
27394 dd_collection_count (dynamic_data_of (0)), cardw, cardw_end));
27396 if (card_bundles_enabled())
27398 size_t cardb = cardw_card_bundle (cardw);
27399 size_t end_cardb = cardw_card_bundle (align_cardw_on_bundle (cardw_end));
27402 // Find a non-zero bundle
27403 while ((cardb < end_cardb) && (card_bundle_set_p (cardb) == 0))
27408 if (cardb == end_cardb)
27411 // We found a bundle, so go through its words and find a non-zero card word
27412 uint32_t* card_word = &card_table[max(card_bundle_cardw (cardb),cardw)];
27413 uint32_t* card_word_end = &card_table[min(card_bundle_cardw (cardb+1),cardw_end)];
27414 while ((card_word < card_word_end) && !(*card_word))
27419 if (card_word != card_word_end)
27421 cardw = (card_word - &card_table[0]);
27424 else if ((cardw <= card_bundle_cardw (cardb)) &&
27425 (card_word == &card_table [card_bundle_cardw (cardb+1)]))
27427 // a whole bundle was explored and is empty
27428 dprintf (3, ("gc: %d, find_card_dword clear bundle: %Ix cardw:[%Ix,%Ix[",
27429 dd_collection_count (dynamic_data_of (0)),
27430 cardb, card_bundle_cardw (cardb),
27431 card_bundle_cardw (cardb+1)));
27432 card_bundle_clear (cardb);
27440 uint32_t* card_word = &card_table[cardw];
27441 uint32_t* card_word_end = &card_table [cardw_end];
27443 while (card_word < card_word_end)
27445 if (*card_word != 0)
27447 cardw = (card_word - &card_table [0]);
27458 #endif //CARD_BUNDLE
27460 // Find cards that are set between two points in a card table.
27462 // card_table : The card table.
27463 // card : [in/out] As input, the card to start searching from.
27464 // As output, the first card that's set.
27465 // card_word_end : The card word at which to stop looking.
27466 // end_card : [out] The last card which is set.
27467 BOOL gc_heap::find_card(uint32_t* card_table,
27469 size_t card_word_end,
27472 uint32_t* last_card_word;
27473 uint32_t card_word_value;
27474 uint32_t bit_position;
27476 // Find the first card which is set
27477 last_card_word = &card_table [card_word (card)];
27478 bit_position = card_bit (card);
27479 card_word_value = (*last_card_word) >> bit_position;
27480 if (!card_word_value)
27484 // Using the card bundle, go through the remaining card words between here and
27485 // card_word_end until we find one that is non-zero.
27486 size_t lcw = card_word(card) + 1;
27487 if (gc_heap::find_card_dword (lcw, card_word_end) == FALSE)
27493 last_card_word = &card_table [lcw];
27494 card_word_value = *last_card_word;
27497 #else //CARD_BUNDLE
27498 // Go through the remaining card words between here and card_word_end until we find
27499 // one that is non-zero.
27504 while ((last_card_word < &card_table [card_word_end]) && !(*last_card_word));
27506 if (last_card_word < &card_table [card_word_end])
27508 card_word_value = *last_card_word;
27512 // We failed to find any non-zero card words before we got to card_word_end
27515 #endif //CARD_BUNDLE
27518 // Look for the lowest bit set
27519 if (card_word_value)
27521 while (!(card_word_value & 1))
27524 card_word_value = card_word_value / 2;
27528 // card is the card word index * card size + the bit index within the card
27529 card = (last_card_word - &card_table[0]) * card_word_width + bit_position;
27533 // Keep going until we get to an un-set card.
27535 card_word_value = card_word_value / 2;
27537 // If we reach the end of the card word and haven't hit a 0 yet, start going
27538 // card word by card word until we get to one that's not fully set (0xFFFF...)
27539 // or we reach card_word_end.
27540 if ((bit_position == card_word_width) && (last_card_word < &card_table [card_word_end]))
27544 card_word_value = *(++last_card_word);
27545 } while ((last_card_word < &card_table [card_word_end]) &&
27548 (card_word_value == (1 << card_word_width)-1)
27550 // if left shift count >= width of type,
27551 // gcc reports error.
27552 (card_word_value == ~0u)
27557 } while (card_word_value & 1);
27559 end_card = (last_card_word - &card_table [0])* card_word_width + bit_position;
27561 //dprintf (3, ("find_card: [%Ix, %Ix[ set", card, end_card));
27562 dprintf (3, ("fc: [%Ix, %Ix[", card, end_card));
27567 //because of heap expansion, computing end is complicated.
27568 uint8_t* compute_next_end (heap_segment* seg, uint8_t* low)
27570 if ((low >= heap_segment_mem (seg)) &&
27571 (low < heap_segment_allocated (seg)))
27574 return heap_segment_allocated (seg);
27578 gc_heap::compute_next_boundary (uint8_t* low, int gen_number,
27581 UNREFERENCED_PARAMETER(low);
27583 //when relocating, the fault line is the plan start of the younger
27584 //generation because the generation is promoted.
27585 if (relocating && (gen_number == (settings.condemned_generation + 1)))
27587 generation* gen = generation_of (gen_number - 1);
27588 uint8_t* gen_alloc = generation_plan_allocation_start (gen);
27589 assert (gen_alloc);
27594 assert (gen_number > settings.condemned_generation);
27595 return generation_allocation_start (generation_of (gen_number - 1 ));
27601 gc_heap::keep_card_live (uint8_t* o, size_t& n_gen,
27602 size_t& cg_pointers_found)
27605 if ((gc_low <= o) && (gc_high > o))
27609 #ifdef MULTIPLE_HEAPS
27612 gc_heap* hp = heap_of (o);
27615 if ((hp->gc_low <= o) &&
27622 #endif //MULTIPLE_HEAPS
27623 cg_pointers_found ++;
27624 dprintf (4, ("keep card live for %Ix", o));
27628 gc_heap::mark_through_cards_helper (uint8_t** poo, size_t& n_gen,
27629 size_t& cg_pointers_found,
27630 card_fn fn, uint8_t* nhigh,
27631 uint8_t* next_boundary)
27634 if ((gc_low <= *poo) && (gc_high > *poo))
27637 call_fn(fn) (poo THREAD_NUMBER_ARG);
27639 #ifdef MULTIPLE_HEAPS
27642 gc_heap* hp = heap_of_gc (*poo);
27645 if ((hp->gc_low <= *poo) &&
27646 (hp->gc_high > *poo))
27649 call_fn(fn) (poo THREAD_NUMBER_ARG);
27651 if ((fn == &gc_heap::relocate_address) ||
27652 ((hp->ephemeral_low <= *poo) &&
27653 (hp->ephemeral_high > *poo)))
27655 cg_pointers_found++;
27659 #endif //MULTIPLE_HEAPS
27660 if ((next_boundary <= *poo) && (nhigh > *poo))
27662 cg_pointers_found ++;
27663 dprintf (4, ("cg pointer %Ix found, %Id so far",
27664 (size_t)*poo, cg_pointers_found ));
27669 BOOL gc_heap::card_transition (uint8_t* po, uint8_t* end, size_t card_word_end,
27670 size_t& cg_pointers_found,
27671 size_t& n_eph, size_t& n_card_set,
27672 size_t& card, size_t& end_card,
27673 BOOL& foundp, uint8_t*& start_address,
27674 uint8_t*& limit, size_t& n_cards_cleared)
27676 dprintf (3, ("pointer %Ix past card %Ix", (size_t)po, (size_t)card));
27677 dprintf (3, ("ct: %Id cg", cg_pointers_found));
27678 BOOL passed_end_card_p = FALSE;
27681 if (cg_pointers_found == 0)
27683 //dprintf(3,(" Clearing cards [%Ix, %Ix[ ",
27684 dprintf(3,(" CC [%Ix, %Ix[ ",
27685 (size_t)card_address(card), (size_t)po));
27686 clear_cards (card, card_of(po));
27687 n_card_set -= (card_of (po) - card);
27688 n_cards_cleared += (card_of (po) - card);
27691 n_eph +=cg_pointers_found;
27692 cg_pointers_found = 0;
27693 card = card_of (po);
27694 if (card >= end_card)
27696 passed_end_card_p = TRUE;
27697 dprintf (3, ("card %Ix exceeding end_card %Ix",
27698 (size_t)card, (size_t)end_card));
27699 foundp = find_card (card_table, card, card_word_end, end_card);
27702 n_card_set+= end_card - card;
27703 start_address = card_address (card);
27704 dprintf (3, ("NewC: %Ix, start: %Ix, end: %Ix",
27705 (size_t)card, (size_t)start_address,
27706 (size_t)card_address (end_card)));
27708 limit = min (end, card_address (end_card));
27710 assert (!((limit == card_address (end_card))&&
27711 card_set_p (end_card)));
27714 return passed_end_card_p;
27717 void gc_heap::mark_through_cards_for_segments (card_fn fn, BOOL relocating)
27719 #ifdef BACKGROUND_GC
27720 dprintf (3, ("current_sweep_pos is %Ix, saved_sweep_ephemeral_seg is %Ix(%Ix)",
27721 current_sweep_pos, saved_sweep_ephemeral_seg, saved_sweep_ephemeral_start));
27723 heap_segment* soh_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
27724 PREFIX_ASSUME(soh_seg != NULL);
27728 dprintf (3, ("seg %Ix, bgc_alloc: %Ix, alloc: %Ix",
27730 heap_segment_background_allocated (soh_seg),
27731 heap_segment_allocated (soh_seg)));
27733 soh_seg = heap_segment_next_rw (soh_seg);
27735 #endif //BACKGROUND_GC
27737 uint8_t* low = gc_low;
27738 uint8_t* high = gc_high;
27739 size_t end_card = 0;
27741 generation* oldest_gen = generation_of (max_generation);
27742 int curr_gen_number = max_generation;
27743 uint8_t* gen_boundary = generation_allocation_start(generation_of(curr_gen_number - 1));
27744 uint8_t* next_boundary = compute_next_boundary(gc_low, curr_gen_number, relocating);
27746 heap_segment* seg = heap_segment_rw (generation_start_segment (oldest_gen));
27747 PREFIX_ASSUME(seg != NULL);
27749 uint8_t* beg = generation_allocation_start (oldest_gen);
27750 uint8_t* end = compute_next_end (seg, low);
27751 uint8_t* last_object = beg;
27753 size_t cg_pointers_found = 0;
27755 size_t card_word_end = (card_of (align_on_card_word (end)) / card_word_width);
27759 size_t n_card_set = 0;
27760 uint8_t* nhigh = (relocating ? heap_segment_plan_allocated (ephemeral_heap_segment) : high);
27762 BOOL foundp = FALSE;
27763 uint8_t* start_address = 0;
27764 uint8_t* limit = 0;
27765 size_t card = card_of (beg);
27766 #ifdef BACKGROUND_GC
27767 BOOL consider_bgc_mark_p = FALSE;
27768 BOOL check_current_sweep_p = FALSE;
27769 BOOL check_saved_sweep_p = FALSE;
27770 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
27771 #endif //BACKGROUND_GC
27773 dprintf(3, ("CMs: %Ix->%Ix", (size_t)beg, (size_t)end));
27774 size_t total_cards_cleared = 0;
27778 if (card_of(last_object) > card)
27780 // cg means cross-generational
27781 dprintf (3, ("Found %Id cg pointers", cg_pointers_found));
27782 if (cg_pointers_found == 0)
27784 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card), (size_t)last_object));
27785 clear_cards (card, card_of(last_object));
27786 n_card_set -= (card_of (last_object) - card);
27787 total_cards_cleared += (card_of (last_object) - card);
27790 n_eph += cg_pointers_found;
27791 cg_pointers_found = 0;
27792 card = card_of (last_object);
27795 if (card >= end_card)
27797 // Find the first card that's set (between card and card_word_end)
27798 foundp = find_card(card_table, card, card_word_end, end_card);
27801 // We found card(s) set.
27802 n_card_set += end_card - card;
27803 start_address = max (beg, card_address (card));
27806 limit = min (end, card_address (end_card));
27809 if (!foundp || (last_object >= end) || (card_address (card) >= end))
27811 if (foundp && (cg_pointers_found == 0))
27813 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card),
27815 clear_cards (card, card_of (end));
27816 n_card_set -= (card_of (end) - card);
27817 total_cards_cleared += (card_of (end) - card);
27820 n_eph += cg_pointers_found;
27821 cg_pointers_found = 0;
27823 if ((seg = heap_segment_next_in_range (seg)) != 0)
27825 #ifdef BACKGROUND_GC
27826 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
27827 #endif //BACKGROUND_GC
27828 beg = heap_segment_mem (seg);
27829 end = compute_next_end (seg, low);
27830 card_word_end = card_of (align_on_card_word (end)) / card_word_width;
27831 card = card_of (beg);
27842 // We've found a card and will now go through the objects in it.
27843 assert (card_set_p (card));
27845 uint8_t* o = last_object;
27846 o = find_first_object (start_address, last_object);
27847 // Never visit an object twice.
27848 assert (o >= last_object);
27850 //dprintf(3,("Considering card %Ix start object: %Ix, %Ix[ boundary: %Ix",
27851 dprintf(3, ("c: %Ix, o: %Ix, l: %Ix[ boundary: %Ix",
27852 card, (size_t)o, (size_t)limit, (size_t)gen_boundary));
27856 assert (Align (size (o)) >= Align (min_obj_size));
27857 size_t s = size (o);
27859 uint8_t* next_o = o + Align (s);
27862 if ((o >= gen_boundary) &&
27863 (seg == ephemeral_heap_segment))
27865 dprintf (3, ("switching gen boundary %Ix", (size_t)gen_boundary));
27867 assert ((curr_gen_number > 0));
27868 gen_boundary = generation_allocation_start
27869 (generation_of (curr_gen_number - 1));
27870 next_boundary = (compute_next_boundary
27871 (low, curr_gen_number, relocating));
27874 dprintf (4, ("|%Ix|", (size_t)o));
27876 if (next_o < start_address)
27881 #ifdef BACKGROUND_GC
27882 if (!fgc_should_consider_object (o, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p))
27886 #endif //BACKGROUND_GC
27888 #ifdef COLLECTIBLE_CLASS
27889 if (is_collectible(o))
27891 BOOL passed_end_card_p = FALSE;
27893 if (card_of (o) > card)
27895 passed_end_card_p = card_transition (o, end, card_word_end,
27899 foundp, start_address,
27900 limit, total_cards_cleared);
27903 if ((!passed_end_card_p || foundp) && (card_of (o) == card))
27905 // card is valid and it covers the head of the object
27906 if (fn == &gc_heap::relocate_address)
27908 keep_card_live (o, n_gen, cg_pointers_found);
27912 uint8_t* class_obj = get_class_object (o);
27913 mark_through_cards_helper (&class_obj, n_gen,
27914 cg_pointers_found, fn,
27915 nhigh, next_boundary);
27919 if (passed_end_card_p)
27921 if (foundp && (card_address (card) < next_o))
27923 goto go_through_refs;
27925 else if (foundp && (start_address < limit))
27927 next_o = find_first_object (start_address, o);
27936 #endif //COLLECTIBLE_CLASS
27938 if (contain_pointers (o))
27940 dprintf(3,("Going through %Ix start_address: %Ix", (size_t)o, (size_t)start_address));
27943 dprintf (4, ("normal object path"));
27945 (method_table(o), o, s, poo,
27946 start_address, use_start, (o + s),
27948 dprintf (4, ("<%Ix>:%Ix", (size_t)poo, (size_t)*poo));
27949 if (card_of ((uint8_t*)poo) > card)
27951 BOOL passed_end_card_p = card_transition ((uint8_t*)poo, end,
27956 foundp, start_address,
27957 limit, total_cards_cleared);
27959 if (passed_end_card_p)
27961 if (foundp && (card_address (card) < next_o))
27965 if (ppstop <= (uint8_t**)start_address)
27967 else if (poo < (uint8_t**)start_address)
27968 {poo = (uint8_t**)start_address;}
27971 else if (foundp && (start_address < limit))
27973 next_o = find_first_object (start_address, o);
27981 mark_through_cards_helper (poo, n_gen,
27982 cg_pointers_found, fn,
27983 nhigh, next_boundary);
27990 if (((size_t)next_o / brick_size) != ((size_t) o / brick_size))
27992 if (brick_table [brick_of (o)] <0)
27993 fix_brick_to_highest (o, next_o);
28001 // compute the efficiency ratio of the card table
28004 generation_skip_ratio = ((n_eph > 400)? (int)(((float)n_gen / (float)n_eph) * 100) : 100);
28005 dprintf (3, ("Msoh: cross: %Id, useful: %Id, cards set: %Id, cards cleared: %Id, ratio: %d",
28006 n_eph, n_gen , n_card_set, total_cards_cleared, generation_skip_ratio));
28010 dprintf (3, ("R: Msoh: cross: %Id, useful: %Id, cards set: %Id, cards cleared: %Id, ratio: %d",
28011 n_gen, n_eph, n_card_set, total_cards_cleared, generation_skip_ratio));
28015 #ifdef SEG_REUSE_STATS
28016 size_t gc_heap::dump_buckets (size_t* ordered_indices, int count, size_t* total_size)
28018 size_t total_items = 0;
28020 for (int i = 0; i < count; i++)
28022 total_items += ordered_indices[i];
28023 *total_size += ordered_indices[i] << (MIN_INDEX_POWER2 + i);
28024 dprintf (SEG_REUSE_LOG_0, ("[%d]%4d 2^%2d", heap_number, ordered_indices[i], (MIN_INDEX_POWER2 + i)));
28026 dprintf (SEG_REUSE_LOG_0, ("[%d]Total %d items, total size is 0x%Ix", heap_number, total_items, *total_size));
28027 return total_items;
28029 #endif // SEG_REUSE_STATS
28031 void gc_heap::count_plug (size_t last_plug_size, uint8_t*& last_plug)
28033 // detect pinned plugs
28034 if (!pinned_plug_que_empty_p() && (last_plug == pinned_plug (oldest_pin())))
28036 deque_pinned_plug();
28037 update_oldest_pinned_plug();
28038 dprintf (3, ("dequed pin,now oldest pin is %Ix", pinned_plug (oldest_pin())));
28042 size_t plug_size = last_plug_size + Align(min_obj_size);
28043 BOOL is_padded = FALSE;
28046 plug_size += Align (min_obj_size);
28048 #endif //SHORT_PLUGS
28050 #ifdef RESPECT_LARGE_ALIGNMENT
28051 plug_size += switch_alignment_size (is_padded);
28052 #endif //RESPECT_LARGE_ALIGNMENT
28054 total_ephemeral_plugs += plug_size;
28055 size_t plug_size_power2 = round_up_power2 (plug_size);
28056 ordered_plug_indices[relative_index_power2_plug (plug_size_power2)]++;
28057 dprintf (SEG_REUSE_LOG_1, ("[%d]count_plug: adding 0x%Ix - %Id (2^%d) to ordered plug array",
28061 (relative_index_power2_plug (plug_size_power2) + MIN_INDEX_POWER2)));
28065 void gc_heap::count_plugs_in_brick (uint8_t* tree, uint8_t*& last_plug)
28067 assert ((tree != NULL));
28068 if (node_left_child (tree))
28070 count_plugs_in_brick (tree + node_left_child (tree), last_plug);
28073 if (last_plug != 0)
28075 uint8_t* plug = tree;
28076 size_t gap_size = node_gap_size (plug);
28077 uint8_t* gap = (plug - gap_size);
28078 uint8_t* last_plug_end = gap;
28079 size_t last_plug_size = (last_plug_end - last_plug);
28080 dprintf (3, ("tree: %Ix, last plug: %Ix, gap size: %Ix, gap: %Ix, last plug size: %Ix",
28081 tree, last_plug, gap_size, gap, last_plug_size));
28083 if (tree == oldest_pinned_plug)
28085 dprintf (3, ("tree %Ix is pinned, last plug is %Ix, size is %Ix",
28086 tree, last_plug, last_plug_size));
28087 mark* m = oldest_pin();
28088 if (m->has_pre_plug_info())
28090 last_plug_size += sizeof (gap_reloc_pair);
28091 dprintf (3, ("pin %Ix has pre plug, adjusting plug size to %Ix", tree, last_plug_size));
28094 // Can't assert here - if it's a pinned plug it can be less.
28095 //assert (last_plug_size >= Align (min_obj_size));
28097 count_plug (last_plug_size, last_plug);
28102 if (node_right_child (tree))
28104 count_plugs_in_brick (tree + node_right_child (tree), last_plug);
28108 void gc_heap::build_ordered_plug_indices ()
28110 memset (ordered_plug_indices, 0, sizeof(ordered_plug_indices));
28111 memset (saved_ordered_plug_indices, 0, sizeof(saved_ordered_plug_indices));
28113 uint8_t* start_address = generation_limit (max_generation);
28114 uint8_t* end_address = heap_segment_allocated (ephemeral_heap_segment);
28115 size_t current_brick = brick_of (start_address);
28116 size_t end_brick = brick_of (end_address - 1);
28117 uint8_t* last_plug = 0;
28119 //Look for the right pinned plug to start from.
28120 reset_pinned_queue_bos();
28121 while (!pinned_plug_que_empty_p())
28123 mark* m = oldest_pin();
28124 if ((m->first >= start_address) && (m->first < end_address))
28126 dprintf (3, ("found a pin %Ix between %Ix and %Ix", m->first, start_address, end_address));
28131 deque_pinned_plug();
28134 update_oldest_pinned_plug();
28136 while (current_brick <= end_brick)
28138 int brick_entry = brick_table [ current_brick ];
28139 if (brick_entry >= 0)
28141 count_plugs_in_brick (brick_address (current_brick) + brick_entry -1, last_plug);
28149 count_plug (end_address - last_plug, last_plug);
28152 // we need to make sure that after fitting all the existing plugs, we
28153 // have big enough free space left to guarantee that the next allocation
28155 size_t extra_size = END_SPACE_AFTER_GC + Align (min_obj_size);
28156 total_ephemeral_plugs += extra_size;
28157 dprintf (SEG_REUSE_LOG_0, ("Making sure we can fit a large object after fitting all plugs"));
28158 ordered_plug_indices[relative_index_power2_plug (round_up_power2 (extra_size))]++;
28160 memcpy (saved_ordered_plug_indices, ordered_plug_indices, sizeof(ordered_plug_indices));
28162 #ifdef SEG_REUSE_STATS
28163 dprintf (SEG_REUSE_LOG_0, ("Plugs:"));
28164 size_t total_plug_power2 = 0;
28165 dump_buckets (ordered_plug_indices, MAX_NUM_BUCKETS, &total_plug_power2);
28166 dprintf (SEG_REUSE_LOG_0, ("plugs: 0x%Ix (rounded up to 0x%Ix (%d%%))",
28167 total_ephemeral_plugs,
28169 (total_ephemeral_plugs ?
28170 (total_plug_power2 * 100 / total_ephemeral_plugs) :
28172 dprintf (SEG_REUSE_LOG_0, ("-------------------"));
28173 #endif // SEG_REUSE_STATS
28176 void gc_heap::init_ordered_free_space_indices ()
28178 memset (ordered_free_space_indices, 0, sizeof(ordered_free_space_indices));
28179 memset (saved_ordered_free_space_indices, 0, sizeof(saved_ordered_free_space_indices));
28182 void gc_heap::trim_free_spaces_indices ()
28184 trimmed_free_space_index = -1;
28185 size_t max_count = max_free_space_items - 1;
28188 for (i = (MAX_NUM_BUCKETS - 1); i >= 0; i--)
28190 count += ordered_free_space_indices[i];
28192 if (count >= max_count)
28198 ptrdiff_t extra_free_space_items = count - max_count;
28200 if (extra_free_space_items > 0)
28202 ordered_free_space_indices[i] -= extra_free_space_items;
28203 free_space_items = max_count;
28204 trimmed_free_space_index = i;
28208 free_space_items = count;
28216 free_space_buckets = MAX_NUM_BUCKETS - i;
28218 for (--i; i >= 0; i--)
28220 ordered_free_space_indices[i] = 0;
28223 memcpy (saved_ordered_free_space_indices,
28224 ordered_free_space_indices,
28225 sizeof(ordered_free_space_indices));
28228 // We fit as many plugs as we can and update the number of plugs left and the number
28229 // of free spaces left.
28230 BOOL gc_heap::can_fit_in_spaces_p (size_t* ordered_blocks, int small_index, size_t* ordered_spaces, int big_index)
28232 assert (small_index <= big_index);
28233 assert (big_index < MAX_NUM_BUCKETS);
28235 size_t small_blocks = ordered_blocks[small_index];
28237 if (small_blocks == 0)
28242 size_t big_spaces = ordered_spaces[big_index];
28244 if (big_spaces == 0)
28249 dprintf (SEG_REUSE_LOG_1, ("[%d]Fitting %Id 2^%d plugs into %Id 2^%d free spaces",
28251 small_blocks, (small_index + MIN_INDEX_POWER2),
28252 big_spaces, (big_index + MIN_INDEX_POWER2)));
28254 size_t big_to_small = big_spaces << (big_index - small_index);
28256 ptrdiff_t extra_small_spaces = big_to_small - small_blocks;
28257 dprintf (SEG_REUSE_LOG_1, ("[%d]%d 2^%d spaces can fit %d 2^%d blocks",
28259 big_spaces, (big_index + MIN_INDEX_POWER2), big_to_small, (small_index + MIN_INDEX_POWER2)));
28260 BOOL can_fit = (extra_small_spaces >= 0);
28264 dprintf (SEG_REUSE_LOG_1, ("[%d]Can fit with %d 2^%d extras blocks",
28266 extra_small_spaces, (small_index + MIN_INDEX_POWER2)));
28271 dprintf (SEG_REUSE_LOG_1, ("[%d]Setting # of 2^%d spaces to 0", heap_number, (big_index + MIN_INDEX_POWER2)));
28272 ordered_spaces[big_index] = 0;
28273 if (extra_small_spaces > 0)
28275 dprintf (SEG_REUSE_LOG_1, ("[%d]Setting # of 2^%d blocks to 0", heap_number, (small_index + MIN_INDEX_POWER2)));
28276 ordered_blocks[small_index] = 0;
28277 for (i = small_index; i < big_index; i++)
28279 if (extra_small_spaces & 1)
28281 dprintf (SEG_REUSE_LOG_1, ("[%d]Increasing # of 2^%d spaces from %d to %d",
28283 (i + MIN_INDEX_POWER2), ordered_spaces[i], (ordered_spaces[i] + 1)));
28284 ordered_spaces[i] += 1;
28286 extra_small_spaces >>= 1;
28289 dprintf (SEG_REUSE_LOG_1, ("[%d]Finally increasing # of 2^%d spaces from %d to %d",
28291 (i + MIN_INDEX_POWER2), ordered_spaces[i], (ordered_spaces[i] + extra_small_spaces)));
28292 ordered_spaces[i] += extra_small_spaces;
28296 dprintf (SEG_REUSE_LOG_1, ("[%d]Decreasing # of 2^%d blocks from %d to %d",
28298 (small_index + MIN_INDEX_POWER2),
28299 ordered_blocks[small_index],
28300 (ordered_blocks[small_index] - big_to_small)));
28301 ordered_blocks[small_index] -= big_to_small;
28304 #ifdef SEG_REUSE_STATS
28306 dprintf (SEG_REUSE_LOG_1, ("[%d]Plugs became:", heap_number));
28307 dump_buckets (ordered_blocks, MAX_NUM_BUCKETS, &temp);
28309 dprintf (SEG_REUSE_LOG_1, ("[%d]Free spaces became:", heap_number));
28310 dump_buckets (ordered_spaces, MAX_NUM_BUCKETS, &temp);
28311 #endif //SEG_REUSE_STATS
28316 // space_index gets updated to the biggest available space index.
28317 BOOL gc_heap::can_fit_blocks_p (size_t* ordered_blocks, int block_index, size_t* ordered_spaces, int* space_index)
28319 assert (*space_index >= block_index);
28321 while (!can_fit_in_spaces_p (ordered_blocks, block_index, ordered_spaces, *space_index))
28324 if (*space_index < block_index)
28333 BOOL gc_heap::can_fit_all_blocks_p (size_t* ordered_blocks, size_t* ordered_spaces, int count)
28335 #ifdef FEATURE_STRUCTALIGN
28336 // BARTOKTODO (4841): reenable when can_fit_in_spaces_p takes alignment requirements into account
28338 #endif // FEATURE_STRUCTALIGN
28339 int space_index = count - 1;
28340 for (int block_index = (count - 1); block_index >= 0; block_index--)
28342 if (!can_fit_blocks_p (ordered_blocks, block_index, ordered_spaces, &space_index))
28351 void gc_heap::build_ordered_free_spaces (heap_segment* seg)
28353 assert (bestfit_seg);
28355 //bestfit_seg->add_buckets (MAX_NUM_BUCKETS - free_space_buckets + MIN_INDEX_POWER2,
28356 // ordered_free_space_indices + (MAX_NUM_BUCKETS - free_space_buckets),
28357 // free_space_buckets,
28358 // free_space_items);
28360 bestfit_seg->add_buckets (MIN_INDEX_POWER2,
28361 ordered_free_space_indices,
28365 assert (settings.condemned_generation == max_generation);
28367 uint8_t* first_address = heap_segment_mem (seg);
28368 uint8_t* end_address = heap_segment_reserved (seg);
28369 //look through the pinned plugs for relevant ones.
28370 //Look for the right pinned plug to start from.
28371 reset_pinned_queue_bos();
28373 // See comment in can_expand_into_p why we need (max_generation + 1).
28374 size_t eph_gen_starts = (Align (min_obj_size)) * (max_generation + 1);
28375 BOOL has_fit_gen_starts = FALSE;
28377 while (!pinned_plug_que_empty_p())
28380 if ((pinned_plug (m) >= first_address) &&
28381 (pinned_plug (m) < end_address) &&
28382 (pinned_len (m) >= eph_gen_starts))
28385 assert ((pinned_plug (m) - pinned_len (m)) == bestfit_first_pin);
28390 deque_pinned_plug();
28394 if (!pinned_plug_que_empty_p())
28396 bestfit_seg->add ((void*)m, TRUE, TRUE);
28397 deque_pinned_plug();
28399 has_fit_gen_starts = TRUE;
28402 while (!pinned_plug_que_empty_p() &&
28403 ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address)))
28405 bestfit_seg->add ((void*)m, TRUE, FALSE);
28406 deque_pinned_plug();
28410 if (commit_end_of_seg)
28412 if (!has_fit_gen_starts)
28414 assert (bestfit_first_pin == heap_segment_plan_allocated (seg));
28416 bestfit_seg->add ((void*)seg, FALSE, (!has_fit_gen_starts));
28420 bestfit_seg->check();
28424 BOOL gc_heap::try_best_fit (BOOL end_of_segment_p)
28426 if (!end_of_segment_p)
28428 trim_free_spaces_indices ();
28431 BOOL can_bestfit = can_fit_all_blocks_p (ordered_plug_indices,
28432 ordered_free_space_indices,
28435 return can_bestfit;
28438 BOOL gc_heap::best_fit (size_t free_space,
28439 size_t largest_free_space,
28440 size_t additional_space,
28441 BOOL* use_additional_space)
28443 dprintf (SEG_REUSE_LOG_0, ("gen%d: trying best fit mechanism", settings.condemned_generation));
28445 assert (!additional_space || (additional_space && use_additional_space));
28446 if (use_additional_space)
28448 *use_additional_space = FALSE;
28451 if (ordered_plug_indices_init == FALSE)
28453 total_ephemeral_plugs = 0;
28454 build_ordered_plug_indices();
28455 ordered_plug_indices_init = TRUE;
28459 memcpy (ordered_plug_indices, saved_ordered_plug_indices, sizeof(ordered_plug_indices));
28462 if (total_ephemeral_plugs == (END_SPACE_AFTER_GC + Align (min_obj_size)))
28464 dprintf (SEG_REUSE_LOG_0, ("No ephemeral plugs to realloc, done"));
28465 size_t empty_eph = (END_SPACE_AFTER_GC + Align (min_obj_size) + (Align (min_obj_size)) * (max_generation + 1));
28466 BOOL can_fit_empty_eph = (largest_free_space >= empty_eph);
28467 if (!can_fit_empty_eph)
28469 can_fit_empty_eph = (additional_space >= empty_eph);
28471 if (can_fit_empty_eph)
28473 *use_additional_space = TRUE;
28477 return can_fit_empty_eph;
28480 if ((total_ephemeral_plugs + approximate_new_allocation()) >= (free_space + additional_space))
28482 dprintf (SEG_REUSE_LOG_0, ("We won't have enough free space left in this segment after fitting, done"));
28486 if ((free_space + additional_space) == 0)
28488 dprintf (SEG_REUSE_LOG_0, ("No free space in this segment, done"));
28492 #ifdef SEG_REUSE_STATS
28493 dprintf (SEG_REUSE_LOG_0, ("Free spaces:"));
28494 size_t total_free_space_power2 = 0;
28495 size_t total_free_space_items =
28496 dump_buckets (ordered_free_space_indices,
28498 &total_free_space_power2);
28499 dprintf (SEG_REUSE_LOG_0, ("currently max free spaces is %Id", max_free_space_items));
28501 dprintf (SEG_REUSE_LOG_0, ("Ephemeral plugs: 0x%Ix, free space: 0x%Ix (rounded down to 0x%Ix (%Id%%)), additional free_space: 0x%Ix",
28502 total_ephemeral_plugs,
28504 total_free_space_power2,
28505 (free_space ? (total_free_space_power2 * 100 / free_space) : 0),
28506 additional_space));
28508 size_t saved_all_free_space_indices[MAX_NUM_BUCKETS];
28509 memcpy (saved_all_free_space_indices,
28510 ordered_free_space_indices,
28511 sizeof(saved_all_free_space_indices));
28513 #endif // SEG_REUSE_STATS
28515 if (total_ephemeral_plugs > (free_space + additional_space))
28520 use_bestfit = try_best_fit(FALSE);
28522 if (!use_bestfit && additional_space)
28524 int relative_free_space_index = relative_index_power2_free_space (round_down_power2 (additional_space));
28526 if (relative_free_space_index != -1)
28528 int relative_plug_index = 0;
28529 size_t plugs_to_fit = 0;
28531 for (relative_plug_index = (MAX_NUM_BUCKETS - 1); relative_plug_index >= 0; relative_plug_index--)
28533 plugs_to_fit = ordered_plug_indices[relative_plug_index];
28534 if (plugs_to_fit != 0)
28540 if ((relative_plug_index > relative_free_space_index) ||
28541 ((relative_plug_index == relative_free_space_index) &&
28542 (plugs_to_fit > 1)))
28544 #ifdef SEG_REUSE_STATS
28545 dprintf (SEG_REUSE_LOG_0, ("additional space is 2^%d but we stopped at %d 2^%d plug(s)",
28546 (relative_free_space_index + MIN_INDEX_POWER2),
28548 (relative_plug_index + MIN_INDEX_POWER2)));
28549 #endif // SEG_REUSE_STATS
28553 dprintf (SEG_REUSE_LOG_0, ("Adding end of segment (2^%d)", (relative_free_space_index + MIN_INDEX_POWER2)));
28554 ordered_free_space_indices[relative_free_space_index]++;
28555 use_bestfit = try_best_fit(TRUE);
28558 free_space_items++;
28559 // Since we might've trimmed away some of the free spaces we had, we should see
28560 // if we really need to use end of seg space - if it's the same or smaller than
28561 // the largest space we trimmed we can just add that one back instead of
28562 // using end of seg.
28563 if (relative_free_space_index > trimmed_free_space_index)
28565 *use_additional_space = TRUE;
28569 // If the addition space is <= than the last trimmed space, we
28570 // should just use that last trimmed space instead.
28571 saved_ordered_free_space_indices[trimmed_free_space_index]++;
28581 dprintf (SEG_REUSE_LOG_0, ("couldn't fit..."));
28583 #ifdef SEG_REUSE_STATS
28584 size_t saved_max = max_free_space_items;
28585 BOOL temp_bestfit = FALSE;
28587 dprintf (SEG_REUSE_LOG_0, ("----Starting experiment process----"));
28588 dprintf (SEG_REUSE_LOG_0, ("----Couldn't fit with max free items %Id", max_free_space_items));
28590 // TODO: need to take the end of segment into consideration.
28591 while (max_free_space_items <= total_free_space_items)
28593 max_free_space_items += max_free_space_items / 2;
28594 dprintf (SEG_REUSE_LOG_0, ("----Temporarily increasing max free spaces to %Id", max_free_space_items));
28595 memcpy (ordered_free_space_indices,
28596 saved_all_free_space_indices,
28597 sizeof(ordered_free_space_indices));
28598 if (try_best_fit(FALSE))
28600 temp_bestfit = TRUE;
28607 dprintf (SEG_REUSE_LOG_0, ("----With %Id max free spaces we could fit", max_free_space_items));
28611 dprintf (SEG_REUSE_LOG_0, ("----Tried all free spaces and still couldn't fit, lost too much space"));
28614 dprintf (SEG_REUSE_LOG_0, ("----Restoring max free spaces to %Id", saved_max));
28615 max_free_space_items = saved_max;
28616 #endif // SEG_REUSE_STATS
28617 if (free_space_items)
28619 max_free_space_items = min (MAX_NUM_FREE_SPACES, free_space_items * 2);
28620 max_free_space_items = max (max_free_space_items, MIN_NUM_FREE_SPACES);
28624 max_free_space_items = MAX_NUM_FREE_SPACES;
28628 dprintf (SEG_REUSE_LOG_0, ("Adjusted number of max free spaces to %Id", max_free_space_items));
28629 dprintf (SEG_REUSE_LOG_0, ("------End of best fitting process------\n"));
28631 return use_bestfit;
28634 BOOL gc_heap::process_free_space (heap_segment* seg,
28636 size_t min_free_size,
28637 size_t min_cont_size,
28638 size_t* total_free_space,
28639 size_t* largest_free_space)
28641 *total_free_space += free_space;
28642 *largest_free_space = max (*largest_free_space, free_space);
28644 #ifdef SIMPLE_DPRINTF
28645 dprintf (SEG_REUSE_LOG_1, ("free space len: %Ix, total free space: %Ix, largest free space: %Ix",
28646 free_space, *total_free_space, *largest_free_space));
28647 #endif //SIMPLE_DPRINTF
28649 if ((*total_free_space >= min_free_size) && (*largest_free_space >= min_cont_size))
28651 #ifdef SIMPLE_DPRINTF
28652 dprintf (SEG_REUSE_LOG_0, ("(gen%d)total free: %Ix(min: %Ix), largest free: %Ix(min: %Ix). Found segment %Ix to reuse without bestfit",
28653 settings.condemned_generation,
28654 *total_free_space, min_free_size, *largest_free_space, min_cont_size,
28657 UNREFERENCED_PARAMETER(seg);
28658 #endif //SIMPLE_DPRINTF
28662 int free_space_index = relative_index_power2_free_space (round_down_power2 (free_space));
28663 if (free_space_index != -1)
28665 ordered_free_space_indices[free_space_index]++;
28670 BOOL gc_heap::expand_reused_seg_p()
28672 BOOL reused_seg = FALSE;
28673 int heap_expand_mechanism = gc_data_per_heap.get_mechanism (gc_heap_expand);
28674 if ((heap_expand_mechanism == expand_reuse_bestfit) ||
28675 (heap_expand_mechanism == expand_reuse_normal))
28683 BOOL gc_heap::can_expand_into_p (heap_segment* seg, size_t min_free_size, size_t min_cont_size,
28684 allocator* gen_allocator)
28686 min_cont_size += END_SPACE_AFTER_GC;
28687 use_bestfit = FALSE;
28688 commit_end_of_seg = FALSE;
28689 bestfit_first_pin = 0;
28690 uint8_t* first_address = heap_segment_mem (seg);
28691 uint8_t* end_address = heap_segment_reserved (seg);
28692 size_t end_extra_space = end_space_after_gc();
28694 if ((heap_segment_reserved (seg) - end_extra_space) <= heap_segment_plan_allocated (seg))
28696 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: can't use segment [%Ix %Ix, has less than %d bytes at the end",
28697 first_address, end_address, end_extra_space));
28701 end_address -= end_extra_space;
28703 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p(gen%d): min free: %Ix, min continuous: %Ix",
28704 settings.condemned_generation, min_free_size, min_cont_size));
28705 size_t eph_gen_starts = eph_gen_starts_size;
28707 if (settings.condemned_generation == max_generation)
28709 size_t free_space = 0;
28710 size_t largest_free_space = free_space;
28711 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: gen2: testing segment [%Ix %Ix", first_address, end_address));
28712 //Look through the pinned plugs for relevant ones and Look for the right pinned plug to start from.
28713 //We are going to allocate the generation starts in the 1st free space,
28714 //so start from the first free space that's big enough for gen starts and a min object size.
28715 // If we see a free space that is >= gen starts but < gen starts + min obj size we just don't use it -
28716 // we could use it by allocating the last generation start a bit bigger but
28717 // the complexity isn't worth the effort (those plugs are from gen2
28718 // already anyway).
28719 reset_pinned_queue_bos();
28721 BOOL has_fit_gen_starts = FALSE;
28723 init_ordered_free_space_indices ();
28724 while (!pinned_plug_que_empty_p())
28727 if ((pinned_plug (m) >= first_address) &&
28728 (pinned_plug (m) < end_address) &&
28729 (pinned_len (m) >= (eph_gen_starts + Align (min_obj_size))))
28735 deque_pinned_plug();
28739 if (!pinned_plug_que_empty_p())
28741 bestfit_first_pin = pinned_plug (m) - pinned_len (m);
28743 if (process_free_space (seg,
28744 pinned_len (m) - eph_gen_starts,
28745 min_free_size, min_cont_size,
28746 &free_space, &largest_free_space))
28751 deque_pinned_plug();
28753 has_fit_gen_starts = TRUE;
28756 dprintf (3, ("first pin is %Ix", pinned_plug (m)));
28758 //tally up free space
28759 while (!pinned_plug_que_empty_p() &&
28760 ((pinned_plug (m) >= first_address) && (pinned_plug (m) < end_address)))
28762 dprintf (3, ("looking at pin %Ix", pinned_plug (m)));
28763 if (process_free_space (seg,
28765 min_free_size, min_cont_size,
28766 &free_space, &largest_free_space))
28771 deque_pinned_plug();
28775 //try to find space at the end of the segment.
28776 size_t end_space = (end_address - heap_segment_plan_allocated (seg));
28777 size_t additional_space = ((min_free_size > free_space) ? (min_free_size - free_space) : 0);
28778 dprintf (SEG_REUSE_LOG_0, ("end space: %Ix; additional: %Ix", end_space, additional_space));
28779 if (end_space >= additional_space)
28781 BOOL can_fit = TRUE;
28782 commit_end_of_seg = TRUE;
28784 if (largest_free_space < min_cont_size)
28786 if (end_space >= min_cont_size)
28788 additional_space = max (min_cont_size, additional_space);
28789 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse without bestfit, with committing end of seg for eph",
28794 if (settings.concurrent)
28797 commit_end_of_seg = FALSE;
28801 size_t additional_space_bestfit = additional_space;
28802 if (!has_fit_gen_starts)
28804 if (additional_space_bestfit < (eph_gen_starts + Align (min_obj_size)))
28806 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, gen starts not allocated yet and end space is too small: %Id",
28807 additional_space_bestfit));
28811 bestfit_first_pin = heap_segment_plan_allocated (seg);
28812 additional_space_bestfit -= eph_gen_starts;
28815 can_fit = best_fit (free_space,
28816 largest_free_space,
28817 additional_space_bestfit,
28818 &commit_end_of_seg);
28822 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse with bestfit, %s committing end of seg",
28823 seg, (commit_end_of_seg ? "with" : "without")));
28827 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, total free space is %Ix", (free_space + end_space)));
28834 dprintf (SEG_REUSE_LOG_0, ("(gen2)Found segment %Ix to reuse without bestfit, with committing end of seg", seg));
28837 assert (additional_space <= end_space);
28838 if (commit_end_of_seg)
28840 if (!grow_heap_segment (seg, heap_segment_plan_allocated (seg) + additional_space))
28842 dprintf (2, ("Couldn't commit end of segment?!"));
28843 use_bestfit = FALSE;
28850 // We increase the index here because growing heap segment could create a discrepency with
28851 // the additional space we used (could be bigger).
28852 size_t free_space_end_of_seg =
28853 heap_segment_committed (seg) - heap_segment_plan_allocated (seg);
28854 int relative_free_space_index = relative_index_power2_free_space (round_down_power2 (free_space_end_of_seg));
28855 saved_ordered_free_space_indices[relative_free_space_index]++;
28861 memcpy (ordered_free_space_indices,
28862 saved_ordered_free_space_indices,
28863 sizeof(ordered_free_space_indices));
28864 max_free_space_items = max (MIN_NUM_FREE_SPACES, free_space_items * 3 / 2);
28865 max_free_space_items = min (MAX_NUM_FREE_SPACES, max_free_space_items);
28866 dprintf (SEG_REUSE_LOG_0, ("could fit! %Id free spaces, %Id max", free_space_items, max_free_space_items));
28872 dprintf (SEG_REUSE_LOG_0, ("(gen2)Couldn't fit, total free space is %Ix", (free_space + end_space)));
28877 assert (settings.condemned_generation == (max_generation-1));
28878 size_t free_space = (end_address - heap_segment_plan_allocated (seg));
28879 size_t largest_free_space = free_space;
28880 dprintf (SEG_REUSE_LOG_0, ("can_expand_into_p: gen1: testing segment [%Ix %Ix", first_address, end_address));
28881 //find the first free list in range of the current segment
28882 size_t sz_list = gen_allocator->first_bucket_size();
28883 unsigned int a_l_idx = 0;
28884 uint8_t* free_list = 0;
28885 for (; a_l_idx < gen_allocator->number_of_buckets(); a_l_idx++)
28887 if ((eph_gen_starts <= sz_list) || (a_l_idx == (gen_allocator->number_of_buckets()-1)))
28889 free_list = gen_allocator->alloc_list_head_of (a_l_idx);
28892 if ((free_list >= first_address) &&
28893 (free_list < end_address) &&
28894 (unused_array_size (free_list) >= eph_gen_starts))
28900 free_list = free_list_slot (free_list);
28908 init_ordered_free_space_indices ();
28909 if (process_free_space (seg,
28910 unused_array_size (free_list) - eph_gen_starts + Align (min_obj_size),
28911 min_free_size, min_cont_size,
28912 &free_space, &largest_free_space))
28917 free_list = free_list_slot (free_list);
28921 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, no free list"));
28925 //tally up free space
28931 if ((free_list >= first_address) && (free_list < end_address) &&
28932 process_free_space (seg,
28933 unused_array_size (free_list),
28934 min_free_size, min_cont_size,
28935 &free_space, &largest_free_space))
28940 free_list = free_list_slot (free_list);
28943 if (a_l_idx < gen_allocator->number_of_buckets())
28945 free_list = gen_allocator->alloc_list_head_of (a_l_idx);
28951 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, total free space is %Ix", free_space));
28955 BOOL can_fit = best_fit (free_space, 0, NULL);
28958 dprintf (SEG_REUSE_LOG_0, ("(gen1)Found segment %Ix to reuse with bestfit", seg));
28962 dprintf (SEG_REUSE_LOG_0, ("(gen1)Couldn't fit, total free space is %Ix", free_space));
28970 void gc_heap::realloc_plug (size_t last_plug_size, uint8_t*& last_plug,
28971 generation* gen, uint8_t* start_address,
28972 unsigned int& active_new_gen_number,
28973 uint8_t*& last_pinned_gap, BOOL& leftp,
28976 , mark* pinned_plug_entry
28977 #endif //SHORT_PLUGS
28980 // detect generation boundaries
28981 // make sure that active_new_gen_number is not the youngest generation.
28982 // because the generation_limit wouldn't return the right thing in this case.
28985 if ((active_new_gen_number > 1) &&
28986 (last_plug >= generation_limit (active_new_gen_number)))
28988 assert (last_plug >= start_address);
28989 active_new_gen_number--;
28990 realloc_plan_generation_start (generation_of (active_new_gen_number), gen);
28991 assert (generation_plan_allocation_start (generation_of (active_new_gen_number)));
28996 // detect pinned plugs
28997 if (!pinned_plug_que_empty_p() && (last_plug == pinned_plug (oldest_pin())))
28999 size_t entry = deque_pinned_plug();
29000 mark* m = pinned_plug_of (entry);
29002 size_t saved_pinned_len = pinned_len(m);
29003 pinned_len(m) = last_plug - last_pinned_gap;
29004 //dprintf (3,("Adjusting pinned gap: [%Ix, %Ix[", (size_t)last_pinned_gap, (size_t)last_plug));
29006 if (m->has_post_plug_info())
29008 last_plug_size += sizeof (gap_reloc_pair);
29009 dprintf (3, ("ra pinned %Ix was shortened, adjusting plug size to %Ix", last_plug, last_plug_size))
29012 last_pinned_gap = last_plug + last_plug_size;
29013 dprintf (3, ("ra found pin %Ix, len: %Ix->%Ix, last_p: %Ix, last_p_size: %Ix",
29014 pinned_plug (m), saved_pinned_len, pinned_len (m), last_plug, last_plug_size));
29017 //we are creating a generation fault. set the cards.
29019 size_t end_card = card_of (align_on_card (last_plug + last_plug_size));
29020 size_t card = card_of (last_plug);
29021 while (card != end_card)
29028 else if (last_plug >= start_address)
29030 #ifdef FEATURE_STRUCTALIGN
29031 int requiredAlignment;
29033 node_aligninfo (last_plug, requiredAlignment, pad);
29035 // from how we previously aligned the plug's destination address,
29036 // compute the actual alignment offset.
29037 uint8_t* reloc_plug = last_plug + node_relocation_distance (last_plug);
29038 ptrdiff_t alignmentOffset = ComputeStructAlignPad(reloc_plug, requiredAlignment, 0);
29039 if (!alignmentOffset)
29041 // allocate_in_expanded_heap doesn't expect alignmentOffset to be zero.
29042 alignmentOffset = requiredAlignment;
29045 //clear the alignment info because we are reallocating
29046 clear_node_aligninfo (last_plug);
29047 #else // FEATURE_STRUCTALIGN
29048 //clear the realignment flag because we are reallocating
29049 clear_node_realigned (last_plug);
29050 #endif // FEATURE_STRUCTALIGN
29051 BOOL adjacentp = FALSE;
29052 BOOL set_padding_on_saved_p = FALSE;
29056 last_plug_size += sizeof (gap_reloc_pair);
29059 assert (pinned_plug_entry != NULL);
29060 if (last_plug_size <= sizeof (plug_and_gap))
29062 set_padding_on_saved_p = TRUE;
29064 #endif //SHORT_PLUGS
29066 dprintf (3, ("ra plug %Ix was shortened, adjusting plug size to %Ix", last_plug, last_plug_size))
29070 clear_padding_in_expand (last_plug, set_padding_on_saved_p, pinned_plug_entry);
29071 #endif //SHORT_PLUGS
29073 uint8_t* new_address = allocate_in_expanded_heap(gen, last_plug_size, adjacentp, last_plug,
29075 set_padding_on_saved_p,
29077 #endif //SHORT_PLUGS
29078 TRUE, active_new_gen_number REQD_ALIGN_AND_OFFSET_ARG);
29080 dprintf (3, ("ra NA: [%Ix, %Ix[: %Ix", new_address, (new_address + last_plug_size), last_plug_size));
29081 assert (new_address);
29082 set_node_relocation_distance (last_plug, new_address - last_plug);
29083 #ifdef FEATURE_STRUCTALIGN
29084 if (leftp && node_alignpad (last_plug) == 0)
29085 #else // FEATURE_STRUCTALIGN
29086 if (leftp && !node_realigned (last_plug))
29087 #endif // FEATURE_STRUCTALIGN
29089 // TODO - temporarily disable L optimization because of a bug in it.
29090 //set_node_left (last_plug);
29092 dprintf (3,(" Re-allocating %Ix->%Ix len %Id", (size_t)last_plug, (size_t)new_address, last_plug_size));
29097 void gc_heap::realloc_in_brick (uint8_t* tree, uint8_t*& last_plug,
29098 uint8_t* start_address,
29100 unsigned int& active_new_gen_number,
29101 uint8_t*& last_pinned_gap, BOOL& leftp)
29103 assert (tree != NULL);
29104 int left_node = node_left_child (tree);
29105 int right_node = node_right_child (tree);
29107 dprintf (3, ("ra: tree: %Ix, last_pin_gap: %Ix, last_p: %Ix, L: %d, R: %d",
29108 tree, last_pinned_gap, last_plug, left_node, right_node));
29112 dprintf (3, ("LN: realloc %Ix(%Ix)", (tree + left_node), last_plug));
29113 realloc_in_brick ((tree + left_node), last_plug, start_address,
29114 gen, active_new_gen_number, last_pinned_gap,
29118 if (last_plug != 0)
29120 uint8_t* plug = tree;
29122 BOOL has_pre_plug_info_p = FALSE;
29123 BOOL has_post_plug_info_p = FALSE;
29124 mark* pinned_plug_entry = get_next_pinned_entry (tree,
29125 &has_pre_plug_info_p,
29126 &has_post_plug_info_p,
29129 // We only care about the pre plug info 'cause that's what decides if the last plug is shortened.
29130 // The pinned plugs are handled in realloc_plug.
29131 size_t gap_size = node_gap_size (plug);
29132 uint8_t* gap = (plug - gap_size);
29133 uint8_t* last_plug_end = gap;
29134 size_t last_plug_size = (last_plug_end - last_plug);
29135 // Cannot assert this - a plug could be less than that due to the shortened ones.
29136 //assert (last_plug_size >= Align (min_obj_size));
29137 dprintf (3, ("ra: plug %Ix, gap size: %Ix, last_pin_gap: %Ix, last_p: %Ix, last_p_end: %Ix, shortened: %d",
29138 plug, gap_size, last_pinned_gap, last_plug, last_plug_end, (has_pre_plug_info_p ? 1 : 0)));
29139 realloc_plug (last_plug_size, last_plug, gen, start_address,
29140 active_new_gen_number, last_pinned_gap,
29141 leftp, has_pre_plug_info_p
29143 , pinned_plug_entry
29144 #endif //SHORT_PLUGS
29152 dprintf (3, ("RN: realloc %Ix(%Ix)", (tree + right_node), last_plug));
29153 realloc_in_brick ((tree + right_node), last_plug, start_address,
29154 gen, active_new_gen_number, last_pinned_gap,
29160 gc_heap::realloc_plugs (generation* consing_gen, heap_segment* seg,
29161 uint8_t* start_address, uint8_t* end_address,
29162 unsigned active_new_gen_number)
29164 dprintf (3, ("--- Reallocing ---"));
29168 //make sure that every generation has a planned allocation start
29169 int gen_number = max_generation - 1;
29170 while (gen_number >= 0)
29172 generation* gen = generation_of (gen_number);
29173 if (0 == generation_plan_allocation_start (gen))
29175 generation_plan_allocation_start (gen) =
29176 bestfit_first_pin + (max_generation - gen_number - 1) * Align (min_obj_size);
29177 generation_plan_allocation_start_size (gen) = Align (min_obj_size);
29178 assert (generation_plan_allocation_start (gen));
29184 uint8_t* first_address = start_address;
29185 //Look for the right pinned plug to start from.
29186 reset_pinned_queue_bos();
29187 uint8_t* planned_ephemeral_seg_end = heap_segment_plan_allocated (seg);
29188 while (!pinned_plug_que_empty_p())
29190 mark* m = oldest_pin();
29191 if ((pinned_plug (m) >= planned_ephemeral_seg_end) && (pinned_plug (m) < end_address))
29193 if (pinned_plug (m) < first_address)
29195 first_address = pinned_plug (m);
29200 deque_pinned_plug();
29203 size_t current_brick = brick_of (first_address);
29204 size_t end_brick = brick_of (end_address-1);
29205 uint8_t* last_plug = 0;
29207 uint8_t* last_pinned_gap = heap_segment_plan_allocated (seg);
29208 BOOL leftp = FALSE;
29210 dprintf (3, ("start addr: %Ix, first addr: %Ix, current oldest pin: %Ix",
29211 start_address, first_address, pinned_plug (oldest_pin())));
29213 while (current_brick <= end_brick)
29215 int brick_entry = brick_table [ current_brick ];
29216 if (brick_entry >= 0)
29218 realloc_in_brick ((brick_address (current_brick) + brick_entry - 1),
29219 last_plug, start_address, consing_gen,
29220 active_new_gen_number, last_pinned_gap,
29226 if (last_plug != 0)
29228 realloc_plug (end_address - last_plug, last_plug, consing_gen,
29230 active_new_gen_number, last_pinned_gap,
29234 #endif //SHORT_PLUGS
29238 //Fix the old segment allocated size
29239 assert (last_pinned_gap >= heap_segment_mem (seg));
29240 assert (last_pinned_gap <= heap_segment_committed (seg));
29241 heap_segment_plan_allocated (seg) = last_pinned_gap;
29244 void gc_heap::verify_no_pins (uint8_t* start, uint8_t* end)
29247 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
29249 BOOL contains_pinned_plugs = FALSE;
29252 while (mi != mark_stack_tos)
29254 m = pinned_plug_of (mi);
29255 if ((pinned_plug (m) >= start) && (pinned_plug (m) < end))
29257 contains_pinned_plugs = TRUE;
29264 if (contains_pinned_plugs)
29269 #endif //VERIFY_HEAP
29272 void gc_heap::set_expand_in_full_gc (int condemned_gen_number)
29274 if (!should_expand_in_full_gc)
29276 if ((condemned_gen_number != max_generation) &&
29277 (settings.pause_mode != pause_low_latency) &&
29278 (settings.pause_mode != pause_sustained_low_latency))
29280 should_expand_in_full_gc = TRUE;
29285 void gc_heap::save_ephemeral_generation_starts()
29287 for (int ephemeral_generation = 0; ephemeral_generation < max_generation; ephemeral_generation++)
29289 saved_ephemeral_plan_start[ephemeral_generation] =
29290 generation_plan_allocation_start (generation_of (ephemeral_generation));
29291 saved_ephemeral_plan_start_size[ephemeral_generation] =
29292 generation_plan_allocation_start_size (generation_of (ephemeral_generation));
29296 generation* gc_heap::expand_heap (int condemned_generation,
29297 generation* consing_gen,
29298 heap_segment* new_heap_segment)
29300 UNREFERENCED_PARAMETER(condemned_generation);
29301 assert (condemned_generation >= (max_generation -1));
29302 unsigned int active_new_gen_number = max_generation; //Set one too high to get generation gap
29303 uint8_t* start_address = generation_limit (max_generation);
29304 uint8_t* end_address = heap_segment_allocated (ephemeral_heap_segment);
29305 BOOL should_promote_ephemeral = FALSE;
29306 ptrdiff_t eph_size = total_ephemeral_size;
29307 #ifdef BACKGROUND_GC
29308 dprintf(2,("%s: ---- Heap Expansion ----", (recursive_gc_sync::background_running_p() ? "FGC" : "NGC")));
29309 #endif //BACKGROUND_GC
29310 settings.heap_expansion = TRUE;
29312 #ifdef BACKGROUND_GC
29313 if (cm_in_progress)
29315 if (!expanded_in_fgc)
29317 expanded_in_fgc = TRUE;
29320 #endif //BACKGROUND_GC
29322 //reset the elevation state for next time.
29323 dprintf (2, ("Elevation: elevation = el_none"));
29324 if (settings.should_lock_elevation && !expand_reused_seg_p())
29325 settings.should_lock_elevation = FALSE;
29327 heap_segment* new_seg = new_heap_segment;
29330 return consing_gen;
29332 //copy the card and brick tables
29333 if (g_gc_card_table!= card_table)
29334 copy_brick_card_table();
29336 BOOL new_segment_p = (heap_segment_next (new_seg) == 0);
29337 dprintf (2, ("new_segment_p %Ix", (size_t)new_segment_p));
29339 assert (generation_plan_allocation_start (generation_of (max_generation-1)));
29340 assert (generation_plan_allocation_start (generation_of (max_generation-1)) >=
29341 heap_segment_mem (ephemeral_heap_segment));
29342 assert (generation_plan_allocation_start (generation_of (max_generation-1)) <=
29343 heap_segment_committed (ephemeral_heap_segment));
29345 assert (generation_plan_allocation_start (youngest_generation));
29346 assert (generation_plan_allocation_start (youngest_generation) <
29347 heap_segment_plan_allocated (ephemeral_heap_segment));
29349 if (settings.pause_mode == pause_no_gc)
29351 // We don't reuse for no gc, so the size used on the new eph seg is eph_size.
29352 if ((size_t)(heap_segment_reserved (new_seg) - heap_segment_mem (new_seg)) < (eph_size + soh_allocation_no_gc))
29353 should_promote_ephemeral = TRUE;
29359 should_promote_ephemeral = dt_low_ephemeral_space_p (tuning_deciding_promote_ephemeral);
29363 if (should_promote_ephemeral)
29365 ephemeral_promotion = TRUE;
29366 get_gc_data_per_heap()->set_mechanism (gc_heap_expand, expand_new_seg_ep);
29367 dprintf (2, ("promoting ephemeral"));
29368 save_ephemeral_generation_starts();
29372 // commit the new ephemeral segment all at once if it is a new one.
29373 if ((eph_size > 0) && new_segment_p)
29375 #ifdef FEATURE_STRUCTALIGN
29376 // The destination may require a larger alignment padding than the source.
29377 // Assume the worst possible alignment padding.
29378 eph_size += ComputeStructAlignPad(heap_segment_mem (new_seg), MAX_STRUCTALIGN, OBJECT_ALIGNMENT_OFFSET);
29379 #endif // FEATURE_STRUCTALIGN
29380 #ifdef RESPECT_LARGE_ALIGNMENT
29381 //Since the generation start can be larger than min_obj_size
29382 //The alignment could be switched.
29383 eph_size += switch_alignment_size(FALSE);
29384 #endif //RESPECT_LARGE_ALIGNMENT
29385 //Since the generation start can be larger than min_obj_size
29386 //Compare the alignment of the first object in gen1
29387 if (grow_heap_segment (new_seg, heap_segment_mem (new_seg) + eph_size) == 0)
29389 fgm_result.set_fgm (fgm_commit_eph_segment, eph_size, FALSE);
29390 return consing_gen;
29392 heap_segment_used (new_seg) = heap_segment_committed (new_seg);
29395 //Fix the end of the old ephemeral heap segment
29396 heap_segment_plan_allocated (ephemeral_heap_segment) =
29397 generation_plan_allocation_start (generation_of (max_generation-1));
29399 dprintf (3, ("Old ephemeral allocated set to %Ix",
29400 (size_t)heap_segment_plan_allocated (ephemeral_heap_segment)));
29405 // TODO - Is this really necessary? We should think about it.
29406 //initialize the first brick
29407 size_t first_brick = brick_of (heap_segment_mem (new_seg));
29408 set_brick (first_brick,
29409 heap_segment_mem (new_seg) - brick_address (first_brick));
29412 //From this point on, we cannot run out of memory
29414 //reset the allocation of the consing generation back to the end of the
29415 //old ephemeral segment
29416 generation_allocation_limit (consing_gen) =
29417 heap_segment_plan_allocated (ephemeral_heap_segment);
29418 generation_allocation_pointer (consing_gen) = generation_allocation_limit (consing_gen);
29419 generation_allocation_segment (consing_gen) = ephemeral_heap_segment;
29421 //clear the generation gap for all of the ephemeral generations
29423 int generation_num = max_generation-1;
29424 while (generation_num >= 0)
29426 generation* gen = generation_of (generation_num);
29427 generation_plan_allocation_start (gen) = 0;
29432 heap_segment* old_seg = ephemeral_heap_segment;
29433 ephemeral_heap_segment = new_seg;
29435 //Note: the ephemeral segment shouldn't be threaded onto the segment chain
29436 //because the relocation and compact phases shouldn't see it
29438 // set the generation members used by allocate_in_expanded_heap
29439 // and switch to ephemeral generation
29440 consing_gen = ensure_ephemeral_heap_segment (consing_gen);
29442 if (!should_promote_ephemeral)
29444 realloc_plugs (consing_gen, old_seg, start_address, end_address,
29445 active_new_gen_number);
29450 repair_allocation_in_expanded_heap (consing_gen);
29453 // assert that the generation gap for all of the ephemeral generations were allocated.
29456 int generation_num = max_generation-1;
29457 while (generation_num >= 0)
29459 generation* gen = generation_of (generation_num);
29460 assert (generation_plan_allocation_start (gen));
29466 if (!new_segment_p)
29468 dprintf (2, ("Demoting ephemeral segment"));
29469 //demote the entire segment.
29470 settings.demotion = TRUE;
29471 get_gc_data_per_heap()->set_mechanism_bit (gc_demotion_bit);
29472 demotion_low = heap_segment_mem (ephemeral_heap_segment);
29473 demotion_high = heap_segment_reserved (ephemeral_heap_segment);
29477 demotion_low = MAX_PTR;
29479 #ifndef MULTIPLE_HEAPS
29480 settings.demotion = FALSE;
29481 get_gc_data_per_heap()->clear_mechanism_bit (gc_demotion_bit);
29482 #endif //!MULTIPLE_HEAPS
29484 ptrdiff_t eph_size1 = total_ephemeral_size;
29485 MAYBE_UNUSED_VAR(eph_size1);
29487 if (!should_promote_ephemeral && new_segment_p)
29489 assert (eph_size1 <= eph_size);
29492 if (heap_segment_mem (old_seg) == heap_segment_plan_allocated (old_seg))
29494 // This is to catch when we accidently delete a segment that has pins.
29495 verify_no_pins (heap_segment_mem (old_seg), heap_segment_reserved (old_seg));
29498 verify_no_pins (heap_segment_plan_allocated (old_seg), heap_segment_reserved(old_seg));
29500 dprintf(2,("---- End of Heap Expansion ----"));
29501 return consing_gen;
29504 void gc_heap::set_static_data()
29506 static_data* pause_mode_sdata = static_data_table[latency_level];
29507 for (int i = 0; i < NUMBERGENERATIONS; i++)
29509 dynamic_data* dd = dynamic_data_of (i);
29510 static_data* sdata = &pause_mode_sdata[i];
29513 dd->min_size = sdata->min_size;
29515 dprintf (GTC_LOG, ("PM: %d - min: %Id, max: %Id, fr_l: %Id, fr_b: %d%%",
29516 settings.pause_mode,
29517 dd->min_size, dd_max_size,
29518 dd->fragmentation_limit, (int)(dd->fragmentation_burden_limit * 100)));
29522 // Initialize the values that are not const.
29523 void gc_heap::init_static_data()
29525 size_t gen0size = GCHeap::GetValidGen0MaxSize(get_valid_segment_size());
29526 size_t gen0_min_size = Align(gen0size / 8 * 5);
29528 size_t gen0_max_size =
29529 #ifdef MULTIPLE_HEAPS
29530 max (6*1024*1024, min ( Align(soh_segment_size/2), 200*1024*1024));
29531 #else //MULTIPLE_HEAPS
29532 (gc_can_use_concurrent ?
29534 max (6*1024*1024, min ( Align(soh_segment_size/2), 200*1024*1024)));
29535 #endif //MULTIPLE_HEAPS
29537 // TODO: gen0_max_size has a 200mb cap; gen1_max_size should also have a cap.
29538 size_t gen1_max_size =
29539 #ifdef MULTIPLE_HEAPS
29540 max (6*1024*1024, Align(soh_segment_size/2));
29541 #else //MULTIPLE_HEAPS
29542 (gc_can_use_concurrent ?
29544 max (6*1024*1024, Align(soh_segment_size/2)));
29545 #endif //MULTIPLE_HEAPS
29547 dprintf (GTC_LOG, ("gen0size: %Id, gen0 min: %Id, max: %Id, gen1 max: %Id",
29548 gen0size, gen0_min_size, gen0_max_size, gen1_max_size));
29550 for (int i = latency_level_first; i <= latency_level_last; i++)
29552 static_data_table[i][0].min_size = gen0_min_size;
29553 static_data_table[i][0].max_size = gen0_max_size;
29554 static_data_table[i][1].max_size = gen1_max_size;
29558 bool gc_heap::init_dynamic_data()
29560 qpf = GCToOSInterface::QueryPerformanceFrequency();
29562 uint32_t now = (uint32_t)GetHighPrecisionTimeStamp();
29566 for (int i = 0; i <= max_generation+1; i++)
29568 dynamic_data* dd = dynamic_data_of (i);
29570 dd->time_clock = now;
29571 dd->current_size = 0;
29572 dd->promoted_size = 0;
29573 dd->collection_count = 0;
29574 dd->new_allocation = dd->min_size;
29575 dd->gc_new_allocation = dd->new_allocation;
29576 dd->desired_allocation = dd->new_allocation;
29577 dd->fragmentation = 0;
29580 #ifdef GC_CONFIG_DRIVEN
29581 if (heap_number == 0)
29583 #endif //GC_CONFIG_DRIVEN
29588 float gc_heap::surv_to_growth (float cst, float limit, float max_limit)
29590 if (cst < ((max_limit - limit ) / (limit * (max_limit-1.0f))))
29591 return ((limit - limit*cst) / (1.0f - (cst * limit)));
29597 //if the allocation budget wasn't exhausted, the new budget may be wrong because the survival may
29598 //not be correct (collection happened too soon). Correct with a linear estimation based on the previous
29599 //value of the budget
29600 static size_t linear_allocation_model (float allocation_fraction, size_t new_allocation,
29601 size_t previous_desired_allocation, size_t collection_count)
29603 if ((allocation_fraction < 0.95) && (allocation_fraction > 0.0))
29605 dprintf (2, ("allocation fraction: %d", (int)(allocation_fraction/100.0)));
29606 new_allocation = (size_t)(allocation_fraction*new_allocation + (1.0-allocation_fraction)*previous_desired_allocation);
29609 size_t smoothing = 3; // exponential smoothing factor
29610 if (smoothing > collection_count)
29611 smoothing = collection_count;
29612 new_allocation = new_allocation / smoothing + ((previous_desired_allocation / smoothing) * (smoothing-1));
29614 UNREFERENCED_PARAMETER(collection_count);
29616 return new_allocation;
29619 size_t gc_heap::desired_new_allocation (dynamic_data* dd,
29620 size_t out, int gen_number,
29623 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
29625 if (dd_begin_data_size (dd) == 0)
29627 size_t new_allocation = dd_min_size (dd);
29628 current_gc_data_per_heap->gen_data[gen_number].new_allocation = new_allocation;
29629 return new_allocation;
29634 size_t previous_desired_allocation = dd_desired_allocation (dd);
29635 size_t current_size = dd_current_size (dd);
29636 float max_limit = dd_max_limit (dd);
29637 float limit = dd_limit (dd);
29638 size_t min_gc_size = dd_min_size (dd);
29640 size_t max_size = dd_max_size (dd);
29641 size_t new_allocation = 0;
29642 float allocation_fraction = (float) (dd_desired_allocation (dd) - dd_gc_new_allocation (dd)) / (float) (dd_desired_allocation (dd));
29643 if (gen_number >= max_generation)
29645 size_t new_size = 0;
29647 cst = min (1.0f, float (out) / float (dd_begin_data_size (dd)));
29649 f = surv_to_growth (cst, limit, max_limit);
29650 size_t max_growth_size = (size_t)(max_size / f);
29651 if (current_size >= max_growth_size)
29653 new_size = max_size;
29657 new_size = (size_t) min (max ( (f * current_size), min_gc_size), max_size);
29660 assert ((new_size >= current_size) || (new_size == max_size));
29662 if (gen_number == max_generation)
29664 new_allocation = max((new_size - current_size), min_gc_size);
29666 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
29667 dd_desired_allocation (dd), dd_collection_count (dd));
29669 if ((dd_fragmentation (dd) > ((size_t)((f-1)*current_size))))
29671 //reducing allocation in case of fragmentation
29672 size_t new_allocation1 = max (min_gc_size,
29674 (size_t)((float)new_allocation * current_size /
29675 ((float)current_size + 2*dd_fragmentation (dd))));
29676 dprintf (2, ("Reducing max_gen allocation due to fragmentation from %Id to %Id",
29677 new_allocation, new_allocation1));
29678 new_allocation = new_allocation1;
29681 else //large object heap
29683 uint32_t memory_load = 0;
29684 uint64_t available_physical = 0;
29685 get_memory_info (&memory_load, &available_physical);
29686 if (heap_number == 0)
29687 settings.exit_memory_load = memory_load;
29688 if (available_physical > 1024*1024)
29689 available_physical -= 1024*1024;
29691 uint64_t available_free = available_physical + (uint64_t)generation_free_list_space (generation_of (gen_number));
29692 if (available_free > (uint64_t)MAX_PTR)
29694 available_free = (uint64_t)MAX_PTR;
29697 //try to avoid OOM during large object allocation
29698 new_allocation = max (min(max((new_size - current_size), dd_desired_allocation (dynamic_data_of (max_generation))),
29699 (size_t)available_free),
29700 max ((current_size/4), min_gc_size));
29702 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
29703 dd_desired_allocation (dd), dd_collection_count (dd));
29709 size_t survivors = out;
29710 cst = float (survivors) / float (dd_begin_data_size (dd));
29711 f = surv_to_growth (cst, limit, max_limit);
29712 new_allocation = (size_t) min (max ((f * (survivors)), min_gc_size), max_size);
29714 new_allocation = linear_allocation_model (allocation_fraction, new_allocation,
29715 dd_desired_allocation (dd), dd_collection_count (dd));
29717 if (gen_number == 0)
29722 //printf ("%f, %Id\n", cst, new_allocation);
29723 size_t free_space = generation_free_list_space (generation_of (gen_number));
29724 // DTREVIEW - is min_gc_size really a good choice?
29725 // on 64-bit this will almost always be true.
29726 dprintf (GTC_LOG, ("frag: %Id, min: %Id", free_space, min_gc_size));
29727 if (free_space > min_gc_size)
29729 settings.gen0_reduction_count = 2;
29733 if (settings.gen0_reduction_count > 0)
29734 settings.gen0_reduction_count--;
29737 if (settings.gen0_reduction_count > 0)
29739 dprintf (2, ("Reducing new allocation based on fragmentation"));
29740 new_allocation = min (new_allocation,
29741 max (min_gc_size, (max_size/3)));
29746 size_t new_allocation_ret =
29747 Align (new_allocation, get_alignment_constant (!(gen_number == (max_generation+1))));
29748 int gen_data_index = gen_number;
29749 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_data_index]);
29750 gen_data->new_allocation = new_allocation_ret;
29752 dd_surv (dd) = cst;
29754 #ifdef SIMPLE_DPRINTF
29755 dprintf (1, ("h%d g%d surv: %Id current: %Id alloc: %Id (%d%%) f: %d%% new-size: %Id new-alloc: %Id",
29756 heap_number, gen_number, out, current_size, (dd_desired_allocation (dd) - dd_gc_new_allocation (dd)),
29757 (int)(cst*100), (int)(f*100), current_size + new_allocation, new_allocation));
29759 dprintf (1,("gen: %d in: %Id out: %Id ", gen_number, generation_allocation_size (generation_of (gen_number)), out));
29760 dprintf (1,("current: %Id alloc: %Id ", current_size, (dd_desired_allocation (dd) - dd_gc_new_allocation (dd))));
29761 dprintf (1,(" surv: %d%% f: %d%% new-size: %Id new-alloc: %Id",
29762 (int)(cst*100), (int)(f*100), current_size + new_allocation, new_allocation));
29763 #endif //SIMPLE_DPRINTF
29765 return new_allocation_ret;
29769 //returns the planned size of a generation (including free list element)
29770 size_t gc_heap::generation_plan_size (int gen_number)
29772 if (0 == gen_number)
29773 return max((heap_segment_plan_allocated (ephemeral_heap_segment) -
29774 generation_plan_allocation_start (generation_of (gen_number))),
29775 (int)Align (min_obj_size));
29778 generation* gen = generation_of (gen_number);
29779 if (heap_segment_rw (generation_start_segment (gen)) == ephemeral_heap_segment)
29780 return (generation_plan_allocation_start (generation_of (gen_number - 1)) -
29781 generation_plan_allocation_start (generation_of (gen_number)));
29784 size_t gensize = 0;
29785 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
29787 PREFIX_ASSUME(seg != NULL);
29789 while (seg && (seg != ephemeral_heap_segment))
29791 gensize += heap_segment_plan_allocated (seg) -
29792 heap_segment_mem (seg);
29793 seg = heap_segment_next_rw (seg);
29797 gensize += (generation_plan_allocation_start (generation_of (gen_number - 1)) -
29798 heap_segment_mem (ephemeral_heap_segment));
29806 //returns the size of a generation (including free list element)
29807 size_t gc_heap::generation_size (int gen_number)
29809 if (0 == gen_number)
29810 return max((heap_segment_allocated (ephemeral_heap_segment) -
29811 generation_allocation_start (generation_of (gen_number))),
29812 (int)Align (min_obj_size));
29815 generation* gen = generation_of (gen_number);
29816 if (heap_segment_rw (generation_start_segment (gen)) == ephemeral_heap_segment)
29817 return (generation_allocation_start (generation_of (gen_number - 1)) -
29818 generation_allocation_start (generation_of (gen_number)));
29821 size_t gensize = 0;
29822 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
29824 PREFIX_ASSUME(seg != NULL);
29826 while (seg && (seg != ephemeral_heap_segment))
29828 gensize += heap_segment_allocated (seg) -
29829 heap_segment_mem (seg);
29830 seg = heap_segment_next_rw (seg);
29834 gensize += (generation_allocation_start (generation_of (gen_number - 1)) -
29835 heap_segment_mem (ephemeral_heap_segment));
29844 size_t gc_heap::compute_in (int gen_number)
29846 assert (gen_number != 0);
29847 dynamic_data* dd = dynamic_data_of (gen_number);
29849 size_t in = generation_allocation_size (generation_of (gen_number));
29851 if (gen_number == max_generation && ephemeral_promotion)
29854 for (int i = 0; i <= max_generation; i++)
29856 dynamic_data* dd = dynamic_data_of (i);
29857 in += dd_survived_size (dd);
29858 if (i != max_generation)
29860 generation_condemned_allocated (generation_of (gen_number)) += dd_survived_size (dd);
29865 dd_gc_new_allocation (dd) -= in;
29866 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
29868 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
29869 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_number]);
29872 generation_allocation_size (generation_of (gen_number)) = 0;
29876 void gc_heap::compute_promoted_allocation (int gen_number)
29878 compute_in (gen_number);
29883 size_t gc_heap::trim_youngest_desired (uint32_t memory_load,
29884 size_t total_new_allocation,
29885 size_t total_min_allocation)
29887 if (memory_load < MAX_ALLOWED_MEM_LOAD)
29889 // If the total of memory load and gen0 budget exceeds
29890 // our max memory load limit, trim the gen0 budget so the total
29891 // is the max memory load limit.
29892 size_t remain_memory_load = (MAX_ALLOWED_MEM_LOAD - memory_load) * mem_one_percent;
29893 return min (total_new_allocation, remain_memory_load);
29897 return max (mem_one_percent, total_min_allocation);
29901 size_t gc_heap::joined_youngest_desired (size_t new_allocation)
29903 dprintf (2, ("Entry memory load: %d; gen0 new_alloc: %Id", settings.entry_memory_load, new_allocation));
29905 size_t final_new_allocation = new_allocation;
29906 if (new_allocation > MIN_YOUNGEST_GEN_DESIRED)
29908 uint32_t num_heaps = 1;
29910 #ifdef MULTIPLE_HEAPS
29911 num_heaps = gc_heap::n_heaps;
29912 #endif //MULTIPLE_HEAPS
29914 size_t total_new_allocation = new_allocation * num_heaps;
29915 size_t total_min_allocation = MIN_YOUNGEST_GEN_DESIRED * num_heaps;
29917 if ((settings.entry_memory_load >= MAX_ALLOWED_MEM_LOAD) ||
29918 (total_new_allocation > max (youngest_gen_desired_th, total_min_allocation)))
29920 uint32_t memory_load = 0;
29921 get_memory_info (&memory_load);
29922 settings.exit_memory_load = memory_load;
29923 dprintf (2, ("Current emory load: %d", memory_load));
29925 size_t final_total =
29926 trim_youngest_desired (memory_load, total_new_allocation, total_min_allocation);
29927 size_t max_new_allocation =
29928 #ifdef MULTIPLE_HEAPS
29929 dd_max_size (g_heaps[0]->dynamic_data_of (0));
29930 #else //MULTIPLE_HEAPS
29931 dd_max_size (dynamic_data_of (0));
29932 #endif //MULTIPLE_HEAPS
29934 final_new_allocation = min (Align ((final_total / num_heaps), get_alignment_constant (TRUE)), max_new_allocation);
29938 if (final_new_allocation < new_allocation)
29940 settings.gen0_reduction_count = 2;
29943 return final_new_allocation;
29948 gc_history_per_heap* gc_heap::get_gc_data_per_heap()
29950 #ifdef BACKGROUND_GC
29951 return (settings.concurrent ? &bgc_data_per_heap : &gc_data_per_heap);
29953 return &gc_data_per_heap;
29954 #endif //BACKGROUND_GC
29957 void gc_heap::compute_new_dynamic_data (int gen_number)
29959 PREFIX_ASSUME(gen_number >= 0);
29960 PREFIX_ASSUME(gen_number <= max_generation);
29962 dynamic_data* dd = dynamic_data_of (gen_number);
29963 generation* gen = generation_of (gen_number);
29964 size_t in = (gen_number==0) ? 0 : compute_in (gen_number);
29966 size_t total_gen_size = generation_size (gen_number);
29967 //keep track of fragmentation
29968 dd_fragmentation (dd) = generation_free_list_space (gen) + generation_free_obj_space (gen);
29969 dd_current_size (dd) = total_gen_size - dd_fragmentation (dd);
29971 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
29973 size_t out = dd_survived_size (dd);
29975 gc_generation_data* gen_data = &(current_gc_data_per_heap->gen_data[gen_number]);
29976 gen_data->size_after = total_gen_size;
29977 gen_data->free_list_space_after = generation_free_list_space (gen);
29978 gen_data->free_obj_space_after = generation_free_obj_space (gen);
29980 if ((settings.pause_mode == pause_low_latency) && (gen_number <= 1))
29982 // When we are in the low latency mode, we can still be
29983 // condemning more than gen1's 'cause of induced GCs.
29984 dd_desired_allocation (dd) = low_latency_alloc;
29988 if (gen_number == 0)
29990 //compensate for dead finalizable objects promotion.
29991 //they shoudn't be counted for growth.
29992 size_t final_promoted = 0;
29993 final_promoted = min (promoted_bytes (heap_number), out);
29994 // Prefast: this is clear from above but prefast needs to be told explicitly
29995 PREFIX_ASSUME(final_promoted <= out);
29997 dprintf (2, ("gen: %d final promoted: %Id", gen_number, final_promoted));
29998 dd_freach_previous_promotion (dd) = final_promoted;
29999 size_t lower_bound = desired_new_allocation (dd, out-final_promoted, gen_number, 0);
30001 if (settings.condemned_generation == 0)
30003 //there is no noise.
30004 dd_desired_allocation (dd) = lower_bound;
30008 size_t higher_bound = desired_new_allocation (dd, out, gen_number, 1);
30010 // <TODO>This assert was causing AppDomains\unload\test1n\test1nrun.bat to fail</TODO>
30011 //assert ( lower_bound <= higher_bound);
30013 //discount the noise. Change the desired allocation
30014 //only if the previous value is outside of the range.
30015 if (dd_desired_allocation (dd) < lower_bound)
30017 dd_desired_allocation (dd) = lower_bound;
30019 else if (dd_desired_allocation (dd) > higher_bound)
30021 dd_desired_allocation (dd) = higher_bound;
30023 #if defined (BIT64) && !defined (MULTIPLE_HEAPS)
30024 dd_desired_allocation (dd) = joined_youngest_desired (dd_desired_allocation (dd));
30025 #endif // BIT64 && !MULTIPLE_HEAPS
30026 trim_youngest_desired_low_memory();
30027 dprintf (2, ("final gen0 new_alloc: %Id", dd_desired_allocation (dd)));
30032 dd_desired_allocation (dd) = desired_new_allocation (dd, out, gen_number, 0);
30036 gen_data->pinned_surv = dd_pinned_survived_size (dd);
30037 gen_data->npinned_surv = dd_survived_size (dd) - dd_pinned_survived_size (dd);
30039 dd_gc_new_allocation (dd) = dd_desired_allocation (dd);
30040 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
30043 dd_promoted_size (dd) = out;
30044 if (gen_number == max_generation)
30046 dd = dynamic_data_of (max_generation+1);
30047 total_gen_size = generation_size (max_generation + 1);
30048 dd_fragmentation (dd) = generation_free_list_space (large_object_generation) +
30049 generation_free_obj_space (large_object_generation);
30050 dd_current_size (dd) = total_gen_size - dd_fragmentation (dd);
30051 dd_survived_size (dd) = dd_current_size (dd);
30053 out = dd_current_size (dd);
30054 dd_desired_allocation (dd) = desired_new_allocation (dd, out, max_generation+1, 0);
30055 dd_gc_new_allocation (dd) = Align (dd_desired_allocation (dd),
30056 get_alignment_constant (FALSE));
30057 dd_new_allocation (dd) = dd_gc_new_allocation (dd);
30059 gen_data = &(current_gc_data_per_heap->gen_data[max_generation+1]);
30060 gen_data->size_after = total_gen_size;
30061 gen_data->free_list_space_after = generation_free_list_space (large_object_generation);
30062 gen_data->free_obj_space_after = generation_free_obj_space (large_object_generation);
30063 gen_data->npinned_surv = out;
30064 #ifdef BACKGROUND_GC
30065 end_loh_size = total_gen_size;
30066 #endif //BACKGROUND_GC
30068 dd_promoted_size (dd) = out;
30072 void gc_heap::trim_youngest_desired_low_memory()
30074 if (g_low_memory_status)
30076 size_t committed_mem = 0;
30077 heap_segment* seg = generation_start_segment (generation_of (max_generation));
30080 committed_mem += heap_segment_committed (seg) - heap_segment_mem (seg);
30081 seg = heap_segment_next (seg);
30083 seg = generation_start_segment (generation_of (max_generation + 1));
30086 committed_mem += heap_segment_committed (seg) - heap_segment_mem (seg);
30087 seg = heap_segment_next (seg);
30090 dynamic_data* dd = dynamic_data_of (0);
30091 size_t current = dd_desired_allocation (dd);
30092 size_t candidate = max (Align ((committed_mem / 10), get_alignment_constant(FALSE)), dd_min_size (dd));
30094 dd_desired_allocation (dd) = min (current, candidate);
30098 void gc_heap::decommit_ephemeral_segment_pages()
30100 if (settings.concurrent)
30105 size_t slack_space = heap_segment_committed (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment);
30106 dynamic_data* dd = dynamic_data_of (0);
30108 #ifndef MULTIPLE_HEAPS
30109 size_t extra_space = (g_low_memory_status ? 0 : (512 * 1024));
30110 size_t decommit_timeout = (g_low_memory_status ? 0 : GC_EPHEMERAL_DECOMMIT_TIMEOUT);
30111 size_t ephemeral_elapsed = dd_time_clock(dd) - gc_last_ephemeral_decommit_time;
30113 if (dd_desired_allocation (dd) > gc_gen0_desired_high)
30115 gc_gen0_desired_high = dd_desired_allocation (dd) + extra_space;
30118 if (ephemeral_elapsed >= decommit_timeout)
30120 slack_space = min (slack_space, gc_gen0_desired_high);
30122 gc_last_ephemeral_decommit_time = dd_time_clock(dd);
30123 gc_gen0_desired_high = 0;
30125 #endif //!MULTIPLE_HEAPS
30127 if (settings.condemned_generation >= (max_generation-1))
30129 size_t new_slack_space =
30131 max(min(min(soh_segment_size/32, dd_max_size(dd)), (generation_size (max_generation) / 10)), dd_desired_allocation(dd));
30133 #ifdef FEATURE_CORECLR
30134 dd_desired_allocation (dd);
30137 #endif //FEATURE_CORECLR
30140 slack_space = min (slack_space, new_slack_space);
30143 decommit_heap_segment_pages (ephemeral_heap_segment, slack_space);
30145 gc_history_per_heap* current_gc_data_per_heap = get_gc_data_per_heap();
30146 current_gc_data_per_heap->extra_gen0_committed = heap_segment_committed (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment);
30149 size_t gc_heap::new_allocation_limit (size_t size, size_t free_size, int gen_number)
30151 dynamic_data* dd = dynamic_data_of (gen_number);
30152 ptrdiff_t new_alloc = dd_new_allocation (dd);
30153 assert (new_alloc == (ptrdiff_t)Align (new_alloc,
30154 get_alignment_constant (!(gen_number == (max_generation+1)))));
30155 size_t limit = min (max (new_alloc, (ptrdiff_t)size), (ptrdiff_t)free_size);
30156 assert (limit == Align (limit, get_alignment_constant (!(gen_number == (max_generation+1)))));
30157 dd_new_allocation (dd) = (new_alloc - limit );
30161 //This is meant to be called by decide_on_compacting.
30163 size_t gc_heap::generation_fragmentation (generation* gen,
30164 generation* consing_gen,
30168 uint8_t* alloc = generation_allocation_pointer (consing_gen);
30169 // If the allocation pointer has reached the ephemeral segment
30170 // fine, otherwise the whole ephemeral segment is considered
30172 if (in_range_for_segment (alloc, ephemeral_heap_segment))
30174 if (alloc <= heap_segment_allocated(ephemeral_heap_segment))
30175 frag = end - alloc;
30178 // case when no survivors, allocated set to beginning
30181 dprintf (3, ("ephemeral frag: %Id", frag));
30184 frag = (heap_segment_allocated (ephemeral_heap_segment) -
30185 heap_segment_mem (ephemeral_heap_segment));
30186 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
30188 PREFIX_ASSUME(seg != NULL);
30190 while (seg != ephemeral_heap_segment)
30192 frag += (heap_segment_allocated (seg) -
30193 heap_segment_plan_allocated (seg));
30194 dprintf (3, ("seg: %Ix, frag: %Id", (size_t)seg,
30195 (heap_segment_allocated (seg) -
30196 heap_segment_plan_allocated (seg))));
30198 seg = heap_segment_next_rw (seg);
30201 dprintf (3, ("frag: %Id discounting pinned plugs", frag));
30202 //add the length of the dequeued plug free space
30204 while (bos < mark_stack_bos)
30206 frag += (pinned_len (pinned_plug_of (bos)));
30213 // for SOH this returns the total sizes of the generation and its
30214 // younger generation(s).
30215 // for LOH this returns just LOH size.
30216 size_t gc_heap::generation_sizes (generation* gen)
30219 if (generation_start_segment (gen ) == ephemeral_heap_segment)
30220 result = (heap_segment_allocated (ephemeral_heap_segment) -
30221 generation_allocation_start (gen));
30224 heap_segment* seg = heap_segment_in_range (generation_start_segment (gen));
30226 PREFIX_ASSUME(seg != NULL);
30230 result += (heap_segment_allocated (seg) -
30231 heap_segment_mem (seg));
30232 seg = heap_segment_next_in_range (seg);
30239 BOOL gc_heap::decide_on_compacting (int condemned_gen_number,
30240 size_t fragmentation,
30241 BOOL& should_expand)
30243 BOOL should_compact = FALSE;
30244 should_expand = FALSE;
30245 generation* gen = generation_of (condemned_gen_number);
30246 dynamic_data* dd = dynamic_data_of (condemned_gen_number);
30247 size_t gen_sizes = generation_sizes(gen);
30248 float fragmentation_burden = ( ((0 == fragmentation) || (0 == gen_sizes)) ? (0.0f) :
30249 (float (fragmentation) / gen_sizes) );
30251 dprintf (GTC_LOG, ("fragmentation: %Id (%d%%)", fragmentation, (int)(fragmentation_burden * 100.0)));
30254 // for pure GC stress runs we need compaction, for GC stress "mix"
30255 // we need to ensure a better mix of compacting and sweeping collections
30256 if (GCStress<cfg_any>::IsEnabled() && !settings.concurrent
30257 && !g_pConfig->IsGCStressMix())
30258 should_compact = TRUE;
30261 // in GC stress "mix" mode, for stress induced collections make sure we
30262 // keep sweeps and compactions relatively balanced. do not (yet) force sweeps
30263 // against the GC's determination, as it may lead to premature OOMs.
30264 if (g_pConfig->IsGCStressMix() && settings.stress_induced)
30266 int compactions = g_GCStatistics.cntCompactFGC+g_GCStatistics.cntCompactNGC;
30267 int sweeps = g_GCStatistics.cntFGC + g_GCStatistics.cntNGC - compactions;
30268 if (compactions < sweeps / 10)
30270 should_compact = TRUE;
30274 #endif //STRESS_HEAP
30276 if (GCConfig::GetForceCompact())
30277 should_compact = TRUE;
30279 if ((condemned_gen_number == max_generation) && last_gc_before_oom)
30281 should_compact = TRUE;
30282 last_gc_before_oom = FALSE;
30283 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_last_gc);
30286 if (settings.reason == reason_induced_compacting)
30288 dprintf (2, ("induced compacting GC"));
30289 should_compact = TRUE;
30290 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_induced_compacting);
30293 dprintf (2, ("Fragmentation: %d Fragmentation burden %d%%",
30294 fragmentation, (int) (100*fragmentation_burden)));
30296 if (!should_compact)
30298 if (dt_low_ephemeral_space_p (tuning_deciding_compaction))
30300 dprintf(GTC_LOG, ("compacting due to low ephemeral"));
30301 should_compact = TRUE;
30302 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_low_ephemeral);
30306 if (should_compact)
30308 if ((condemned_gen_number >= (max_generation - 1)))
30310 if (dt_low_ephemeral_space_p (tuning_deciding_expansion))
30312 dprintf (GTC_LOG,("Not enough space for all ephemeral generations with compaction"));
30313 should_expand = TRUE;
30319 BOOL high_memory = FALSE;
30322 if (!should_compact)
30324 // We are not putting this in dt_high_frag_p because it's not exactly
30325 // high fragmentation - it's just enough planned fragmentation for us to
30326 // want to compact. Also the "fragmentation" we are talking about here
30327 // is different from anywhere else.
30328 BOOL frag_exceeded = ((fragmentation >= dd_fragmentation_limit (dd)) &&
30329 (fragmentation_burden >= dd_fragmentation_burden_limit (dd)));
30333 #ifdef BACKGROUND_GC
30334 // do not force compaction if this was a stress-induced GC
30335 IN_STRESS_HEAP(if (!settings.stress_induced))
30337 #endif // BACKGROUND_GC
30338 assert (settings.concurrent == FALSE);
30339 should_compact = TRUE;
30340 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_high_frag);
30341 #ifdef BACKGROUND_GC
30343 #endif // BACKGROUND_GC
30347 // check for high memory situation
30348 if(!should_compact)
30350 uint32_t num_heaps = 1;
30351 #ifdef MULTIPLE_HEAPS
30352 num_heaps = gc_heap::n_heaps;
30353 #endif // MULTIPLE_HEAPS
30355 ptrdiff_t reclaim_space = generation_size(max_generation) - generation_plan_size(max_generation);
30356 if((settings.entry_memory_load >= high_memory_load_th) && (settings.entry_memory_load < v_high_memory_load_th))
30358 if(reclaim_space > (int64_t)(min_high_fragmentation_threshold (entry_available_physical_mem, num_heaps)))
30360 dprintf(GTC_LOG,("compacting due to fragmentation in high memory"));
30361 should_compact = TRUE;
30362 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_high_mem_frag);
30364 high_memory = TRUE;
30366 else if(settings.entry_memory_load >= v_high_memory_load_th)
30368 if(reclaim_space > (ptrdiff_t)(min_reclaim_fragmentation_threshold (num_heaps)))
30370 dprintf(GTC_LOG,("compacting due to fragmentation in very high memory"));
30371 should_compact = TRUE;
30372 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_vhigh_mem_frag);
30374 high_memory = TRUE;
30380 // The purpose of calling ensure_gap_allocation here is to make sure
30381 // that we actually are able to commit the memory to allocate generation
30383 if ((should_compact == FALSE) &&
30384 (ensure_gap_allocation (condemned_gen_number) == FALSE))
30386 should_compact = TRUE;
30387 get_gc_data_per_heap()->set_mechanism (gc_heap_compact, compact_no_gaps);
30390 if (settings.condemned_generation == max_generation)
30392 //check the progress
30395 (high_memory && !should_compact) ||
30397 (generation_plan_allocation_start (generation_of (max_generation - 1)) >=
30398 generation_allocation_start (generation_of (max_generation - 1))))
30400 dprintf (2, (" Elevation: gen2 size: %d, gen2 plan size: %d, no progress, elevation = locked",
30401 generation_size (max_generation),
30402 generation_plan_size (max_generation)));
30403 //no progress -> lock
30404 settings.should_lock_elevation = TRUE;
30408 if (settings.pause_mode == pause_no_gc)
30410 should_compact = TRUE;
30411 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_plan_allocated (ephemeral_heap_segment))
30412 < soh_allocation_no_gc)
30414 should_expand = TRUE;
30418 dprintf (2, ("will %s", (should_compact ? "compact" : "sweep")));
30419 return should_compact;
30422 size_t align_lower_good_size_allocation (size_t size)
30424 return (size/64)*64;
30427 size_t gc_heap::approximate_new_allocation()
30429 dynamic_data* dd0 = dynamic_data_of (0);
30430 return max (2*dd_min_size (dd0), ((dd_desired_allocation (dd0)*2)/3));
30433 // After we did a GC we expect to have at least this
30434 // much space at the end of the segment to satisfy
30435 // a reasonable amount of allocation requests.
30436 size_t gc_heap::end_space_after_gc()
30438 return max ((dd_min_size (dynamic_data_of (0))/2), (END_SPACE_AFTER_GC + Align (min_obj_size)));
30441 BOOL gc_heap::ephemeral_gen_fit_p (gc_tuning_point tp)
30443 uint8_t* start = 0;
30445 if ((tp == tuning_deciding_condemned_gen) ||
30446 (tp == tuning_deciding_compaction))
30448 start = (settings.concurrent ? alloc_allocated : heap_segment_allocated (ephemeral_heap_segment));
30449 if (settings.concurrent)
30451 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment (alloc_allocated)",
30452 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated)));
30456 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment (allocated)",
30457 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - heap_segment_allocated (ephemeral_heap_segment))));
30460 else if (tp == tuning_deciding_expansion)
30462 start = heap_segment_plan_allocated (ephemeral_heap_segment);
30463 dprintf (GTC_LOG, ("%Id left at the end of ephemeral segment based on plan",
30464 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - start)));
30468 assert (tp == tuning_deciding_full_gc);
30469 dprintf (GTC_LOG, ("FGC: %Id left at the end of ephemeral segment (alloc_allocated)",
30470 (size_t)(heap_segment_reserved (ephemeral_heap_segment) - alloc_allocated)));
30471 start = alloc_allocated;
30474 if (start == 0) // empty ephemeral generations
30476 assert (tp == tuning_deciding_expansion);
30477 // if there are no survivors in the ephemeral segment,
30478 // this should be the beginning of ephemeral segment.
30479 start = generation_allocation_pointer (generation_of (max_generation));
30480 assert (start == heap_segment_mem (ephemeral_heap_segment));
30483 if (tp == tuning_deciding_expansion)
30485 assert (settings.condemned_generation >= (max_generation-1));
30486 size_t gen0size = approximate_new_allocation();
30487 size_t eph_size = gen0size;
30489 for (int j = 1; j <= max_generation-1; j++)
30491 eph_size += 2*dd_min_size (dynamic_data_of(j));
30494 // We must find room for one large object and enough room for gen0size
30495 if ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - start) > eph_size)
30497 dprintf (3, ("Enough room before end of segment"));
30502 size_t room = align_lower_good_size_allocation
30503 (heap_segment_reserved (ephemeral_heap_segment) - start);
30504 size_t end_seg = room;
30506 //look at the plug free space
30507 size_t largest_alloc = END_SPACE_AFTER_GC + Align (min_obj_size);
30508 bool large_chunk_found = FALSE;
30510 uint8_t* gen0start = generation_plan_allocation_start (youngest_generation);
30511 dprintf (3, ("ephemeral_gen_fit_p: gen0 plan start: %Ix", (size_t)gen0start));
30512 if (gen0start == 0)
30514 dprintf (3, ("ephemeral_gen_fit_p: room before free list search %Id, needed: %Id",
30516 while ((bos < mark_stack_bos) &&
30517 !((room >= gen0size) && large_chunk_found))
30519 uint8_t* plug = pinned_plug (pinned_plug_of (bos));
30520 if (in_range_for_segment (plug, ephemeral_heap_segment))
30522 if (plug >= gen0start)
30524 size_t chunk = align_lower_good_size_allocation (pinned_len (pinned_plug_of (bos)));
30526 if (!large_chunk_found)
30528 large_chunk_found = (chunk >= largest_alloc);
30530 dprintf (3, ("ephemeral_gen_fit_p: room now %Id, large chunk: %Id",
30531 room, large_chunk_found));
30537 if (room >= gen0size)
30539 if (large_chunk_found)
30541 dprintf (3, ("Enough room"));
30546 // now we need to find largest_alloc at the end of the segment.
30547 if (end_seg >= end_space_after_gc())
30549 dprintf (3, ("Enough room (may need end of seg)"));
30555 dprintf (3, ("Not enough room"));
30561 size_t end_space = 0;
30562 dynamic_data* dd = dynamic_data_of (0);
30563 if ((tp == tuning_deciding_condemned_gen) ||
30564 (tp == tuning_deciding_full_gc))
30566 end_space = 2*dd_min_size (dd);
30570 assert (tp == tuning_deciding_compaction);
30571 end_space = approximate_new_allocation();
30574 if (!((size_t)(heap_segment_reserved (ephemeral_heap_segment) - start) > end_space))
30576 dprintf (GTC_LOG, ("ephemeral_gen_fit_p: does not fit without compaction"));
30578 return ((size_t)(heap_segment_reserved (ephemeral_heap_segment) - start) > end_space);
30582 CObjectHeader* gc_heap::allocate_large_object (size_t jsize, int64_t& alloc_bytes)
30584 //create a new alloc context because gen3context is shared.
30585 alloc_context acontext;
30586 acontext.alloc_ptr = 0;
30587 acontext.alloc_limit = 0;
30588 acontext.alloc_bytes = 0;
30589 #ifdef MULTIPLE_HEAPS
30590 acontext.set_alloc_heap(vm_heap);
30591 #endif //MULTIPLE_HEAPS
30594 uint8_t* current_lowest_address = lowest_address;
30595 uint8_t* current_highest_address = highest_address;
30596 #ifdef BACKGROUND_GC
30597 if (recursive_gc_sync::background_running_p())
30599 current_lowest_address = background_saved_lowest_address;
30600 current_highest_address = background_saved_highest_address;
30602 #endif //BACKGROUND_GC
30603 #endif // MARK_ARRAY
30606 size_t maxObjectSize = (INT64_MAX - 7 - Align(min_obj_size));
30608 size_t maxObjectSize = (INT32_MAX - 7 - Align(min_obj_size));
30611 if (jsize >= maxObjectSize)
30613 if (GCConfig::GetBreakOnOOM())
30615 GCToOSInterface::DebugBreak();
30620 size_t size = AlignQword (jsize);
30621 int align_const = get_alignment_constant (FALSE);
30622 #ifdef FEATURE_LOH_COMPACTION
30623 size_t pad = Align (loh_padding_obj_size, align_const);
30626 #endif //FEATURE_LOH_COMPACTION
30628 assert (size >= Align (min_obj_size, align_const));
30630 #pragma inline_depth(0)
30632 if (! allocate_more_space (&acontext, (size + pad), max_generation+1))
30638 #pragma inline_depth(20)
30641 #ifdef FEATURE_LOH_COMPACTION
30642 // The GC allocator made a free object already in this alloc context and
30643 // adjusted the alloc_ptr accordingly.
30644 #endif //FEATURE_LOH_COMPACTION
30646 uint8_t* result = acontext.alloc_ptr;
30648 assert ((size_t)(acontext.alloc_limit - acontext.alloc_ptr) == size);
30649 alloc_bytes += size;
30651 CObjectHeader* obj = (CObjectHeader*)result;
30654 if (recursive_gc_sync::background_running_p())
30656 if ((result < current_highest_address) && (result >= current_lowest_address))
30658 dprintf (3, ("Clearing mark bit at address %Ix",
30659 (size_t)(&mark_array [mark_word_of (result)])));
30661 mark_array_clear_marked (result);
30663 #ifdef BACKGROUND_GC
30664 //the object has to cover one full mark uint32_t
30665 assert (size > mark_word_size);
30666 if (current_c_gc_state == c_gc_state_marking)
30668 dprintf (3, ("Concurrent allocation of a large object %Ix",
30670 //mark the new block specially so we know it is a new object
30671 if ((result < current_highest_address) && (result >= current_lowest_address))
30673 dprintf (3, ("Setting mark bit at address %Ix",
30674 (size_t)(&mark_array [mark_word_of (result)])));
30676 mark_array_set_marked (result);
30679 #endif //BACKGROUND_GC
30681 #endif //MARK_ARRAY
30684 assert ((size_t)obj == Align ((size_t)obj, align_const));
30689 void reset_memory (uint8_t* o, size_t sizeo)
30691 if (sizeo > 128 * 1024)
30693 // We cannot reset the memory for the useful part of a free object.
30694 size_t size_to_skip = min_free_list - plug_skew;
30696 size_t page_start = align_on_page ((size_t)(o + size_to_skip));
30697 size_t size = align_lower_page ((size_t)o + sizeo - size_to_skip - plug_skew) - page_start;
30698 // Note we need to compensate for an OS bug here. This bug would cause the MEM_RESET to fail
30699 // on write watched memory.
30702 reset_mm_p = GCToOSInterface::VirtualReset((void*)page_start, size, true /* unlock */);
30707 void gc_heap::reset_large_object (uint8_t* o)
30709 // If it's a large object, allow the O/S to discard the backing store for these pages.
30710 reset_memory (o, size(o));
30713 BOOL gc_heap::large_object_marked (uint8_t* o, BOOL clearp)
30716 // It shouldn't be necessary to do these comparisons because this is only used for blocking
30717 // GCs and LOH segments cannot be out of range.
30718 if ((o >= lowest_address) && (o < highest_address))
30738 void gc_heap::walk_survivors_relocation (void* profiling_context, record_surv_fn fn)
30740 // Now walk the portion of memory that is actually being relocated.
30741 walk_relocation (profiling_context, fn);
30743 #ifdef FEATURE_LOH_COMPACTION
30744 if (loh_compacted_p)
30746 walk_relocation_for_loh (profiling_context, fn);
30748 #endif //FEATURE_LOH_COMPACTION
30751 void gc_heap::walk_survivors_for_loh (void* profiling_context, record_surv_fn fn)
30753 generation* gen = large_object_generation;
30754 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));;
30756 PREFIX_ASSUME(seg != NULL);
30758 uint8_t* o = generation_allocation_start (gen);
30759 uint8_t* plug_end = o;
30760 uint8_t* plug_start = o;
30764 if (o >= heap_segment_allocated (seg))
30766 seg = heap_segment_next (seg);
30770 o = heap_segment_mem (seg);
30772 if (large_object_marked(o, FALSE))
30779 o = o + AlignQword (size (o));
30780 if (o >= heap_segment_allocated (seg))
30784 m = large_object_marked (o, FALSE);
30789 fn (plug_start, plug_end, 0, profiling_context, false, false);
30793 while (o < heap_segment_allocated (seg) && !large_object_marked(o, FALSE))
30795 o = o + AlignQword (size (o));
30801 #ifdef BACKGROUND_GC
30803 BOOL gc_heap::background_object_marked (uint8_t* o, BOOL clearp)
30806 if ((o >= background_saved_lowest_address) && (o < background_saved_highest_address))
30808 if (mark_array_marked (o))
30812 mark_array_clear_marked (o);
30813 //dprintf (3, ("mark array bit for object %Ix is cleared", o));
30814 dprintf (3, ("CM: %Ix", o));
30824 dprintf (3, ("o %Ix(%d) %s", o, size(o), (m ? "was bm" : "was NOT bm")));
30828 uint8_t* gc_heap::background_next_end (heap_segment* seg, BOOL large_objects_p)
30831 (large_objects_p ? heap_segment_allocated (seg) : heap_segment_background_allocated (seg));
30834 void gc_heap::set_mem_verify (uint8_t* start, uint8_t* end, uint8_t b)
30839 if ((GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC) &&
30840 !(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_NO_MEM_FILL))
30842 dprintf (3, ("setting mem to %c [%Ix, [%Ix", b, start, end));
30843 memset (start, b, (end - start));
30846 #endif //VERIFY_HEAP
30849 void gc_heap::generation_delete_heap_segment (generation* gen,
30851 heap_segment* prev_seg,
30852 heap_segment* next_seg)
30854 dprintf (3, ("bgc sweep: deleting seg %Ix", seg));
30855 if (gen == large_object_generation)
30857 heap_segment_next (prev_seg) = next_seg;
30859 dprintf (3, ("Preparing empty large segment %Ix for deletion", (size_t)seg));
30861 heap_segment_next (seg) = freeable_large_heap_segment;
30862 freeable_large_heap_segment = seg;
30866 if (seg == ephemeral_heap_segment)
30871 heap_segment_next (next_seg) = prev_seg;
30873 dprintf (3, ("Preparing empty small segment %Ix for deletion", (size_t)seg));
30874 heap_segment_next (seg) = freeable_small_heap_segment;
30875 freeable_small_heap_segment = seg;
30878 decommit_heap_segment (seg);
30879 seg->flags |= heap_segment_flags_decommitted;
30881 set_mem_verify (heap_segment_allocated (seg) - plug_skew, heap_segment_used (seg), 0xbb);
30884 void gc_heap::process_background_segment_end (heap_segment* seg,
30886 uint8_t* last_plug_end,
30887 heap_segment* start_seg,
30891 uint8_t* allocated = heap_segment_allocated (seg);
30892 uint8_t* background_allocated = heap_segment_background_allocated (seg);
30894 dprintf (3, ("Processing end of background segment [%Ix, %Ix[(%Ix[)",
30895 (size_t)heap_segment_mem (seg), background_allocated, allocated));
30898 if (allocated != background_allocated)
30900 if (gen == large_object_generation)
30905 dprintf (3, ("Make a free object before newly promoted objects [%Ix, %Ix[",
30906 (size_t)last_plug_end, background_allocated));
30907 thread_gap (last_plug_end, background_allocated - last_plug_end, generation_of (max_generation));
30909 fix_brick_to_highest (last_plug_end, background_allocated);
30911 // When we allowed fgc's during going through gaps, we could have erased the brick
30912 // that corresponds to bgc_allocated 'cause we had to update the brick there,
30913 // recover it here.
30914 fix_brick_to_highest (background_allocated, background_allocated);
30918 // by default, if allocated == background_allocated, it can't
30919 // be the ephemeral segment.
30920 if (seg == ephemeral_heap_segment)
30925 if (allocated == heap_segment_mem (seg))
30927 // this can happen with LOH segments when multiple threads
30928 // allocate new segments and not all of them were needed to
30929 // satisfy allocation requests.
30930 assert (gen == large_object_generation);
30933 if (last_plug_end == heap_segment_mem (seg))
30935 dprintf (3, ("Segment allocated is %Ix (beginning of this seg) - %s be deleted",
30936 (size_t)allocated, (*delete_p ? "should" : "should not")));
30938 if (seg != start_seg)
30945 dprintf (3, ("Trimming seg to %Ix[", (size_t)last_plug_end));
30946 heap_segment_allocated (seg) = last_plug_end;
30947 set_mem_verify (heap_segment_allocated (seg) - plug_skew, heap_segment_used (seg), 0xbb);
30949 decommit_heap_segment_pages (seg, 0);
30953 dprintf (3, ("verifying seg %Ix's mark array was completely cleared", seg));
30954 bgc_verify_mark_array_cleared (seg);
30957 void gc_heap::process_n_background_segments (heap_segment* seg,
30958 heap_segment* prev_seg,
30961 assert (gen != large_object_generation);
30965 dprintf (2, ("processing seg %Ix (not seen by bgc mark)", seg));
30966 heap_segment* next_seg = heap_segment_next (seg);
30968 if (heap_segment_read_only_p (seg))
30974 if (heap_segment_allocated (seg) == heap_segment_mem (seg))
30976 // This can happen - if we have a LOH segment where nothing survived
30977 // or a SOH segment allocated by a gen1 GC when BGC was going where
30978 // nothing survived last time we did a gen1 GC.
30979 generation_delete_heap_segment (gen, seg, prev_seg, next_seg);
30987 verify_soh_segment_list();
30993 BOOL gc_heap::fgc_should_consider_object (uint8_t* o,
30995 BOOL consider_bgc_mark_p,
30996 BOOL check_current_sweep_p,
30997 BOOL check_saved_sweep_p)
30999 // the logic for this function must be kept in sync with the analogous function
31000 // in ToolBox\SOS\Strike\gc.cpp
31002 // TRUE means we don't need to check the bgc mark bit
31003 // FALSE means we do.
31004 BOOL no_bgc_mark_p = FALSE;
31006 if (consider_bgc_mark_p)
31008 if (check_current_sweep_p && (o < current_sweep_pos))
31010 dprintf (3, ("no bgc mark - o: %Ix < cs: %Ix", o, current_sweep_pos));
31011 no_bgc_mark_p = TRUE;
31014 if (!no_bgc_mark_p)
31016 if(check_saved_sweep_p && (o >= saved_sweep_ephemeral_start))
31018 dprintf (3, ("no bgc mark - o: %Ix >= ss: %Ix", o, saved_sweep_ephemeral_start));
31019 no_bgc_mark_p = TRUE;
31022 if (!check_saved_sweep_p)
31024 uint8_t* background_allocated = heap_segment_background_allocated (seg);
31025 // if this was the saved ephemeral segment, check_saved_sweep_p
31026 // would've been true.
31027 assert (heap_segment_background_allocated (seg) != saved_sweep_ephemeral_start);
31028 // background_allocated could be 0 for the new segments acquired during bgc
31029 // sweep and we still want no_bgc_mark_p to be true.
31030 if (o >= background_allocated)
31032 dprintf (3, ("no bgc mark - o: %Ix >= ba: %Ix", o, background_allocated));
31033 no_bgc_mark_p = TRUE;
31040 no_bgc_mark_p = TRUE;
31043 dprintf (3, ("bgc mark %Ix: %s (bm: %s)", o, (no_bgc_mark_p ? "no" : "yes"), (background_object_marked (o, FALSE) ? "yes" : "no")));
31044 return (no_bgc_mark_p ? TRUE : background_object_marked (o, FALSE));
31047 // consider_bgc_mark_p tells you if you need to care about the bgc mark bit at all
31048 // if it's TRUE, check_current_sweep_p tells you if you should consider the
31049 // current sweep position or not.
31050 void gc_heap::should_check_bgc_mark (heap_segment* seg,
31051 BOOL* consider_bgc_mark_p,
31052 BOOL* check_current_sweep_p,
31053 BOOL* check_saved_sweep_p)
31055 // the logic for this function must be kept in sync with the analogous function
31056 // in ToolBox\SOS\Strike\gc.cpp
31057 *consider_bgc_mark_p = FALSE;
31058 *check_current_sweep_p = FALSE;
31059 *check_saved_sweep_p = FALSE;
31061 if (current_c_gc_state == c_gc_state_planning)
31063 // We are doing the current_sweep_pos comparison here because we have yet to
31064 // turn on the swept flag for the segment but in_range_for_segment will return
31065 // FALSE if the address is the same as reserved.
31066 if ((seg->flags & heap_segment_flags_swept) || (current_sweep_pos == heap_segment_reserved (seg)))
31068 dprintf (3, ("seg %Ix is already swept by bgc", seg));
31072 *consider_bgc_mark_p = TRUE;
31074 dprintf (3, ("seg %Ix hasn't been swept by bgc", seg));
31076 if (seg == saved_sweep_ephemeral_seg)
31078 dprintf (3, ("seg %Ix is the saved ephemeral seg", seg));
31079 *check_saved_sweep_p = TRUE;
31082 if (in_range_for_segment (current_sweep_pos, seg))
31084 dprintf (3, ("current sweep pos is %Ix and within seg %Ix",
31085 current_sweep_pos, seg));
31086 *check_current_sweep_p = TRUE;
31092 void gc_heap::background_ephemeral_sweep()
31094 dprintf (3, ("bgc ephemeral sweep"));
31096 int align_const = get_alignment_constant (TRUE);
31098 saved_sweep_ephemeral_seg = ephemeral_heap_segment;
31099 saved_sweep_ephemeral_start = generation_allocation_start (generation_of (max_generation - 1));
31101 // Since we don't want to interfere with gen0 allocation while we are threading gen0 free list,
31102 // we thread onto a list first then publish it when we are done.
31103 allocator youngest_free_list;
31104 size_t youngest_free_list_space = 0;
31105 size_t youngest_free_obj_space = 0;
31107 youngest_free_list.clear();
31109 for (int i = 0; i <= (max_generation - 1); i++)
31111 generation* gen_to_reset = generation_of (i);
31112 assert (generation_free_list_space (gen_to_reset) == 0);
31113 // Can only assert free_list_space is 0, not free_obj_space as the allocator could have added
31114 // something there.
31117 for (int i = (max_generation - 1); i >= 0; i--)
31119 generation* current_gen = generation_of (i);
31120 uint8_t* o = generation_allocation_start (current_gen);
31121 //Skip the generation gap object
31122 o = o + Align(size (o), align_const);
31123 uint8_t* end = ((i > 0) ?
31124 generation_allocation_start (generation_of (i - 1)) :
31125 heap_segment_allocated (ephemeral_heap_segment));
31127 uint8_t* plug_end = o;
31128 uint8_t* plug_start = o;
31129 BOOL marked_p = FALSE;
31133 marked_p = background_object_marked (o, TRUE);
31137 size_t plug_size = plug_start - plug_end;
31141 thread_gap (plug_end, plug_size, current_gen);
31147 make_unused_array (plug_end, plug_size);
31148 if (plug_size >= min_free_list)
31150 youngest_free_list_space += plug_size;
31151 youngest_free_list.thread_item (plug_end, plug_size);
31155 youngest_free_obj_space += plug_size;
31160 fix_brick_to_highest (plug_end, plug_start);
31161 fix_brick_to_highest (plug_start, plug_start);
31166 o = o + Align (size (o), align_const);
31172 m = background_object_marked (o, TRUE);
31175 dprintf (3, ("bgs: plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
31179 while ((o < end) && !background_object_marked (o, FALSE))
31181 o = o + Align (size (o), align_const);
31186 if (plug_end != end)
31190 thread_gap (plug_end, end - plug_end, current_gen);
31191 fix_brick_to_highest (plug_end, end);
31195 heap_segment_allocated (ephemeral_heap_segment) = plug_end;
31196 // the following line is temporary.
31197 heap_segment_saved_bg_allocated (ephemeral_heap_segment) = plug_end;
31199 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
31201 make_unused_array (plug_end, (end - plug_end));
31203 #endif //VERIFY_HEAP
31207 dd_fragmentation (dynamic_data_of (i)) =
31208 generation_free_list_space (current_gen) + generation_free_obj_space (current_gen);
31211 generation* youngest_gen = generation_of (0);
31212 generation_free_list_space (youngest_gen) = youngest_free_list_space;
31213 generation_free_obj_space (youngest_gen) = youngest_free_obj_space;
31214 dd_fragmentation (dynamic_data_of (0)) = youngest_free_list_space + youngest_free_obj_space;
31215 generation_allocator (youngest_gen)->copy_with_no_repair (&youngest_free_list);
31218 void gc_heap::background_sweep()
31220 generation* gen = generation_of (max_generation);
31221 dynamic_data* dd = dynamic_data_of (max_generation);
31222 // For SOH segments we go backwards.
31223 heap_segment* start_seg = ephemeral_heap_segment;
31224 PREFIX_ASSUME(start_seg != NULL);
31225 heap_segment* fseg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
31226 heap_segment* seg = start_seg;
31227 uint8_t* o = heap_segment_mem (seg);
31229 heap_segment* prev_seg = heap_segment_next (seg);
31230 int align_const = get_alignment_constant (TRUE);
31233 assert (o == generation_allocation_start (generation_of (max_generation)));
31234 o = o + Align(size (o), align_const);
31237 uint8_t* plug_end = o;
31238 uint8_t* plug_start = o;
31239 next_sweep_obj = o;
31240 current_sweep_pos = o;
31242 //uint8_t* end = background_next_end (seg, (gen == large_object_generation));
31243 uint8_t* end = heap_segment_background_allocated (seg);
31244 BOOL delete_p = FALSE;
31246 //concurrent_print_time_delta ("finished with mark and start with sweep");
31247 concurrent_print_time_delta ("Sw");
31248 dprintf (2, ("---- (GC%d)Background Sweep Phase ----", VolatileLoad(&settings.gc_index)));
31250 //block concurrent allocation for large objects
31251 dprintf (3, ("lh state: planning"));
31252 if (gc_lh_block_event.IsValid())
31254 gc_lh_block_event.Reset();
31257 for (int i = 0; i <= (max_generation + 1); i++)
31259 generation* gen_to_reset = generation_of (i);
31260 generation_allocator (gen_to_reset)->clear();
31261 generation_free_list_space (gen_to_reset) = 0;
31262 generation_free_obj_space (gen_to_reset) = 0;
31263 generation_free_list_allocated (gen_to_reset) = 0;
31264 generation_end_seg_allocated (gen_to_reset) = 0;
31265 generation_condemned_allocated (gen_to_reset) = 0;
31266 //reset the allocation so foreground gc can allocate into older generation
31267 generation_allocation_pointer (gen_to_reset)= 0;
31268 generation_allocation_limit (gen_to_reset) = 0;
31269 generation_allocation_segment (gen_to_reset) = heap_segment_rw (generation_start_segment (gen_to_reset));
31272 FIRE_EVENT(BGC2ndNonConEnd);
31274 current_bgc_state = bgc_sweep_soh;
31275 verify_soh_segment_list();
31277 #ifdef FEATURE_BASICFREEZE
31278 if ((generation_start_segment (gen) != ephemeral_heap_segment) &&
31279 ro_segments_in_range)
31281 sweep_ro_segments (generation_start_segment (gen));
31283 #endif // FEATURE_BASICFREEZE
31285 //TODO BACKGROUND_GC: can we move this to where we switch to the LOH?
31286 if (current_c_gc_state != c_gc_state_planning)
31288 current_c_gc_state = c_gc_state_planning;
31291 concurrent_print_time_delta ("Swe");
31293 heap_segment* loh_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation + 1)));
31294 PREFIX_ASSUME(loh_seg != NULL);
31297 loh_seg->flags &= ~heap_segment_flags_swept;
31298 heap_segment_background_allocated (loh_seg) = heap_segment_allocated (loh_seg);
31299 loh_seg = heap_segment_next_rw (loh_seg);
31302 #ifdef MULTIPLE_HEAPS
31303 bgc_t_join.join(this, gc_join_restart_ee);
31304 if (bgc_t_join.joined())
31305 #endif //MULTIPLE_HEAPS
31307 #ifdef MULTIPLE_HEAPS
31308 dprintf(2, ("Starting BGC threads for resuming EE"));
31309 bgc_t_join.restart();
31310 #endif //MULTIPLE_HEAPS
31313 if (heap_number == 0)
31318 FIRE_EVENT(BGC2ndConBegin);
31320 background_ephemeral_sweep();
31322 #ifdef MULTIPLE_HEAPS
31323 bgc_t_join.join(this, gc_join_after_ephemeral_sweep);
31324 if (bgc_t_join.joined())
31325 #endif //MULTIPLE_HEAPS
31327 #ifdef FEATURE_EVENT_TRACE
31328 bgc_heap_walk_for_etw_p = GCEventStatus::IsEnabled(GCEventProvider_Default,
31329 GCEventKeyword_GCHeapSurvivalAndMovement,
31330 GCEventLevel_Information);
31331 #endif //FEATURE_EVENT_TRACE
31333 leave_spin_lock (&gc_lock);
31335 #ifdef MULTIPLE_HEAPS
31336 dprintf(2, ("Starting BGC threads for BGC sweeping"));
31337 bgc_t_join.restart();
31338 #endif //MULTIPLE_HEAPS
31341 disable_preemptive (true);
31343 dprintf (2, ("bgs: sweeping gen2 objects"));
31344 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
31345 (size_t)heap_segment_mem (seg),
31346 (size_t)heap_segment_allocated (seg),
31347 (size_t)heap_segment_background_allocated (seg)));
31349 int num_objs = 256;
31350 int current_num_objs = 0;
31351 heap_segment* next_seg = 0;
31357 if (gen == large_object_generation)
31359 next_seg = heap_segment_next (seg);
31363 next_seg = heap_segment_prev (fseg, seg);
31368 if (!heap_segment_read_only_p (seg))
31370 if (gen == large_object_generation)
31372 // we can treat all LOH segments as in the bgc domain
31373 // regardless of whether we saw in bgc mark or not
31374 // because we don't allow LOH allocations during bgc
31375 // sweep anyway - the LOH segments can't change.
31376 process_background_segment_end (seg, gen, plug_end,
31377 start_seg, &delete_p);
31381 assert (heap_segment_background_allocated (seg) != 0);
31382 process_background_segment_end (seg, gen, plug_end,
31383 start_seg, &delete_p);
31385 assert (next_seg || !delete_p);
31391 generation_delete_heap_segment (gen, seg, prev_seg, next_seg);
31396 dprintf (2, ("seg %Ix has been swept", seg));
31397 seg->flags |= heap_segment_flags_swept;
31400 verify_soh_segment_list();
31404 dprintf (GTC_LOG, ("seg: %Ix, next_seg: %Ix, prev_seg: %Ix", seg, next_seg, prev_seg));
31408 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
31410 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
31412 if (gen != large_object_generation)
31414 dprintf (2, ("bgs: sweeping gen3 objects"));
31415 current_bgc_state = bgc_sweep_loh;
31416 gen = generation_of (max_generation+1);
31417 start_seg = heap_segment_rw (generation_start_segment (gen));
31419 PREFIX_ASSUME(start_seg != NULL);
31423 o = generation_allocation_start (gen);
31424 assert (method_table (o) == g_gc_pFreeObjectMethodTable);
31425 align_const = get_alignment_constant (FALSE);
31426 o = o + Align(size (o), align_const);
31428 end = heap_segment_allocated (seg);
31429 dprintf (2, ("sweeping gen3 objects"));
31430 generation_free_obj_space (gen) = 0;
31431 generation_allocator (gen)->clear();
31432 generation_free_list_space (gen) = 0;
31434 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
31435 (size_t)heap_segment_mem (seg),
31436 (size_t)heap_segment_allocated (seg),
31437 (size_t)heap_segment_background_allocated (seg)));
31444 o = heap_segment_mem (seg);
31447 assert (gen != large_object_generation);
31448 assert (o == generation_allocation_start (generation_of (max_generation)));
31449 align_const = get_alignment_constant (TRUE);
31450 o = o + Align(size (o), align_const);
31454 current_sweep_pos = o;
31455 next_sweep_obj = o;
31458 end = background_next_end (seg, (gen == large_object_generation));
31459 dprintf (2, ("bgs: seg: %Ix, [%Ix, %Ix[%Ix", (size_t)seg,
31460 (size_t)heap_segment_mem (seg),
31461 (size_t)heap_segment_allocated (seg),
31462 (size_t)heap_segment_background_allocated (seg)));
31466 if ((o < end) && background_object_marked (o, TRUE))
31469 if (gen == large_object_generation)
31471 dprintf (2, ("loh fr: [%Ix-%Ix[(%Id)", plug_end, plug_start, plug_start-plug_end));
31474 thread_gap (plug_end, plug_start-plug_end, gen);
31475 if (gen != large_object_generation)
31477 add_gen_free (max_generation, plug_start-plug_end);
31478 fix_brick_to_highest (plug_end, plug_start);
31479 // we need to fix the brick for the next plug here 'cause an FGC can
31480 // happen and can't read a stale brick.
31481 fix_brick_to_highest (plug_start, plug_start);
31488 next_sweep_obj = o + Align(size (o), align_const);
31489 current_num_objs++;
31490 if (current_num_objs >= num_objs)
31492 current_sweep_pos = next_sweep_obj;
31495 current_num_objs = 0;
31498 o = next_sweep_obj;
31504 m = background_object_marked (o, TRUE);
31507 if (gen != large_object_generation)
31509 add_gen_plug (max_generation, plug_end-plug_start);
31510 dd_survived_size (dd) += (plug_end - plug_start);
31512 dprintf (3, ("bgs: plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
31516 while ((o < end) && !background_object_marked (o, FALSE))
31518 next_sweep_obj = o + Align(size (o), align_const);;
31519 current_num_objs++;
31520 if (current_num_objs >= num_objs)
31522 current_sweep_pos = plug_end;
31523 dprintf (1234, ("f: swept till %Ix", current_sweep_pos));
31525 current_num_objs = 0;
31528 o = next_sweep_obj;
31533 size_t total_loh_size = generation_size (max_generation + 1);
31534 size_t total_soh_size = generation_sizes (generation_of (max_generation));
31536 dprintf (GTC_LOG, ("h%d: S: loh: %Id, soh: %Id", heap_number, total_loh_size, total_soh_size));
31538 dprintf (GTC_LOG, ("end of bgc sweep: gen2 FL: %Id, FO: %Id",
31539 generation_free_list_space (generation_of (max_generation)),
31540 generation_free_obj_space (generation_of (max_generation))));
31541 dprintf (GTC_LOG, ("h%d: end of bgc sweep: gen3 FL: %Id, FO: %Id",
31543 generation_free_list_space (generation_of (max_generation + 1)),
31544 generation_free_obj_space (generation_of (max_generation + 1))));
31546 FIRE_EVENT(BGC2ndConEnd);
31547 concurrent_print_time_delta ("background sweep");
31549 heap_segment* reset_seg = heap_segment_rw (generation_start_segment (generation_of (max_generation)));
31550 PREFIX_ASSUME(reset_seg != NULL);
31554 heap_segment_saved_bg_allocated (reset_seg) = heap_segment_background_allocated (reset_seg);
31555 heap_segment_background_allocated (reset_seg) = 0;
31556 reset_seg = heap_segment_next_rw (reset_seg);
31559 // We calculate dynamic data here because if we wait till we signal the lh event,
31560 // the allocation thread can change the fragmentation and we may read an intermediate
31561 // value (which can be greater than the generation size). Plus by that time it won't
31563 compute_new_dynamic_data (max_generation);
31565 enable_preemptive ();
31567 #ifdef MULTIPLE_HEAPS
31568 bgc_t_join.join(this, gc_join_set_state_free);
31569 if (bgc_t_join.joined())
31570 #endif //MULTIPLE_HEAPS
31572 // TODO: We are using this join just to set the state. Should
31573 // look into eliminating it - check to make sure things that use
31574 // this state can live with per heap state like should_check_bgc_mark.
31575 current_c_gc_state = c_gc_state_free;
31577 #ifdef MULTIPLE_HEAPS
31578 dprintf(2, ("Starting BGC threads after background sweep phase"));
31579 bgc_t_join.restart();
31580 #endif //MULTIPLE_HEAPS
31583 disable_preemptive (true);
31585 if (gc_lh_block_event.IsValid())
31587 gc_lh_block_event.Set();
31590 //dprintf (GTC_LOG, ("---- (GC%d)End Background Sweep Phase ----", VolatileLoad(&settings.gc_index)));
31591 dprintf (GTC_LOG, ("---- (GC%d)ESw ----", VolatileLoad(&settings.gc_index)));
31593 #endif //BACKGROUND_GC
31595 void gc_heap::sweep_large_objects ()
31597 //this min value is for the sake of the dynamic tuning.
31598 //so we know that we are not starting even if we have no
31600 generation* gen = large_object_generation;
31601 heap_segment* start_seg = heap_segment_rw (generation_start_segment (gen));
31603 PREFIX_ASSUME(start_seg != NULL);
31605 heap_segment* seg = start_seg;
31606 heap_segment* prev_seg = 0;
31607 uint8_t* o = generation_allocation_start (gen);
31608 int align_const = get_alignment_constant (FALSE);
31610 //Skip the generation gap object
31611 o = o + Align(size (o), align_const);
31613 uint8_t* plug_end = o;
31614 uint8_t* plug_start = o;
31616 generation_allocator (gen)->clear();
31617 generation_free_list_space (gen) = 0;
31618 generation_free_obj_space (gen) = 0;
31621 dprintf (3, ("sweeping large objects"));
31622 dprintf (3, ("seg: %Ix, [%Ix, %Ix[, starting from %Ix",
31624 (size_t)heap_segment_mem (seg),
31625 (size_t)heap_segment_allocated (seg),
31630 if (o >= heap_segment_allocated (seg))
31632 heap_segment* next_seg = heap_segment_next (seg);
31633 //delete the empty segment if not the only one
31634 if ((plug_end == heap_segment_mem (seg)) &&
31635 (seg != start_seg) && !heap_segment_read_only_p (seg))
31637 //prepare for deletion
31638 dprintf (3, ("Preparing empty large segment %Ix", (size_t)seg));
31640 heap_segment_next (prev_seg) = next_seg;
31641 heap_segment_next (seg) = freeable_large_heap_segment;
31642 freeable_large_heap_segment = seg;
31646 if (!heap_segment_read_only_p (seg))
31648 dprintf (3, ("Trimming seg to %Ix[", (size_t)plug_end));
31649 heap_segment_allocated (seg) = plug_end;
31650 decommit_heap_segment_pages (seg, 0);
31659 o = heap_segment_mem (seg);
31661 dprintf (3, ("seg: %Ix, [%Ix, %Ix[", (size_t)seg,
31662 (size_t)heap_segment_mem (seg),
31663 (size_t)heap_segment_allocated (seg)));
31666 if (large_object_marked(o, TRUE))
31669 //everything between plug_end and plug_start is free
31670 thread_gap (plug_end, plug_start-plug_end, gen);
31675 o = o + AlignQword (size (o));
31676 if (o >= heap_segment_allocated (seg))
31680 m = large_object_marked (o, TRUE);
31683 dprintf (3, ("plug [%Ix, %Ix[", (size_t)plug_start, (size_t)plug_end));
31687 while (o < heap_segment_allocated (seg) && !large_object_marked(o, FALSE))
31689 o = o + AlignQword (size (o));
31694 generation_allocation_segment (gen) = heap_segment_rw (generation_start_segment (gen));
31696 PREFIX_ASSUME(generation_allocation_segment(gen) != NULL);
31699 void gc_heap::relocate_in_large_objects ()
31701 relocate_args args;
31703 args.high = gc_high;
31704 args.last_plug = 0;
31706 generation* gen = large_object_generation;
31708 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
31710 PREFIX_ASSUME(seg != NULL);
31712 uint8_t* o = generation_allocation_start (gen);
31716 if (o >= heap_segment_allocated (seg))
31718 seg = heap_segment_next_rw (seg);
31723 o = heap_segment_mem (seg);
31726 while (o < heap_segment_allocated (seg))
31728 check_class_object_demotion (o);
31729 if (contain_pointers (o))
31731 dprintf(3, ("Relocating through large object %Ix", (size_t)o));
31732 go_through_object_nostart (method_table (o), o, size(o), pval,
31734 reloc_survivor_helper (pval);
31737 o = o + AlignQword (size (o));
31742 void gc_heap::mark_through_cards_for_large_objects (card_fn fn,
31745 uint8_t* low = gc_low;
31746 size_t end_card = 0;
31747 generation* oldest_gen = generation_of (max_generation+1);
31748 heap_segment* seg = heap_segment_rw (generation_start_segment (oldest_gen));
31750 PREFIX_ASSUME(seg != NULL);
31752 uint8_t* beg = generation_allocation_start (oldest_gen);
31753 uint8_t* end = heap_segment_allocated (seg);
31755 size_t cg_pointers_found = 0;
31757 size_t card_word_end = (card_of (align_on_card_word (end)) /
31762 size_t n_card_set = 0;
31763 uint8_t* next_boundary = (relocating ?
31764 generation_plan_allocation_start (generation_of (max_generation -1)) :
31767 uint8_t* nhigh = (relocating ?
31768 heap_segment_plan_allocated (ephemeral_heap_segment) :
31771 BOOL foundp = FALSE;
31772 uint8_t* start_address = 0;
31773 uint8_t* limit = 0;
31774 size_t card = card_of (beg);
31776 #ifdef BACKGROUND_GC
31777 BOOL consider_bgc_mark_p = FALSE;
31778 BOOL check_current_sweep_p = FALSE;
31779 BOOL check_saved_sweep_p = FALSE;
31780 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
31781 #endif //BACKGROUND_GC
31783 size_t total_cards_cleared = 0;
31785 //dprintf(3,( "scanning large objects from %Ix to %Ix", (size_t)beg, (size_t)end));
31786 dprintf(3, ("CMl: %Ix->%Ix", (size_t)beg, (size_t)end));
31789 if ((o < end) && (card_of(o) > card))
31791 dprintf (3, ("Found %Id cg pointers", cg_pointers_found));
31792 if (cg_pointers_found == 0)
31794 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card), (size_t)o));
31795 clear_cards (card, card_of((uint8_t*)o));
31796 total_cards_cleared += (card_of((uint8_t*)o) - card);
31798 n_eph +=cg_pointers_found;
31799 cg_pointers_found = 0;
31800 card = card_of ((uint8_t*)o);
31802 if ((o < end) &&(card >= end_card))
31804 foundp = find_card (card_table, card, card_word_end, end_card);
31807 n_card_set+= end_card - card;
31808 start_address = max (beg, card_address (card));
31810 limit = min (end, card_address (end_card));
31812 if ((!foundp) || (o >= end) || (card_address (card) >= end))
31814 if ((foundp) && (cg_pointers_found == 0))
31816 dprintf(3,(" Clearing cards [%Ix, %Ix[ ", (size_t)card_address(card),
31817 (size_t)card_address(card+1)));
31818 clear_cards (card, card+1);
31819 total_cards_cleared += 1;
31821 n_eph +=cg_pointers_found;
31822 cg_pointers_found = 0;
31823 if ((seg = heap_segment_next_rw (seg)) != 0)
31825 #ifdef BACKGROUND_GC
31826 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
31827 #endif //BACKGROUND_GC
31828 beg = heap_segment_mem (seg);
31829 end = compute_next_end (seg, low);
31830 card_word_end = card_of (align_on_card_word (end)) / card_word_width;
31831 card = card_of (beg);
31842 assert (card_set_p (card));
31844 dprintf(3,("card %Ix: o: %Ix, l: %Ix[ ",
31845 card, (size_t)o, (size_t)limit));
31847 assert (Align (size (o)) >= Align (min_obj_size));
31848 size_t s = size (o);
31849 uint8_t* next_o = o + AlignQword (s);
31855 assert (Align (s) >= Align (min_obj_size));
31856 next_o = o + AlignQword (s);
31859 dprintf (4, ("|%Ix|", (size_t)o));
31860 if (next_o < start_address)
31865 #ifdef BACKGROUND_GC
31866 if (!fgc_should_consider_object (o, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p))
31870 #endif //BACKGROUND_GC
31872 #ifdef COLLECTIBLE_CLASS
31873 if (is_collectible(o))
31875 BOOL passed_end_card_p = FALSE;
31877 if (card_of (o) > card)
31879 passed_end_card_p = card_transition (o, end, card_word_end,
31883 foundp, start_address,
31884 limit, total_cards_cleared);
31887 if ((!passed_end_card_p || foundp) && (card_of (o) == card))
31889 // card is valid and it covers the head of the object
31890 if (fn == &gc_heap::relocate_address)
31892 keep_card_live (o, n_gen, cg_pointers_found);
31896 uint8_t* class_obj = get_class_object (o);
31897 mark_through_cards_helper (&class_obj, n_gen,
31898 cg_pointers_found, fn,
31899 nhigh, next_boundary);
31903 if (passed_end_card_p)
31905 if (foundp && (card_address (card) < next_o))
31907 goto go_through_refs;
31917 #endif //COLLECTIBLE_CLASS
31919 if (contain_pointers (o))
31921 dprintf(3,("Going through %Ix", (size_t)o));
31923 go_through_object (method_table(o), o, s, poo,
31924 start_address, use_start, (o + s),
31926 if (card_of ((uint8_t*)poo) > card)
31928 BOOL passed_end_card_p = card_transition ((uint8_t*)poo, end,
31933 foundp, start_address,
31934 limit, total_cards_cleared);
31936 if (passed_end_card_p)
31938 if (foundp && (card_address (card) < next_o))
31942 if (ppstop <= (uint8_t**)start_address)
31944 else if (poo < (uint8_t**)start_address)
31945 {poo = (uint8_t**)start_address;}
31955 mark_through_cards_helper (poo, n_gen,
31956 cg_pointers_found, fn,
31957 nhigh, next_boundary);
31969 // compute the efficiency ratio of the card table
31972 generation_skip_ratio = min (((n_eph > 800) ?
31973 (int)(((float)n_gen / (float)n_eph) * 100) : 100),
31974 generation_skip_ratio);
31976 dprintf (3, ("Mloh: cross: %Id, useful: %Id, cards cleared: %Id, cards set: %Id, ratio: %d",
31977 n_eph, n_gen, total_cards_cleared, n_card_set, generation_skip_ratio));
31981 dprintf (3, ("R: Mloh: cross: %Id, useful: %Id, cards set: %Id, ratio: %d",
31982 n_eph, n_gen, n_card_set, generation_skip_ratio));
31986 void gc_heap::descr_segment (heap_segment* seg )
31989 uint8_t* x = heap_segment_mem (seg);
31990 while (x < heap_segment_allocated (seg))
31992 dprintf(2, ( "%Ix: %d ", (size_t)x, size (x)));
31993 x = x + Align(size (x));
31996 UNREFERENCED_PARAMETER(seg);
32000 void gc_heap::descr_generations_to_profiler (gen_walk_fn fn, void *context)
32002 #ifdef MULTIPLE_HEAPS
32003 int n_heaps = g_theGCHeap->GetNumberOfHeaps ();
32004 for (int i = 0; i < n_heaps; i++)
32006 gc_heap* hp = GCHeap::GetHeap(i)->pGenGCHeap;
32007 #else //MULTIPLE_HEAPS
32009 gc_heap* hp = NULL;
32011 // prefix complains about us dereferencing hp in wks build even though we only access static members
32012 // this way. not sure how to shut it up except for this ugly workaround:
32013 PREFIX_ASSUME(hp != NULL);
32014 #endif // _PREFAST_
32015 #endif //MULTIPLE_HEAPS
32017 int curr_gen_number0 = max_generation+1;
32018 while (curr_gen_number0 >= 0)
32020 generation* gen = hp->generation_of (curr_gen_number0);
32021 heap_segment* seg = generation_start_segment (gen);
32022 while (seg && (seg != hp->ephemeral_heap_segment))
32024 assert (curr_gen_number0 > 0);
32026 // report bounds from heap_segment_mem (seg) to
32027 // heap_segment_allocated (seg);
32028 // for generation # curr_gen_number0
32029 // for heap # heap_no
32031 fn(context, curr_gen_number0, heap_segment_mem (seg),
32032 heap_segment_allocated (seg),
32033 curr_gen_number0 == max_generation+1 ? heap_segment_reserved (seg) : heap_segment_allocated (seg));
32035 seg = heap_segment_next (seg);
32039 assert (seg == hp->ephemeral_heap_segment);
32040 assert (curr_gen_number0 <= max_generation);
32042 if (curr_gen_number0 == max_generation)
32044 if (heap_segment_mem (seg) < generation_allocation_start (hp->generation_of (max_generation-1)))
32046 // report bounds from heap_segment_mem (seg) to
32047 // generation_allocation_start (generation_of (max_generation-1))
32048 // for heap # heap_number
32050 fn(context, curr_gen_number0, heap_segment_mem (seg),
32051 generation_allocation_start (hp->generation_of (max_generation-1)),
32052 generation_allocation_start (hp->generation_of (max_generation-1)) );
32055 else if (curr_gen_number0 != 0)
32057 //report bounds from generation_allocation_start (generation_of (curr_gen_number0))
32058 // to generation_allocation_start (generation_of (curr_gen_number0-1))
32059 // for heap # heap_number
32061 fn(context, curr_gen_number0, generation_allocation_start (hp->generation_of (curr_gen_number0)),
32062 generation_allocation_start (hp->generation_of (curr_gen_number0-1)),
32063 generation_allocation_start (hp->generation_of (curr_gen_number0-1)));
32067 //report bounds from generation_allocation_start (generation_of (curr_gen_number0))
32068 // to heap_segment_allocated (ephemeral_heap_segment);
32069 // for heap # heap_number
32071 fn(context, curr_gen_number0, generation_allocation_start (hp->generation_of (curr_gen_number0)),
32072 heap_segment_allocated (hp->ephemeral_heap_segment),
32073 heap_segment_reserved (hp->ephemeral_heap_segment) );
32076 curr_gen_number0--;
32082 // Note that when logging is on it can take a long time to go through the free items.
32083 void gc_heap::print_free_list (int gen, heap_segment* seg)
32085 UNREFERENCED_PARAMETER(gen);
32086 UNREFERENCED_PARAMETER(seg);
32088 if (settings.concurrent == FALSE)
32090 uint8_t* seg_start = heap_segment_mem (seg);
32091 uint8_t* seg_end = heap_segment_allocated (seg);
32093 dprintf (3, ("Free list in seg %Ix:", seg_start));
32095 size_t total_free_item = 0;
32097 allocator* gen_allocator = generation_allocator (generation_of (gen));
32098 for (unsigned int b = 0; b < gen_allocator->number_of_buckets(); b++)
32100 uint8_t* fo = gen_allocator->alloc_list_head_of (b);
32103 if (fo >= seg_start && fo < seg_end)
32107 size_t free_item_len = size(fo);
32109 dprintf (3, ("[%Ix, %Ix[:%Id",
32111 (size_t)(fo + free_item_len),
32115 fo = free_list_slot (fo);
32119 dprintf (3, ("total %Id free items", total_free_item));
32125 void gc_heap::descr_generations (BOOL begin_gc_p)
32127 UNREFERENCED_PARAMETER(begin_gc_p);
32129 if (StressLog::StressLogOn(LF_GC, LL_INFO10))
32132 #ifdef MULTIPLE_HEAPS
32134 #endif //MULTIPLE_HEAPS
32136 STRESS_LOG1(LF_GC, LL_INFO10, "GC Heap %p\n", hp);
32137 for (int n = max_generation; n >= 0; --n)
32139 STRESS_LOG4(LF_GC, LL_INFO10, " Generation %d [%p, %p] cur = %p\n",
32141 generation_allocation_start(generation_of(n)),
32142 generation_allocation_limit(generation_of(n)),
32143 generation_allocation_pointer(generation_of(n)));
32145 heap_segment* seg = generation_start_segment(generation_of(n));
32148 STRESS_LOG4(LF_GC, LL_INFO10, " Segment mem %p alloc = %p used %p committed %p\n",
32149 heap_segment_mem(seg),
32150 heap_segment_allocated(seg),
32151 heap_segment_used(seg),
32152 heap_segment_committed(seg));
32153 seg = heap_segment_next(seg);
32157 #endif // STRESS_LOG
32160 dprintf (2, ("lowest_address: %Ix highest_address: %Ix",
32161 (size_t) lowest_address, (size_t) highest_address));
32162 #ifdef BACKGROUND_GC
32163 dprintf (2, ("bgc lowest_address: %Ix bgc highest_address: %Ix",
32164 (size_t) background_saved_lowest_address, (size_t) background_saved_highest_address));
32165 #endif //BACKGROUND_GC
32167 if (heap_number == 0)
32169 dprintf (1, ("total heap size: %Id, commit size: %Id", get_total_heap_size(), get_total_committed_size()));
32172 int curr_gen_number = max_generation+1;
32173 while (curr_gen_number >= 0)
32175 size_t total_gen_size = generation_size (curr_gen_number);
32176 #ifdef SIMPLE_DPRINTF
32177 dprintf (GTC_LOG, ("[%s][g%d]gen %d:, size: %Id, frag: %Id(L: %Id, O: %Id), f: %d%% %s %s %s",
32178 (begin_gc_p ? "BEG" : "END"),
32179 settings.condemned_generation,
32182 dd_fragmentation (dynamic_data_of (curr_gen_number)),
32183 generation_free_list_space (generation_of (curr_gen_number)),
32184 generation_free_obj_space (generation_of (curr_gen_number)),
32186 (int)(((double)dd_fragmentation (dynamic_data_of (curr_gen_number)) / (double)total_gen_size) * 100) :
32188 (begin_gc_p ? ("") : (settings.compaction ? "(compact)" : "(sweep)")),
32189 (settings.heap_expansion ? "(EX)" : " "),
32190 (settings.promotion ? "Promotion" : "NoPromotion")));
32192 dprintf (2, ( "Generation %d: gap size: %d, generation size: %Id, fragmentation: %Id",
32194 size (generation_allocation_start (generation_of (curr_gen_number))),
32196 dd_fragmentation (dynamic_data_of (curr_gen_number))));
32197 #endif //SIMPLE_DPRINTF
32199 generation* gen = generation_of (curr_gen_number);
32200 heap_segment* seg = generation_start_segment (gen);
32201 while (seg && (seg != ephemeral_heap_segment))
32203 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[-%Ix[ (%Id) (%Id)",
32205 (size_t)heap_segment_mem (seg),
32206 (size_t)heap_segment_allocated (seg),
32207 (size_t)heap_segment_committed (seg),
32208 (size_t)(heap_segment_allocated (seg) - heap_segment_mem (seg)),
32209 (size_t)(heap_segment_committed (seg) - heap_segment_allocated (seg))));
32210 print_free_list (curr_gen_number, seg);
32211 seg = heap_segment_next (seg);
32213 if (seg && (seg != generation_start_segment (gen)))
32215 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[",
32217 (size_t)heap_segment_mem (seg),
32218 (size_t)generation_allocation_start (generation_of (curr_gen_number-1))));
32219 print_free_list (curr_gen_number, seg);
32224 dprintf (GTC_LOG, ("g%d: [%Ix %Ix[",
32226 (size_t)generation_allocation_start (generation_of (curr_gen_number)),
32227 (size_t)(((curr_gen_number == 0)) ?
32228 (heap_segment_allocated
32229 (generation_start_segment
32230 (generation_of (curr_gen_number)))) :
32231 (generation_allocation_start
32232 (generation_of (curr_gen_number - 1))))
32234 print_free_list (curr_gen_number, seg);
32246 //-----------------------------------------------------------------------------
32248 // VM Specific support
32250 //-----------------------------------------------------------------------------
32255 unsigned int PromotedObjectCount = 0;
32256 unsigned int CreatedObjectCount = 0;
32257 unsigned int AllocDuration = 0;
32258 unsigned int AllocCount = 0;
32259 unsigned int AllocBigCount = 0;
32260 unsigned int AllocSmallCount = 0;
32261 unsigned int AllocStart = 0;
32264 //Static member variables.
32265 VOLATILE(BOOL) GCHeap::GcInProgress = FALSE;
32267 //CMCSafeLock* GCHeap::fGcLock;
32268 GCEvent *GCHeap::WaitForGCEvent = NULL;
32271 unsigned int GCHeap::GcDuration;
32273 unsigned GCHeap::GcCondemnedGeneration = 0;
32274 size_t GCHeap::totalSurvivedSize = 0;
32275 #ifdef FEATURE_PREMORTEM_FINALIZATION
32276 CFinalize* GCHeap::m_Finalize = 0;
32277 BOOL GCHeap::GcCollectClasses = FALSE;
32278 VOLATILE(int32_t) GCHeap::m_GCFLock = 0;
32280 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
32282 #ifdef BACKGROUND_GC
32283 int GCHeap::gc_stress_fgcs_in_bgc = 0;
32284 #endif // BACKGROUND_GC
32285 #ifndef MULTIPLE_HEAPS
32286 OBJECTHANDLE GCHeap::m_StressObjs[NUM_HEAP_STRESS_OBJS];
32287 int GCHeap::m_CurStressObj = 0;
32288 #endif // !MULTIPLE_HEAPS
32289 #endif // STRESS_HEAP
32290 #endif // FEATURE_REDHAWK
32292 #endif //FEATURE_PREMORTEM_FINALIZATION
32294 class NoGCRegionLockHolder
32297 NoGCRegionLockHolder()
32299 enter_spin_lock_noinstru(&g_no_gc_lock);
32302 ~NoGCRegionLockHolder()
32304 leave_spin_lock_noinstru(&g_no_gc_lock);
32308 // An explanation of locking for finalization:
32310 // Multiple threads allocate objects. During the allocation, they are serialized by
32311 // the AllocLock above. But they release that lock before they register the object
32312 // for finalization. That's because there is much contention for the alloc lock, but
32313 // finalization is presumed to be a rare case.
32315 // So registering an object for finalization must be protected by the FinalizeLock.
32317 // There is another logical queue that involves finalization. When objects registered
32318 // for finalization become unreachable, they are moved from the "registered" queue to
32319 // the "unreachable" queue. Note that this only happens inside a GC, so no other
32320 // threads can be manipulating either queue at that time. Once the GC is over and
32321 // threads are resumed, the Finalizer thread will dequeue objects from the "unreachable"
32322 // queue and call their finalizers. This dequeue operation is also protected with
32323 // the finalize lock.
32325 // At first, this seems unnecessary. Only one thread is ever enqueuing or dequeuing
32326 // on the unreachable queue (either the GC thread during a GC or the finalizer thread
32327 // when a GC is not in progress). The reason we share a lock with threads enqueuing
32328 // on the "registered" queue is that the "registered" and "unreachable" queues are
32331 // They are actually two regions of a longer list, which can only grow at one end.
32332 // So to enqueue an object to the "registered" list, you actually rotate an unreachable
32333 // object at the boundary between the logical queues, out to the other end of the
32334 // unreachable queue -- where all growing takes place. Then you move the boundary
32335 // pointer so that the gap we created at the boundary is now on the "registered"
32336 // side rather than the "unreachable" side. Now the object can be placed into the
32337 // "registered" side at that point. This is much more efficient than doing moves
32338 // of arbitrarily long regions, but it causes the two queues to require a shared lock.
32340 // Notice that Enter/LeaveFinalizeLock is not a GC-aware spin lock. Instead, it relies
32341 // on the fact that the lock will only be taken for a brief period and that it will
32342 // never provoke or allow a GC while the lock is held. This is critical. If the
32343 // FinalizeLock used enter_spin_lock (and thus sometimes enters preemptive mode to
32344 // allow a GC), then the Alloc client would have to GC protect a finalizable object
32345 // to protect against that eventuality. That is too slow!
32349 BOOL IsValidObject99(uint8_t *pObject)
32352 if (!((CObjectHeader*)pObject)->IsFree())
32353 ((CObjectHeader *) pObject)->Validate();
32354 #endif //VERIFY_HEAP
32358 #ifdef BACKGROUND_GC
32359 BOOL gc_heap::bgc_mark_array_range (heap_segment* seg,
32361 uint8_t** range_beg,
32362 uint8_t** range_end)
32364 uint8_t* seg_start = heap_segment_mem (seg);
32365 uint8_t* seg_end = (whole_seg_p ? heap_segment_reserved (seg) : align_on_mark_word (heap_segment_allocated (seg)));
32367 if ((seg_start < background_saved_highest_address) &&
32368 (seg_end > background_saved_lowest_address))
32370 *range_beg = max (seg_start, background_saved_lowest_address);
32371 *range_end = min (seg_end, background_saved_highest_address);
32380 void gc_heap::bgc_verify_mark_array_cleared (heap_segment* seg)
32382 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32383 if (recursive_gc_sync::background_running_p() && GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
32385 uint8_t* range_beg = 0;
32386 uint8_t* range_end = 0;
32388 if (bgc_mark_array_range (seg, TRUE, &range_beg, &range_end))
32390 size_t markw = mark_word_of (range_beg);
32391 size_t markw_end = mark_word_of (range_end);
32392 while (markw < markw_end)
32394 if (mark_array [markw])
32396 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
32397 markw, mark_array [markw], mark_word_address (markw)));
32402 uint8_t* p = mark_word_address (markw_end);
32403 while (p < range_end)
32405 assert (!(mark_array_marked (p)));
32410 #endif //VERIFY_HEAP && MARK_ARRAY
32413 void gc_heap::verify_mark_bits_cleared (uint8_t* obj, size_t s)
32415 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32416 size_t start_mark_bit = mark_bit_of (obj) + 1;
32417 size_t end_mark_bit = mark_bit_of (obj + s);
32418 unsigned int startbit = mark_bit_bit (start_mark_bit);
32419 unsigned int endbit = mark_bit_bit (end_mark_bit);
32420 size_t startwrd = mark_bit_word (start_mark_bit);
32421 size_t endwrd = mark_bit_word (end_mark_bit);
32422 unsigned int result = 0;
32424 unsigned int firstwrd = ~(lowbits (~0, startbit));
32425 unsigned int lastwrd = ~(highbits (~0, endbit));
32427 if (startwrd == endwrd)
32429 unsigned int wrd = firstwrd & lastwrd;
32430 result = mark_array[startwrd] & wrd;
32438 // verify the first mark word is cleared.
32441 result = mark_array[startwrd] & firstwrd;
32449 for (size_t wrdtmp = startwrd; wrdtmp < endwrd; wrdtmp++)
32451 result = mark_array[wrdtmp];
32458 // set the last mark word.
32461 result = mark_array[endwrd] & lastwrd;
32467 #endif //VERIFY_HEAP && MARK_ARRAY
32470 void gc_heap::clear_all_mark_array()
32473 //size_t num_dwords_written = 0;
32474 //size_t begin_time = GetHighPrecisionTimeStamp();
32476 generation* gen = generation_of (max_generation);
32477 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32483 if (gen != large_object_generation)
32485 gen = generation_of (max_generation+1);
32486 seg = heap_segment_rw (generation_start_segment (gen));
32494 uint8_t* range_beg = 0;
32495 uint8_t* range_end = 0;
32497 if (bgc_mark_array_range (seg, (seg == ephemeral_heap_segment), &range_beg, &range_end))
32499 size_t markw = mark_word_of (range_beg);
32500 size_t markw_end = mark_word_of (range_end);
32501 size_t size_total = (markw_end - markw) * sizeof (uint32_t);
32502 //num_dwords_written = markw_end - markw;
32504 size_t size_left = 0;
32506 assert (((size_t)&mark_array[markw] & (sizeof(PTR_PTR)-1)) == 0);
32508 if ((size_total & (sizeof(PTR_PTR) - 1)) != 0)
32510 size = (size_total & ~(sizeof(PTR_PTR) - 1));
32511 size_left = size_total - size;
32512 assert ((size_left & (sizeof (uint32_t) - 1)) == 0);
32519 memclr ((uint8_t*)&mark_array[markw], size);
32521 if (size_left != 0)
32523 uint32_t* markw_to_clear = &mark_array[markw + size / sizeof (uint32_t)];
32524 for (size_t i = 0; i < (size_left / sizeof (uint32_t)); i++)
32526 *markw_to_clear = 0;
32532 seg = heap_segment_next_rw (seg);
32535 //size_t end_time = GetHighPrecisionTimeStamp() - begin_time;
32537 //printf ("took %Id ms to clear %Id bytes\n", end_time, num_dwords_written*sizeof(uint32_t));
32539 #endif //MARK_ARRAY
32542 #endif //BACKGROUND_GC
32544 void gc_heap::verify_mark_array_cleared (heap_segment* seg)
32546 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32547 assert (card_table == g_gc_card_table);
32548 size_t markw = mark_word_of (heap_segment_mem (seg));
32549 size_t markw_end = mark_word_of (heap_segment_reserved (seg));
32551 while (markw < markw_end)
32553 if (mark_array [markw])
32555 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
32556 markw, mark_array [markw], mark_word_address (markw)));
32561 #endif //VERIFY_HEAP && MARK_ARRAY
32564 void gc_heap::verify_mark_array_cleared ()
32566 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32567 if (recursive_gc_sync::background_running_p() && GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
32569 generation* gen = generation_of (max_generation);
32570 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32576 if (gen != large_object_generation)
32578 gen = generation_of (max_generation+1);
32579 seg = heap_segment_rw (generation_start_segment (gen));
32587 bgc_verify_mark_array_cleared (seg);
32588 seg = heap_segment_next_rw (seg);
32591 #endif //VERIFY_HEAP && MARK_ARRAY
32594 void gc_heap::verify_seg_end_mark_array_cleared()
32596 #if defined (VERIFY_HEAP) && defined (MARK_ARRAY)
32597 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
32599 generation* gen = generation_of (max_generation);
32600 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32606 if (gen != large_object_generation)
32608 gen = generation_of (max_generation+1);
32609 seg = heap_segment_rw (generation_start_segment (gen));
32617 // We already cleared all mark array bits for ephemeral generations
32618 // at the beginning of bgc sweep
32619 uint8_t* from = ((seg == ephemeral_heap_segment) ?
32620 generation_allocation_start (generation_of (max_generation - 1)) :
32621 heap_segment_allocated (seg));
32622 size_t markw = mark_word_of (align_on_mark_word (from));
32623 size_t markw_end = mark_word_of (heap_segment_reserved (seg));
32625 while (from < mark_word_address (markw))
32627 if (is_mark_bit_set (from))
32629 dprintf (3, ("mark bit for %Ix was not cleared", from));
32633 from += mark_bit_pitch;
32636 while (markw < markw_end)
32638 if (mark_array [markw])
32640 dprintf (3, ("The mark bits at 0x%Ix:0x%Ix(addr: 0x%Ix) were not cleared",
32641 markw, mark_array [markw], mark_word_address (markw)));
32646 seg = heap_segment_next_rw (seg);
32649 #endif //VERIFY_HEAP && MARK_ARRAY
32652 // This function is called to make sure we don't mess up the segment list
32653 // in SOH. It's called by:
32654 // 1) begin and end of ephemeral GCs
32655 // 2) during bgc sweep when we switch segments.
32656 void gc_heap::verify_soh_segment_list()
32659 if (GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_GC)
32661 generation* gen = generation_of (max_generation);
32662 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32663 heap_segment* last_seg = 0;
32667 seg = heap_segment_next_rw (seg);
32669 if (last_seg != ephemeral_heap_segment)
32674 #endif //VERIFY_HEAP
32677 // This function can be called at any foreground GCs or blocking GCs. For background GCs,
32678 // it can be called at the end of the final marking; and at any point during background
32680 // NOTE - to be able to call this function during background sweep, we need to temporarily
32681 // NOT clear the mark array bits as we go.
32682 void gc_heap::verify_partial ()
32684 #ifdef BACKGROUND_GC
32685 //printf ("GC#%d: Verifying loh during sweep\n", settings.gc_index);
32686 //generation* gen = large_object_generation;
32687 generation* gen = generation_of (max_generation);
32688 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
32689 int align_const = get_alignment_constant (gen != large_object_generation);
32695 // Different ways to fail.
32696 BOOL mark_missed_p = FALSE;
32697 BOOL bad_ref_p = FALSE;
32698 BOOL free_ref_p = FALSE;
32704 if (gen != large_object_generation)
32707 gen = large_object_generation;
32708 align_const = get_alignment_constant (gen != large_object_generation);
32709 seg = heap_segment_rw (generation_start_segment (gen));
32718 o = heap_segment_mem (seg);
32719 end = heap_segment_allocated (seg);
32720 //printf ("validating [%Ix-[%Ix\n", o, end);
32725 BOOL marked_p = background_object_marked (o, FALSE);
32729 go_through_object_cl (method_table (o), o, s, oo,
32733 //dprintf (3, ("VOM: verifying member %Ix in obj %Ix", (size_t)*oo, o));
32734 MethodTable *pMT = method_table (*oo);
32736 if (pMT == g_gc_pFreeObjectMethodTable)
32742 if (!pMT->SanityCheck())
32745 dprintf (3, ("Bad member of %Ix %Ix",
32746 (size_t)oo, (size_t)*oo));
32750 if (current_bgc_state == bgc_final_marking)
32752 if (marked_p && !background_object_marked (*oo, FALSE))
32754 mark_missed_p = TRUE;
32763 o = o + Align(s, align_const);
32765 seg = heap_segment_next_rw (seg);
32768 //printf ("didn't find any large object large enough...\n");
32769 //printf ("finished verifying loh\n");
32770 #endif //BACKGROUND_GC
32776 gc_heap::verify_free_lists ()
32778 for (int gen_num = 0; gen_num <= max_generation+1; gen_num++)
32780 dprintf (3, ("Verifying free list for gen:%d", gen_num));
32781 allocator* gen_alloc = generation_allocator (generation_of (gen_num));
32782 size_t sz = gen_alloc->first_bucket_size();
32783 bool verify_undo_slot = (gen_num != 0) && (gen_num != max_generation+1) && !gen_alloc->discard_if_no_fit_p();
32785 for (unsigned int a_l_number = 0; a_l_number < gen_alloc->number_of_buckets(); a_l_number++)
32787 uint8_t* free_list = gen_alloc->alloc_list_head_of (a_l_number);
32791 if (!((CObjectHeader*)free_list)->IsFree())
32793 dprintf (3, ("Verifiying Heap: curr free list item %Ix isn't a free object)",
32794 (size_t)free_list));
32797 if (((a_l_number < (gen_alloc->number_of_buckets()-1))&& (unused_array_size (free_list) >= sz))
32798 || ((a_l_number != 0) && (unused_array_size (free_list) < sz/2)))
32800 dprintf (3, ("Verifiying Heap: curr free list item %Ix isn't in the right bucket",
32801 (size_t)free_list));
32804 if (verify_undo_slot && (free_list_undo (free_list) != UNDO_EMPTY))
32806 dprintf (3, ("Verifiying Heap: curr free list item %Ix has non empty undo slot",
32807 (size_t)free_list));
32810 if ((gen_num != max_generation+1)&&(object_gennum (free_list)!= gen_num))
32812 dprintf (3, ("Verifiying Heap: curr free list item %Ix is in the wrong generation free list",
32813 (size_t)free_list));
32818 free_list = free_list_slot (free_list);
32820 //verify the sanity of the tail
32821 uint8_t* tail = gen_alloc->alloc_list_tail_of (a_l_number);
32822 if (!((tail == 0) || (tail == prev)))
32824 dprintf (3, ("Verifying Heap: tail of free list is not correct"));
32829 uint8_t* head = gen_alloc->alloc_list_head_of (a_l_number);
32830 if ((head != 0) && (free_list_slot (head) != 0))
32832 dprintf (3, ("Verifying Heap: tail of free list is not correct"));
32843 gc_heap::verify_heap (BOOL begin_gc_p)
32845 int heap_verify_level = static_cast<int>(GCConfig::GetHeapVerifyLevel());
32846 size_t last_valid_brick = 0;
32847 BOOL bCurrentBrickInvalid = FALSE;
32848 BOOL large_brick_p = TRUE;
32849 size_t curr_brick = 0;
32850 size_t prev_brick = (size_t)-1;
32851 int curr_gen_num = max_generation+1;
32852 heap_segment* seg = heap_segment_in_range (generation_start_segment (generation_of (curr_gen_num ) ));
32854 PREFIX_ASSUME(seg != NULL);
32856 uint8_t* curr_object = heap_segment_mem (seg);
32857 uint8_t* prev_object = 0;
32858 uint8_t* begin_youngest = generation_allocation_start(generation_of(0));
32859 uint8_t* end_youngest = heap_segment_allocated (ephemeral_heap_segment);
32860 uint8_t* next_boundary = generation_allocation_start (generation_of (max_generation - 1));
32861 int align_const = get_alignment_constant (FALSE);
32862 size_t total_objects_verified = 0;
32863 size_t total_objects_verified_deep = 0;
32865 #ifdef BACKGROUND_GC
32866 BOOL consider_bgc_mark_p = FALSE;
32867 BOOL check_current_sweep_p = FALSE;
32868 BOOL check_saved_sweep_p = FALSE;
32869 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
32870 #endif //BACKGROUND_GC
32872 #ifdef MULTIPLE_HEAPS
32873 t_join* current_join = &gc_t_join;
32874 #ifdef BACKGROUND_GC
32875 if (settings.concurrent && (bgc_thread_id.IsCurrentThread()))
32877 // We always call verify_heap on entry of GC on the SVR GC threads.
32878 current_join = &bgc_t_join;
32880 #endif //BACKGROUND_GC
32881 #endif //MULTIPLE_HEAPS
32883 UNREFERENCED_PARAMETER(begin_gc_p);
32884 #ifdef BACKGROUND_GC
32885 dprintf (2,("[%s]GC#%d(%s): Verifying heap - begin",
32886 (begin_gc_p ? "BEG" : "END"),
32887 VolatileLoad(&settings.gc_index),
32888 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC"))));
32890 dprintf (2,("[%s]GC#%d: Verifying heap - begin",
32891 (begin_gc_p ? "BEG" : "END"), VolatileLoad(&settings.gc_index)));
32892 #endif //BACKGROUND_GC
32894 #ifndef MULTIPLE_HEAPS
32895 if ((ephemeral_low != generation_allocation_start (generation_of (max_generation - 1))) ||
32896 (ephemeral_high != heap_segment_reserved (ephemeral_heap_segment)))
32900 #endif //MULTIPLE_HEAPS
32902 #ifdef BACKGROUND_GC
32903 //don't touch the memory because the program is allocating from it.
32904 if (!settings.concurrent)
32905 #endif //BACKGROUND_GC
32907 if (!(heap_verify_level & GCConfig::HEAPVERIFY_NO_MEM_FILL))
32909 //uninit the unused portions of segments.
32910 generation* gen1 = large_object_generation;
32911 heap_segment* seg1 = heap_segment_rw (generation_start_segment (gen1));
32912 PREFIX_ASSUME(seg1 != NULL);
32918 uint8_t* clear_start = heap_segment_allocated (seg1) - plug_skew;
32919 if (heap_segment_used (seg1) > clear_start)
32921 dprintf (3, ("setting end of seg %Ix: [%Ix-[%Ix to 0xaa",
32922 heap_segment_mem (seg1),
32924 heap_segment_used (seg1)));
32925 memset (heap_segment_allocated (seg1) - plug_skew, 0xaa,
32926 (heap_segment_used (seg1) - clear_start));
32928 seg1 = heap_segment_next_rw (seg1);
32932 if (gen1 == large_object_generation)
32934 gen1 = generation_of (max_generation);
32935 seg1 = heap_segment_rw (generation_start_segment (gen1));
32936 PREFIX_ASSUME(seg1 != NULL);
32947 #ifdef MULTIPLE_HEAPS
32948 current_join->join(this, gc_join_verify_copy_table);
32949 if (current_join->joined())
32951 // in concurrent GC, new segment could be allocated when GC is working so the card brick table might not be updated at this point
32952 for (int i = 0; i < n_heaps; i++)
32954 //copy the card and brick tables
32955 if (g_gc_card_table != g_heaps[i]->card_table)
32957 g_heaps[i]->copy_brick_card_table();
32961 current_join->restart();
32964 if (g_gc_card_table != card_table)
32965 copy_brick_card_table();
32966 #endif //MULTIPLE_HEAPS
32968 //verify that the generation structures makes sense
32970 generation* gen = generation_of (max_generation);
32972 assert (generation_allocation_start (gen) ==
32973 heap_segment_mem (heap_segment_rw (generation_start_segment (gen))));
32974 int gen_num = max_generation-1;
32975 generation* prev_gen = gen;
32976 while (gen_num >= 0)
32978 gen = generation_of (gen_num);
32979 assert (generation_allocation_segment (gen) == ephemeral_heap_segment);
32980 assert (generation_allocation_start (gen) >= heap_segment_mem (ephemeral_heap_segment));
32981 assert (generation_allocation_start (gen) < heap_segment_allocated (ephemeral_heap_segment));
32983 if (generation_start_segment (prev_gen ) ==
32984 generation_start_segment (gen))
32986 assert (generation_allocation_start (prev_gen) <
32987 generation_allocation_start (gen));
32996 // Handle segment transitions
32997 if (curr_object >= heap_segment_allocated (seg))
32999 if (curr_object > heap_segment_allocated(seg))
33001 dprintf (3, ("Verifiying Heap: curr_object: %Ix > heap_segment_allocated (seg: %Ix)",
33002 (size_t)curr_object, (size_t)seg));
33005 seg = heap_segment_next_in_range (seg);
33008 #ifdef BACKGROUND_GC
33009 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
33010 #endif //BACKGROUND_GC
33011 curr_object = heap_segment_mem(seg);
33017 if (curr_gen_num == (max_generation+1))
33020 seg = heap_segment_in_range (generation_start_segment (generation_of (curr_gen_num)));
33022 PREFIX_ASSUME(seg != NULL);
33024 #ifdef BACKGROUND_GC
33025 should_check_bgc_mark (seg, &consider_bgc_mark_p, &check_current_sweep_p, &check_saved_sweep_p);
33026 #endif //BACKGROUND_GC
33027 curr_object = heap_segment_mem (seg);
33029 large_brick_p = FALSE;
33030 align_const = get_alignment_constant (TRUE);
33033 break; // Done Verifying Heap -- no more segments
33037 // Are we at the end of the youngest_generation?
33038 if (seg == ephemeral_heap_segment)
33040 if (curr_object >= end_youngest)
33042 // prev_object length is too long if we hit this int3
33043 if (curr_object > end_youngest)
33045 dprintf (3, ("Verifiying Heap: curr_object: %Ix > end_youngest: %Ix",
33046 (size_t)curr_object, (size_t)end_youngest));
33052 if ((curr_object >= next_boundary) && (curr_gen_num > 0))
33055 if (curr_gen_num > 0)
33057 next_boundary = generation_allocation_start (generation_of (curr_gen_num - 1));
33062 //if (is_mark_set (curr_object))
33064 // printf ("curr_object: %Ix is marked!",(size_t)curr_object);
33065 // FATAL_GC_ERROR();
33068 size_t s = size (curr_object);
33069 dprintf (3, ("o: %Ix, s: %d", (size_t)curr_object, s));
33072 dprintf (3, ("Verifying Heap: size of current object %Ix == 0", curr_object));
33076 // If object is not in the youngest generation, then lets
33077 // verify that the brick table is correct....
33078 if (((seg != ephemeral_heap_segment) ||
33079 (brick_of(curr_object) < brick_of(begin_youngest))))
33081 curr_brick = brick_of(curr_object);
33083 // Brick Table Verification...
33085 // On brick transition
33086 // if brick is negative
33087 // verify that brick indirects to previous valid brick
33089 // set current brick invalid flag to be flipped if we
33090 // encounter an object at the correct place
33092 if (curr_brick != prev_brick)
33094 // If the last brick we were examining had positive
33095 // entry but we never found the matching object, then
33096 // we have a problem
33097 // If prev_brick was the last one of the segment
33098 // it's ok for it to be invalid because it is never looked at
33099 if (bCurrentBrickInvalid &&
33100 (curr_brick != brick_of (heap_segment_mem (seg))) &&
33101 !heap_segment_read_only_p (seg))
33103 dprintf (3, ("curr brick %Ix invalid", curr_brick));
33109 //large objects verify the table only if they are in
33111 if ((heap_segment_reserved (seg) <= highest_address) &&
33112 (heap_segment_mem (seg) >= lowest_address) &&
33113 brick_table [curr_brick] != 0)
33115 dprintf (3, ("curr_brick %Ix for large object %Ix not set to -32768",
33116 curr_brick, (size_t)curr_object));
33121 bCurrentBrickInvalid = FALSE;
33126 // If the current brick contains a negative value make sure
33127 // that the indirection terminates at the last valid brick
33128 if (brick_table [curr_brick] <= 0)
33130 if (brick_table [curr_brick] == 0)
33132 dprintf(3, ("curr_brick %Ix for object %Ix set to 0",
33133 curr_brick, (size_t)curr_object));
33136 ptrdiff_t i = curr_brick;
33137 while ((i >= ((ptrdiff_t) brick_of (heap_segment_mem (seg)))) &&
33138 (brick_table[i] < 0))
33140 i = i + brick_table[i];
33142 if (i < ((ptrdiff_t)(brick_of (heap_segment_mem (seg))) - 1))
33144 dprintf (3, ("ptrdiff i: %Ix < brick_of (heap_segment_mem (seg)):%Ix - 1. curr_brick: %Ix",
33145 i, brick_of (heap_segment_mem (seg)),
33149 // if (i != last_valid_brick)
33150 // FATAL_GC_ERROR();
33151 bCurrentBrickInvalid = FALSE;
33153 else if (!heap_segment_read_only_p (seg))
33155 bCurrentBrickInvalid = TRUE;
33160 if (bCurrentBrickInvalid)
33162 if (curr_object == (brick_address(curr_brick) + brick_table[curr_brick] - 1))
33164 bCurrentBrickInvalid = FALSE;
33165 last_valid_brick = curr_brick;
33170 if (*((uint8_t**)curr_object) != (uint8_t *) g_gc_pFreeObjectMethodTable)
33172 #ifdef FEATURE_LOH_COMPACTION
33173 if ((curr_gen_num == (max_generation+1)) && (prev_object != 0))
33175 assert (method_table (prev_object) == g_gc_pFreeObjectMethodTable);
33177 #endif //FEATURE_LOH_COMPACTION
33179 total_objects_verified++;
33181 BOOL can_verify_deep = TRUE;
33182 #ifdef BACKGROUND_GC
33183 can_verify_deep = fgc_should_consider_object (curr_object, seg, consider_bgc_mark_p, check_current_sweep_p, check_saved_sweep_p);
33184 #endif //BACKGROUND_GC
33186 BOOL deep_verify_obj = can_verify_deep;
33187 if ((heap_verify_level & GCConfig::HEAPVERIFY_DEEP_ON_COMPACT) && !settings.compaction)
33188 deep_verify_obj = FALSE;
33190 ((CObjectHeader*)curr_object)->ValidateHeap((Object*)curr_object, deep_verify_obj);
33192 if (can_verify_deep)
33194 if (curr_gen_num > 0)
33196 BOOL need_card_p = FALSE;
33197 if (contain_pointers_or_collectible (curr_object))
33199 dprintf (4, ("curr_object: %Ix", (size_t)curr_object));
33200 size_t crd = card_of (curr_object);
33201 BOOL found_card_p = card_set_p (crd);
33203 #ifdef COLLECTIBLE_CLASS
33204 if (is_collectible(curr_object))
33206 uint8_t* class_obj = get_class_object (curr_object);
33207 if ((class_obj < ephemeral_high) && (class_obj >= next_boundary))
33211 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix pointing to class object %Ix",
33212 card_of (curr_object), (size_t)curr_object, class_obj));
33218 #endif //COLLECTIBLE_CLASS
33220 if (contain_pointers(curr_object))
33222 go_through_object_nostart
33223 (method_table(curr_object), curr_object, s, oo,
33225 if ((crd != card_of ((uint8_t*)oo)) && !found_card_p)
33227 crd = card_of ((uint8_t*)oo);
33228 found_card_p = card_set_p (crd);
33229 need_card_p = FALSE;
33231 if ((*oo < ephemeral_high) && (*oo >= next_boundary))
33233 need_card_p = TRUE;
33236 if (need_card_p && !found_card_p)
33239 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix, %Ix:%Ix[",
33240 card_of (curr_object), (size_t)curr_object,
33241 card_of (curr_object+Align(s, align_const)), (size_t)curr_object+Align(s, align_const)));
33247 if (need_card_p && !found_card_p)
33249 dprintf (3, ("Card not set, curr_object = [%Ix:%Ix, %Ix:%Ix[",
33250 card_of (curr_object), (size_t)curr_object,
33251 card_of (curr_object+Align(s, align_const)), (size_t)curr_object+Align(s, align_const)));
33256 total_objects_verified_deep++;
33260 prev_object = curr_object;
33261 prev_brick = curr_brick;
33262 curr_object = curr_object + Align(s, align_const);
33263 if (curr_object < prev_object)
33265 dprintf (3, ("overflow because of a bad object size: %Ix size %Ix", prev_object, s));
33270 #ifdef BACKGROUND_GC
33271 dprintf (2, ("(%s)(%s)(%s) total_objects_verified is %Id, total_objects_verified_deep is %Id",
33272 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p () ? "FGC" : "NGC")),
33273 (begin_gc_p ? "BEG" : "END"),
33274 ((current_c_gc_state == c_gc_state_planning) ? "in plan" : "not in plan"),
33275 total_objects_verified, total_objects_verified_deep));
33276 if (current_c_gc_state != c_gc_state_planning)
33278 assert (total_objects_verified == total_objects_verified_deep);
33280 #endif //BACKGROUND_GC
33282 verify_free_lists();
33284 #ifdef FEATURE_PREMORTEM_FINALIZATION
33285 finalize_queue->CheckFinalizerObjects();
33286 #endif // FEATURE_PREMORTEM_FINALIZATION
33289 // to be consistent with handle table APIs pass a ScanContext*
33290 // to provide the heap number. the SC isn't complete though so
33291 // limit its scope to handle table verification.
33293 sc.thread_number = heap_number;
33294 GCScan::VerifyHandleTable(max_generation, max_generation, &sc);
33297 #ifdef MULTIPLE_HEAPS
33298 current_join->join(this, gc_join_verify_objects_done);
33299 if (current_join->joined())
33300 #endif //MULTIPLE_HEAPS
33302 SyncBlockCache::GetSyncBlockCache()->VerifySyncTableEntry();
33303 #ifdef MULTIPLE_HEAPS
33304 current_join->restart();
33305 #endif //MULTIPLE_HEAPS
33308 #ifdef BACKGROUND_GC
33309 if (!settings.concurrent)
33311 if (current_c_gc_state == c_gc_state_planning)
33313 // temporarily commenting this out 'cause an FGC
33314 // could be triggered before we sweep ephemeral.
33315 //verify_seg_end_mark_array_cleared();
33319 if (settings.concurrent)
33321 verify_mark_array_cleared();
33323 dprintf (2,("GC%d(%s): Verifying heap - end",
33324 VolatileLoad(&settings.gc_index),
33325 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC"))));
33327 dprintf (2,("GC#d: Verifying heap - end", VolatileLoad(&settings.gc_index)));
33328 #endif //BACKGROUND_GC
33331 #endif //VERIFY_HEAP
33334 void GCHeap::ValidateObjectMember (Object* obj)
33337 size_t s = size (obj);
33338 uint8_t* o = (uint8_t*)obj;
33340 go_through_object_cl (method_table (obj), o, s, oo,
33342 uint8_t* child_o = *oo;
33345 dprintf (3, ("VOM: m: %Ix obj %Ix", (size_t)child_o, o));
33346 MethodTable *pMT = method_table (child_o);
33348 if (!pMT->SanityCheck()) {
33349 dprintf (3, ("Bad member of %Ix %Ix",
33350 (size_t)oo, (size_t)child_o));
33355 #endif // VERIFY_HEAP
33358 void DestructObject (CObjectHeader* hdr)
33360 UNREFERENCED_PARAMETER(hdr); // compiler bug? -- this *is*, indeed, referenced
33361 hdr->~CObjectHeader();
33364 HRESULT GCHeap::Shutdown ()
33368 GCScan::GcRuntimeStructuresValid (FALSE);
33370 // Cannot assert this, since we use SuspendEE as the mechanism to quiesce all
33371 // threads except the one performing the shutdown.
33372 // ASSERT( !GcInProgress );
33374 // Guard against any more GC occurring and against any threads blocking
33375 // for GC to complete when the GC heap is gone. This fixes a race condition
33376 // where a thread in GC is destroyed as part of process destruction and
33377 // the remaining threads block for GC complete.
33380 //EnterAllocLock();
33382 //EnterFinalizeLock();
33385 // during shutdown lot of threads are suspended
33386 // on this even, we don't want to wake them up just yet
33387 //CloseHandle (WaitForGCEvent);
33389 //find out if the global card table hasn't been used yet
33390 uint32_t* ct = &g_gc_card_table[card_word (gcard_of (g_gc_lowest_address))];
33391 if (card_table_refcount (ct) == 0)
33393 destroy_card_table (ct);
33394 g_gc_card_table = nullptr;
33396 #ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
33397 g_gc_card_bundle_table = nullptr;
33399 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
33400 SoftwareWriteWatch::StaticClose();
33401 #endif // FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
33404 //destroy all segments on the standby list
33405 while(gc_heap::segment_standby_list != 0)
33407 heap_segment* next_seg = heap_segment_next (gc_heap::segment_standby_list);
33408 #ifdef MULTIPLE_HEAPS
33409 (gc_heap::g_heaps[0])->delete_heap_segment (gc_heap::segment_standby_list, FALSE);
33410 #else //MULTIPLE_HEAPS
33411 pGenGCHeap->delete_heap_segment (gc_heap::segment_standby_list, FALSE);
33412 #endif //MULTIPLE_HEAPS
33413 gc_heap::segment_standby_list = next_seg;
33417 #ifdef MULTIPLE_HEAPS
33419 for (int i = 0; i < gc_heap::n_heaps; i ++)
33421 delete gc_heap::g_heaps[i]->vm_heap;
33422 //destroy pure GC stuff
33423 gc_heap::destroy_gc_heap (gc_heap::g_heaps[i]);
33426 gc_heap::destroy_gc_heap (pGenGCHeap);
33428 #endif //MULTIPLE_HEAPS
33429 gc_heap::shutdown_gc();
33434 // Wait until a garbage collection is complete
33435 // returns NOERROR if wait was OK, other error code if failure.
33436 // WARNING: This will not undo the must complete state. If you are
33437 // in a must complete when you call this, you'd better know what you're
33440 #ifdef FEATURE_PREMORTEM_FINALIZATION
33442 HRESULT AllocateCFinalize(CFinalize **pCFinalize)
33444 *pCFinalize = new (nothrow) CFinalize();
33445 if (*pCFinalize == NULL || !(*pCFinalize)->Initialize())
33446 return E_OUTOFMEMORY;
33450 #endif // FEATURE_PREMORTEM_FINALIZATION
33452 // init the instance heap
33453 HRESULT GCHeap::Init(size_t hn)
33455 HRESULT hres = S_OK;
33457 #ifdef MULTIPLE_HEAPS
33458 if ((pGenGCHeap = gc_heap::make_gc_heap(this, (int)hn)) == 0)
33459 hres = E_OUTOFMEMORY;
33461 UNREFERENCED_PARAMETER(hn);
33462 if (!gc_heap::make_gc_heap())
33463 hres = E_OUTOFMEMORY;
33464 #endif //MULTIPLE_HEAPS
33470 //System wide initialization
33471 HRESULT GCHeap::Initialize ()
33475 g_gc_pFreeObjectMethodTable = GCToEEInterface::GetFreeObjectMethodTable();
33476 g_num_processors = GCToOSInterface::GetTotalProcessorCount();
33477 assert(g_num_processors != 0);
33479 //Initialize the static members.
33482 CreatedObjectCount = 0;
33485 size_t seg_size = get_valid_segment_size();
33486 gc_heap::soh_segment_size = seg_size;
33487 size_t large_seg_size = get_valid_segment_size(TRUE);
33488 gc_heap::min_loh_segment_size = large_seg_size;
33489 gc_heap::min_segment_size = min (seg_size, large_seg_size);
33490 #ifdef SEG_MAPPING_TABLE
33491 gc_heap::min_segment_size_shr = index_of_set_bit (gc_heap::min_segment_size);
33492 #endif //SEG_MAPPING_TABLE
33494 #ifdef MULTIPLE_HEAPS
33495 if (GCConfig::GetNoAffinitize())
33496 gc_heap::gc_thread_no_affinitize_p = true;
33498 uint32_t nhp_from_config = static_cast<uint32_t>(GCConfig::GetHeapCount());
33500 uint32_t nhp_from_process = GCToOSInterface::GetCurrentProcessCpuCount();
33502 uint32_t nhp = ((nhp_from_config == 0) ? nhp_from_process :
33503 (min (nhp_from_config, nhp_from_process)));
33505 nhp = min (nhp, MAX_SUPPORTED_CPUS);
33507 hr = gc_heap::initialize_gc (seg_size, large_seg_size /*LHEAP_ALLOC*/, nhp);
33509 hr = gc_heap::initialize_gc (seg_size, large_seg_size /*LHEAP_ALLOC*/);
33510 #endif //MULTIPLE_HEAPS
33515 gc_heap::total_physical_mem = GCToOSInterface::GetPhysicalMemoryLimit();
33517 gc_heap::mem_one_percent = gc_heap::total_physical_mem / 100;
33518 #ifndef MULTIPLE_HEAPS
33519 gc_heap::mem_one_percent /= g_num_processors;
33520 #endif //!MULTIPLE_HEAPS
33522 // We should only use this if we are in the "many process" mode which really is only applicable
33523 // to very powerful machines - before that's implemented, temporarily I am only enabling this for 80GB+ memory.
33524 // For now I am using an estimate to calculate these numbers but this should really be obtained
33525 // programmatically going forward.
33526 // I am assuming 47 processes using WKS GC and 3 using SVR GC.
33527 // I am assuming 3 in part due to the "very high memory load" is 97%.
33528 int available_mem_th = 10;
33529 if (gc_heap::total_physical_mem >= ((uint64_t)80 * 1024 * 1024 * 1024))
33531 int adjusted_available_mem_th = 3 + (int)((float)47 / (float)(g_num_processors));
33532 available_mem_th = min (available_mem_th, adjusted_available_mem_th);
33535 gc_heap::high_memory_load_th = 100 - available_mem_th;
33538 gc_heap::youngest_gen_desired_th = gc_heap::mem_one_percent;
33541 WaitForGCEvent = new (nothrow) GCEvent;
33543 if (!WaitForGCEvent)
33545 return E_OUTOFMEMORY;
33548 if (!WaitForGCEvent->CreateManualEventNoThrow(TRUE))
33553 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
33554 #if defined (STRESS_HEAP) && !defined (MULTIPLE_HEAPS)
33555 if (GCStress<cfg_any>::IsEnabled()) {
33556 for(int i = 0; i < GCHeap::NUM_HEAP_STRESS_OBJS; i++)
33557 m_StressObjs[i] = CreateGlobalHandle(0);
33558 m_CurStressObj = 0;
33560 #endif //STRESS_HEAP && !MULTIPLE_HEAPS
33561 #endif // FEATURE_REDHAWK
33563 initGCShadow(); // If we are debugging write barriers, initialize heap shadow
33565 #ifdef MULTIPLE_HEAPS
33567 for (unsigned i = 0; i < nhp; i++)
33569 GCHeap* Hp = new (nothrow) GCHeap();
33571 return E_OUTOFMEMORY;
33573 if ((hr = Hp->Init (i))!= S_OK)
33578 // initialize numa node to heap map
33579 heap_select::init_numa_node_to_heap_map(nhp);
33582 #endif //MULTIPLE_HEAPS
33586 GCScan::GcRuntimeStructuresValid (TRUE);
33588 GCToEEInterface::DiagUpdateGenerationBounds();
33595 // GC callback functions
33596 bool GCHeap::IsPromoted(Object* object)
33599 ((CObjectHeader*)object)->Validate();
33602 uint8_t* o = (uint8_t*)object;
33604 if (gc_heap::settings.condemned_generation == max_generation)
33606 #ifdef MULTIPLE_HEAPS
33607 gc_heap* hp = gc_heap::g_heaps[0];
33609 gc_heap* hp = pGenGCHeap;
33610 #endif //MULTIPLE_HEAPS
33612 #ifdef BACKGROUND_GC
33613 if (gc_heap::settings.concurrent)
33615 bool is_marked = (!((o < hp->background_saved_highest_address) && (o >= hp->background_saved_lowest_address))||
33616 hp->background_marked (o));
33620 #endif //BACKGROUND_GC
33622 return (!((o < hp->highest_address) && (o >= hp->lowest_address))
33623 || hp->is_mark_set (o));
33628 gc_heap* hp = gc_heap::heap_of (o);
33629 return (!((o < hp->gc_high) && (o >= hp->gc_low))
33630 || hp->is_mark_set (o));
33634 size_t GCHeap::GetPromotedBytes(int heap_index)
33636 #ifdef BACKGROUND_GC
33637 if (gc_heap::settings.concurrent)
33639 return gc_heap::bpromoted_bytes (heap_index);
33642 #endif //BACKGROUND_GC
33644 return gc_heap::promoted_bytes (heap_index);
33648 unsigned int GCHeap::WhichGeneration (Object* object)
33650 gc_heap* hp = gc_heap::heap_of ((uint8_t*)object);
33651 unsigned int g = hp->object_gennum ((uint8_t*)object);
33652 dprintf (3, ("%Ix is in gen %d", (size_t)object, g));
33656 bool GCHeap::IsEphemeral (Object* object)
33658 uint8_t* o = (uint8_t*)object;
33659 gc_heap* hp = gc_heap::heap_of (o);
33660 return !!hp->ephemeral_pointer_p (o);
33663 // Return NULL if can't find next object. When EE is not suspended,
33664 // the result is not accurate: if the input arg is in gen0, the function could
33665 // return zeroed out memory as next object
33666 Object * GCHeap::NextObj (Object * object)
33669 uint8_t* o = (uint8_t*)object;
33671 #ifndef FEATURE_BASICFREEZE
33672 if (!((o < g_gc_highest_address) && (o >= g_gc_lowest_address)))
33676 #endif //!FEATURE_BASICFREEZE
33678 heap_segment * hs = gc_heap::find_segment (o, FALSE);
33684 BOOL large_object_p = heap_segment_loh_p (hs);
33685 if (large_object_p)
33686 return NULL; //could be racing with another core allocating.
33687 #ifdef MULTIPLE_HEAPS
33688 gc_heap* hp = heap_segment_heap (hs);
33689 #else //MULTIPLE_HEAPS
33691 #endif //MULTIPLE_HEAPS
33692 unsigned int g = hp->object_gennum ((uint8_t*)object);
33693 if ((g == 0) && hp->settings.demotion)
33694 return NULL;//could be racing with another core allocating.
33695 int align_const = get_alignment_constant (!large_object_p);
33696 uint8_t* nextobj = o + Align (size (o), align_const);
33697 if (nextobj <= o) // either overflow or 0 sized object.
33702 if ((nextobj < heap_segment_mem(hs)) ||
33703 (nextobj >= heap_segment_allocated(hs) && hs != hp->ephemeral_heap_segment) ||
33704 (nextobj >= hp->alloc_allocated))
33709 return (Object *)nextobj;
33712 #endif // VERIFY_HEAP
33717 #ifdef FEATURE_BASICFREEZE
33718 BOOL GCHeap::IsInFrozenSegment (Object * object)
33720 uint8_t* o = (uint8_t*)object;
33721 heap_segment * hs = gc_heap::find_segment (o, FALSE);
33722 //We create a frozen object for each frozen segment before the segment is inserted
33723 //to segment list; during ngen, we could also create frozen objects in segments which
33724 //don't belong to current GC heap.
33725 //So we return true if hs is NULL. It might create a hole about detecting invalidate
33726 //object. But given all other checks present, the hole should be very small
33727 return !hs || heap_segment_read_only_p (hs);
33729 #endif //FEATURE_BASICFREEZE
33731 #endif //VERIFY_HEAP
33733 // returns TRUE if the pointer is in one of the GC heaps.
33734 bool GCHeap::IsHeapPointer (void* vpObject, bool small_heap_only)
33736 STATIC_CONTRACT_SO_TOLERANT;
33738 // removed STATIC_CONTRACT_CAN_TAKE_LOCK here because find_segment
33739 // no longer calls GCEvent::Wait which eventually takes a lock.
33741 uint8_t* object = (uint8_t*) vpObject;
33742 #ifndef FEATURE_BASICFREEZE
33743 if (!((object < g_gc_highest_address) && (object >= g_gc_lowest_address)))
33745 #endif //!FEATURE_BASICFREEZE
33747 heap_segment * hs = gc_heap::find_segment (object, small_heap_only);
33751 #ifdef STRESS_PINNING
33752 static n_promote = 0;
33753 #endif //STRESS_PINNING
33754 // promote an object
33755 void GCHeap::Promote(Object** ppObject, ScanContext* sc, uint32_t flags)
33757 THREAD_NUMBER_FROM_CONTEXT;
33758 #ifndef MULTIPLE_HEAPS
33759 const int thread = 0;
33760 #endif //!MULTIPLE_HEAPS
33762 uint8_t* o = (uint8_t*)*ppObject;
33767 #ifdef DEBUG_DestroyedHandleValue
33768 // we can race with destroy handle during concurrent scan
33769 if (o == (uint8_t*)DEBUG_DestroyedHandleValue)
33771 #endif //DEBUG_DestroyedHandleValue
33775 gc_heap* hp = gc_heap::heap_of (o);
33777 dprintf (3, ("Promote %Ix", (size_t)o));
33779 #ifdef INTERIOR_POINTERS
33780 if (flags & GC_CALL_INTERIOR)
33782 if ((o < hp->gc_low) || (o >= hp->gc_high))
33786 if ( (o = hp->find_object (o, hp->gc_low)) == 0)
33792 #endif //INTERIOR_POINTERS
33794 #ifdef FEATURE_CONSERVATIVE_GC
33795 // For conservative GC, a value on stack may point to middle of a free object.
33796 // In this case, we don't need to promote the pointer.
33797 if (GCConfig::GetConservativeGC()
33798 && ((CObjectHeader*)o)->IsFree())
33805 ((CObjectHeader*)o)->ValidatePromote(sc, flags);
33807 UNREFERENCED_PARAMETER(sc);
33810 if (flags & GC_CALL_PINNED)
33811 hp->pin_object (o, (uint8_t**) ppObject, hp->gc_low, hp->gc_high);
33813 #ifdef STRESS_PINNING
33814 if ((++n_promote % 20) == 1)
33815 hp->pin_object (o, (uint8_t**) ppObject, hp->gc_low, hp->gc_high);
33816 #endif //STRESS_PINNING
33818 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
33819 size_t promoted_size_begin = hp->promoted_bytes (thread);
33820 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
33822 if ((o >= hp->gc_low) && (o < hp->gc_high))
33824 hpt->mark_object_simple (&o THREAD_NUMBER_ARG);
33827 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
33828 size_t promoted_size_end = hp->promoted_bytes (thread);
33831 if (sc->pCurrentDomain)
33833 sc->pCurrentDomain->RecordSurvivedBytes ((promoted_size_end - promoted_size_begin), thread);
33836 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
33838 STRESS_LOG_ROOT_PROMOTE(ppObject, o, o ? header(o)->GetMethodTable() : NULL);
33841 void GCHeap::Relocate (Object** ppObject, ScanContext* sc,
33844 UNREFERENCED_PARAMETER(sc);
33846 uint8_t* object = (uint8_t*)(Object*)(*ppObject);
33848 THREAD_NUMBER_FROM_CONTEXT;
33850 //dprintf (3, ("Relocate location %Ix\n", (size_t)ppObject));
33851 dprintf (3, ("R: %Ix", (size_t)ppObject));
33856 gc_heap* hp = gc_heap::heap_of (object);
33859 if (!(flags & GC_CALL_INTERIOR))
33861 // We cannot validate this object if it's in the condemned gen because it could
33862 // be one of the objects that were overwritten by an artificial gap due to a pinned plug.
33863 if (!((object >= hp->gc_low) && (object < hp->gc_high)))
33865 ((CObjectHeader*)object)->Validate(FALSE);
33870 dprintf (3, ("Relocate %Ix\n", (size_t)object));
33874 if ((flags & GC_CALL_INTERIOR) && gc_heap::settings.loh_compaction)
33876 if (!((object >= hp->gc_low) && (object < hp->gc_high)))
33881 if (gc_heap::loh_object_p (object))
33883 pheader = hp->find_object (object, 0);
33889 ptrdiff_t ref_offset = object - pheader;
33890 hp->relocate_address(&pheader THREAD_NUMBER_ARG);
33891 *ppObject = (Object*)(pheader + ref_offset);
33898 hp->relocate_address(&pheader THREAD_NUMBER_ARG);
33899 *ppObject = (Object*)pheader;
33902 STRESS_LOG_ROOT_RELOCATE(ppObject, object, pheader, ((!(flags & GC_CALL_INTERIOR)) ? ((Object*)object)->GetGCSafeMethodTable() : 0));
33905 /*static*/ bool GCHeap::IsObjectInFixedHeap(Object *pObj)
33907 // For now we simply look at the size of the object to determine if it in the
33908 // fixed heap or not. If the bit indicating this gets set at some point
33909 // we should key off that instead.
33910 return size( pObj ) >= LARGE_OBJECT_SIZE;
33913 #ifndef FEATURE_REDHAWK // Redhawk forces relocation a different way
33916 void StressHeapDummy ();
33918 static int32_t GCStressStartCount = -1;
33919 static int32_t GCStressCurCount = 0;
33920 static int32_t GCStressStartAtJit = -1;
33922 // the maximum number of foreground GCs we'll induce during one BGC
33923 // (this number does not include "naturally" occuring GCs).
33924 static int32_t GCStressMaxFGCsPerBGC = -1;
33926 // CLRRandom implementation can produce FPU exceptions if
33927 // the test/application run by CLR is enabling any FPU exceptions.
33928 // We want to avoid any unexpected exception coming from stress
33929 // infrastructure, so CLRRandom is not an option.
33930 // The code below is a replicate of CRT rand() implementation.
33931 // Using CRT rand() is not an option because we will interfere with the user application
33932 // that may also use it.
33933 int StressRNG(int iMaxValue)
33935 static BOOL bisRandInit = FALSE;
33936 static int lHoldrand = 1L;
33940 lHoldrand = (int)time(NULL);
33941 bisRandInit = TRUE;
33943 int randValue = (((lHoldrand = lHoldrand * 214013L + 2531011L) >> 16) & 0x7fff);
33944 return randValue % iMaxValue;
33946 #endif // STRESS_HEAP
33947 #endif // !FEATURE_REDHAWK
33949 // free up object so that things will move and then do a GC
33950 //return TRUE if GC actually happens, otherwise FALSE
33951 bool GCHeap::StressHeap(gc_alloc_context * context)
33953 #if defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
33954 alloc_context* acontext = static_cast<alloc_context*>(context);
33955 assert(context != nullptr);
33957 // if GC stress was dynamically disabled during this run we return FALSE
33958 if (!GCStressPolicy::IsEnabled())
33962 if (g_pConfig->FastGCStressLevel() && !GCToEEInterface::GetThread()->StressHeapIsEnabled()) {
33968 if ((g_pConfig->GetGCStressLevel() & EEConfig::GCSTRESS_UNIQUE)
33970 || g_pConfig->FastGCStressLevel() > 1
33973 if (!Thread::UniqueStack(&acontext)) {
33978 #ifdef BACKGROUND_GC
33979 // don't trigger a GC from the GC threads but still trigger GCs from user threads.
33980 if (GCToEEInterface::WasCurrentThreadCreatedByGC())
33984 #endif //BACKGROUND_GC
33986 if (GCStressStartAtJit == -1 || GCStressStartCount == -1)
33988 GCStressStartCount = CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_GCStressStart);
33989 GCStressStartAtJit = CLRConfig::GetConfigValue(CLRConfig::INTERNAL_GCStressStartAtJit);
33992 if (GCStressMaxFGCsPerBGC == -1)
33994 GCStressMaxFGCsPerBGC = CLRConfig::GetConfigValue(CLRConfig::INTERNAL_GCStressMaxFGCsPerBGC);
33995 if (g_pConfig->IsGCStressMix() && GCStressMaxFGCsPerBGC == -1)
33996 GCStressMaxFGCsPerBGC = 6;
34000 if (g_JitCount < GCStressStartAtJit)
34004 // Allow programmer to skip the first N Stress GCs so that you can
34005 // get to the interesting ones faster.
34006 Interlocked::Increment(&GCStressCurCount);
34007 if (GCStressCurCount < GCStressStartCount)
34010 // throttle the number of stress-induced GCs by a factor given by GCStressStep
34011 if ((GCStressCurCount % g_pConfig->GetGCStressStep()) != 0)
34016 #ifdef BACKGROUND_GC
34017 if (IsConcurrentGCEnabled() && IsConcurrentGCInProgress())
34019 // allow a maximum number of stress induced FGCs during one BGC
34020 if (gc_stress_fgcs_in_bgc >= GCStressMaxFGCsPerBGC)
34022 ++gc_stress_fgcs_in_bgc;
34024 #endif // BACKGROUND_GC
34026 if (g_pStringClass == 0)
34028 // If the String class has not been loaded, dont do any stressing. This should
34029 // be kept to a minimum to get as complete coverage as possible.
34030 _ASSERTE(g_fEEInit);
34034 #ifndef MULTIPLE_HEAPS
34035 static int32_t OneAtATime = -1;
34037 // Only bother with this if the stress level is big enough and if nobody else is
34038 // doing it right now. Note that some callers are inside the AllocLock and are
34039 // guaranteed synchronized. But others are using AllocationContexts and have no
34040 // particular synchronization.
34042 // For this latter case, we want a very high-speed way of limiting this to one
34043 // at a time. A secondary advantage is that we release part of our StressObjs
34044 // buffer sparingly but just as effectively.
34046 if (Interlocked::Increment(&OneAtATime) == 0 &&
34047 !TrackAllocations()) // Messing with object sizes can confuse the profiler (see ICorProfilerInfo::GetObjectSize)
34051 // If the current string is used up
34052 if (HndFetchHandle(m_StressObjs[m_CurStressObj]) == 0)
34054 // Populate handles with strings
34055 int i = m_CurStressObj;
34056 while(HndFetchHandle(m_StressObjs[i]) == 0)
34058 _ASSERTE(m_StressObjs[i] != 0);
34059 unsigned strLen = (LARGE_OBJECT_SIZE - 32) / sizeof(WCHAR);
34060 unsigned strSize = PtrAlign(StringObject::GetSize(strLen));
34062 // update the cached type handle before allocating
34063 SetTypeHandleOnThreadForAlloc(TypeHandle(g_pStringClass));
34064 str = (StringObject*) pGenGCHeap->allocate (strSize, acontext);
34067 str->SetMethodTable (g_pStringClass);
34068 str->SetStringLength (strLen);
34070 HndAssignHandle(m_StressObjs[i], ObjectToOBJECTREF(str));
34072 i = (i + 1) % NUM_HEAP_STRESS_OBJS;
34073 if (i == m_CurStressObj) break;
34076 // advance the current handle to the next string
34077 m_CurStressObj = (m_CurStressObj + 1) % NUM_HEAP_STRESS_OBJS;
34080 // Get the current string
34081 str = (StringObject*) OBJECTREFToObject(HndFetchHandle(m_StressObjs[m_CurStressObj]));
34084 // Chop off the end of the string and form a new object out of it.
34085 // This will 'free' an object at the begining of the heap, which will
34086 // force data movement. Note that we can only do this so many times.
34087 // before we have to move on to the next string.
34088 unsigned sizeOfNewObj = (unsigned)Align(min_obj_size * 31);
34089 if (str->GetStringLength() > sizeOfNewObj / sizeof(WCHAR))
34091 unsigned sizeToNextObj = (unsigned)Align(size(str));
34092 uint8_t* freeObj = ((uint8_t*) str) + sizeToNextObj - sizeOfNewObj;
34093 pGenGCHeap->make_unused_array (freeObj, sizeOfNewObj);
34094 str->SetStringLength(str->GetStringLength() - (sizeOfNewObj / sizeof(WCHAR)));
34098 // Let the string itself become garbage.
34099 // will be realloced next time around
34100 HndAssignHandle(m_StressObjs[m_CurStressObj], 0);
34104 Interlocked::Decrement(&OneAtATime);
34105 #endif // !MULTIPLE_HEAPS
34106 if (IsConcurrentGCEnabled())
34108 int rgen = StressRNG(10);
34110 // gen0:gen1:gen2 distribution: 40:40:20
34113 else if (rgen >= 4)
34118 GarbageCollectTry (rgen, FALSE, collection_gcstress);
34122 GarbageCollect(max_generation, FALSE, collection_gcstress);
34127 UNREFERENCED_PARAMETER(context);
34129 #endif // defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
34133 #ifdef FEATURE_PREMORTEM_FINALIZATION
34134 #define REGISTER_FOR_FINALIZATION(_object, _size) \
34135 hp->finalize_queue->RegisterForFinalization (0, (_object), (_size))
34136 #else // FEATURE_PREMORTEM_FINALIZATION
34137 #define REGISTER_FOR_FINALIZATION(_object, _size) true
34138 #endif // FEATURE_PREMORTEM_FINALIZATION
34140 #define CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(_object, _size, _register) do { \
34141 if ((_object) == NULL || ((_register) && !REGISTER_FOR_FINALIZATION(_object, _size))) \
34143 STRESS_LOG_OOM_STACK(_size); \
34149 // Small Object Allocator
34152 // Allocate small object with an alignment requirement of 8-bytes.
34154 GCHeap::AllocAlign8(gc_alloc_context* ctx, size_t size, uint32_t flags )
34156 #ifdef FEATURE_64BIT_ALIGNMENT
34162 alloc_context* acontext = static_cast<alloc_context*>(ctx);
34164 #ifdef MULTIPLE_HEAPS
34165 if (acontext->get_alloc_heap() == 0)
34167 AssignHeap (acontext);
34168 assert (acontext->get_alloc_heap());
34171 gc_heap* hp = acontext->get_alloc_heap()->pGenGCHeap;
34173 gc_heap* hp = pGenGCHeap;
34174 #endif //MULTIPLE_HEAPS
34176 return AllocAlign8Common(hp, acontext, size, flags);
34178 UNREFERENCED_PARAMETER(ctx);
34179 UNREFERENCED_PARAMETER(size);
34180 UNREFERENCED_PARAMETER(flags);
34181 assert(!"should not call GCHeap::AllocAlign8 without FEATURE_64BIT_ALIGNMENT defined!");
34183 #endif //FEATURE_64BIT_ALIGNMENT
34186 // Common code used by both variants of AllocAlign8 above.
34188 GCHeap::AllocAlign8Common(void* _hp, alloc_context* acontext, size_t size, uint32_t flags)
34190 #ifdef FEATURE_64BIT_ALIGNMENT
34196 gc_heap* hp = (gc_heap*)_hp;
34200 Object* newAlloc = NULL;
34203 #ifdef COUNT_CYCLES
34204 AllocStart = GetCycleCount32();
34206 #elif defined(ENABLE_INSTRUMENTATION)
34207 unsigned AllocStart = GetInstLogTime();
34209 #endif //COUNT_CYCLES
34212 if (size < LARGE_OBJECT_SIZE)
34218 // Depending on where in the object the payload requiring 8-byte alignment resides we might have to
34219 // align the object header on an 8-byte boundary or midway between two such boundaries. The unaligned
34220 // case is indicated to the GC via the GC_ALLOC_ALIGN8_BIAS flag.
34221 size_t desiredAlignment = (flags & GC_ALLOC_ALIGN8_BIAS) ? 4 : 0;
34223 // Retrieve the address of the next allocation from the context (note that we're inside the alloc
34224 // lock at this point).
34225 uint8_t* result = acontext->alloc_ptr;
34227 // Will an allocation at this point yield the correct alignment and fit into the remainder of the
34229 if ((((size_t)result & 7) == desiredAlignment) && ((result + size) <= acontext->alloc_limit))
34231 // Yes, we can just go ahead and make the allocation.
34232 newAlloc = (Object*) hp->allocate (size, acontext);
34233 ASSERT(((size_t)newAlloc & 7) == desiredAlignment);
34237 // No, either the next available address is not aligned in the way we require it or there's
34238 // not enough space to allocate an object of the required size. In both cases we allocate a
34239 // padding object (marked as a free object). This object's size is such that it will reverse
34240 // the alignment of the next header (asserted below).
34242 // We allocate both together then decide based on the result whether we'll format the space as
34243 // free object + real object or real object + free object.
34244 ASSERT((Align(min_obj_size) & 7) == 4);
34245 CObjectHeader *freeobj = (CObjectHeader*) hp->allocate (Align(size) + Align(min_obj_size), acontext);
34248 if (((size_t)freeobj & 7) == desiredAlignment)
34250 // New allocation has desired alignment, return this one and place the free object at the
34251 // end of the allocated space.
34252 newAlloc = (Object*)freeobj;
34253 freeobj = (CObjectHeader*)((uint8_t*)freeobj + Align(size));
34257 // New allocation is still mis-aligned, format the initial space as a free object and the
34258 // rest of the space should be correctly aligned for the real object.
34259 newAlloc = (Object*)((uint8_t*)freeobj + Align(min_obj_size));
34260 ASSERT(((size_t)newAlloc & 7) == desiredAlignment);
34262 freeobj->SetFree(min_obj_size);
34268 // The LOH always guarantees at least 8-byte alignment, regardless of platform. Moreover it doesn't
34269 // support mis-aligned object headers so we can't support biased headers as above. Luckily for us
34270 // we've managed to arrange things so the only case where we see a bias is for boxed value types and
34271 // these can never get large enough to be allocated on the LOH.
34272 ASSERT(65536 < LARGE_OBJECT_SIZE);
34273 ASSERT((flags & GC_ALLOC_ALIGN8_BIAS) == 0);
34275 alloc_context* acontext = generation_alloc_context (hp->generation_of (max_generation+1));
34277 newAlloc = (Object*) hp->allocate_large_object (size, acontext->alloc_bytes_loh);
34278 ASSERT(((size_t)newAlloc & 7) == 0);
34281 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
34284 #ifdef COUNT_CYCLES
34285 finish = GetCycleCount32();
34286 #elif defined(ENABLE_INSTRUMENTATION)
34287 finish = GetInstLogTime();
34288 #endif //COUNT_CYCLES
34289 AllocDuration += finish - AllocStart;
34294 UNREFERENCED_PARAMETER(_hp);
34295 UNREFERENCED_PARAMETER(acontext);
34296 UNREFERENCED_PARAMETER(size);
34297 UNREFERENCED_PARAMETER(flags);
34298 assert(!"Should not call GCHeap::AllocAlign8Common without FEATURE_64BIT_ALIGNMENT defined!");
34300 #endif // FEATURE_64BIT_ALIGNMENT
34304 GCHeap::AllocLHeap( size_t size, uint32_t flags REQD_ALIGN_DCL)
34313 Object* newAlloc = NULL;
34316 #ifdef COUNT_CYCLES
34317 AllocStart = GetCycleCount32();
34319 #elif defined(ENABLE_INSTRUMENTATION)
34320 unsigned AllocStart = GetInstLogTime();
34322 #endif //COUNT_CYCLES
34325 #ifdef MULTIPLE_HEAPS
34326 //take the first heap....
34327 gc_heap* hp = gc_heap::g_heaps[0];
34329 gc_heap* hp = pGenGCHeap;
34331 // prefix complains about us dereferencing hp in wks build even though we only access static members
34332 // this way. not sure how to shut it up except for this ugly workaround:
34333 PREFIX_ASSUME(hp != NULL);
34335 #endif //MULTIPLE_HEAPS
34337 alloc_context* acontext = generation_alloc_context (hp->generation_of (max_generation+1));
34339 newAlloc = (Object*) hp->allocate_large_object (size + ComputeMaxStructAlignPadLarge(requiredAlignment), acontext->alloc_bytes_loh);
34340 #ifdef FEATURE_STRUCTALIGN
34341 newAlloc = (Object*) hp->pad_for_alignment_large ((uint8_t*) newAlloc, requiredAlignment, size);
34342 #endif // FEATURE_STRUCTALIGN
34343 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
34346 #ifdef COUNT_CYCLES
34347 finish = GetCycleCount32();
34348 #elif defined(ENABLE_INSTRUMENTATION)
34349 finish = GetInstLogTime();
34350 #endif //COUNT_CYCLES
34351 AllocDuration += finish - AllocStart;
34358 GCHeap::Alloc(gc_alloc_context* context, size_t size, uint32_t flags REQD_ALIGN_DCL)
34367 Object* newAlloc = NULL;
34368 alloc_context* acontext = static_cast<alloc_context*>(context);
34371 #ifdef COUNT_CYCLES
34372 AllocStart = GetCycleCount32();
34374 #elif defined(ENABLE_INSTRUMENTATION)
34375 unsigned AllocStart = GetInstLogTime();
34377 #endif //COUNT_CYCLES
34380 #ifdef MULTIPLE_HEAPS
34381 if (acontext->get_alloc_heap() == 0)
34383 AssignHeap (acontext);
34384 assert (acontext->get_alloc_heap());
34386 #endif //MULTIPLE_HEAPS
34388 #ifdef MULTIPLE_HEAPS
34389 gc_heap* hp = acontext->get_alloc_heap()->pGenGCHeap;
34391 gc_heap* hp = pGenGCHeap;
34393 // prefix complains about us dereferencing hp in wks build even though we only access static members
34394 // this way. not sure how to shut it up except for this ugly workaround:
34395 PREFIX_ASSUME(hp != NULL);
34397 #endif //MULTIPLE_HEAPS
34399 if (size < LARGE_OBJECT_SIZE)
34405 newAlloc = (Object*) hp->allocate (size + ComputeMaxStructAlignPad(requiredAlignment), acontext);
34406 #ifdef FEATURE_STRUCTALIGN
34407 newAlloc = (Object*) hp->pad_for_alignment ((uint8_t*) newAlloc, requiredAlignment, size, acontext);
34408 #endif // FEATURE_STRUCTALIGN
34409 // ASSERT (newAlloc);
34413 newAlloc = (Object*) hp->allocate_large_object (size + ComputeMaxStructAlignPadLarge(requiredAlignment), acontext->alloc_bytes_loh);
34414 #ifdef FEATURE_STRUCTALIGN
34415 newAlloc = (Object*) hp->pad_for_alignment_large ((uint8_t*) newAlloc, requiredAlignment, size);
34416 #endif // FEATURE_STRUCTALIGN
34419 CHECK_ALLOC_AND_POSSIBLY_REGISTER_FOR_FINALIZATION(newAlloc, size, flags & GC_ALLOC_FINALIZE);
34422 #ifdef COUNT_CYCLES
34423 finish = GetCycleCount32();
34424 #elif defined(ENABLE_INSTRUMENTATION)
34425 finish = GetInstLogTime();
34426 #endif //COUNT_CYCLES
34427 AllocDuration += finish - AllocStart;
34434 GCHeap::FixAllocContext (gc_alloc_context* context, bool lockp, void* arg, void *heap)
34436 alloc_context* acontext = static_cast<alloc_context*>(context);
34437 #ifdef MULTIPLE_HEAPS
34440 acontext->alloc_count = 0;
34442 uint8_t * alloc_ptr = acontext->alloc_ptr;
34447 // The acontext->alloc_heap can be out of sync with the ptrs because
34448 // of heap re-assignment in allocate
34449 gc_heap* hp = gc_heap::heap_of (alloc_ptr);
34451 gc_heap* hp = pGenGCHeap;
34452 #endif //MULTIPLE_HEAPS
34454 if (heap == NULL || heap == hp)
34458 enter_spin_lock (&hp->more_space_lock);
34460 hp->fix_allocation_context (acontext, ((arg != 0)? TRUE : FALSE),
34461 get_alignment_constant(TRUE));
34464 leave_spin_lock (&hp->more_space_lock);
34470 GCHeap::GetContainingObject (void *pInteriorPtr, bool fCollectedGenOnly)
34472 uint8_t *o = (uint8_t*)pInteriorPtr;
34474 gc_heap* hp = gc_heap::heap_of (o);
34476 uint8_t* lowest = (fCollectedGenOnly ? hp->gc_low : hp->lowest_address);
34477 uint8_t* highest = (fCollectedGenOnly ? hp->gc_high : hp->highest_address);
34479 if (o >= lowest && o < highest)
34481 o = hp->find_object (o, lowest);
34488 return (Object *)o;
34491 BOOL should_collect_optimized (dynamic_data* dd, BOOL low_memory_p)
34493 if (dd_new_allocation (dd) < 0)
34498 if (((float)(dd_new_allocation (dd)) / (float)dd_desired_allocation (dd)) < (low_memory_p ? 0.7 : 0.3))
34506 //----------------------------------------------------------------------------
34507 // #GarbageCollector
34509 // API to ensure that a complete new garbage collection takes place
34512 GCHeap::GarbageCollect (int generation, bool low_memory_p, int mode)
34517 size_t total_allocated = 0;
34518 size_t total_desired = 0;
34519 #ifdef MULTIPLE_HEAPS
34521 for (hn = 0; hn < gc_heap::n_heaps; hn++)
34523 gc_heap* hp = gc_heap::g_heaps [hn];
34524 total_desired += dd_desired_allocation (hp->dynamic_data_of (0));
34525 total_allocated += dd_desired_allocation (hp->dynamic_data_of (0))-
34526 dd_new_allocation (hp->dynamic_data_of (0));
34529 gc_heap* hp = pGenGCHeap;
34530 total_desired = dd_desired_allocation (hp->dynamic_data_of (0));
34531 total_allocated = dd_desired_allocation (hp->dynamic_data_of (0))-
34532 dd_new_allocation (hp->dynamic_data_of (0));
34533 #endif //MULTIPLE_HEAPS
34535 if ((total_desired > gc_heap::mem_one_percent) && (total_allocated < gc_heap::mem_one_percent))
34537 dprintf (2, ("Async low mem but we've only allocated %d (< 10%% of physical mem) out of %d, returning",
34538 total_allocated, total_desired));
34545 #ifdef MULTIPLE_HEAPS
34546 gc_heap* hpt = gc_heap::g_heaps[0];
34549 #endif //MULTIPLE_HEAPS
34551 generation = (generation < 0) ? max_generation : min (generation, max_generation);
34552 dynamic_data* dd = hpt->dynamic_data_of (generation);
34554 #ifdef BACKGROUND_GC
34555 if (recursive_gc_sync::background_running_p())
34557 if ((mode == collection_optimized) || (mode & collection_non_blocking))
34561 if (mode & collection_blocking)
34563 pGenGCHeap->background_gc_wait();
34564 if (mode & collection_optimized)
34570 #endif //BACKGROUND_GC
34572 if (mode & collection_optimized)
34574 if (pGenGCHeap->gc_started)
34580 BOOL should_collect = FALSE;
34581 BOOL should_check_loh = (generation == max_generation);
34582 #ifdef MULTIPLE_HEAPS
34583 for (int i = 0; i < gc_heap::n_heaps; i++)
34585 dynamic_data* dd1 = gc_heap::g_heaps [i]->dynamic_data_of (generation);
34586 dynamic_data* dd2 = (should_check_loh ?
34587 (gc_heap::g_heaps [i]->dynamic_data_of (max_generation + 1)) :
34590 if (should_collect_optimized (dd1, low_memory_p))
34592 should_collect = TRUE;
34595 if (dd2 && should_collect_optimized (dd2, low_memory_p))
34597 should_collect = TRUE;
34602 should_collect = should_collect_optimized (dd, low_memory_p);
34603 if (!should_collect && should_check_loh)
34606 should_collect_optimized (hpt->dynamic_data_of (max_generation + 1), low_memory_p);
34608 #endif //MULTIPLE_HEAPS
34609 if (!should_collect)
34616 size_t CollectionCountAtEntry = dd_collection_count (dd);
34617 size_t BlockingCollectionCountAtEntry = gc_heap::full_gc_counts[gc_type_blocking];
34618 size_t CurrentCollectionCount = 0;
34622 CurrentCollectionCount = GarbageCollectTry(generation, low_memory_p, mode);
34624 if ((mode & collection_blocking) &&
34625 (generation == max_generation) &&
34626 (gc_heap::full_gc_counts[gc_type_blocking] == BlockingCollectionCountAtEntry))
34628 #ifdef BACKGROUND_GC
34629 if (recursive_gc_sync::background_running_p())
34631 pGenGCHeap->background_gc_wait();
34633 #endif //BACKGROUND_GC
34638 if (CollectionCountAtEntry == CurrentCollectionCount)
34647 GCHeap::GarbageCollectTry (int generation, BOOL low_memory_p, int mode)
34649 int gen = (generation < 0) ?
34650 max_generation : min (generation, max_generation);
34652 gc_reason reason = reason_empty;
34656 if (mode & collection_blocking)
34657 reason = reason_lowmemory_blocking;
34659 reason = reason_lowmemory;
34662 reason = reason_induced;
34664 if (reason == reason_induced)
34666 if (mode & collection_compacting)
34668 reason = reason_induced_compacting;
34670 else if (mode & collection_non_blocking)
34672 reason = reason_induced_noforce;
34675 else if (mode & collection_gcstress)
34677 reason = reason_gcstress;
34682 return GarbageCollectGeneration (gen, reason);
34685 void gc_heap::do_pre_gc()
34687 STRESS_LOG_GC_STACK;
34690 STRESS_LOG_GC_START(VolatileLoad(&settings.gc_index),
34691 (uint32_t)settings.condemned_generation,
34692 (uint32_t)settings.reason);
34693 #endif // STRESS_LOG
34695 #ifdef MULTIPLE_HEAPS
34696 gc_heap* hp = g_heaps[0];
34699 #endif //MULTIPLE_HEAPS
34701 #ifdef BACKGROUND_GC
34702 settings.b_state = hp->current_bgc_state;
34703 #endif //BACKGROUND_GC
34705 #ifdef BACKGROUND_GC
34706 dprintf (1, ("*GC* %d(gen0:%d)(%d)(%s)(%d)",
34707 VolatileLoad(&settings.gc_index),
34708 dd_collection_count (hp->dynamic_data_of (0)),
34709 settings.condemned_generation,
34710 (settings.concurrent ? "BGC" : (recursive_gc_sync::background_running_p() ? "FGC" : "NGC")),
34711 settings.b_state));
34713 dprintf (1, ("*GC* %d(gen0:%d)(%d)",
34714 VolatileLoad(&settings.gc_index),
34715 dd_collection_count(hp->dynamic_data_of(0)),
34716 settings.condemned_generation));
34717 #endif //BACKGROUND_GC
34719 // TODO: this can happen...it's because of the way we are calling
34720 // do_pre_gc, will fix later.
34721 //if (last_gc_index > VolatileLoad(&settings.gc_index))
34723 // FATAL_GC_ERROR();
34726 last_gc_index = VolatileLoad(&settings.gc_index);
34727 GCHeap::UpdatePreGCCounters();
34729 if (settings.concurrent)
34731 #ifdef BACKGROUND_GC
34732 full_gc_counts[gc_type_background]++;
34733 #if defined(STRESS_HEAP) && !defined(FEATURE_REDHAWK)
34734 GCHeap::gc_stress_fgcs_in_bgc = 0;
34735 #endif // STRESS_HEAP && !FEATURE_REDHAWK
34736 #endif // BACKGROUND_GC
34740 if (settings.condemned_generation == max_generation)
34742 full_gc_counts[gc_type_blocking]++;
34746 #ifdef BACKGROUND_GC
34747 if (settings.background_p)
34749 ephemeral_fgc_counts[settings.condemned_generation]++;
34751 #endif //BACKGROUND_GC
34755 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
34758 SystemDomain::ResetADSurvivedBytes();
34760 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
34763 #ifdef GC_CONFIG_DRIVEN
34764 void gc_heap::record_interesting_info_per_heap()
34766 // datapoints are always from the last blocking GC so don't record again
34768 if (!(settings.concurrent))
34770 for (int i = 0; i < max_idp_count; i++)
34772 interesting_data_per_heap[i] += interesting_data_per_gc[i];
34776 int compact_reason = get_gc_data_per_heap()->get_mechanism (gc_heap_compact);
34777 if (compact_reason >= 0)
34778 (compact_reasons_per_heap[compact_reason])++;
34779 int expand_mechanism = get_gc_data_per_heap()->get_mechanism (gc_heap_expand);
34780 if (expand_mechanism >= 0)
34781 (expand_mechanisms_per_heap[expand_mechanism])++;
34783 for (int i = 0; i < max_gc_mechanism_bits_count; i++)
34785 if (get_gc_data_per_heap()->is_mechanism_bit_set ((gc_mechanism_bit_per_heap)i))
34786 (interesting_mechanism_bits_per_heap[i])++;
34789 // h# | GC | gen | C | EX | NF | BF | ML | DM || PreS | PostS | Merge | Conv | Pre | Post | PrPo | PreP | PostP |
34790 cprintf (("%2d | %6d | %1d | %1s | %2s | %2s | %2s | %2s | %2s || %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id | %5Id |",
34792 (size_t)settings.gc_index,
34793 settings.condemned_generation,
34794 // TEMP - I am just doing this for wks GC 'cuase I wanna see the pattern of doing C/S GCs.
34795 (settings.compaction ? (((compact_reason >= 0) && gc_heap_compact_reason_mandatory_p[compact_reason]) ? "M" : "W") : ""), // compaction
34796 ((expand_mechanism >= 0)? "X" : ""), // EX
34797 ((expand_mechanism == expand_reuse_normal) ? "X" : ""), // NF
34798 ((expand_mechanism == expand_reuse_bestfit) ? "X" : ""), // BF
34799 (get_gc_data_per_heap()->is_mechanism_bit_set (gc_mark_list_bit) ? "X" : ""), // ML
34800 (get_gc_data_per_heap()->is_mechanism_bit_set (gc_demotion_bit) ? "X" : ""), // DM
34801 interesting_data_per_gc[idp_pre_short],
34802 interesting_data_per_gc[idp_post_short],
34803 interesting_data_per_gc[idp_merged_pin],
34804 interesting_data_per_gc[idp_converted_pin],
34805 interesting_data_per_gc[idp_pre_pin],
34806 interesting_data_per_gc[idp_post_pin],
34807 interesting_data_per_gc[idp_pre_and_post_pin],
34808 interesting_data_per_gc[idp_pre_short_padded],
34809 interesting_data_per_gc[idp_post_short_padded]));
34812 void gc_heap::record_global_mechanisms()
34814 for (int i = 0; i < max_global_mechanisms_count; i++)
34816 if (gc_data_global.get_mechanism_p ((gc_global_mechanism_p)i))
34818 ::record_global_mechanism (i);
34823 BOOL gc_heap::should_do_sweeping_gc (BOOL compact_p)
34825 if (!compact_ratio)
34826 return (!compact_p);
34828 size_t compact_count = compact_or_sweep_gcs[0];
34829 size_t sweep_count = compact_or_sweep_gcs[1];
34831 size_t total_count = compact_count + sweep_count;
34832 BOOL should_compact = compact_p;
34833 if (total_count > 3)
34837 int temp_ratio = (int)((compact_count + 1) * 100 / (total_count + 1));
34838 if (temp_ratio > compact_ratio)
34840 // cprintf (("compact would be: %d, total_count: %d, ratio would be %d%% > target\n",
34841 // (compact_count + 1), (total_count + 1), temp_ratio));
34842 should_compact = FALSE;
34847 int temp_ratio = (int)((sweep_count + 1) * 100 / (total_count + 1));
34848 if (temp_ratio > (100 - compact_ratio))
34850 // cprintf (("sweep would be: %d, total_count: %d, ratio would be %d%% > target\n",
34851 // (sweep_count + 1), (total_count + 1), temp_ratio));
34852 should_compact = TRUE;
34857 return !should_compact;
34859 #endif //GC_CONFIG_DRIVEN
34861 void gc_heap::do_post_gc()
34863 if (!settings.concurrent)
34869 #ifdef COUNT_CYCLES
34870 AllocStart = GetCycleCount32();
34872 AllocStart = clock();
34873 #endif //COUNT_CYCLES
34876 #ifdef MULTIPLE_HEAPS
34877 gc_heap* hp = g_heaps[0];
34880 #endif //MULTIPLE_HEAPS
34882 GCToEEInterface::GcDone(settings.condemned_generation);
34884 GCToEEInterface::DiagGCEnd(VolatileLoad(&settings.gc_index),
34885 (uint32_t)settings.condemned_generation,
34886 (uint32_t)settings.reason,
34887 !!settings.concurrent);
34889 //dprintf (1, (" ****end of Garbage Collection**** %d(gen0:%d)(%d)",
34890 dprintf (1, ("*EGC* %d(gen0:%d)(%d)(%s)",
34891 VolatileLoad(&settings.gc_index),
34892 dd_collection_count(hp->dynamic_data_of(0)),
34893 settings.condemned_generation,
34894 (settings.concurrent ? "BGC" : "GC")));
34896 if (settings.exit_memory_load != 0)
34897 last_gc_memory_load = settings.exit_memory_load;
34898 else if (settings.entry_memory_load != 0)
34899 last_gc_memory_load = settings.entry_memory_load;
34901 last_gc_heap_size = get_total_heap_size();
34902 last_gc_fragmentation = get_total_fragmentation();
34904 GCHeap::UpdatePostGCCounters();
34905 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
34906 //if (g_fEnableARM)
34908 // SystemDomain::GetADSurvivedBytes();
34910 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
34913 STRESS_LOG_GC_END(VolatileLoad(&settings.gc_index),
34914 (uint32_t)settings.condemned_generation,
34915 (uint32_t)settings.reason);
34916 #endif // STRESS_LOG
34918 #ifdef GC_CONFIG_DRIVEN
34919 if (!settings.concurrent)
34921 if (settings.compaction)
34922 (compact_or_sweep_gcs[0])++;
34924 (compact_or_sweep_gcs[1])++;
34927 #ifdef MULTIPLE_HEAPS
34928 for (int i = 0; i < n_heaps; i++)
34929 g_heaps[i]->record_interesting_info_per_heap();
34931 record_interesting_info_per_heap();
34932 #endif //MULTIPLE_HEAPS
34933 record_global_mechanisms();
34934 #endif //GC_CONFIG_DRIVEN
34937 unsigned GCHeap::GetGcCount()
34939 return (unsigned int)VolatileLoad(&pGenGCHeap->settings.gc_index);
34943 GCHeap::GarbageCollectGeneration (unsigned int gen, gc_reason reason)
34945 dprintf (2, ("triggered a GC!"));
34947 #ifdef MULTIPLE_HEAPS
34948 gc_heap* hpt = gc_heap::g_heaps[0];
34951 #endif //MULTIPLE_HEAPS
34952 bool cooperative_mode = true;
34953 dynamic_data* dd = hpt->dynamic_data_of (gen);
34954 size_t localCount = dd_collection_count (dd);
34956 enter_spin_lock (&gc_heap::gc_lock);
34957 dprintf (SPINLOCK_LOG, ("GC Egc"));
34958 ASSERT_HOLDING_SPIN_LOCK(&gc_heap::gc_lock);
34960 //don't trigger another GC if one was already in progress
34961 //while waiting for the lock
34963 size_t col_count = dd_collection_count (dd);
34965 if (localCount != col_count)
34967 #ifdef SYNCHRONIZATION_STATS
34968 gc_lock_contended++;
34969 #endif //SYNCHRONIZATION_STATS
34970 dprintf (SPINLOCK_LOG, ("no need GC Lgc"));
34971 leave_spin_lock (&gc_heap::gc_lock);
34973 // We don't need to release msl here 'cause this means a GC
34974 // has happened and would have release all msl's.
34979 #ifdef COUNT_CYCLES
34980 int gc_start = GetCycleCount32();
34981 #endif //COUNT_CYCLES
34984 #ifdef COUNT_CYCLES
34985 AllocDuration += GetCycleCount32() - AllocStart;
34987 AllocDuration += clock() - AllocStart;
34988 #endif //COUNT_CYCLES
34991 gc_heap::g_low_memory_status = (reason == reason_lowmemory) ||
34992 (reason == reason_lowmemory_blocking) ||
34993 (gc_heap::latency_level == latency_level_memory_footprint);
34995 gc_trigger_reason = reason;
34997 #ifdef MULTIPLE_HEAPS
34998 for (int i = 0; i < gc_heap::n_heaps; i++)
35000 gc_heap::g_heaps[i]->reset_gc_done();
35003 gc_heap::reset_gc_done();
35004 #endif //MULTIPLE_HEAPS
35006 gc_heap::gc_started = TRUE;
35009 init_sync_log_stats();
35011 #ifndef MULTIPLE_HEAPS
35012 cooperative_mode = gc_heap::enable_preemptive ();
35014 dprintf (2, ("Suspending EE"));
35015 BEGIN_TIMING(suspend_ee_during_log);
35016 GCToEEInterface::SuspendEE(SUSPEND_FOR_GC);
35017 END_TIMING(suspend_ee_during_log);
35018 gc_heap::proceed_with_gc_p = gc_heap::should_proceed_with_gc();
35019 gc_heap::disable_preemptive (cooperative_mode);
35020 if (gc_heap::proceed_with_gc_p)
35021 pGenGCHeap->settings.init_mechanisms();
35023 gc_heap::update_collection_counts_for_no_gc();
35025 #endif //!MULTIPLE_HEAPS
35028 // MAP_EVENT_MONITORS(EE_MONITOR_GARBAGE_COLLECTIONS, NotifyEvent(EE_EVENT_TYPE_GC_STARTED, 0));
35031 #ifdef COUNT_CYCLES
35034 start = GetCycleCount32();
35039 #endif //COUNT_CYCLES
35040 PromotedObjectCount = 0;
35043 unsigned int condemned_generation_number = gen;
35045 // We want to get a stack from the user thread that triggered the GC
35046 // instead of on the GC thread which is the case for Server GC.
35047 // But we are doing it for Workstation GC as well to be uniform.
35048 FIRE_EVENT(GCTriggered, static_cast<uint32_t>(reason));
35050 #ifdef MULTIPLE_HEAPS
35051 GcCondemnedGeneration = condemned_generation_number;
35053 cooperative_mode = gc_heap::enable_preemptive ();
35055 BEGIN_TIMING(gc_during_log);
35056 gc_heap::ee_suspend_event.Set();
35057 gc_heap::wait_for_gc_done();
35058 END_TIMING(gc_during_log);
35060 gc_heap::disable_preemptive (cooperative_mode);
35062 condemned_generation_number = GcCondemnedGeneration;
35064 if (gc_heap::proceed_with_gc_p)
35066 BEGIN_TIMING(gc_during_log);
35067 pGenGCHeap->garbage_collect (condemned_generation_number);
35068 END_TIMING(gc_during_log);
35070 #endif //MULTIPLE_HEAPS
35073 #ifdef COUNT_CYCLES
35074 finish = GetCycleCount32();
35077 #endif //COUNT_CYCLES
35078 GcDuration += finish - start;
35080 ("<GC# %d> Condemned: %d, Duration: %d, total: %d Alloc Avg: %d, Small Objects:%d Large Objects:%d",
35081 VolatileLoad(&pGenGCHeap->settings.gc_index), condemned_generation_number,
35082 finish - start, GcDuration,
35083 AllocCount ? (AllocDuration / AllocCount) : 0,
35084 AllocSmallCount, AllocBigCount));
35089 #ifdef BACKGROUND_GC
35090 // We are deciding whether we should fire the alloc wait end event here
35091 // because in begin_foreground we could be calling end_foreground
35092 // if we need to retry.
35093 if (gc_heap::alloc_wait_event_p)
35095 hpt->fire_alloc_wait_event_end (awr_fgc_wait_for_bgc);
35096 gc_heap::alloc_wait_event_p = FALSE;
35098 #endif //BACKGROUND_GC
35100 #ifndef MULTIPLE_HEAPS
35101 #ifdef BACKGROUND_GC
35102 if (!gc_heap::dont_restart_ee_p)
35104 #endif //BACKGROUND_GC
35105 BEGIN_TIMING(restart_ee_during_log);
35106 GCToEEInterface::RestartEE(TRUE);
35107 END_TIMING(restart_ee_during_log);
35108 #ifdef BACKGROUND_GC
35110 #endif //BACKGROUND_GC
35111 #endif //!MULTIPLE_HEAPS
35113 #ifdef COUNT_CYCLES
35114 printf ("GC: %d Time: %d\n", GcCondemnedGeneration,
35115 GetCycleCount32() - gc_start);
35116 #endif //COUNT_CYCLES
35118 #ifndef MULTIPLE_HEAPS
35119 process_sync_log_stats();
35120 gc_heap::gc_started = FALSE;
35121 gc_heap::set_gc_done();
35122 dprintf (SPINLOCK_LOG, ("GC Lgc"));
35123 leave_spin_lock (&gc_heap::gc_lock);
35124 #endif //!MULTIPLE_HEAPS
35126 #ifdef FEATURE_PREMORTEM_FINALIZATION
35127 GCToEEInterface::EnableFinalization(!pGenGCHeap->settings.concurrent && pGenGCHeap->settings.found_finalizers);
35128 #endif // FEATURE_PREMORTEM_FINALIZATION
35130 return dd_collection_count (dd);
35133 size_t GCHeap::GetTotalBytesInUse ()
35135 #ifdef MULTIPLE_HEAPS
35136 //enumarate all the heaps and get their size.
35137 size_t tot_size = 0;
35138 for (int i = 0; i < gc_heap::n_heaps; i++)
35140 GCHeap* Hp = gc_heap::g_heaps [i]->vm_heap;
35141 tot_size += Hp->ApproxTotalBytesInUse (FALSE);
35145 return ApproxTotalBytesInUse ();
35146 #endif //MULTIPLE_HEAPS
35149 int GCHeap::CollectionCount (int generation, int get_bgc_fgc_count)
35151 if (get_bgc_fgc_count != 0)
35153 #ifdef BACKGROUND_GC
35154 if (generation == max_generation)
35156 return (int)(gc_heap::full_gc_counts[gc_type_background]);
35160 return (int)(gc_heap::ephemeral_fgc_counts[generation]);
35164 #endif //BACKGROUND_GC
35167 #ifdef MULTIPLE_HEAPS
35168 gc_heap* hp = gc_heap::g_heaps [0];
35169 #else //MULTIPLE_HEAPS
35170 gc_heap* hp = pGenGCHeap;
35171 #endif //MULTIPLE_HEAPS
35172 if (generation > max_generation)
35175 return (int)dd_collection_count (hp->dynamic_data_of (generation));
35178 size_t GCHeap::ApproxTotalBytesInUse(BOOL small_heap_only)
35180 size_t totsize = 0;
35182 //ASSERT(InMustComplete());
35183 enter_spin_lock (&pGenGCHeap->gc_lock);
35185 heap_segment* eph_seg = generation_allocation_segment (pGenGCHeap->generation_of (0));
35186 // Get small block heap size info
35187 totsize = (pGenGCHeap->alloc_allocated - heap_segment_mem (eph_seg));
35188 heap_segment* seg1 = generation_start_segment (pGenGCHeap->generation_of (max_generation));
35189 while (seg1 != eph_seg)
35191 totsize += heap_segment_allocated (seg1) -
35192 heap_segment_mem (seg1);
35193 seg1 = heap_segment_next (seg1);
35196 //discount the fragmentation
35197 for (int i = 0; i <= max_generation; i++)
35199 generation* gen = pGenGCHeap->generation_of (i);
35200 totsize -= (generation_free_list_space (gen) + generation_free_obj_space (gen));
35203 if (!small_heap_only)
35205 heap_segment* seg2 = generation_start_segment (pGenGCHeap->generation_of (max_generation+1));
35209 totsize += heap_segment_allocated (seg2) -
35210 heap_segment_mem (seg2);
35211 seg2 = heap_segment_next (seg2);
35214 //discount the fragmentation
35215 generation* loh_gen = pGenGCHeap->generation_of (max_generation+1);
35216 size_t frag = generation_free_list_space (loh_gen) + generation_free_obj_space (loh_gen);
35219 leave_spin_lock (&pGenGCHeap->gc_lock);
35223 #ifdef MULTIPLE_HEAPS
35224 void GCHeap::AssignHeap (alloc_context* acontext)
35226 // Assign heap based on processor
35227 acontext->set_alloc_heap(GetHeap(heap_select::select_heap(acontext, 0)));
35228 acontext->set_home_heap(acontext->get_alloc_heap());
35230 GCHeap* GCHeap::GetHeap (int n)
35232 assert (n < gc_heap::n_heaps);
35233 return gc_heap::g_heaps [n]->vm_heap;
35235 #endif //MULTIPLE_HEAPS
35237 bool GCHeap::IsThreadUsingAllocationContextHeap(gc_alloc_context* context, int thread_number)
35239 alloc_context* acontext = static_cast<alloc_context*>(context);
35240 #ifdef MULTIPLE_HEAPS
35241 return ((acontext->get_home_heap() == GetHeap(thread_number)) ||
35242 ((acontext->get_home_heap() == 0) && (thread_number == 0)));
35244 UNREFERENCED_PARAMETER(acontext);
35245 UNREFERENCED_PARAMETER(thread_number);
35247 #endif //MULTIPLE_HEAPS
35250 // Returns the number of processors required to trigger the use of thread based allocation contexts
35251 int GCHeap::GetNumberOfHeaps ()
35253 #ifdef MULTIPLE_HEAPS
35254 return gc_heap::n_heaps;
35257 #endif //MULTIPLE_HEAPS
35261 in this way we spend extra time cycling through all the heaps while create the handle
35262 it ought to be changed by keeping alloc_context.home_heap as number (equals heap_number)
35264 int GCHeap::GetHomeHeapNumber ()
35266 #ifdef MULTIPLE_HEAPS
35267 Thread *pThread = GCToEEInterface::GetThread();
35268 for (int i = 0; i < gc_heap::n_heaps; i++)
35272 gc_alloc_context* ctx = GCToEEInterface::GetAllocContext();
35273 GCHeap *hp = static_cast<alloc_context*>(ctx)->get_home_heap();
35274 if (hp == gc_heap::g_heaps[i]->vm_heap) return i;
35280 #endif //MULTIPLE_HEAPS
35283 unsigned int GCHeap::GetCondemnedGeneration()
35285 return gc_heap::settings.condemned_generation;
35288 void GCHeap::GetMemoryInfo(uint32_t* highMemLoadThreshold,
35289 uint64_t* totalPhysicalMem,
35290 uint32_t* lastRecordedMemLoad,
35291 size_t* lastRecordedHeapSize,
35292 size_t* lastRecordedFragmentation)
35294 *highMemLoadThreshold = gc_heap::high_memory_load_th;
35295 *totalPhysicalMem = gc_heap::total_physical_mem;
35296 *lastRecordedMemLoad = gc_heap::last_gc_memory_load;
35297 *lastRecordedHeapSize = gc_heap::last_gc_heap_size;
35298 *lastRecordedFragmentation = gc_heap::last_gc_fragmentation;
35301 int GCHeap::GetGcLatencyMode()
35303 return (int)(pGenGCHeap->settings.pause_mode);
35306 int GCHeap::SetGcLatencyMode (int newLatencyMode)
35308 if (gc_heap::settings.pause_mode == pause_no_gc)
35309 return (int)set_pause_mode_no_gc;
35311 gc_pause_mode new_mode = (gc_pause_mode)newLatencyMode;
35313 if (new_mode == pause_low_latency)
35315 #ifndef MULTIPLE_HEAPS
35316 pGenGCHeap->settings.pause_mode = new_mode;
35317 #endif //!MULTIPLE_HEAPS
35319 else if (new_mode == pause_sustained_low_latency)
35321 #ifdef BACKGROUND_GC
35322 if (gc_heap::gc_can_use_concurrent)
35324 pGenGCHeap->settings.pause_mode = new_mode;
35326 #endif //BACKGROUND_GC
35330 pGenGCHeap->settings.pause_mode = new_mode;
35333 #ifdef BACKGROUND_GC
35334 if (recursive_gc_sync::background_running_p())
35336 // If we get here, it means we are doing an FGC. If the pause
35337 // mode was altered we will need to save it in the BGC settings.
35338 if (gc_heap::saved_bgc_settings.pause_mode != new_mode)
35340 gc_heap::saved_bgc_settings.pause_mode = new_mode;
35343 #endif //BACKGROUND_GC
35345 return (int)set_pause_mode_success;
35348 int GCHeap::GetLOHCompactionMode()
35350 return pGenGCHeap->loh_compaction_mode;
35353 void GCHeap::SetLOHCompactionMode (int newLOHCompactionyMode)
35355 #ifdef FEATURE_LOH_COMPACTION
35356 pGenGCHeap->loh_compaction_mode = (gc_loh_compaction_mode)newLOHCompactionyMode;
35357 #endif //FEATURE_LOH_COMPACTION
35360 bool GCHeap::RegisterForFullGCNotification(uint32_t gen2Percentage,
35361 uint32_t lohPercentage)
35363 #ifdef MULTIPLE_HEAPS
35364 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35366 gc_heap* hp = gc_heap::g_heaps [hn];
35367 hp->fgn_last_alloc = dd_new_allocation (hp->dynamic_data_of (0));
35369 #else //MULTIPLE_HEAPS
35370 pGenGCHeap->fgn_last_alloc = dd_new_allocation (pGenGCHeap->dynamic_data_of (0));
35371 #endif //MULTIPLE_HEAPS
35373 pGenGCHeap->full_gc_approach_event.Reset();
35374 pGenGCHeap->full_gc_end_event.Reset();
35375 pGenGCHeap->full_gc_approach_event_set = false;
35377 pGenGCHeap->fgn_maxgen_percent = gen2Percentage;
35378 pGenGCHeap->fgn_loh_percent = lohPercentage;
35383 bool GCHeap::CancelFullGCNotification()
35385 pGenGCHeap->fgn_maxgen_percent = 0;
35386 pGenGCHeap->fgn_loh_percent = 0;
35388 pGenGCHeap->full_gc_approach_event.Set();
35389 pGenGCHeap->full_gc_end_event.Set();
35394 int GCHeap::WaitForFullGCApproach(int millisecondsTimeout)
35396 dprintf (2, ("WFGA: Begin wait"));
35397 int result = gc_heap::full_gc_wait (&(pGenGCHeap->full_gc_approach_event), millisecondsTimeout);
35398 dprintf (2, ("WFGA: End wait"));
35402 int GCHeap::WaitForFullGCComplete(int millisecondsTimeout)
35404 dprintf (2, ("WFGE: Begin wait"));
35405 int result = gc_heap::full_gc_wait (&(pGenGCHeap->full_gc_end_event), millisecondsTimeout);
35406 dprintf (2, ("WFGE: End wait"));
35410 int GCHeap::StartNoGCRegion(uint64_t totalSize, bool lohSizeKnown, uint64_t lohSize, bool disallowFullBlockingGC)
35412 NoGCRegionLockHolder lh;
35414 dprintf (1, ("begin no gc called"));
35415 start_no_gc_region_status status = gc_heap::prepare_for_no_gc_region (totalSize, lohSizeKnown, lohSize, disallowFullBlockingGC);
35416 if (status == start_no_gc_success)
35418 GarbageCollect (max_generation);
35419 status = gc_heap::get_start_no_gc_region_status();
35422 if (status != start_no_gc_success)
35423 gc_heap::handle_failure_for_no_gc();
35425 return (int)status;
35428 int GCHeap::EndNoGCRegion()
35430 NoGCRegionLockHolder lh;
35431 return (int)gc_heap::end_no_gc_region();
35434 void GCHeap::PublishObject (uint8_t* Obj)
35436 #ifdef BACKGROUND_GC
35437 gc_heap* hp = gc_heap::heap_of (Obj);
35438 hp->bgc_alloc_lock->loh_alloc_done (Obj);
35439 #endif //BACKGROUND_GC
35442 // The spec for this one isn't clear. This function
35443 // returns the size that can be allocated without
35444 // triggering a GC of any kind.
35445 size_t GCHeap::ApproxFreeBytes()
35448 //ASSERT(InMustComplete());
35449 enter_spin_lock (&pGenGCHeap->gc_lock);
35451 generation* gen = pGenGCHeap->generation_of (0);
35452 size_t res = generation_allocation_limit (gen) - generation_allocation_pointer (gen);
35454 leave_spin_lock (&pGenGCHeap->gc_lock);
35459 HRESULT GCHeap::GetGcCounters(int gen, gc_counters* counters)
35461 if ((gen < 0) || (gen > max_generation))
35463 #ifdef MULTIPLE_HEAPS
35464 counters->current_size = 0;
35465 counters->promoted_size = 0;
35466 counters->collection_count = 0;
35468 //enumarate all the heaps and get their counters.
35469 for (int i = 0; i < gc_heap::n_heaps; i++)
35471 dynamic_data* dd = gc_heap::g_heaps [i]->dynamic_data_of (gen);
35473 counters->current_size += dd_current_size (dd);
35474 counters->promoted_size += dd_promoted_size (dd);
35476 counters->collection_count += dd_collection_count (dd);
35479 dynamic_data* dd = pGenGCHeap->dynamic_data_of (gen);
35480 counters->current_size = dd_current_size (dd);
35481 counters->promoted_size = dd_promoted_size (dd);
35482 counters->collection_count = dd_collection_count (dd);
35483 #endif //MULTIPLE_HEAPS
35487 // Get the segment size to use, making sure it conforms.
35488 size_t GCHeap::GetValidSegmentSize(bool large_seg)
35490 return get_valid_segment_size (large_seg);
35493 // Get the max gen0 heap size, making sure it conforms.
35494 size_t GCHeap::GetValidGen0MaxSize(size_t seg_size)
35496 size_t gen0size = static_cast<size_t>(GCConfig::GetGen0Size());
35498 if ((gen0size == 0) || !g_theGCHeap->IsValidGen0MaxSize(gen0size))
35501 // performance data seems to indicate halving the size results
35502 // in optimal perf. Ask for adjusted gen0 size.
35503 gen0size = max(GCToOSInterface::GetCacheSizePerLogicalCpu(FALSE),(256*1024));
35505 // if gen0 size is too large given the available memory, reduce it.
35506 // Get true cache size, as we don't want to reduce below this.
35507 size_t trueSize = max(GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE),(256*1024));
35508 dprintf (2, ("cache: %Id-%Id, cpu: %Id",
35509 GCToOSInterface::GetCacheSizePerLogicalCpu(FALSE),
35510 GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE)));
35512 int n_heaps = gc_heap::n_heaps;
35514 size_t trueSize = GCToOSInterface::GetCacheSizePerLogicalCpu(TRUE);
35515 gen0size = max((4*trueSize/5),(256*1024));
35516 trueSize = max(trueSize, (256*1024));
35520 // if the total min GC across heaps will exceed 1/6th of available memory,
35521 // then reduce the min GC size until it either fits or has been reduced to cache size.
35522 while ((gen0size * n_heaps) > GCToOSInterface::GetPhysicalMemoryLimit() / 6)
35524 gen0size = gen0size / 2;
35525 if (gen0size <= trueSize)
35527 gen0size = trueSize;
35533 // Generation 0 must never be more than 1/2 the segment size.
35534 if (gen0size >= (seg_size / 2))
35535 gen0size = seg_size / 2;
35540 void GCHeap::SetReservedVMLimit (size_t vmlimit)
35542 gc_heap::reserved_memory_limit = vmlimit;
35546 //versions of same method on each heap
35548 #ifdef FEATURE_PREMORTEM_FINALIZATION
35550 Object* GCHeap::GetNextFinalizableObject()
35553 #ifdef MULTIPLE_HEAPS
35555 //return the first non critical one in the first queue.
35556 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35558 gc_heap* hp = gc_heap::g_heaps [hn];
35559 Object* O = hp->finalize_queue->GetNextFinalizableObject(TRUE);
35563 //return the first non crtitical/critical one in the first queue.
35564 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35566 gc_heap* hp = gc_heap::g_heaps [hn];
35567 Object* O = hp->finalize_queue->GetNextFinalizableObject(FALSE);
35574 #else //MULTIPLE_HEAPS
35575 return pGenGCHeap->finalize_queue->GetNextFinalizableObject();
35576 #endif //MULTIPLE_HEAPS
35580 size_t GCHeap::GetNumberFinalizableObjects()
35582 #ifdef MULTIPLE_HEAPS
35584 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35586 gc_heap* hp = gc_heap::g_heaps [hn];
35587 cnt += hp->finalize_queue->GetNumberFinalizableObjects();
35592 #else //MULTIPLE_HEAPS
35593 return pGenGCHeap->finalize_queue->GetNumberFinalizableObjects();
35594 #endif //MULTIPLE_HEAPS
35597 size_t GCHeap::GetFinalizablePromotedCount()
35599 #ifdef MULTIPLE_HEAPS
35602 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35604 gc_heap* hp = gc_heap::g_heaps [hn];
35605 cnt += hp->finalize_queue->GetPromotedCount();
35609 #else //MULTIPLE_HEAPS
35610 return pGenGCHeap->finalize_queue->GetPromotedCount();
35611 #endif //MULTIPLE_HEAPS
35614 bool GCHeap::FinalizeAppDomain(void *pDomain, bool fRunFinalizers)
35616 #ifdef MULTIPLE_HEAPS
35617 bool foundp = false;
35618 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35620 gc_heap* hp = gc_heap::g_heaps [hn];
35621 if (hp->finalize_queue->FinalizeAppDomain (pDomain, fRunFinalizers))
35626 #else //MULTIPLE_HEAPS
35627 return pGenGCHeap->finalize_queue->FinalizeAppDomain (pDomain, fRunFinalizers);
35628 #endif //MULTIPLE_HEAPS
35631 bool GCHeap::ShouldRestartFinalizerWatchDog()
35633 // This condition was historically used as part of the condition to detect finalizer thread timeouts
35634 return gc_heap::gc_lock.lock != -1;
35637 void GCHeap::SetFinalizeQueueForShutdown(bool fHasLock)
35639 #ifdef MULTIPLE_HEAPS
35640 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
35642 gc_heap* hp = gc_heap::g_heaps [hn];
35643 hp->finalize_queue->SetSegForShutDown(fHasLock);
35646 #else //MULTIPLE_HEAPS
35647 pGenGCHeap->finalize_queue->SetSegForShutDown(fHasLock);
35648 #endif //MULTIPLE_HEAPS
35651 //---------------------------------------------------------------------------
35652 // Finalized class tracking
35653 //---------------------------------------------------------------------------
35655 bool GCHeap::RegisterForFinalization (int gen, Object* obj)
35659 if (((((CObjectHeader*)obj)->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN))
35661 //just reset the bit
35662 ((CObjectHeader*)obj)->GetHeader()->ClrBit(BIT_SBLK_FINALIZER_RUN);
35667 gc_heap* hp = gc_heap::heap_of ((uint8_t*)obj);
35668 return hp->finalize_queue->RegisterForFinalization (gen, obj);
35672 void GCHeap::SetFinalizationRun (Object* obj)
35674 ((CObjectHeader*)obj)->GetHeader()->SetBit(BIT_SBLK_FINALIZER_RUN);
35678 //--------------------------------------------------------------------
35680 // Support for finalization
35682 //--------------------------------------------------------------------
35685 unsigned int gen_segment (int gen)
35687 assert (((signed)NUMBERGENERATIONS - gen - 1)>=0);
35688 return (NUMBERGENERATIONS - gen - 1);
35691 bool CFinalize::Initialize()
35698 m_Array = new (nothrow)(Object*[100]);
35703 STRESS_LOG_OOM_STACK(sizeof(Object*[100]));
35704 if (GCConfig::GetBreakOnOOM())
35706 GCToOSInterface::DebugBreak();
35710 m_EndArray = &m_Array[100];
35712 for (int i =0; i < FreeList; i++)
35714 SegQueueLimit (i) = m_Array;
35716 m_PromotedCount = 0;
35719 lockowner_threadid.Clear();
35725 CFinalize::~CFinalize()
35730 size_t CFinalize::GetPromotedCount ()
35732 return m_PromotedCount;
35736 void CFinalize::EnterFinalizeLock()
35738 _ASSERTE(dbgOnly_IsSpecialEEThread() ||
35739 GCToEEInterface::GetThread() == 0 ||
35740 GCToEEInterface::IsPreemptiveGCDisabled());
35743 if (Interlocked::CompareExchange(&lock, 0, -1) >= 0)
35745 unsigned int i = 0;
35748 YieldProcessor(); // indicate to the processor that we are spining
35750 GCToOSInterface::YieldThread (0);
35752 GCToOSInterface::Sleep (5);
35758 lockowner_threadid.SetToCurrentThread();
35763 void CFinalize::LeaveFinalizeLock()
35765 _ASSERTE(dbgOnly_IsSpecialEEThread() ||
35766 GCToEEInterface::GetThread() == 0 ||
35767 GCToEEInterface::IsPreemptiveGCDisabled());
35770 lockowner_threadid.Clear();
35776 CFinalize::RegisterForFinalization (int gen, Object* obj, size_t size)
35783 EnterFinalizeLock();
35785 unsigned int dest = 0;
35787 if (g_fFinalizerRunOnShutDown)
35789 //no method table available yet,
35790 //put it in the finalizer queue and sort out when
35792 dest = FinalizerListSeg;
35796 dest = gen_segment (gen);
35798 // Adjust boundary for segments so that GC will keep objects alive.
35799 Object*** s_i = &SegQueue (FreeList);
35800 if ((*s_i) == m_EndArray)
35804 LeaveFinalizeLock();
35805 if (method_table(obj) == NULL)
35807 // If the object is uninitialized, a valid size should have been passed.
35808 assert (size >= Align (min_obj_size));
35809 dprintf (3, ("Making unused array [%Ix, %Ix[", (size_t)obj, (size_t)(obj+size)));
35810 ((CObjectHeader*)obj)->SetFree(size);
35812 STRESS_LOG_OOM_STACK(0);
35813 if (GCConfig::GetBreakOnOOM())
35815 GCToOSInterface::DebugBreak();
35820 Object*** end_si = &SegQueueLimit (dest);
35823 //is the segment empty?
35824 if (!(*s_i == *(s_i-1)))
35826 //no, swap the end elements.
35827 *(*s_i) = *(*(s_i-1));
35829 //increment the fill pointer
35831 //go to the next segment.
35833 } while (s_i > end_si);
35835 // We have reached the destination segment
35836 // store the object
35838 // increment the fill pointer
35841 LeaveFinalizeLock();
35847 CFinalize::GetNextFinalizableObject (BOOL only_non_critical)
35851 EnterFinalizeLock();
35854 if (!IsSegEmpty(FinalizerListSeg))
35856 if (g_fFinalizerRunOnShutDown)
35858 obj = *(SegQueueLimit (FinalizerListSeg)-1);
35859 if (method_table(obj)->HasCriticalFinalizer())
35861 MoveItem ((SegQueueLimit (FinalizerListSeg)-1),
35862 FinalizerListSeg, CriticalFinalizerListSeg);
35866 --SegQueueLimit (FinalizerListSeg);
35869 obj = *(--SegQueueLimit (FinalizerListSeg));
35872 else if (!only_non_critical && !IsSegEmpty(CriticalFinalizerListSeg))
35874 //the FinalizerList is empty, we can adjust both
35875 // limit instead of moving the object to the free list
35876 obj = *(--SegQueueLimit (CriticalFinalizerListSeg));
35877 --SegQueueLimit (FinalizerListSeg);
35881 dprintf (3, ("running finalizer for %Ix (mt: %Ix)", obj, method_table (obj)));
35883 LeaveFinalizeLock();
35888 CFinalize::SetSegForShutDown(BOOL fHasLock)
35893 EnterFinalizeLock();
35894 for (i = 0; i <= max_generation; i++)
35896 unsigned int seg = gen_segment (i);
35897 Object** startIndex = SegQueueLimit (seg)-1;
35898 Object** stopIndex = SegQueue (seg);
35899 for (Object** po = startIndex; po >= stopIndex; po--)
35902 if (method_table(obj)->HasCriticalFinalizer())
35904 MoveItem (po, seg, CriticalFinalizerListSeg);
35908 MoveItem (po, seg, FinalizerListSeg);
35913 LeaveFinalizeLock();
35917 CFinalize::DiscardNonCriticalObjects()
35919 //empty the finalization queue
35920 Object** startIndex = SegQueueLimit (FinalizerListSeg)-1;
35921 Object** stopIndex = SegQueue (FinalizerListSeg);
35922 for (Object** po = startIndex; po >= stopIndex; po--)
35924 MoveItem (po, FinalizerListSeg, FreeList);
35929 CFinalize::GetNumberFinalizableObjects()
35931 return SegQueueLimit (FinalizerListSeg) -
35932 (g_fFinalizerRunOnShutDown ? m_Array : SegQueue(FinalizerListSeg));
35936 CFinalize::FinalizeSegForAppDomain (void *pDomain,
35937 BOOL fRunFinalizers,
35940 BOOL finalizedFound = FALSE;
35941 Object** endIndex = SegQueue (Seg);
35942 for (Object** i = SegQueueLimit (Seg)-1; i >= endIndex ;i--)
35944 CObjectHeader* obj = (CObjectHeader*)*i;
35946 // Objects are put into the finalization queue before they are complete (ie their methodtable
35947 // may be null) so we must check that the object we found has a method table before checking
35948 // if it has the index we are looking for. If the methodtable is null, it can't be from the
35949 // unloading domain, so skip it.
35950 if (method_table(obj) == NULL)
35955 // does the EE actually want us to finalize this object?
35956 if (!GCToEEInterface::ShouldFinalizeObjectForUnload(pDomain, obj))
35961 if (!fRunFinalizers || (obj->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN)
35963 //remove the object because we don't want to
35964 //run the finalizer
35965 MoveItem (i, Seg, FreeList);
35966 //Reset the bit so it will be put back on the queue
35967 //if resurrected and re-registered.
35968 obj->GetHeader()->ClrBit (BIT_SBLK_FINALIZER_RUN);
35972 if (method_table(obj)->HasCriticalFinalizer())
35974 finalizedFound = TRUE;
35975 MoveItem (i, Seg, CriticalFinalizerListSeg);
35979 if (GCToEEInterface::AppDomainIsRudeUnload(pDomain))
35981 MoveItem (i, Seg, FreeList);
35985 finalizedFound = TRUE;
35986 MoveItem (i, Seg, FinalizerListSeg);
35992 return finalizedFound;
35996 CFinalize::FinalizeAppDomain (void *pDomain, bool fRunFinalizers)
35998 bool finalizedFound = false;
36000 unsigned int startSeg = gen_segment (max_generation);
36002 EnterFinalizeLock();
36004 for (unsigned int Seg = startSeg; Seg <= gen_segment (0); Seg++)
36006 if (FinalizeSegForAppDomain (pDomain, fRunFinalizers, Seg))
36008 finalizedFound = true;
36012 LeaveFinalizeLock();
36014 return finalizedFound;
36018 CFinalize::MoveItem (Object** fromIndex,
36019 unsigned int fromSeg,
36020 unsigned int toSeg)
36024 ASSERT (fromSeg != toSeg);
36025 if (fromSeg > toSeg)
36029 // Place the element at the boundary closest to dest
36030 Object** srcIndex = fromIndex;
36031 for (unsigned int i = fromSeg; i != toSeg; i+= step)
36033 Object**& destFill = m_FillPointers[i+(step - 1 )/2];
36034 Object** destIndex = destFill - (step + 1)/2;
36035 if (srcIndex != destIndex)
36037 Object* tmp = *srcIndex;
36038 *srcIndex = *destIndex;
36042 srcIndex = destIndex;
36047 CFinalize::GcScanRoots (promote_func* fn, int hn, ScanContext *pSC)
36053 pSC->thread_number = hn;
36055 //scan the finalization queue
36056 Object** startIndex = SegQueue (CriticalFinalizerListSeg);
36057 Object** stopIndex = SegQueueLimit (FinalizerListSeg);
36059 for (Object** po = startIndex; po < stopIndex; po++)
36062 //dprintf (3, ("scan freacheable %Ix", (size_t)o));
36063 dprintf (3, ("scan f %Ix", (size_t)o));
36064 #ifdef FEATURE_APPDOMAIN_RESOURCE_MONITORING
36067 pSC->pCurrentDomain = SystemDomain::GetAppDomainAtIndex(o->GetAppDomainIndex());
36069 #endif //FEATURE_APPDOMAIN_RESOURCE_MONITORING
36075 void CFinalize::WalkFReachableObjects (fq_walk_fn fn)
36077 Object** startIndex = SegQueue (CriticalFinalizerListSeg);
36078 Object** stopCriticalIndex = SegQueueLimit (CriticalFinalizerListSeg);
36079 Object** stopIndex = SegQueueLimit (FinalizerListSeg);
36080 for (Object** po = startIndex; po < stopIndex; po++)
36083 fn(po < stopCriticalIndex, *po);
36088 CFinalize::ScanForFinalization (promote_func* pfn, int gen, BOOL mark_only_p,
36092 sc.promotion = TRUE;
36093 #ifdef MULTIPLE_HEAPS
36094 sc.thread_number = hp->heap_number;
36096 UNREFERENCED_PARAMETER(hp);
36097 #endif //MULTIPLE_HEAPS
36099 BOOL finalizedFound = FALSE;
36101 //start with gen and explore all the younger generations.
36102 unsigned int startSeg = gen_segment (gen);
36104 m_PromotedCount = 0;
36105 for (unsigned int Seg = startSeg; Seg <= gen_segment(0); Seg++)
36107 Object** endIndex = SegQueue (Seg);
36108 for (Object** i = SegQueueLimit (Seg)-1; i >= endIndex ;i--)
36110 CObjectHeader* obj = (CObjectHeader*)*i;
36111 dprintf (3, ("scanning: %Ix", (size_t)obj));
36112 if (!g_theGCHeap->IsPromoted (obj))
36114 dprintf (3, ("freacheable: %Ix", (size_t)obj));
36116 assert (method_table(obj)->HasFinalizer());
36118 if (GCToEEInterface::EagerFinalized(obj))
36120 MoveItem (i, Seg, FreeList);
36122 else if ((obj->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN)
36124 //remove the object because we don't want to
36125 //run the finalizer
36127 MoveItem (i, Seg, FreeList);
36129 //Reset the bit so it will be put back on the queue
36130 //if resurrected and re-registered.
36131 obj->GetHeader()->ClrBit (BIT_SBLK_FINALIZER_RUN);
36138 if (method_table(obj)->HasCriticalFinalizer())
36140 MoveItem (i, Seg, CriticalFinalizerListSeg);
36144 MoveItem (i, Seg, FinalizerListSeg);
36148 #ifdef BACKGROUND_GC
36151 if ((gen == max_generation) && (recursive_gc_sync::background_running_p()))
36153 // TODO - fix the following line.
36154 //assert (gc_heap::background_object_marked ((uint8_t*)obj, FALSE));
36155 dprintf (3, ("%Ix is marked", (size_t)obj));
36158 #endif //BACKGROUND_GC
36162 finalizedFound = !IsSegEmpty(FinalizerListSeg) ||
36163 !IsSegEmpty(CriticalFinalizerListSeg);
36165 if (finalizedFound)
36167 //Promote the f-reachable objects
36169 #ifdef MULTIPLE_HEAPS
36173 #endif //MULTIPLE_HEAPS
36176 hp->settings.found_finalizers = TRUE;
36178 #ifdef BACKGROUND_GC
36179 if (hp->settings.concurrent)
36181 hp->settings.found_finalizers = !(IsSegEmpty(FinalizerListSeg) && IsSegEmpty(CriticalFinalizerListSeg));
36183 #endif //BACKGROUND_GC
36184 if (hp->settings.concurrent && hp->settings.found_finalizers)
36187 GCToEEInterface::EnableFinalization(true);
36191 return finalizedFound;
36194 //Relocates all of the objects in the finalization array
36196 CFinalize::RelocateFinalizationData (int gen, gc_heap* hp)
36199 sc.promotion = FALSE;
36200 #ifdef MULTIPLE_HEAPS
36201 sc.thread_number = hp->heap_number;
36203 UNREFERENCED_PARAMETER(hp);
36204 #endif //MULTIPLE_HEAPS
36206 unsigned int Seg = gen_segment (gen);
36208 Object** startIndex = SegQueue (Seg);
36209 for (Object** po = startIndex; po < SegQueue (FreeList);po++)
36211 GCHeap::Relocate (po, &sc);
36216 CFinalize::UpdatePromotedGenerations (int gen, BOOL gen_0_empty_p)
36218 // update the generation fill pointers.
36219 // if gen_0_empty is FALSE, test each object to find out if
36220 // it was promoted or not
36223 for (int i = min (gen+1, max_generation); i > 0; i--)
36225 m_FillPointers [gen_segment(i)] = m_FillPointers [gen_segment(i-1)];
36230 //Look for demoted or promoted plugs
36232 for (int i = gen; i >= 0; i--)
36234 unsigned int Seg = gen_segment (i);
36235 Object** startIndex = SegQueue (Seg);
36237 for (Object** po = startIndex;
36238 po < SegQueueLimit (gen_segment(i)); po++)
36240 int new_gen = g_theGCHeap->WhichGeneration (*po);
36246 MoveItem (po, gen_segment (i), gen_segment (new_gen));
36251 MoveItem (po, gen_segment (i), gen_segment (new_gen));
36252 //back down in order to see all objects.
36263 CFinalize::GrowArray()
36265 size_t oldArraySize = (m_EndArray - m_Array);
36266 size_t newArraySize = (size_t)(((float)oldArraySize / 10) * 12);
36268 Object** newArray = new (nothrow) Object*[newArraySize];
36271 // It's not safe to throw here, because of the FinalizeLock. Tell our caller
36272 // to throw for us.
36273 // ASSERT (newArray);
36276 memcpy (newArray, m_Array, oldArraySize*sizeof(Object*));
36278 //adjust the fill pointers
36279 for (int i = 0; i < FreeList; i++)
36281 m_FillPointers [i] += (newArray - m_Array);
36284 m_Array = newArray;
36285 m_EndArray = &m_Array [newArraySize];
36291 void CFinalize::CheckFinalizerObjects()
36293 for (int i = 0; i <= max_generation; i++)
36295 Object **startIndex = SegQueue (gen_segment (i));
36296 Object **stopIndex = SegQueueLimit (gen_segment (i));
36298 for (Object **po = startIndex; po < stopIndex; po++)
36300 if ((int)g_theGCHeap->WhichGeneration (*po) < i)
36302 ((CObjectHeader*)*po)->Validate();
36306 #endif //VERIFY_HEAP
36308 #endif // FEATURE_PREMORTEM_FINALIZATION
36311 //------------------------------------------------------------------------------
36313 // End of VM specific support
36315 //------------------------------------------------------------------------------
36316 void gc_heap::walk_heap_per_heap (walk_fn fn, void* context, int gen_number, BOOL walk_large_object_heap_p)
36318 generation* gen = gc_heap::generation_of (gen_number);
36319 heap_segment* seg = generation_start_segment (gen);
36320 uint8_t* x = ((gen_number == max_generation) ? heap_segment_mem (seg) :
36321 generation_allocation_start (gen));
36323 uint8_t* end = heap_segment_allocated (seg);
36324 BOOL small_object_segments = TRUE;
36325 int align_const = get_alignment_constant (small_object_segments);
36332 if ((seg = heap_segment_next (seg)) != 0)
36334 x = heap_segment_mem (seg);
36335 end = heap_segment_allocated (seg);
36340 if (small_object_segments && walk_large_object_heap_p)
36343 small_object_segments = FALSE;
36344 align_const = get_alignment_constant (small_object_segments);
36345 seg = generation_start_segment (large_object_generation);
36346 x = heap_segment_mem (seg);
36347 end = heap_segment_allocated (seg);
36357 size_t s = size (x);
36358 CObjectHeader* o = (CObjectHeader*)x;
36363 _ASSERTE(((size_t)o & 0x3) == 0); // Last two bits should never be set at this point
36365 if (!fn (o->GetObjectBase(), context))
36368 x = x + Align (s, align_const);
36372 void gc_heap::walk_finalize_queue (fq_walk_fn fn)
36374 #ifdef FEATURE_PREMORTEM_FINALIZATION
36375 finalize_queue->WalkFReachableObjects (fn);
36376 #endif //FEATURE_PREMORTEM_FINALIZATION
36379 void gc_heap::walk_heap (walk_fn fn, void* context, int gen_number, BOOL walk_large_object_heap_p)
36381 #ifdef MULTIPLE_HEAPS
36382 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36384 gc_heap* hp = gc_heap::g_heaps [hn];
36386 hp->walk_heap_per_heap (fn, context, gen_number, walk_large_object_heap_p);
36389 walk_heap_per_heap(fn, context, gen_number, walk_large_object_heap_p);
36390 #endif //MULTIPLE_HEAPS
36393 void GCHeap::DiagWalkObject (Object* obj, walk_fn fn, void* context)
36395 uint8_t* o = (uint8_t*)obj;
36398 go_through_object_cl (method_table (o), o, size(o), oo,
36402 Object *oh = (Object*)*oo;
36403 if (!fn (oh, context))
36411 void GCHeap::DiagWalkSurvivorsWithType (void* gc_context, record_surv_fn fn, void* diag_context, walk_surv_type type)
36413 gc_heap* hp = (gc_heap*)gc_context;
36414 hp->walk_survivors (fn, diag_context, type);
36417 void GCHeap::DiagWalkHeap (walk_fn fn, void* context, int gen_number, bool walk_large_object_heap_p)
36419 gc_heap::walk_heap (fn, context, gen_number, walk_large_object_heap_p);
36422 void GCHeap::DiagWalkFinalizeQueue (void* gc_context, fq_walk_fn fn)
36424 gc_heap* hp = (gc_heap*)gc_context;
36425 hp->walk_finalize_queue (fn);
36428 void GCHeap::DiagScanFinalizeQueue (fq_scan_fn fn, ScanContext* sc)
36430 #ifdef MULTIPLE_HEAPS
36431 for (int hn = 0; hn < gc_heap::n_heaps; hn++)
36433 gc_heap* hp = gc_heap::g_heaps [hn];
36434 hp->finalize_queue->GcScanRoots(fn, hn, sc);
36437 pGenGCHeap->finalize_queue->GcScanRoots(fn, 0, sc);
36438 #endif //MULTIPLE_HEAPS
36441 void GCHeap::DiagScanHandles (handle_scan_fn fn, int gen_number, ScanContext* context)
36443 UNREFERENCED_PARAMETER(gen_number);
36444 GCScan::GcScanHandlesForProfilerAndETW (max_generation, context, fn);
36447 void GCHeap::DiagScanDependentHandles (handle_scan_fn fn, int gen_number, ScanContext* context)
36449 UNREFERENCED_PARAMETER(gen_number);
36450 GCScan::GcScanDependentHandlesForProfilerAndETW (max_generation, context, fn);
36453 // Go through and touch (read) each page straddled by a memory block.
36454 void TouchPages(void * pStart, size_t cb)
36456 const uint32_t pagesize = OS_PAGE_SIZE;
36457 _ASSERTE(0 == (pagesize & (pagesize-1))); // Must be a power of 2.
36460 VOLATILE(char)* pEnd = (VOLATILE(char)*)(cb + (char*)pStart);
36461 VOLATILE(char)* p = (VOLATILE(char)*)(((char*)pStart) - (((size_t)pStart) & (pagesize-1)));
36465 a = VolatileLoad(p);
36466 //printf("Touching page %lxh\n", (uint32_t)p);
36472 #if defined(WRITE_BARRIER_CHECK) && !defined (SERVER_GC)
36473 // This code is designed to catch the failure to update the write barrier
36474 // The way it works is to copy the whole heap right after every GC. The write
36475 // barrier code has been modified so that it updates the shadow as well as the
36476 // real GC heap. Before doing the next GC, we walk the heap, looking for pointers
36477 // that were updated in the real heap, but not the shadow. A mismatch indicates
36478 // an error. The offending code can be found by breaking after the correct GC,
36479 // and then placing a data breakpoint on the Heap location that was updated without
36480 // going through the write barrier.
36482 // Called at process shutdown
36483 void deleteGCShadow()
36485 if (g_GCShadow != 0)
36486 GCToOSInterface::VirtualRelease (g_GCShadow, g_GCShadowEnd - g_GCShadow);
36491 // Called at startup and right after a GC, get a snapshot of the GC Heap
36492 void initGCShadow()
36494 if (!(GCConfig::GetHeapVerifyLevel() & GCConfig::HEAPVERIFY_BARRIERCHECK))
36497 size_t len = g_gc_highest_address - g_gc_lowest_address;
36498 if (len > (size_t)(g_GCShadowEnd - g_GCShadow))
36501 g_GCShadowEnd = g_GCShadow = (uint8_t *)GCToOSInterface::VirtualReserve(len, 0, VirtualReserveFlags::None);
36502 if (g_GCShadow == NULL || !GCToOSInterface::VirtualCommit(g_GCShadow, len))
36504 _ASSERTE(!"Not enough memory to run HeapVerify level 2");
36505 // If after the assert we decide to allow the program to continue
36506 // running we need to be in a state that will not trigger any
36507 // additional AVs while we fail to allocate a shadow segment, i.e.
36508 // ensure calls to updateGCShadow() checkGCWriteBarrier() don't AV
36513 g_GCShadowEnd += len;
36516 // save the value of g_gc_lowest_address at this time. If this value changes before
36517 // the next call to checkGCWriteBarrier() it means we extended the heap (with a
36518 // large object segment most probably), and the whole shadow segment is inconsistent.
36519 g_shadow_lowest_address = g_gc_lowest_address;
36521 //****** Copy the whole GC heap ******
36523 // NOTE: This is the one situation where the combination of heap_segment_rw(gen_start_segment())
36524 // can produce a NULL result. This is because the initialization has not completed.
36526 generation* gen = gc_heap::generation_of (max_generation);
36527 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
36529 ptrdiff_t delta = g_GCShadow - g_gc_lowest_address;
36530 BOOL small_object_segments = TRUE;
36535 if (small_object_segments)
36537 small_object_segments = FALSE;
36538 seg = heap_segment_rw (generation_start_segment (gc_heap::generation_of (max_generation+1)));
36544 // Copy the segment
36545 uint8_t* start = heap_segment_mem(seg);
36546 uint8_t* end = heap_segment_allocated (seg);
36547 memcpy(start + delta, start, end - start);
36548 seg = heap_segment_next_rw (seg);
36552 #define INVALIDGCVALUE (void*)((size_t)0xcccccccd)
36554 // test to see if 'ptr' was only updated via the write barrier.
36555 inline void testGCShadow(Object** ptr)
36557 Object** shadow = (Object**) &g_GCShadow[((uint8_t*) ptr - g_gc_lowest_address)];
36558 if (*ptr != 0 && (uint8_t*) shadow < g_GCShadowEnd && *ptr != *shadow)
36561 // If you get this assertion, someone updated a GC pointer in the heap without
36562 // using the write barrier. To find out who, check the value of
36563 // dd_collection_count (dynamic_data_of (0)). Also
36564 // note the value of 'ptr'. Rerun the App that the previous GC just occurred.
36565 // Then put a data breakpoint for the value of 'ptr' Then check every write
36566 // to pointer between the two GCs. The last one is not using the write barrier.
36568 // If the memory of interest does not exist at system startup,
36569 // you need to set the data breakpoint right after the memory gets committed
36570 // Set a breakpoint at the end of grow_heap_segment, and put the value of 'ptr'
36571 // in the memory window. run until the memory gets mapped. Then you can set
36574 // Note a recent change, we've identified race conditions when updating the gc shadow.
36575 // Throughout the runtime, code will update an address in the gc heap, then erect the
36576 // write barrier, which calls updateGCShadow. With an app that pounds one heap location
36577 // from multiple threads, you can hit this assert even though all involved are using the
36578 // write barrier properly. Thusly, we detect the race and set this location to INVALIDGCVALUE.
36579 // TODO: the code in jithelp.asm doesn't call updateGCShadow, and hasn't been
36580 // TODO: fixed to detect the race. We've only seen this race from VolatileWritePtr,
36581 // TODO: so elect not to fix jithelp.asm at this time. It should be done if we start hitting
36582 // TODO: erroneous asserts in here.
36584 if(*shadow!=INVALIDGCVALUE)
36586 #ifdef FEATURE_BASICFREEZE
36587 // Write barriers for stores of references to frozen objects may be optimized away.
36588 if (!gc_heap::frozen_object_p(*ptr))
36589 #endif // FEATURE_BASICFREEZE
36591 _ASSERTE(!"Pointer updated without using write barrier");
36597 printf("saw a INVALIDGCVALUE. (just to let you know)\n");
36603 void testGCShadowHelper (uint8_t* x)
36605 size_t s = size (x);
36606 if (contain_pointers (x))
36608 go_through_object_nostart (method_table(x), x, s, oo,
36609 { testGCShadow((Object**) oo); });
36613 // Walk the whole heap, looking for pointers that were not updated with the write barrier.
36614 void checkGCWriteBarrier()
36616 // g_shadow_lowest_address != g_gc_lowest_address means the GC heap was extended by a segment
36617 // and the GC shadow segment did not track that change!
36618 if (g_GCShadowEnd <= g_GCShadow || g_shadow_lowest_address != g_gc_lowest_address)
36620 // No shadow stack, nothing to check.
36625 generation* gen = gc_heap::generation_of (max_generation);
36626 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
36628 PREFIX_ASSUME(seg != NULL);
36632 uint8_t* x = heap_segment_mem(seg);
36633 while (x < heap_segment_allocated (seg))
36635 size_t s = size (x);
36636 testGCShadowHelper (x);
36639 seg = heap_segment_next_rw (seg);
36644 // go through large object heap
36645 int alignment = get_alignment_constant(FALSE);
36646 generation* gen = gc_heap::generation_of (max_generation+1);
36647 heap_segment* seg = heap_segment_rw (generation_start_segment (gen));
36649 PREFIX_ASSUME(seg != NULL);
36653 uint8_t* x = heap_segment_mem(seg);
36654 while (x < heap_segment_allocated (seg))
36656 size_t s = size (x);
36657 testGCShadowHelper (x);
36658 x = x + Align (s, alignment);
36660 seg = heap_segment_next_rw (seg);
36664 #endif //WRITE_BARRIER_CHECK && !SERVER_GC
36666 #endif // !DACCESS_COMPILE
36668 #ifdef FEATURE_BASICFREEZE
36669 void gc_heap::walk_read_only_segment(heap_segment *seg, void *pvContext, object_callback_func pfnMethodTable, object_callback_func pfnObjRef)
36671 #ifdef DACCESS_COMPILE
36672 UNREFERENCED_PARAMETER(seg);
36673 UNREFERENCED_PARAMETER(pvContext);
36674 UNREFERENCED_PARAMETER(pfnMethodTable);
36675 UNREFERENCED_PARAMETER(pfnObjRef);
36677 uint8_t *o = heap_segment_mem(seg);
36679 // small heap alignment constant
36680 int alignment = get_alignment_constant(TRUE);
36682 while (o < heap_segment_allocated(seg))
36684 pfnMethodTable(pvContext, o);
36686 if (contain_pointers (o))
36688 go_through_object_nostart (method_table (o), o, size(o), oo,
36691 pfnObjRef(pvContext, oo);
36696 o += Align(size(o), alignment);
36698 #endif //!DACCESS_COMPILE
36700 #endif // FEATURE_BASICFREEZE
36702 #ifndef DACCESS_COMPILE
36703 HRESULT GCHeap::WaitUntilConcurrentGCCompleteAsync(int millisecondsTimeout)
36705 #ifdef BACKGROUND_GC
36706 if (recursive_gc_sync::background_running_p())
36708 uint32_t dwRet = pGenGCHeap->background_gc_wait(awr_ignored, millisecondsTimeout);
36709 if (dwRet == WAIT_OBJECT_0)
36711 else if (dwRet == WAIT_TIMEOUT)
36712 return HRESULT_FROM_WIN32(ERROR_TIMEOUT);
36714 return E_FAIL; // It is not clear if what the last error would be if the wait failed,
36715 // as there are too many layers in between. The best we can do is to return E_FAIL;
36721 #endif // !DACCESS_COMPILE
36723 void GCHeap::TemporaryEnableConcurrentGC()
36725 #ifdef BACKGROUND_GC
36726 gc_heap::temp_disable_concurrent_p = false;
36727 #endif //BACKGROUND_GC
36730 void GCHeap::TemporaryDisableConcurrentGC()
36732 #ifdef BACKGROUND_GC
36733 gc_heap::temp_disable_concurrent_p = true;
36734 #endif //BACKGROUND_GC
36737 bool GCHeap::IsConcurrentGCEnabled()
36739 #ifdef BACKGROUND_GC
36740 return (gc_heap::gc_can_use_concurrent && !(gc_heap::temp_disable_concurrent_p));
36743 #endif //BACKGROUND_GC
36746 void GCHeap::SetFinalizeRunOnShutdown(bool value)
36748 g_fFinalizerRunOnShutDown = value;
36751 void PopulateDacVars(GcDacVars *gcDacVars)
36753 #ifndef DACCESS_COMPILE
36754 assert(gcDacVars != nullptr);
36756 gcDacVars->major_version_number = 1;
36757 gcDacVars->minor_version_number = 0;
36758 gcDacVars->built_with_svr = &g_built_with_svr_gc;
36759 gcDacVars->build_variant = &g_build_variant;
36760 gcDacVars->gc_structures_invalid_cnt = const_cast<int32_t*>(&GCScan::m_GcStructuresInvalidCnt);
36761 gcDacVars->generation_size = sizeof(generation);
36762 gcDacVars->max_gen = &g_max_generation;
36763 #ifndef MULTIPLE_HEAPS
36764 gcDacVars->mark_array = &gc_heap::mark_array;
36765 gcDacVars->ephemeral_heap_segment = reinterpret_cast<dac_heap_segment**>(&gc_heap::ephemeral_heap_segment);
36766 gcDacVars->current_c_gc_state = const_cast<c_gc_state*>(&gc_heap::current_c_gc_state);
36767 gcDacVars->saved_sweep_ephemeral_seg = reinterpret_cast<dac_heap_segment**>(&gc_heap::saved_sweep_ephemeral_seg);
36768 gcDacVars->saved_sweep_ephemeral_start = &gc_heap::saved_sweep_ephemeral_start;
36769 gcDacVars->background_saved_lowest_address = &gc_heap::background_saved_lowest_address;
36770 gcDacVars->background_saved_highest_address = &gc_heap::background_saved_highest_address;
36771 gcDacVars->alloc_allocated = &gc_heap::alloc_allocated;
36772 gcDacVars->next_sweep_obj = &gc_heap::next_sweep_obj;
36773 gcDacVars->oom_info = &gc_heap::oom_info;
36774 gcDacVars->finalize_queue = reinterpret_cast<dac_finalize_queue**>(&gc_heap::finalize_queue);
36775 gcDacVars->generation_table = reinterpret_cast<dac_generation**>(&gc_heap::generation_table);
36776 #ifdef GC_CONFIG_DRIVEN
36777 gcDacVars->gc_global_mechanisms = reinterpret_cast<size_t**>(&gc_global_mechanisms);
36778 gcDacVars->interesting_data_per_heap = reinterpret_cast<size_t**>(&gc_heap::interesting_data_per_heap);
36779 gcDacVars->compact_reasons_per_heap = reinterpret_cast<size_t**>(&gc_heap::compact_reasons_per_heap);
36780 gcDacVars->expand_mechanisms_per_heap = reinterpret_cast<size_t**>(&gc_heap::expand_mechanisms_per_heap);
36781 gcDacVars->interesting_mechanism_bits_per_heap = reinterpret_cast<size_t**>(&gc_heap::interesting_mechanism_bits_per_heap);
36782 #endif // GC_CONFIG_DRIVEN
36783 #ifdef HEAP_ANALYZE
36784 gcDacVars->internal_root_array = &gc_heap::internal_root_array;
36785 gcDacVars->internal_root_array_index = &gc_heap::internal_root_array_index;
36786 gcDacVars->heap_analyze_success = &gc_heap::heap_analyze_success;
36787 #endif // HEAP_ANALYZE
36789 gcDacVars->n_heaps = &gc_heap::n_heaps;
36790 gcDacVars->g_heaps = reinterpret_cast<dac_gc_heap***>(&gc_heap::g_heaps);
36791 #endif // MULTIPLE_HEAPS
36792 #endif // DACCESS_COMPILE